This morning I saw this tweet from Mr Brendan Burns:

And I’m sure that once you also read through it, you’ll learn that you have to take several steps in order to achieve AKS Cost Monitoring and Governance With Kubecost.

I’m going to try and save you some time, providing you with a basic terraform configuration to help you get up and running in a breeze.

If you want to learn more about Kubecost in the context of AKS and Azure please read the Cost Governance section of the AKS enterprise-scale platform security governance and compliance guidelines.

Deploy AKS and Kubecost with Terraform#

1. Create a provider.tf file with the following contents:#

 1terraform {
 2  required_version = "> 0.14"
 3  required_providers {
 4    azurerm = {
 5      version = "= 2.57.0"
 6    }
 7    azuread = {
 8      version = "= 1.4.0"
 9    }
10    kubernetes = {
11      version = "= 2.1.0"
12    }
13    helm = {
14      version = "= 2.1.2"
15    }
16  }
17}
18
19provider "azurerm" {
20  features {}
21}
22
23# Configuring the kubernetes provider
24# AKS resource name is aks: azurerm_kubernetes_cluster.aks
25provider "kubernetes" {
26  host                   = azurerm_kubernetes_cluster.aks.kube_config.0.host
27  client_certificate     = base64decode(azurerm_kubernetes_cluster.aks.kube_config.0.client_certificate)
28  client_key             = base64decode(azurerm_kubernetes_cluster.aks.kube_config.0.client_key)
29  cluster_ca_certificate = base64decode(azurerm_kubernetes_cluster.aks.kube_config.0.cluster_ca_certificate)
30}
31
32# Configuring the helm provider
33# AKS resource name is aks: azurerm_kubernetes_cluster.aks
34provider "helm" {
35  kubernetes {
36    host                   = azurerm_kubernetes_cluster.aks.kube_config.0.host
37    client_certificate     = base64decode(azurerm_kubernetes_cluster.aks.kube_config.0.client_certificate)
38    client_key             = base64decode(azurerm_kubernetes_cluster.aks.kube_config.0.client_key)
39    cluster_ca_certificate = base64decode(azurerm_kubernetes_cluster.aks.kube_config.0.cluster_ca_certificate)
40  }
41}

Note that you’ll be using azurerm to deploy Azure services, azuread to create a Service Principal information and the kubernetes and helm provider to install Kubecost.

2. Create a variables.tf file with the following contents:#

 1# Location of the services
 2variable "location" {
 3  default = "west europe"
 4}
 5
 6# Resource Group Name
 7variable "resource_group" {
 8  default = "aks-kubecost"
 9}
10
11# Name of the AKS cluster
12variable "aks_name" {
13  default = "aksmsftkubecost"
14}
15
16# Name of the Servic Principal used by Kubecost
17variable "kubecost_sp_name" {
18  default = "kubecost"
19}

Note: Replace the the default values with your desired location and names.

3. Create a main.tf file with the following contents:#

  1# Create Resource Group
  2resource "azurerm_resource_group" "rg" {
  3  name     = var.resource_group
  4  location = var.location
  5}
  6
  7# Create VNET for AKS
  8resource "azurerm_virtual_network" "vnet" {
  9  name                = "private-network"
 10  address_space       = ["10.0.0.0/8"]
 11  location            = azurerm_resource_group.rg.location
 12  resource_group_name = azurerm_resource_group.rg.name
 13}
 14
 15# Create the Subnet for AKS.
 16resource "azurerm_subnet" "aks" {
 17  name                 = "aks"
 18  resource_group_name  = azurerm_resource_group.rg.name
 19  virtual_network_name = azurerm_virtual_network.vnet.name
 20  address_prefixes     = ["10.240.0.0/16"]
 21}
 22
 23# Create the AKS cluster.
 24# Cause this is a test node_count is set to 1 
 25resource "azurerm_kubernetes_cluster" "aks" {
 26  name                = var.aks_name
 27  location            = azurerm_resource_group.rg.location
 28  resource_group_name = azurerm_resource_group.rg.name
 29  dns_prefix          = var.aks_name
 30
 31  default_node_pool {
 32    name            = "default"
 33    node_count      = 1
 34    vm_size         = "Standard_D2s_v3"
 35    os_disk_size_gb = 30
 36    os_disk_type    = "Ephemeral"
 37    vnet_subnet_id  = azurerm_subnet.aks.id
 38  }
 39
 40  # Using Managed Identity
 41  identity {
 42    type = "SystemAssigned"
 43  }
 44
 45  network_profile {
 46    network_plugin = "kubenet"
 47    network_policy = "calico"
 48  }
 49
 50  role_based_access_control {
 51    enabled = true
 52  }
 53
 54  addon_profile {
 55    kube_dashboard {
 56      enabled = false
 57    }
 58  }
 59}
 60
 61# Create Application registration for Kubecost
 62resource "azuread_application" "kubecost" {
 63  display_name               = var.kubecost_sp_name
 64  identifier_uris            = ["http://${var.kubecost_sp_name}"]
 65}
 66
 67# Create Service principal for kubecost
 68resource "azuread_service_principal" "kubecost" {
 69  application_id = azuread_application.kubecost.application_id
 70}
 71
 72# Generate password for the Service Principal
 73resource "random_password" "passwd" {
 74  length      = 32
 75  min_upper   = 4
 76  min_lower   = 2
 77  min_numeric = 4
 78   keepers = {
 79    aks_app_id = azuread_application.kubecost.id
 80  }
 81}
 82
 83# Create kubecost's Service principal password
 84resource "azuread_service_principal_password" "main" {
 85  service_principal_id = azuread_service_principal.kubecost.id
 86  value                = random_password.passwd.result
 87  end_date             = "2099-01-01T00:00:00Z"
 88}
 89
 90# Get current Subscription
 91data "azurerm_subscription" "current" {
 92}
 93
 94# Create kubecost custom role
 95resource "azurerm_role_definition" "kubecost" {
 96  name        = "kubecost_rate_card_query"
 97  scope       = data.azurerm_subscription.current.id
 98  description = "kubecost Rate Card query role"
 99
100  permissions {
101    actions     = [
102      "Microsoft.Compute/virtualMachines/vmSizes/read",
103      "Microsoft.Resources/subscriptions/locations/read",
104      "Microsoft.Resources/providers/read",
105      "Microsoft.ContainerService/containerServices/read",
106      "Microsoft.Commerce/RateCard/read",
107    ]
108    not_actions = []
109  }
110
111  assignable_scopes = [
112    data.azurerm_subscription.current.id
113  ]
114}
115
116# Assign kubecost's custom role at the subscription level
117resource "azurerm_role_assignment" "kubecost" {
118  scope                = data.azurerm_subscription.current.id
119  role_definition_name = azurerm_role_definition.kubecost.name
120  principal_id         = azuread_service_principal.kubecost.object_id
121}

4. Create a kubecost.tf file with the following contents:#

 1# Create the kubecost namespace
 2resource "kubernetes_namespace" "kubecost" {
 3  metadata {
 4    name = "kubecost"
 5  }
 6}
 7
 8# Install kubecost using the hem chart
 9resource "helm_release" "kubecost" {
10  name       = "kubecost"
11  chart      = "cost-analyzer"
12  namespace  = "kubecost"
13  version    = "1.79.1"
14  repository = "https://kubecost.github.io/cost-analyzer/"
15
16  # Set the cluster name
17  set {
18    name  = "kubecostProductConfigs.clusterName"
19    value = var.aks_name
20  }
21
22  # Set the currency
23  set {
24    name  = "kubecostProductConfigs.currencyCode"
25    value = "EUR"
26  }
27
28  # Set the region
29  set {
30    name  = "kubecostProductConfigs.azureBillingRegion"
31    value = "NL"
32  }
33  
34  # Generate a secret based on the Azure configuration provided below
35  set {
36    name  = "kubecostProductConfigs.createServiceKeySecret"
37    value = true
38  }
39
40  # Azure Subscription ID
41  set {
42    name  = "kubecostProductConfigs.azureSubscriptionID"
43    value = data.azurerm_subscription.current.id
44  }
45
46  # Azure Client ID
47  set {
48    name  = "kubecostProductConfigs.azureClientID"
49    value = azuread_application.kubecost.application_id
50  }
51
52  # Azure Client Password
53  set {
54    name  = "kubecostProductConfigs.azureClientPassword"
55    value = random_password.passwd.result
56  }
57
58  # Azure Tenant ID
59  set {
60    name  = "kubecostProductConfigs.azureTenantID"
61    value = data.azurerm_subscription.current.tenant_id
62  }
63}

The configuration in the previous file installs Kubecost in the AKS cluster. If you want to learn more about the available configuration options please check the following file: values.yaml

5. Deploy the solution:#

Run the following commands:

1terraform init
2terraform plan -out tf.plan
3terraform apply ./tf.plan

6. Test and browse Kubecost:#

To check the status of the kubecost pods run:

1az aks get-credentials -g aks-kubecost -n aksmsftkubecost
2kubectl get pods -n kubecost

Then run:

1kubectl port-forward -n kubecost svc/kubecost-cost-analyzer 9090:9090

and browse to http://localhost:9090 so you can start learning!

Hope it helps! Please find the complete code here