Spinnaker – Externalising kubeconfig files for Kubernetes accounts

#!/bin/bash
#This script is executed in Clouddriver or its Sidecard container to download the kubeconfig files

#Make sure vault cli is loaded on the Container and is accessible by the scripts
#Ensure the VAULT_ADDR and VAULT_TOKEN is set in the SHELL environment

#Set these variables to your choice
export VK8S_PATH='spinnaker/k8configs' #Prefix V for Vault
export LK8S_PATH='/tmp/k8configs' #Prefix L for Local
export R_FILE=/tmp/remote.txt
export L_FILE=/tmp/local.txt
export NEWKEYS=/tmp/newkeys.txt

#Make sure vault CLI is available
command -v vault > /dev/null 2>&1
if [ $? -ne 0 ]; then
  echo "Vault CLI is not available, exiting..."
  exit
fi

#Create local config path if not exists
[ ! -d $LK8S_PATH ] && (mkdir -p $LK8S_PATH; echo Created k8sconfig directory) #|| echo Content mkdir -p $LK8S_PATH

#Set-1
vault kv list $VK8S_PATH | egrep -iv 'Keys|----' > $R_FILE

#Set-2
ls -1 $LK8S_PATH | sed s/\.yml$// > $L_FILE

#New in Vault
comm -23 $R_FILE $L_FILE > $NEWKEYS
if [ ! -s $NEWKEYS ]; then
  echo "There are no new kubeconfigs to download"
  exit
fi

#Download new kubeconfig file from Vault
for i in `cat $NEWKEYS`; do
  echo -en "Fetching kubeconfig file of Account : $i"
  vault kv get -field=kubeconfig $VK8S_PATH/$i > $LK8S_PATH/$i.yml
  if [ $? -eq 0 ]; then
    echo " [success]"
    echo "  File: $LK8S_PATH/$i.yml"
  else
    echo " [failed with return code $?]"
  fi
done

echo "Completed Kubeconfig downlod syncing"

The Vault address and its token are stored as secret

apiVersion: v1
kind: Secret
type: Opaque
metadata:
  name: vaultsec
  namespace: spinnaker
data:
  #Replace the vaule here http://vault-host as vaultaddress, and token as encrypted using base64
  vaultaddr: abcovLzE3Mi40Mi40Mi4xMTE6ODIwMA==
  vaulttoken: abca054QXhQbW01T3hDUzVNaU1RbWl1Qmc=

Configuring the Sidecar container with the script:

kind: Deployment
metadata:
  labels:
spec:
  template:
    metadata:
    spec:
      containers:
      - name: clouddriver
        volumeMounts:
        - mountPath: /tmp/k8configs
          name: vvault
      - name: vault-c
        image: alpine:latest
        command: ["/bin/sh", " -c"]
        args:
        - |
          while true; do sh /tmp/k8configs-sync.sh; sleep 30m; done
        env:
        - name: VAULT_ADDR
          valueFrom:
            secretKeyRef:
              key: vaultaddr
              name: vaultsec
        - name: VAULT_TOKEN
          valueFrom:
            secretKeyRef:
              key: vaulttoken
              name: vaultsec
        volumeMounts:
        - mountPath: /tmp/k8configs
          name: vvault
        - mountPath: /tmp/k8configs-sync.sh
          name: vcm-vault
          subPath: k8configs-sync.sh
      volumes:
      - emptyDir: {}
        name: vvault
      - name: vcm-vault
        configMap:
          defaultMode: 420
          name: vault-k8s

If you carefully observe the above sidecar container, we are configuring two things: