##
# The worker nodes to be used should be tagged as 'kubernetes'.
# One server should be tagged as 'kubernetes-public-ip', to denote a machine
# configured with a public IP.
# The worker nodes with GPUs should be tagged as 'gpu'.
##
series: xenial
description: An eight-machine Kubernetes cluster, appropriate for production. Includes a three-machine etcd and ceph-mon cluster, a Kubernetes master, three Kubernetes worker nodes and a Kubernetes worker with GPUs. The Ceph (Luminous) cluster uses three nodes for Ceph Monitors and a Ceph OSD in LXD containers on each worker machine. Two Ceph MDSs provide redundant shared Ceph File System.
# ----------------------------------------------------------------------
machines:
"0":
series: xenial
constraints: tags=kubernetes #root-disk=8G
"1":
series: xenial
constraints: tags=kubernetes #root-disk=8G
"2":
series: xenial
constraints: tags=kubernetes #root-disk=8G
# ----------------------------------------------------------------------
services:
defaultgw:
charm: cs:~csd-garr/defaultgw
options:
# ip route | grep default
gateway: 90.147.161.27/25
easyrsa:
charm: cs:~containers/easyrsa-50
bindings:
"": space-os-mgmt
num_units: 1
to:
# an lxd on the unit of kubernetes-master
- "lxd:kubernetes-master"
annotations:
gui-x: '450'
gui-y: '550'
etcd:
charm: cs:~containers/etcd-96
bindings:
"": space-os-mgmt
num_units: 3
# constraints: tags=kubernetes
options:
channel: 3.2/stable
to:
- "lxd:0"
- "lxd:1"
- "lxd:2"
annotations:
gui-x: '800'
gui-y: '550'
flannel:
charm: cs:~containers/flannel-66
# no bindings
options:
cidr: 10.111.0.0/16
annotations:
gui-x: '450'
gui-y: '750'
kubeapi-load-balancer:
charm: cs:~containers/kubeapi-load-balancer-69
bindings:
"": space-os-mgmt
# necessary for getting a floating IP
apiserver: space-pub
expose: true
num_units: 1
to:
- "lxd:kubernetes-master"
annotations:
gui-x: '450'
gui-y: '250'
kubernetes-keystone:
charm: ./kubernetes-keystone
options:
keystone-url: https://keystone.cloud.garr.it:5000/v3
webhook-authn-config: /root/cdk/webhook-authn.onfig
kubernetes-master:
charm: cs:~containers/kubernetes-master-122
bindings:
"": space-os-mgmt
#kube-api-endpoint: space-pub
num_units: 1
constraints: tags=kubernetes-public-ip cores=2 mem=4G #root-disk=16G
options:
channel: 1.11/stable
authorization-mode: "Node,RBAC"
# RBAC to allow access to dashboard to User "system:anonymous"
# Because of this bug https://github.com/kubernetes/kubernetes/issues/39722
# the value must be false, rather than what the documentation says.
api-extra-args: "anonymous-auth=false authentication-token-webhook-config-file=/root/cdk/webhook-authn.onfig"
enable-dashboard-addons: True
annotations:
gui-x: '800'
gui-y: '850'
kubernetes-worker:
charm: cs:~containers/kubernetes-worker-138
bindings:
"": space-os-mgmt
num_units: 3
constraints: tags=kubernetes cores=4 mem=4G #root-disk=16G
expose: true
options:
channel: 1.11/stable
cuda_repo: 9.2.88-1
# Enable Cloud Controller Manager
# https://kubernetes.io/docs/tasks/administer-cluster/running-cloud-controller/
# kubelet-extra-args: --cloud-provider=external
annotations:
gui-x: '100'
gui-y: '850'
kubernetes-worker-gpu:
charm: cs:~containers/kubernetes-worker-138
bindings:
"": space-os-mgmt
# charm: ./kubernetes-worker # patched for cuda-9.0
num_units: 1
constraints: tags=gpu cores=4 mem=4G #root-disk=16G
expose: true
options:
channel: 1.11/stable
cuda_repo: 9.2.88-1
# kubelet-extra-args: --cloud-provider=external
annotations:
gui-x: "520"
gui-y: "226"
ceph-mon:
charm: cs:ceph-mon
num_units: 3
bindings:
"": space-os-mgmt
options:
# source: cloud:xenial-pike
to:
- 'lxd:0' # lxd:etcd
- 'lxd:1'
- 'lxd:2'
annotations:
gui-x: '1200'
gui-y: '400'
ceph-osd:
charm: cs:~csd-garr/ceph-osd
num_units: 3
bindings:
"": space-os-mgmt
cluster: space-os-data
options:
osd-devices: /dev/mapper/mpatha /dev/mapper/mpathb /dev/mapper/mpathc /dev/mapper/mpathd
osd-max-backfills: 1
osd-recovery-max-active: 1
osd-journal-size: 10000
use-syslog: True
# source: cloud:xenial-pike
# bluestore: True
to:
- "kubernetes-worker/0"
- "kubernetes-worker/1"
- "kubernetes-worker/2"
annotations:
gui-x: '1400'
gui-y: '400'
ceph-mds:
charm: cs:ceph-fs
num_units: 1
bindings:
"": space-os-mgmt
to:
- 'lxd:1'
annotations:
gui-x: '1200'
gui-y: '400'
# manila:
# charm: cs:manila
# bindings:
# "": space-os-mgmt
# to:
# - 'lxd:2'
# options:
# debug: True
# manila-generic:
# charm: cs:manila-generic
# bindings:
# "": space-os-mgmt
# to:
# - 'lxd:2'
# options:
# debug: True
# ----------------------------------------------------------------------
relations:
- ["kubernetes-master:kube-api-endpoint", "kubeapi-load-balancer:apiserver"]
- ["kubernetes-master:loadbalancer", "kubeapi-load-balancer:loadbalancer"]
- ["kubernetes-master:certificates", "easyrsa:client"]
- ["kubernetes-master:juju-info", "defaultgw:juju-info"]
- ["kubernetes-master:juju-info", "kubernetes-keystone:juju-info"]
- ["kubernetes-master:etcd", "etcd:db"]
# Kube load balancer
- ["kubeapi-load-balancer:certificates", "easyrsa:client"]
- ["kubeapi-load-balancer:juju-info", "defaultgw:juju-info"]
- ["etcd:certificates", "easyrsa:client"]
# Kube worker
- ["kubernetes-master:kube-control", "kubernetes-worker:kube-control"]
- ["kubernetes-worker:certificates", "easyrsa:client"]
- ["kubernetes-worker:kube-api-endpoint", "kubeapi-load-balancer:website"]
# Kube GPU worker
- ["kubernetes-master:kube-control", "kubernetes-worker-gpu:kube-control"]
- ["kubernetes-worker-gpu:certificates", "easyrsa:client"]
- ["kubernetes-worker-gpu:kube-api-endpoint", "kubeapi-load-balancer:website"]
# Flannel
- ["flannel:etcd", "etcd:db"]
- ["flannel:cni", "kubernetes-master:cni"]
- ["flannel:cni", "kubernetes-worker:cni"]
- ["flannel:cni", "kubernetes-worker-gpu:cni"]
# Ceph
- ["kubernetes-master:ceph-storage", "ceph-mon:admin"]
- ["ceph-osd:mon", "ceph-mon:osd"]
- ["ceph-mds:ceph-mds", "ceph-mon:mds"]
# # Manila
# - [ manila, keystone ]
# - [ manila, manila-generic ]
# - [ neutron-openvswitch, manila ]