##
# The worker nodes to be used should be tagged as 'kubernetes-worker'.
# The master node should be tagged as "kubernetes-master"
# The worker nodes with GPUs should be tagged as 'gpu'.
##

series: bionic
description: A 9-machine Kubernetes cluster, appropriate for production. Includes a three-machine etcd and ceph-mon cluster, two Kubernetes masters, three Kubernetes worker nodes and a Kubernetes worker with GPUs. The Ceph (Luminous) cluster uses three nodes for Ceph Monitors and a Ceph OSD in LXD containers on each worker machine. Two Ceph MDSs provide redundant shared Ceph File System.

# ----------------------------------------------------------------------
machines:
  "0":
    series: bionic
    constraints: tags=kubernetes-master #root-disk=8G
  "1":
    series: bionic
    constraints: tags=kubernetes-worker #root-disk=8G
  "2":
    series: bionic
    constraints: tags=kubernetes-worker #root-disk=8G
  "3":
    series: bionic
    constraints: tags=kubernetes-worker #root-disk=8G

  "4":
    series: bionic
    constraints: tags=gpu #root-disk=8G


# ----------------------------------------------------------------------
services:

#  canal:
#    annotations:
#      gui-x: '450'
#      gui-y: '750'
#    charm: cs:~containers/canal
#    resources:
#      calico: 73
#      calico-arm64: 69
#      flannel: 73
#      flannel-arm64: 72

  easyrsa:
    charm: cs:~containers/easyrsa
    bindings:
      "": space-os-mgmt
    num_units: 1
    resources:
      easyrsa: 5
    to:
      # an lxd on the unit of kubernetes-master 
      - "lxd:kubernetes-master"
    annotations:
      gui-x: '450'
      gui-y: '550'

  etcd:
    charm: cs:~containers/etcd
    bindings:
      "": space-os-mgmt
    num_units: 3
    # constraints: tags=kubernetes
    options:
      channel: 3.2/stable
    resources:
      etcd: 3
      snapshot: 0
    to:
      - "lxd:0"
      - "lxd:1"
      - "lxd:2"
    annotations:
      gui-x: '800'
      gui-y: '550'

  flannel:
    charm: cs:~containers/flannel
    # no bindings
    options:
      cidr: 10.111.0.0/16
    annotations:
      gui-x: '450'
      gui-y: '750'

  kubeapi-load-balancer:
    charm: cs:~containers/kubeapi-load-balancer
    bindings:
      "": space-os-mgmt
      # necessary for getting a floating IP
      apiserver: space-k8s-pub
    expose: true
    num_units: 1
    resources: {}
    to:
      - "lxd:kubernetes-master"
    annotations:
      gui-x: '450'
      gui-y: '250'

  kubernetes-keystone:
    charm: cs:~csd-garr/kubernetes-keystone
    options:
      keystone-url: https://keystone.cloud.garr.it:5000/v3
      k8s-keystone-auth-url: https://git.garr.it/cloud/charms/kubernetes-keystone/raw/master/bin/linux-amd64/k8s-keystone-auth

  kubernetes-master:
    charm: cs:~containers/kubernetes-master
    constraints: tags=kubernetes-master
    bindings:
      "": space-os-mgmt
      #kube-api-endpoint: space-pub
    num_units: 1
    options:
      channel: 1.13/stable
      authorization-mode: "Node,RBAC,WebHook"
      # RBAC to allow access to dashboard to User "system:anonymous"
      # Because of this bug https://github.com/kubernetes/kubernetes/issues/39722
      # the value must be false, rather than what the documentation says.
      api-extra-args: "anonymous-auth=false"
      enable-dashboard-addons: True
    resources:
      cdk-addons: 0
      kube-apiserver: 0
      kube-controller-manager: 0
      kube-proxy: 0
      kube-scheduler: 0
      kubectl: 0
    annotations:
      gui-x: '800'
      gui-y: '850'
    to:
      - "0"

  kubernetes-worker:
    charm: cs:~containers/kubernetes-worker
    constraints: tags=kubernetes cores=4 mem=4G root-disk=16G
    bindings:
      "": space-os-mgmt
    expose: true
    num_units: 3
    options:
      channel: 1.13/stable
      # Enable Cloud Controller Manager
      # https://kubernetes.io/docs/tasks/administer-cluster/running-cloud-controller/
      # kubelet-extra-args: --cloud-provider=external
    annotations:
      gui-x: '100'
      gui-y: '850'
    to:
      - "1"
      - "2"
      - "3"

  kubernetes-worker-gpu: 
    charm: cs:~containers/kubernetes-worker
    bindings:
      "": space-os-mgmt
    num_units: 1
    constraints: tags=gpu
    expose: true
    options:
      channel: 1.13/stable
      # cuda_repo: 10.0.130-1
      # kubelet-extra-args: --cloud-provider=external
    annotations: 
      gui-x: "520"
      gui-y: "226"
    to:
      - "4"

      
# ----------------------------------------------------------------------
relations:
  - ["kubernetes-master:kube-api-endpoint", "kubeapi-load-balancer:apiserver"]
  - ["kubernetes-master:loadbalancer", "kubeapi-load-balancer:loadbalancer"]
  - ["kubernetes-master:certificates", "easyrsa:client"]
  - ["kubernetes-master:juju-info", "kubernetes-keystone:juju-info"]
  - ["kubernetes-master:etcd", "etcd:db"]
  # Kube load balancer
  - ["kubeapi-load-balancer:certificates", "easyrsa:client"]
  - ["etcd:certificates", "easyrsa:client"]
  # Kube worker
  - ["kubernetes-master:kube-control", "kubernetes-worker:kube-control"]
  - ["kubernetes-worker:certificates", "easyrsa:client"]
  - ["kubernetes-worker:kube-api-endpoint", "kubeapi-load-balancer:website"]
  # Kube GPU worker
  - ["kubernetes-master:kube-control", "kubernetes-worker-gpu:kube-control"]
  - ["kubernetes-worker-gpu:certificates", "easyrsa:client"]
  - ["kubernetes-worker-gpu:kube-api-endpoint", "kubeapi-load-balancer:website"]
  # Canal
#  - ["canal:etcd", "etcd:db"]
#  - ["canal:cni", "kubernetes-master:cni"]
#  - ["canal:cni", "kubernetes-worker:cni"]
#  - ["canal:cni", "kubernetes-worker-gpu:cni"]
  # Flannel
  - ["flannel:etcd", "etcd:db"]
  - ["flannel:cni", "kubernetes-master:cni"]
  - ["flannel:cni", "kubernetes-worker:cni"]
  - ["flannel:cni", "kubernetes-worker-gpu:cni"]