##
# The server nodes to be used should be tagged as 'kubernetes'.
# At least one of them should also be tagged as 'public-ip', to denote a machine configured with
# a public IP.
# The server nodes with GPUs should be tagged also as 'gpu'.
##

series: xenial
description: An eight-machine Kubernetes cluster, appropriate for production. Includes a three-machine etcd and ceph-mon cluster, a Kubernetes master, three Kubernetes worker nodes and a Kubernetes worker with GPUs. The Ceph (Luminous) cluster uses three nodes for Ceph Monitors and a Ceph OSD in LXD containers on each worker machine.

# ----------------------------------------------------------------------
parameters:

  # Constraints
  cluster: &cluster                "kubernetes"
  public-ip: &public_ip            "public-ip"

# ----------------------------------------------------------------------
machines:
  "0":
    series: xenial
    constraints: "tags=*cluster"
  "1":
    series: xenial
    constraints: "tags=*cluster"
  "2":
    series: xenial
    constraints: "tags=*cluster"
  "3":
    series: xenial
    constraints: "tags=*cluster"
  "4":
    series: xenial
    constraints: "tags=*cluster"
  "5":
    series: xenial
    constraints: "tags=*cluster"
  "6":
    series: xenial
    constraints: "tags=*cluster"

# ----------------------------------------------------------------------
services:

  defaultgw:
    charm: cs:~csd-garr/defaultgw
    options:
      # ip route | grep default
      gateway: 90.147.161.1
                          
  easyrsa:
    annotations:
      gui-x: '450'
      gui-y: '550'
    charm: cs:~containers/easyrsa
    # charm: cs:~containers/easyrsa-27
    num_units: 1
    bindings:
      "": space-os-mgmt
    to:
      - "lxd:kubernetes-master"

  etcd:
    annotations:
      gui-x: '800'
      gui-y: '550'
    charm: cs:~containers/etcd
    # charm: cs:~containers/etcd-63
    num_units: 3
    constraints: tags=*cluster
    to:
      - "0"
      - "1"
      - "2"

  flannel:
    annotations:
      gui-x: '450'
      gui-y: '750'
    charm: cs:~containers/flannel
    bindings:
      "": space-os-mgmt

  kubeapi-load-balancer:
    annotations:
      gui-x: '450'
      gui-y: '250'
    charm: cs:~containers/kubeapi-load-balancer
    expose: true
    num_units: 1
    constraints: tags=*cluster
    bindings:
      "": space-os-mgmt
    to:
      - "lxd:0"

  kubernetes-master:
    annotations:
      gui-x: '800'
      gui-y: '850'
    charm: cs:~containers/kubernetes-master
    # charm: cs:~containers/kubernetes-master-78
    # charm: "cs:~tvansteenburgh/kubernetes-master"
    num_units: 1
    constraints: tags=*cluster,*public_ip
    options:
      channel: 1.9/stable
      authorization-mode: Node,RBAC
      enable-dashboard-addons: True
    to:
      - "3"

  kubernetes-worker:
    annotations:
      gui-x: '100'
      gui-y: '850'
    charm: cs:~containers/kubernetes-worker
    # charm: cs:~containers/kubernetes-worker-81
    expose: true
    num_units: 3
    constraints: cores=4 mem=4G tags=*cluster
    options:
      channel: 1.9/stable
    to:
      - "4"
      - "5"
      - "6"

  kubernetes-worker-gpu: 
     charm: "cs:~tvansteenburgh/kubernetes-worker"
     num_units: 1
     constraints: cores=4 mem=4G tags=*cluster,gpu
     expose: true
     annotations: 
       gui-x: "520"
       gui-y": "226"

  ceph-mon:
    charm: cs:ceph-mon
    source: cloud:xenial-pike
    num_units: 3 
    bindings:
      "": space-os-mgmt
    to:
      - 'lxd:0'                         # lxd:etcd
      - 'lxd:1'
      - 'lxd:2'
    annotations:
      gui-x: '1200'
      gui-y: '400'

  ceph-osd:
    charm: cs:ceph-osd
    source: cloud:xenial-pike
    num_units: 3
    bindings:
      "": space-os-mgmt
      cluster: space-os-data
    options:
      osd-devices: /dev/mapper/mpatha /dev/mapper/mpathb /dev/mapper/mpathc /dev/mapper/mpathd
      osd-max-backfills: 1
      osd-recovery-max-active: 1
      osd-journal-size: 10000
      use-syslog: True
    to:
      - "lxd:4" # "lxd:kubernetes-worker"
      - "lxd:5"
      - "lxd:6"
    annotations:
      gui-x: '1400'
      gui-y: '400'

  ceph-mds:
    charm: cs:ceph-fs
    num_units: 1
    bindings:
      "": space-os-mgmt
    to:
      - 'lxd:0'
    annotations:
      gui-x: '1200'
      gui-y: '400'

# ----------------------------------------------------------------------
relations:
  - ["kubernetes-master:kube-api-endpoint", "kubeapi-load-balancer:apiserver"]
  - ["kubernetes-master:loadbalancer", "kubeapi-load-balancer:loadbalancer"]
  - ["kubernetes-master:kube-control", "kubernetes-worker:kube-control"]
  - ["kubernetes-master:certificates", "easyrsa:client"]
  - ["kubernetes-master:juju-info", "defaultgw:juju-info"]
  - ["etcd:certificates", "easyrsa:client"]
  - ["kubernetes-master:etcd", "etcd:db"]
  - ["kubernetes-worker:certificates", "easyrsa:client"]
  - ["kubernetes-worker:kube-api-endpoint", "kubeapi-load-balancer:website"]
  - ["kubeapi-load-balancer:certificates", "easyrsa:client"]
  # Flannel
  - ["flannel:etcd", "etcd:db"]
  - ["flannel:cni", "kubernetes-master:cni"]
  - ["flannel:cni", "kubernetes-worker:cni"]
  # GPU worker
  - ["flannel:cni", "kubernetes-worker-gpu:cni"]
  - ["kubernetes-worker-gpu:certificates", "easyrsa:client"]
  - ["kubernetes-master:kube-control", "kubernetes-worker-gpu:kube-control"]
  - ["kubernetes-master:kube-api-endpoint", "kubernetes-worker-gpu:kube-api-endpoint"]
  # Ceph
  - ["kubernetes-master:ceph-storage", "ceph-mon:admin"]
  - ["ceph-osd:mon", "ceph-mon:osd"]
  - ["ceph-mds:ceph-mds", "ceph-mon:mds"]