##
# The worker nodes to be used should be tagged as 'kubernetes'.
# One server should be tagged as 'kubernetes-public-ip', to denote a machine
# configured with a public IP.
# The worker nodes with GPUs should be tagged as 'gpu'.
##

series: xenial
description: An eight-machine Kubernetes cluster, appropriate for production. Includes a three-machine etcd and ceph-mon cluster, a Kubernetes master, three Kubernetes worker nodes and a Kubernetes worker with GPUs. The Ceph (Luminous) cluster uses three nodes for Ceph Monitors and a Ceph OSD in LXD containers on each worker machine. Two Ceph MDSs provide redundant shared Ceph File System.

# ----------------------------------------------------------------------
machines:
  "0":
    series: xenial
    constraints: tags=kubernetes #root-disk=8G
  "1":
    series: xenial
    constraints: tags=kubernetes #root-disk=8G
  "2":
    series: xenial
    constraints: tags=kubernetes #root-disk=8G

# ----------------------------------------------------------------------
services:

  defaultgw:
    charm: cs:~csd-garr/defaultgw
    options:
      # ip route | grep default
      gateway: 90.147.161.27/25

  easyrsa:
    charm: cs:~containers/easyrsa-50
    bindings:
      "": space-os-mgmt
    num_units: 1
    to:
      # an lxd on the unit of kubernetes-master 
      - "lxd:kubernetes-master"
    annotations:
      gui-x: '450'
      gui-y: '550'

  etcd:
    charm: cs:~containers/etcd-96
    bindings:
      "": space-os-mgmt
    num_units: 3
    # constraints: tags=kubernetes
    options:
      channel: 3.2/stable
    to:
      - "lxd:0"
      - "lxd:1"
      - "lxd:2"
    annotations:
      gui-x: '800'
      gui-y: '550'

  flannel:
    charm: cs:~containers/flannel-66
    # no bindings
    options:
      cidr: 10.111.0.0/16
    annotations:
      gui-x: '450'
      gui-y: '750'

  kubeapi-load-balancer:
    charm: cs:~containers/kubeapi-load-balancer-69
    bindings:
      "": space-os-mgmt
      # necessary for getting a floating IP
      apiserver: space-pub
    expose: true
    num_units: 1
    to:
      - "lxd:kubernetes-master"
    annotations:
      gui-x: '450'
      gui-y: '250'

  kubernetes-keystone:
    charm: ./kubernetes-keystone
    options:
      keystone-url: https://keystone.cloud.garr.it:5000/v3
      webhook-authn-config: /root/cdk/webhook-authn.onfig

  kubernetes-master:
    charm: cs:~containers/kubernetes-master-122
    bindings:
      "": space-os-mgmt
      #kube-api-endpoint: space-pub
    num_units: 1
    constraints: tags=kubernetes-public-ip cores=2 mem=4G #root-disk=16G
    options:
      channel: 1.11/stable
      authorization-mode: "Node,RBAC"
      # RBAC to allow access to dashboard to User "system:anonymous"
      # Because of this bug https://github.com/kubernetes/kubernetes/issues/39722
      # the value must be false, rather than what the documentation says.
      api-extra-args: "anonymous-auth=false authentication-token-webhook-config-file=/root/cdk/webhook-authn.onfig"
      enable-dashboard-addons: True
    annotations:
      gui-x: '800'
      gui-y: '850'

  kubernetes-worker:
    charm: cs:~containers/kubernetes-worker-138
    bindings:
      "": space-os-mgmt
    num_units: 3
    constraints: tags=kubernetes cores=4 mem=4G #root-disk=16G
    expose: true
    options:
      channel: 1.11/stable
      cuda_repo: 9.2.88-1
      # Enable Cloud Controller Manager
      # https://kubernetes.io/docs/tasks/administer-cluster/running-cloud-controller/
      # kubelet-extra-args: --cloud-provider=external
    annotations:
      gui-x: '100'
      gui-y: '850'

  kubernetes-worker-gpu: 
    charm: cs:~containers/kubernetes-worker-138
    bindings:
      "": space-os-mgmt
    # charm: ./kubernetes-worker # patched for cuda-9.0
    num_units: 1
    constraints: tags=gpu cores=4 mem=4G #root-disk=16G
    expose: true
    options:
      channel: 1.11/stable
      cuda_repo: 9.2.88-1
      # kubelet-extra-args: --cloud-provider=external
    annotations: 
      gui-x: "520"
      gui-y: "226"

  ceph-mon:
    charm: cs:ceph-mon
    num_units: 3 
    bindings:
      "": space-os-mgmt
    options:
      # source: cloud:xenial-pike
    to:
      - 'lxd:0'		# lxd:etcd
      - 'lxd:1'
      - 'lxd:2'
    annotations:
      gui-x: '1200'
      gui-y: '400'

  ceph-osd:
    charm: cs:~csd-garr/ceph-osd
    num_units: 3
    bindings:
      "": space-os-mgmt
      cluster: space-os-data
    options:
      osd-devices: /dev/mapper/mpatha /dev/mapper/mpathb /dev/mapper/mpathc /dev/mapper/mpathd
      osd-max-backfills: 1
      osd-recovery-max-active: 1
      osd-journal-size: 10000
      use-syslog: True
      # source: cloud:xenial-pike
      # bluestore: True
    to:
      - "kubernetes-worker/0"
      - "kubernetes-worker/1"
      - "kubernetes-worker/2"
    annotations:
      gui-x: '1400'
      gui-y: '400'

  ceph-mds:
    charm: cs:ceph-fs
    num_units: 1
    bindings:
      "": space-os-mgmt
    to:
      - 'lxd:1'
    annotations:
      gui-x: '1200'
      gui-y: '400'

  # manila:
  #   charm: cs:manila
  #   bindings:
  #     "": space-os-mgmt
  #   to:
  #     - 'lxd:2'
  #   options:
  #     debug: True

  # manila-generic:
  #   charm: cs:manila-generic
  #   bindings:
  #     "": space-os-mgmt
  #   to:
  #     - 'lxd:2'
  #   options:
  #     debug: True
      
# ----------------------------------------------------------------------
relations:
  - ["kubernetes-master:kube-api-endpoint", "kubeapi-load-balancer:apiserver"]
  - ["kubernetes-master:loadbalancer", "kubeapi-load-balancer:loadbalancer"]
  - ["kubernetes-master:certificates", "easyrsa:client"]
  - ["kubernetes-master:juju-info", "defaultgw:juju-info"]
  - ["kubernetes-master:juju-info", "kubernetes-keystone:juju-info"]
  - ["kubernetes-master:etcd", "etcd:db"]
  # Kube load balancer
  - ["kubeapi-load-balancer:certificates", "easyrsa:client"]
  - ["kubeapi-load-balancer:juju-info", "defaultgw:juju-info"]
  - ["etcd:certificates", "easyrsa:client"]
  # Kube worker
  - ["kubernetes-master:kube-control", "kubernetes-worker:kube-control"]
  - ["kubernetes-worker:certificates", "easyrsa:client"]
  - ["kubernetes-worker:kube-api-endpoint", "kubeapi-load-balancer:website"]
  # Kube GPU worker
  - ["kubernetes-master:kube-control", "kubernetes-worker-gpu:kube-control"]
  - ["kubernetes-worker-gpu:certificates", "easyrsa:client"]
  - ["kubernetes-worker-gpu:kube-api-endpoint", "kubeapi-load-balancer:website"]
  # Flannel
  - ["flannel:etcd", "etcd:db"]
  - ["flannel:cni", "kubernetes-master:cni"]
  - ["flannel:cni", "kubernetes-worker:cni"]
  - ["flannel:cni", "kubernetes-worker-gpu:cni"]
  # Ceph
  - ["kubernetes-master:ceph-storage", "ceph-mon:admin"]
  - ["ceph-osd:mon", "ceph-mon:osd"]
  - ["ceph-mds:ceph-mds", "ceph-mon:mds"]
  # # Manila
  # - [ manila, keystone ]
  # - [ manila, manila-generic ]
  # - [ neutron-openvswitch, manila ]
  
