feat: add serviceMonitor configuration for content, payment, notification, central-storage, and authentication services

Signed-off-by: zhenyus <zhenyus@mathmast.com>

cluster/manifests/freeleaps-infra-system/load-watcher/all-in-one.yaml

@@ -0,0 +1,105 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: load-watcher
+  namespace: freeleaps-infra-system
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: load-watcher-role
+rules:
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["metrics.k8s.io"]
+    resources: ["nodes"]
+    verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: load-watcher-binding
+subjects:
+  - kind: ServiceAccount
+    name: load-watcher
+    namespace: freeleaps-infra-system
+roleRef:
+  kind: ClusterRole
+  name: load-watcher-role
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: load-watcher-config
+  namespace: freeleaps-infra-system
+type: Opaque
+data:
+  METRICS_PROVIDER_NAME: UHJvbWV0aGV1cw==
+  METRICS_PROVIDER_ADDRESS: aHR0cDovL2t1YmUtcHJvbWV0aGV1cy1zdGFjay1wcm9tZXRoZXVzLmZyZWVsZWFwcy1tb25pdG9yaW5nLXN5c3RlbS5zdmMuZnJlZWxlYXBzLmNsdXN0ZXI6OTA5MA==
+  INSECURE_SKIP_VERIFY: dHJ1ZQ==
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: load-watcher
+  namespace: freeleaps-infra-system
+  labels:
+    app: load-watcher
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: load-watcher
+  template:
+    metadata:
+      labels:
+        app: load-watcher
+    spec:
+      serviceAccountName: load-watcher
+      containers:
+      - name: load-watcher
+        image: freeleaps/load-watcher:latest
+        env:
+        - name: METRICS_PROVIDER_NAME
+          valueFrom:
+            secretKeyRef:
+              name: load-watcher-config
+              key: METRICS_PROVIDER_NAME
+        - name: METRICS_PROVIDER_ADDRESS
+          valueFrom:
+            secretKeyRef:
+              name: load-watcher-config
+              key: METRICS_PROVIDER_ADDRESS
+        - name: INSECURE_SKIP_VERIFY
+          valueFrom:
+            secretKeyRef:
+              name: load-watcher-config
+              key: INSECURE_SKIP_VERIFY
+        resources:
+          limits:
+            cpu: "500m"
+            memory: "256Mi"
+          requests:
+            cpu: "250m"
+            memory: "128Mi"
+        ports:
+        - containerPort: 2020
+---
+apiVersion: v1
+kind: Service
+metadata:
+  namespace: freeleaps-infra-system
+  name: load-watcher
+  labels:
+    app: load-watcher
+spec:
+  type: ClusterIP
+  ports:
+  - name: http
+    port: 2020
+    targetPort: 2020
+    protocol: TCP
+  selector:
+    app: load-watcher

cluster/manifests/freeleaps-infra-system/scheduler-plugins/0.30.6/all-in-one.yaml

@@ -0,0 +1,93 @@
+# First part
+# Apply extra privileges to system:kube-scheduler.
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: system:kube-scheduler:plugins
+rules:
+- apiGroups: ["scheduling.x-k8s.io"]
+  resources: ["podgroups", "elasticquotas", "podgroups/status", "elasticquotas/status"]
+  verbs: ["get", "list", "watch", "create", "delete", "update", "patch"]
+# for network-aware plugins add the following lines (scheduler-plugins v.0.24.9)
+#- apiGroups: [ "appgroup.diktyo.k8s.io" ]
+#  resources: [ "appgroups" ]
+#  verbs: [ "get", "list", "watch", "create", "delete", "update", "patch" ]
+#- apiGroups: [ "networktopology.diktyo.k8s.io" ]
+#  resources: [ "networktopologies" ]
+#  verbs: [ "get", "list", "watch", "create", "delete", "update", "patch" ]
+#- apiGroups: ["security-profiles-operator.x-k8s.io"]
+#  resources: ["seccompprofiles", "profilebindings"]
+#  verbs: ["get", "list", "watch", "create", "delete", "update", "patch"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: system:kube-scheduler:plugins
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:kube-scheduler:plugins
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: User
+  name: system:kube-scheduler
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: scheduler-plugins-controller
+  namespace: freeleaps-infra-system
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: scheduler-plugins-controller
+rules:
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["scheduling.x-k8s.io"]
+  resources: ["podgroups", "elasticquotas", "podgroups/status", "elasticquotas/status"]
+  verbs: ["get", "list", "watch", "create", "delete", "update", "patch"]
+- apiGroups: [""]
+  resources: ["events"]
+  verbs: ["create", "patch", "update"]
+#- apiGroups: ["security-profiles-operator.x-k8s.io"]
+#  resources: ["seccompprofiles", "profilebindings"]
+#  verbs: ["get", "list", "watch", "create", "delete", "update", "patch"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: scheduler-plugins-controller
+subjects:
+- kind: ServiceAccount
+  name: scheduler-plugins-controller
+  namespace: freeleaps-infra-system
+roleRef:
+  kind: ClusterRole
+  name: scheduler-plugins-controller
+  apiGroup: rbac.authorization.k8s.io
+---
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: scheduler-plugins-controller
+  namespace: freeleaps-infra-system
+  labels:
+    app: scheduler-plugins-controller
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: scheduler-plugins-controller
+  template:
+    metadata:
+      labels:
+        app: scheduler-plugins-controller
+    spec:
+      serviceAccountName: scheduler-plugins-controller
+      containers:
+      - name: scheduler-plugins-controller
+        image: registry.k8s.io/scheduler-plugins/controller:v0.30.6
+        imagePullPolicy: IfNotPresent

cluster/manifests/freeleaps-infra-system/scheduler-plugins/0.30.6/capacityscheduling/crd.yaml

@@ -0,0 +1,100 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    api-approved.kubernetes.io: https://github.com/kubernetes-sigs/scheduler-plugins/pull/52
+    controller-gen.kubebuilder.io/version: v0.16.5
+  name: elasticquotas.scheduling.x-k8s.io
+spec:
+  group: scheduling.x-k8s.io
+  names:
+    kind: ElasticQuota
+    listKind: ElasticQuotaList
+    plural: elasticquotas
+    shortNames:
+    - eq
+    - eqs
+    singular: elasticquota
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: Used is the current observed total usage of the resource in the
+        namespace.
+      jsonPath: .status.used
+      name: Used
+      type: string
+    - description: Max is the set of desired max limits for each named resource.
+      jsonPath: .spec.max
+      name: Max
+      type: string
+    - description: Age is the time ElasticQuota was created.
+      jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: ElasticQuota sets elastic quota restrictions per namespace
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: ElasticQuotaSpec defines the Min and Max for Quota.
+            properties:
+              max:
+                additionalProperties:
+                  anyOf:
+                  - type: integer
+                  - type: string
+                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                  x-kubernetes-int-or-string: true
+                description: |-
+                  Max is the set of desired max limits for each named resource. The usage of max is based on the resource configurations of
+                  successfully scheduled pods.
+                type: object
+              min:
+                additionalProperties:
+                  anyOf:
+                  - type: integer
+                  - type: string
+                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                  x-kubernetes-int-or-string: true
+                description: Min is the set of desired guaranteed limits for each
+                  named resource.
+                type: object
+            type: object
+          status:
+            description: ElasticQuotaStatus defines the observed use.
+            properties:
+              used:
+                additionalProperties:
+                  anyOf:
+                  - type: integer
+                  - type: string
+                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                  x-kubernetes-int-or-string: true
+                description: Used is the current observed total usage of the resource
+                  in the namespace.
+                type: object
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}

cluster/manifests/freeleaps-infra-system/scheduler-plugins/0.30.6/coscheduling/crd.yaml

@@ -0,0 +1,133 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    api-approved.kubernetes.io: https://github.com/kubernetes-sigs/scheduler-plugins/pull/50
+    controller-gen.kubebuilder.io/version: v0.16.5
+  name: podgroups.scheduling.x-k8s.io
+spec:
+  group: scheduling.x-k8s.io
+  names:
+    kind: PodGroup
+    listKind: PodGroupList
+    plural: podgroups
+    shortNames:
+    - pg
+    - pgs
+    singular: podgroup
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: Current phase of PodGroup.
+      jsonPath: .status.phase
+      name: Phase
+      type: string
+    - description: MinMember defines the minimal number of members/tasks to run the
+        pod group.
+      jsonPath: .spec.minMember
+      name: MinMember
+      type: integer
+    - description: The number of actively running pods.
+      jsonPath: .status.running
+      name: Running
+      type: integer
+    - description: The number of pods which reached phase Succeeded.
+      jsonPath: .status.succeeded
+      name: Succeeded
+      type: integer
+    - description: The number of pods which reached phase Failed.
+      jsonPath: .status.failed
+      name: Failed
+      type: integer
+    - description: Age is the time PodGroup was created.
+      jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: PodGroup is a collection of Pod; used for batch workload.
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: Specification of the desired behavior of the pod group.
+            properties:
+              minMember:
+                description: |-
+                  MinMember defines the minimal number of members/tasks to run the pod group;
+                  if there's not enough resources to start all tasks, the scheduler
+                  will not start any.
+                  The minimum is 1
+                format: int32
+                minimum: 1
+                type: integer
+              minResources:
+                additionalProperties:
+                  anyOf:
+                  - type: integer
+                  - type: string
+                  pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                  x-kubernetes-int-or-string: true
+                description: |-
+                  MinResources defines the minimal resource of members/tasks to run the pod group;
+                  if there's not enough resources to start all tasks, the scheduler
+                  will not start any.
+                type: object
+              scheduleTimeoutSeconds:
+                description: ScheduleTimeoutSeconds defines the maximal time of members/tasks
+                  to wait before run the pod group;
+                format: int32
+                type: integer
+            type: object
+          status:
+            description: |-
+              Status represents the current information about a pod group.
+              This data may not be up to date.
+            properties:
+              failed:
+                description: The number of pods which reached phase Failed.
+                format: int32
+                type: integer
+              occupiedBy:
+                description: |-
+                  OccupiedBy marks the workload (e.g., deployment, statefulset) UID that occupy the podgroup.
+                  It is empty if not initialized.
+                type: string
+              phase:
+                description: Current phase of PodGroup.
+                type: string
+              running:
+                description: The number of actively running pods.
+                format: int32
+                type: integer
+              scheduleStartTime:
+                description: ScheduleStartTime of the group
+                format: date-time
+                type: string
+              succeeded:
+                description: The number of pods which reached phase Succeeded.
+                format: int32
+                type: integer
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}

cluster/manifests/freeleaps-infra-system/scheduler-plugins/sched-cc.yaml

@@ -0,0 +1,19 @@
+apiVersion: kubescheduler.config.k8s.io/v1
+kind: KubeSchedulerConfiguration
+leaderElection:
+  leaderElect: true
+clientConnection:
+  kubeconfig: /etc/kubernetes/scheduler.conf
+profiles:
+- schedulerName: default-scheduler
+  plugins:
+    score:
+      enabled:
+        - name: LoadVariationRiskBalancing
+  pluginConfig:
+    - name: LoadVariationRiskBalancing
+      args:
+        safeVarianceMargin: 1
+        safeVarianceSensitivity: 2
+        # MUST BE ACTUAL CLUSTER IP FOR LOAD WATCHER SERVICE
+        watcherAddress: http://10.233.48.49:2020

freeleaps/helm-pkg/authentication/templates/authentication/authentication-config.yaml

@@ -16,4 +16,6 @@ data:
   MONGODB_NAME: {{ .Values.authentication.configs.mongodbName | b64enc | quote }}
   MONGODB_PORT: {{ .Values.authentication.configs.mongodbPort | toString | b64enc }}
   MONGODB_URI: {{ .Values.authentication.configs.mongodbUri | b64enc | quote }}
+  METRICS_ENABLED: {{ if .Values.authentication.serviceMonitor.enabled | default false | toString | b64enc }}
+  PROBES_ENABLED: {{ if .Values.authentication.probes | default false | toString | b64enc }}
   

freeleaps/helm-pkg/authentication/templates/authentication/deployment.yaml

@@ -82,7 +82,7 @@ spec:
             {{- end }}
             {{- if .Values.authentication.probes.readiness.config.failureThreshold }}
             failureThreshold: {{ .Values.authentication.probes.readiness.config.failureThreshold }}
-            {{- end }}:%
+            {{- end }}
             {{- if .Values.authentication.probes.readiness.config.terminationGracePeriodSeconds }}
             terminationGracePeriodSeconds: {{ .Values.authentication.probes.readiness.config.terminationGracePeriodSeconds }}
             {{- end }}

freeleaps/helm-pkg/authentication/templates/authentication/service.yaml

@@ -18,6 +18,11 @@ spec:
   ports:
     - port: {{ $service.port }}
       targetPort: {{ $service.targetPort }}
+    {{- if .Values.authentication.serviceMonitor.enabled }}
+    - port: {{ $service.port }}
+      targetPort: {{ $service.targetPort }}
+      name: metrics
+    {{- end }}
   selector:
     app.kubernetes.io/version: {{ $appVersion }}
     app.kubernetes.io/name: "authentication"

freeleaps/helm-pkg/authentication/templates/authentication/servicemonitor.yaml

@@ -0,0 +1,32 @@
+{{- if .Values.authentication.serviceMonitor.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ .Release.Name }}-service-monitor
+  namespace: {{ .Values.authentication.serviceMonitor.namespace }}
+  labels:
+    app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+    app.kubernetes.io/name: {{ .Release.Name }}-service-monitor
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    {{- if .Values.authentication.serviceMonitor.labels }}
+    {{- toYaml .Values.authentication.serviceMonitor.labels | nindent 4 }}
+    {{- end }}
+spec:
+  endpoints:
+  - path: /api/_/metrics
+    port: http
+    {{- if .Values.authentication.serviceMonitor.interval }}
+    interval: {{ .Values.authentication.serviceMonitor.interval }}
+    {{- end }}
+    {{- if .Values.authentication.serviceMonitor.scrapeTimeout }}
+    scrapeTimeout: {{ .Values.authentication.serviceMonitor.scrapeTimeout }}
+    {{- end }}
+  namespaceSelector:
+    matchNames:
+    - {{ .Release.Namespace | quote }}
+  selector:
+    matchLabels:
+      app.kubernetes.io/managed-by: {{ .Release.Service }}
+      app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}

freeleaps/helm-pkg/authentication/values.prod.yaml

@@ -21,7 +21,34 @@ authentication:
     limits:
       cpu: 300m
       memory: 128Mi
-  probes: {}
+  probes:
+    readiness:
+      config:
+        path: /api/_/readyz
+        port: 8004
+        initialDelaySeconds: 5
+        periodSeconds: 10
+        timeoutSeconds: 3
+        successThreshold: 1
+        failureThreshold: 3
+        terminationGracePeriodSeconds: 30
+    liveness:
+      config:
+        path: /api/_/livez
+        port: 8004
+        initialDelaySeconds: 5
+        periodSeconds: 10
+        timeoutSeconds: 3
+        successThreshold: 1
+        failureThreshold: 3
+        terminationGracePeriodSeconds: 30
+  serviceMonitor:
+    enabled: true
+    labels:
+      release: kube-prometheus-stack
+    namespace: freeleaps-monitoring-system
+    internal: 30s
+    scrapeTimeout: ""
   services:
   - name: authentication-service
     type: ClusterIP

freeleaps/helm-pkg/authentication/values.yaml

@@ -23,6 +23,12 @@ authentication:
       memory: "1Gi"
   # FIXME: Wait until the developers implements the probes APIs
   probes: {}
+  serviceMonitor:
+    enabled: false
+    labels: {}
+    namespace: ""
+    internal: 30s
+    scrapeTimeout: ""
   services:
     - name: authentication-service
       type: ClusterIP

freeleaps/helm-pkg/centralStorage/templates/central-storage/central-storage-config.yaml

@@ -14,4 +14,6 @@ data:
   MONGODB_URI: {{ .Values.centralStorage.configs.mongodbUri | b64enc | quote }}
   AZURE_STORAGE_DOCUMENT_API_KEY: {{ .Values.centralStorage.configs.azureStorageDocumentApiKey | b64enc | quote }}
   AZURE_STORAGE_DOCUMENT_API_ENDPOINT: {{ .Values.centralStorage.configs.azureStorageDocumentApiEndpoint | b64enc | quote }}
+  METRICS_ENABLED: {{ if .Values.centralStorage.serviceMonitor.enabled | default false | toString | b64enc }}
+  PROBES_ENABLED: {{ if .Values.centralStorage.probes | default false | toString | b64enc }}
   

freeleaps/helm-pkg/centralStorage/templates/central-storage/deployment.yaml

@@ -82,7 +82,7 @@ spec:
             {{- end }}
             {{- if .Values.centralStorage.probes.readiness.config.failureThreshold }}
             failureThreshold: {{ .Values.centralStorage.probes.readiness.config.failureThreshold }}
-            {{- end }}:%
+            {{- end }}
             {{- if .Values.centralStorage.probes.readiness.config.terminationGracePeriodSeconds }}
             terminationGracePeriodSeconds: {{ .Values.centralStorage.probes.readiness.config.terminationGracePeriodSeconds }}
             {{- end }}

freeleaps/helm-pkg/centralStorage/templates/central-storage/service.yaml

@@ -18,6 +18,11 @@ spec:
   ports:
     - port: {{ $service.port }}
       targetPort: {{ $service.targetPort }}
+    {{- if .Values.centralStorage.serviceMonitor.enabled }}
+    - port: {{ $service.port }}
+      targetPort: {{ $service.targetPort }}
+      name: metrics
+    {{- end }}
   selector:
     app.kubernetes.io/version: {{ $appVersion }}
     app.kubernetes.io/name: "central-storage"

freeleaps/helm-pkg/centralStorage/templates/central-storage/servicemonitor.yaml

@@ -0,0 +1,32 @@
+{{- if .Values.centralStorage.serviceMonitor.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ .Release.Name }}-service-monitor
+  namespace: {{ .Values.centralStorage.serviceMonitor.namespace }}
+  labels:
+    app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+    app.kubernetes.io/name: {{ .Release.Name }}-service-monitor
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    {{- if .Values.centralStorage.serviceMonitor.labels }}
+    {{- toYaml .Values.centralStorage.serviceMonitor.labels | nindent 4 }}
+    {{- end }}
+spec:
+  endpoints:
+  - path: /api/_/metrics
+    port: http
+    {{- if .Values.centralStorage.serviceMonitor.interval }}
+    interval: {{ .Values.centralStorage.serviceMonitor.interval }}
+    {{- end }}
+    {{- if .Values.centralStorage.serviceMonitor.scrapeTimeout }}
+    scrapeTimeout: {{ .Values.centralStorage.serviceMonitor.scrapeTimeout }}
+    {{- end }}
+  namespaceSelector:
+    matchNames:
+    - {{ .Release.Namespace | quote }}
+  selector:
+    matchLabels:
+      app.kubernetes.io/managed-by: {{ .Release.Service }}
+      app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}

freeleaps/helm-pkg/centralStorage/values.prod.yaml

@@ -21,7 +21,34 @@ centralStorage:
     limits:
       cpu: 300m
       memory: 512Mi
-  probes: {}
+  probes:
+    readiness:
+      config:
+        path: /api/_/readyz
+        port: 8005
+        initialDelaySeconds: 5
+        periodSeconds: 10
+        timeoutSeconds: 3
+        successThreshold: 1
+        failureThreshold: 3
+        terminationGracePeriodSeconds: 30
+    liveness:
+      config:
+        path: /api/_/livez
+        port: 8005
+        initialDelaySeconds: 5
+        periodSeconds: 10
+        timeoutSeconds: 3
+        successThreshold: 1
+        failureThreshold: 3
+        terminationGracePeriodSeconds: 30
+  serviceMonitor:
+    enabled: true
+    labels:
+      release: kube-prometheus-stack
+    namespace: freeleaps-monitoring-system
+    internal: 30s
+    scrapeTimeout: ""
   services:
   - name: central-storage-service
     type: ClusterIP

freeleaps/helm-pkg/centralStorage/values.yaml

@@ -23,6 +23,12 @@ central-storage:
       memory: "1Gi"
   # FIXME: Wait until the developers implements the probes APIs
   probes: {}
+  serviceMonitor:
+    enabled: false
+    labels: {}
+    namespace: ""
+    internal: 30s
+    scrapeTimeout: ""
   services:
     - name: central-storage-service
       type: ClusterIP

freeleaps/helm-pkg/content/templates/content/content-config.yaml

@@ -14,4 +14,6 @@ data:
   MONGODB_URI: {{ .Values.content.configs.mongodbUri | b64enc | quote }}
   FREELEAPS_WWW_AS_AZURE_CLIENT_SECRET: {{ .Values.content.configs.freeleapsWwwAsAzureClientSecret | b64enc | quote }}
   CENTRAL_STORAGE_WEBAPI_URL_BASE: {{ .Values.content.configs.centralStorageWebapiUrlBase | b64enc | quote }}
+  METRICS_ENABLED: {{ if .Values.content.serviceMonitor.enabled | default false | toString | b64enc }}
+  PROBES_ENABLED: {{ if .Values.content.probes | default false | toString | b64enc }}
   

freeleaps/helm-pkg/content/templates/content/deployment.yaml

@@ -82,7 +82,7 @@ spec:
             {{- end }}
             {{- if .Values.content.probes.readiness.config.failureThreshold }}
             failureThreshold: {{ .Values.content.probes.readiness.config.failureThreshold }}
-            {{- end }}:%
+            {{- end }}
             {{- if .Values.content.probes.readiness.config.terminationGracePeriodSeconds }}
             terminationGracePeriodSeconds: {{ .Values.content.probes.readiness.config.terminationGracePeriodSeconds }}
             {{- end }}

freeleaps/helm-pkg/content/templates/content/service.yaml

@@ -18,6 +18,11 @@ spec:
   ports:
     - port: {{ $service.port }}
       targetPort: {{ $service.targetPort }}
+    {{- if .Values.content.serviceMonitor.enabled }}
+    - port: {{ $service.port }}
+      targetPort: {{ $service.targetPort }}
+      name: metrics
+    {{- end }}
   selector:
     app.kubernetes.io/version: {{ $appVersion }}
     app.kubernetes.io/name: "content"

freeleaps/helm-pkg/content/templates/content/servicemonitor.yaml

@@ -0,0 +1,32 @@
+{{- if .Values.content.serviceMonitor.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ .Release.Name }}-service-monitor
+  namespace: {{ .Values.content.serviceMonitor.namespace }}
+  labels:
+    app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+    app.kubernetes.io/name: {{ .Release.Name }}-service-monitor
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    {{- if .Values.content.serviceMonitor.labels }}
+    {{- toYaml .Values.content.serviceMonitor.labels | nindent 4 }}
+    {{- end }}
+spec:
+  endpoints:
+  - path: /api/_/metrics
+    port: http
+    {{- if .Values.content.serviceMonitor.interval }}
+    interval: {{ .Values.content.serviceMonitor.interval }}
+    {{- end }}
+    {{- if .Values.content.serviceMonitor.scrapeTimeout }}
+    scrapeTimeout: {{ .Values.content.serviceMonitor.scrapeTimeout }}
+    {{- end }}
+  namespaceSelector:
+    matchNames:
+    - {{ .Release.Namespace | quote }}
+  selector:
+    matchLabels:
+      app.kubernetes.io/managed-by: {{ .Release.Service }}
+      app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}

freeleaps/helm-pkg/content/values.prod.yaml

@@ -21,7 +21,34 @@ content:
     limits:
       cpu: 300m
       memory: 512Mi
-  probes: {}
+  probes:
+    readiness:
+      config:
+        path: /api/_/readyz
+        port: 8013
+        initialDelaySeconds: 5
+        periodSeconds: 10
+        timeoutSeconds: 3
+        successThreshold: 1
+        failureThreshold: 3
+        terminationGracePeriodSeconds: 30
+    liveness:
+      config:
+        path: /api/_/livez
+        port: 8013
+        initialDelaySeconds: 5
+        periodSeconds: 10
+        timeoutSeconds: 3
+        successThreshold: 1
+        failureThreshold: 3
+        terminationGracePeriodSeconds: 30
+  serviceMonitor:
+    enabled: true
+    labels:
+      release: kube-prometheus-stack
+    namespace: freeleaps-monitoring-system
+    internal: 30s
+    scrapeTimeout: ""
   services:
   - name: content-service
     type: ClusterIP

freeleaps/helm-pkg/content/values.yaml

@@ -23,6 +23,12 @@ content:
       memory: "1Gi"
   # FIXME: Wait until the developers implements the probes APIs
   probes: {}
+  serviceMonitor:
+    enabled: false
+    labels: {}
+    namespace: ""
+    internal: 30s
+    scrapeTimeout: ""
   services:
     - name: content-service
       type: ClusterIP

freeleaps/helm-pkg/notification/templates/notification/deployment.yaml

@@ -82,7 +82,7 @@ spec:
             {{- end }}
             {{- if .Values.notification.probes.readiness.config.failureThreshold }}
             failureThreshold: {{ .Values.notification.probes.readiness.config.failureThreshold }}
-            {{- end }}:%
+            {{- end }}
             {{- if .Values.notification.probes.readiness.config.terminationGracePeriodSeconds }}
             terminationGracePeriodSeconds: {{ .Values.notification.probes.readiness.config.terminationGracePeriodSeconds }}
             {{- end }}

freeleaps/helm-pkg/notification/templates/notification/notification-config.yaml

@@ -21,4 +21,6 @@ data:
   SENDGRID_API_KEY: {{ .Values.notification.configs.sendgridApiKey | b64enc | quote }}
   TWILIO_ACCOUNT_SID: {{ .Values.notification.configs.twilioAccountSid | b64enc | quote }}
   TWILIO_AUTH_TOKEN: {{ .Values.notification.configs.twilioAuthToken | b64enc | quote }}
+  METRICS_ENABLED: {{ if .Values.notification.serviceMonitor.enabled | default false | toString | b64enc }}
+  PROBES_ENABLED: {{ if .Values.notification.probes | default false | toString | b64enc }}
   

freeleaps/helm-pkg/notification/templates/notification/service.yaml

@@ -18,6 +18,11 @@ spec:
   ports:
     - port: {{ $service.port }}
       targetPort: {{ $service.targetPort }}
+    {{- if .Values.content.serviceMonitor.enabled }}
+    - port: {{ $service.port }}
+      targetPort: {{ $service.targetPort }}
+      name: metrics
+    {{- end }}
   selector:
     app.kubernetes.io/version: {{ $appVersion }}
     app.kubernetes.io/name: "notification"

freeleaps/helm-pkg/notification/templates/notification/servicemonitor.yaml

@@ -0,0 +1,32 @@
+{{- if .Values.notification.serviceMonitor.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ .Release.Name }}-service-monitor
+  namespace: {{ .Values.notification.serviceMonitor.namespace }}
+  labels:
+    app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+    app.kubernetes.io/name: {{ .Release.Name }}-service-monitor
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    {{- if .Values.notification.serviceMonitor.labels }}
+    {{- toYaml .Values.notification.serviceMonitor.labels | nindent 4 }}
+    {{- end }}
+spec:
+  endpoints:
+  - path: /api/_/metrics
+    port: http
+    {{- if .Values.notification.serviceMonitor.interval }}
+    interval: {{ .Values.notification.serviceMonitor.interval }}
+    {{- end }}
+    {{- if .Values.notification.serviceMonitor.scrapeTimeout }}
+    scrapeTimeout: {{ .Values.notification.serviceMonitor.scrapeTimeout }}
+    {{- end }}
+  namespaceSelector:
+    matchNames:
+    - {{ .Release.Namespace | quote }}
+  selector:
+    matchLabels:
+      app.kubernetes.io/managed-by: {{ .Release.Service }}
+      app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}

freeleaps/helm-pkg/notification/values.prod.yaml

@@ -21,7 +21,34 @@ notification:
     limits:
       cpu: 300m
       memory: 128Mi
-  probes: {}
+  probes:
+    readiness:
+      config:
+        path: /api/_/readyz
+        port: 8003
+        initialDelaySeconds: 5
+        periodSeconds: 10
+        timeoutSeconds: 3
+        successThreshold: 1
+        failureThreshold: 3
+        terminationGracePeriodSeconds: 30
+    liveness:
+      config:
+        path: /api/_/livez
+        port: 8003
+        initialDelaySeconds: 5
+        periodSeconds: 10
+        timeoutSeconds: 3
+        successThreshold: 1
+        failureThreshold: 3
+        terminationGracePeriodSeconds: 30
+  serviceMonitor:
+    enabled: true
+    labels:
+      release: kube-prometheus-stack
+    namespace: freeleaps-monitoring-system
+    internal: 30s
+    scrapeTimeout: ""
   services:
   - name: notification-service
     type: ClusterIP

freeleaps/helm-pkg/notification/values.yaml

@@ -23,6 +23,12 @@ notification:
       memory: "1Gi"
   # FIXME: Wait until the developers implements the probes APIs
   probes: {}
+  serviceMonitor:
+    enabled: false
+    labels: {}
+    namespace: ""
+    internal: 30s
+    scrapeTimeout: ""
   services:
     - name: notification-service
       type: ClusterIP

freeleaps/helm-pkg/payment/templates/payment/deployment.yaml

@@ -82,7 +82,7 @@ spec:
             {{- end }}
             {{- if .Values.payment.probes.readiness.config.failureThreshold }}
             failureThreshold: {{ .Values.payment.probes.readiness.config.failureThreshold }}
-            {{- end }}:%
+            {{- end }}
             {{- if .Values.payment.probes.readiness.config.terminationGracePeriodSeconds }}
             terminationGracePeriodSeconds: {{ .Values.payment.probes.readiness.config.terminationGracePeriodSeconds }}
             {{- end }}

freeleaps/helm-pkg/payment/templates/payment/payment-config.yaml

@@ -14,4 +14,6 @@ data:
   MONGODB_URI: {{ .Values.payment.configs.mongodbUri | b64enc | quote }}
   STRIPE_API_KEY: {{ .Values.payment.configs.stripeApiKey | b64enc | quote }}
   SITE_URL_ROOT: {{ .Values.payment.configs.siteUrlRoot | b64enc | quote }}
+  METRICS_ENABLED: {{ if .Values.payment.serviceMonitor.enabled | default false | toString | b64enc }}
+  PROBES_ENABLED: {{ if .Values.payment.probes | default false | toString | b64enc }}
   

freeleaps/helm-pkg/payment/templates/payment/service.yaml

@@ -18,6 +18,11 @@ spec:
   ports:
     - port: {{ $service.port }}
       targetPort: {{ $service.targetPort }}
+    {{- if .Values.content.serviceMonitor.enabled }}
+    - port: {{ $service.port }}
+      targetPort: {{ $service.targetPort }}
+      name: metrics
+    {{- end }}
   selector:
     app.kubernetes.io/version: {{ $appVersion }}
     app.kubernetes.io/name: "payment"

freeleaps/helm-pkg/payment/templates/payment/servicemonitor.yaml

@@ -0,0 +1,32 @@
+{{- if .Values.payment.serviceMonitor.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ .Release.Name }}-service-monitor
+  namespace: {{ .Values.payment.serviceMonitor.namespace }}
+  labels:
+    app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+    app.kubernetes.io/name: {{ .Release.Name }}-service-monitor
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    {{- if .Values.payment.serviceMonitor.labels }}
+    {{- toYaml .Values.payment.serviceMonitor.labels | nindent 4 }}
+    {{- end }}
+spec:
+  endpoints:
+  - path: /api/_/metrics
+    port: http
+    {{- if .Values.payment.serviceMonitor.interval }}
+    interval: {{ .Values.payment.serviceMonitor.interval }}
+    {{- end }}
+    {{- if .Values.payment.serviceMonitor.scrapeTimeout }}
+    scrapeTimeout: {{ .Values.payment.serviceMonitor.scrapeTimeout }}
+    {{- end }}
+  namespaceSelector:
+    matchNames:
+    - {{ .Release.Namespace | quote }}
+  selector:
+    matchLabels:
+      app.kubernetes.io/managed-by: {{ .Release.Service }}
+      app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}

freeleaps/helm-pkg/payment/values.prod.yaml

@@ -21,7 +21,34 @@ payment:
     limits:
       cpu: 300m
       memory: 256Mi
-  probes: {}
+  probes:
+    readiness:
+      config:
+        path: /api/_/readyz
+        port: 8006
+        initialDelaySeconds: 5
+        periodSeconds: 10
+        timeoutSeconds: 3
+        successThreshold: 1
+        failureThreshold: 3
+        terminationGracePeriodSeconds: 30
+    liveness:
+      config:
+        path: /api/_/livez
+        port: 8006
+        initialDelaySeconds: 5
+        periodSeconds: 10
+        timeoutSeconds: 3
+        successThreshold: 1
+        failureThreshold: 3
+        terminationGracePeriodSeconds: 30
+  serviceMonitor:
+    enabled: true
+    labels:
+      release: kube-prometheus-stack
+    namespace: freeleaps-monitoring-system
+    internal: 30s
+    scrapeTimeout: ""
   services:
   - name: payment-service
     type: ClusterIP

freeleaps/helm-pkg/payment/values.yaml

@@ -23,6 +23,12 @@ payment:
       memory: "1Gi"
   # FIXME: Wait until the developers implements the probes APIs
   probes: {}
+  serviceMonitor:
+    enabled: false
+    labels: {}
+    namespace: ""
+    internal: 30s
+    scrapeTimeout: ""
   services:
     - name: payment-service
       type: ClusterIP