Add OpenTelemetry logging support across multiple services

- Introduced log ingestion configuration in values files for centralStorage, content, notification, and payment services.
- Updated deployment templates to conditionally include OpenTelemetry annotations and volume mounts based on log ingestion settings.
- Added OpenTelemetry RBAC configurations for service accounts and cluster roles to enable logging.
- Implemented OpenTelemetry collector configuration to process logs and export them to Loki.
- Ensured compatibility with existing Helm chart structure and maintained backward compatibility for services without log ingestion enabled.

Signed-off-by: zhenyus <zhenyus@mathmast.com>

freeleaps/helm-pkg/authentication/templates/authentication/deployment.yaml

@@ -111,8 +111,8 @@ spec:
             - name: app-logs
               mountPath: {{ .Values.logIngest.logPath }}
 {{- end }}
-      volumes:
 {{- if .Values.logIngest.enabled }}
+      volumes:
         - name: app-logs
           emptyDir: {}
 {{- end }}

freeleaps/helm-pkg/centralStorage/templates/central-storage/deployment.yaml

@@ -6,6 +6,10 @@ metadata:
     app.kubernetes.io/name: "central-storage"
     app.kubernetes.io/managed-by: {{ .Release.Service }}
     app.kubernetes.io/instance: {{ .Release.Name }}
+{{- if .Values.logIngest.enabled }}
+  annotations:
+      opentelemetry.io/config-checksum: {{ include (print $.Template.BasePath "/central-storage/opentelemetry.yaml") . | sha256sum }}
+{{- end }}
   name: "central-storage"
   namespace: {{ .Release.Namespace | quote }}
 spec:
@@ -25,7 +29,14 @@ spec:
         app.kubernetes.io/instance: {{ .Release.Name }}
       annotations:
         app.kubernetes.io/config-checksum: {{ include (print $.Template.BasePath "/central-storage/central-storage-config.yaml") . | sha256sum }}
+{{- if .Values.logIngest.enabled }}
+        opentelemetry.io/config-checksum: {{ include (print $.Template.BasePath "/central-storage/opentelemetry.yaml") . | sha256sum }}
+        sidecar.opentelemetry.io/inject: "{{ .Release.Namespace}}/{{ .Release.Name }}-opentelemetry-collector"
+{{- end }}
     spec:
+{{- if .Values.logIngest.enabled }}
+      serviceAccountName: "{{ .Release.Name }}-otel-collector"
+{{- end }}
       containers:
         - name: "central-storage"
           image: "{{ coalesce .Values.centralStorage.image.registry .Values.global.registry "docker.io"}}/{{ coalesce .Values.centralStorage.image.repository .Values.global.repository }}/{{ .Values.centralStorage.image.name }}:{{ .Values.centralStorage.image.tag | default "latest" }}"
@@ -95,3 +106,13 @@ spec:
                   name: central-storage-config
                   key: {{ $key | snakecase | upper }}
             {{- end }}
+{{- if .Values.logIngest.enabled }}
+          volumeMounts:
+            - name: app-logs
+              mountPath: {{ .Values.logIngest.logPath }}
+{{- end }}
+{{- if .Values.logIngest.enabled }}
+      volumes:
+        - name: app-logs
+          emptyDir: {}
+{{- end }}

freeleaps/helm-pkg/centralStorage/templates/central-storage/opentelemetry-rbac.yaml

@@ -0,0 +1,46 @@
+{{- if .Values.logIngest.enabled }}
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ .Release.Name }}-otel-collector
+  namespace: {{ .Release.Namespace }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ .Release.Name }}-otel-collector
+rules:
+  - apiGroups: [""]
+    resources:
+      - pods
+      - namespaces
+      - nodes
+    verbs:
+      - get
+      - watch
+      - list
+  - apiGroups: ["apps"]
+    resources:
+      - replicasets
+      - deployments
+      - statefulsets
+      - daemonsets
+    verbs:
+      - get
+      - watch
+      - list
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ .Release.Name }}-otel-collector
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ .Release.Name }}-otel-collector
+subjects:
+  - kind: ServiceAccount
+    name: {{ .Release.Name }}-otel-collector
+    namespace: {{ .Release.Namespace }}
+{{- end }}

freeleaps/helm-pkg/centralStorage/templates/central-storage/opentelemetry.yaml

@@ -0,0 +1,115 @@
+{{- if .Values.logIngest.enabled }}
+apiVersion: opentelemetry.io/v1beta1
+kind: OpenTelemetryCollector
+metadata:
+  name: {{ .Release.Name }}-opentelemetry-collector
+  namespace: {{ .Release.Namespace }}
+spec:
+  mode: sidecar
+  image: ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib:latest
+  serviceAccount: "{{ .Release.Name }}-otel-collector"
+  volumeMounts:
+    - name: app-logs
+      mountPath: {{ .Values.logIngest.logPath }}
+  securityContext:
+    allowPrivilegeEscalation: true
+    privileged: true
+    runAsUser: 0
+    runAsGroup: 0
+  env:
+    - name: KUBE_META_POD_NAME
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.name
+    - name: KUBE_META_NAMESPACE
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.namespace
+    - name: KUBE_META_NODE_NAME
+      valueFrom:
+        fieldRef:
+          fieldPath: spec.nodeName
+    - name: KUBE_META_POD_IP
+      valueFrom:
+        fieldRef:
+          fieldPath: status.podIP
+    - name: KUBE_META_POD_UID
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.uid
+    - name: KUBE_META_OBJECT_NAME
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.labels['app.kubernetes.io/instance']
+  config:
+    receivers:
+      filelog:
+        include:
+          - {{ .Values.logIngest.logPathPattern }}
+        start_at: end
+        include_file_path: false
+        include_file_name: false
+        operators:
+          - type: json_parser
+            parse_from: body
+            parse_to: attributes
+    processors:
+      resource:
+        attributes:
+          - action: insert
+            key: k8s.node.name
+            value: ${KUBE_META_NODE_NAME}
+          - action: insert
+            key: k8s.pod.name
+            value: ${KUBE_META_POD_NAME}
+          - action: insert
+            key: k8s.pod.ip
+            value: ${KUBE_META_POD_IP}
+          - action: insert
+            key: k8s.pod.uid
+            value: ${KUBE_META_POD_UID}
+          - action: insert
+            key: k8s.namespace.name
+            value: ${KUBE_META_NAMESPACE}
+          - action: insert
+            key: k8s.deployment.name
+            value: ${KUBE_META_OBJECT_NAME}
+      transform:
+        log_statements:
+          - context: log
+            statements:
+              - set(resource.attributes["application"], log.attributes["context"]["app"])
+              - set(resource.attributes["environment"], log.attributes["context"]["env"])
+              - set(resource.attributes["kubernetes_node_name"], resource.attributes["k8s.node.name"])
+              - set(resource.attributes["kubernetes_pod_name"], resource.attributes["k8s.pod.name"])
+              - set(resource.attributes["kubernetes_pod_ip"], resource.attributes["k8s.pod.ip"])
+              - set(resource.attributes["kubernetes_deployment_name"], resource.attributes["k8s.deployment.name"])
+              - set(resource.attributes["kubernetes_namespace"], resource.attributes["k8s.namespace.name"])
+              - set(resource.attributes["body_json"], ParseJSON(log.body))
+              - set(resource.attributes["body_json"]["kubernetes"]["pod"], resource.attributes["k8s.pod.name"])
+              - set(resource.attributes["body_json"]["kubernetes"]["namespace"], resource.attributes["k8s.namespace.name"])
+              - set(resource.attributes["body_json"]["kubernetes"]["pod_ip"], resource.attributes["k8s.pod.ip"])
+              - set(resource.attributes["body_json"]["kubernetes"]["pod_uid"], resource.attributes["k8s.pod.uid"])
+              - set(resource.attributes["body_json"]["kubernetes"]["deployment"], resource.attributes["k8s.deployment.name"])
+              - set(resource.attributes["body_json"]["kubernetes"]["node"], resource.attributes["k8s.node.name"])
+              - set(resource.attributes["body_json"]["kubernetes"]["namespace"], resource.attributes["k8s.namespace.name"])
+              - set(log.body, resource.attributes["body_json"])
+              - delete_key(resource.attributes, "body_json")
+      batch:
+        send_batch_size: 5
+        timeout: 10s  
+    exporters:
+      otlphttp/logs:
+        endpoint: {{ .Values.logIngest.lokiEndpoint }}/otlp
+        tls:
+          insecure: true
+    service:
+      telemetry:
+        logs:
+          level: info
+      pipelines:
+        logs:
+          receivers: [filelog]
+          processors: [resource, transform, batch]
+          exporters: [otlphttp/logs]
+{{- end }}

freeleaps/helm-pkg/centralStorage/values.alpha.yaml

@@ -4,6 +4,11 @@ global:
   nodeSelector: {}
 dashboard:
   enabled: false
+logIngest:
+  enabled: true
+  lokiEndpoint: http://loki-gateway.freeleaps-logging-system
+  logPathPattern: /app/log/central_storage/*.log
+  logPath: /app/log/central_storage
 centralStorage:
   replicas: 1
   image:

freeleaps/helm-pkg/centralStorage/values.yaml

@@ -7,6 +7,11 @@ dashboard:
   name: freeleaps-prod-central-storage-dashboard
   title: Central Storage Service Dashboard
   metricsPrefix: freeleaps_central_storage
+logIngest:
+  enabled: false
+  lokiEndpoint: http://loki-gateway.freeleaps-logging-system/loki/api/v1/push
+  logPathPattern: /app/log/central_storage/*.log
+  logPath: /app/log/central_storage
 central-storage:
   replicas: 1
   image:

freeleaps/helm-pkg/content/templates/content/deployment.yaml

@@ -6,6 +6,10 @@ metadata:
     app.kubernetes.io/name: "content"
     app.kubernetes.io/managed-by: {{ .Release.Service }}
     app.kubernetes.io/instance: {{ .Release.Name }}
+{{- if .Values.logIngest.enabled }}
+  annotations:
+      opentelemetry.io/config-checksum: {{ include (print $.Template.BasePath "/content/opentelemetry.yaml") . | sha256sum }}
+{{- end }}
   name: "content"
   namespace: {{ .Release.Namespace | quote }}
 spec:
@@ -25,7 +29,14 @@ spec:
         app.kubernetes.io/instance: {{ .Release.Name }}
       annotations:
         app.kubernetes.io/config-checksum: {{ include (print $.Template.BasePath "/content/content-config.yaml") . | sha256sum }}
+{{- if .Values.logIngest.enabled }}
+        opentelemetry.io/config-checksum: {{ include (print $.Template.BasePath "/content/opentelemetry.yaml") . | sha256sum }}
+        sidecar.opentelemetry.io/inject: "{{ .Release.Namespace}}/{{ .Release.Name }}-opentelemetry-collector"
+{{- end }}
     spec:
+{{- if .Values.logIngest.enabled }}
+      serviceAccountName: "{{ .Release.Name }}-otel-collector"
+{{- end }}
       containers:
         - name: "content"
           image: "{{ coalesce .Values.content.image.registry .Values.global.registry "docker.io"}}/{{ coalesce .Values.content.image.repository .Values.global.repository }}/{{ .Values.content.image.name }}:{{ .Values.content.image.tag | default "latest" }}"
@@ -95,3 +106,13 @@ spec:
                   name: content-config
                   key: {{ $key | snakecase | upper }}
             {{- end }}
+{{- if .Values.logIngest.enabled }}
+          volumeMounts:
+            - name: app-logs
+              mountPath: {{ .Values.logIngest.logPath }}
+{{- end }}
+{{- if .Values.logIngest.enabled }}
+      volumes:
+        - name: app-logs
+          emptyDir: {}
+{{- end }}

freeleaps/helm-pkg/content/templates/content/opentelemetry-rbac.yaml

@@ -0,0 +1,46 @@
+{{- if .Values.logIngest.enabled }}
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ .Release.Name }}-otel-collector
+  namespace: {{ .Release.Namespace }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ .Release.Name }}-otel-collector
+rules:
+  - apiGroups: [""]
+    resources:
+      - pods
+      - namespaces
+      - nodes
+    verbs:
+      - get
+      - watch
+      - list
+  - apiGroups: ["apps"]
+    resources:
+      - replicasets
+      - deployments
+      - statefulsets
+      - daemonsets
+    verbs:
+      - get
+      - watch
+      - list
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ .Release.Name }}-otel-collector
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ .Release.Name }}-otel-collector
+subjects:
+  - kind: ServiceAccount
+    name: {{ .Release.Name }}-otel-collector
+    namespace: {{ .Release.Namespace }}
+{{- end }}

freeleaps/helm-pkg/content/templates/content/opentelemetry.yaml

@@ -0,0 +1,115 @@
+{{- if .Values.logIngest.enabled }}
+apiVersion: opentelemetry.io/v1beta1
+kind: OpenTelemetryCollector
+metadata:
+  name: {{ .Release.Name }}-opentelemetry-collector
+  namespace: {{ .Release.Namespace }}
+spec:
+  mode: sidecar
+  image: ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib:latest
+  serviceAccount: "{{ .Release.Name }}-otel-collector"
+  volumeMounts:
+    - name: app-logs
+      mountPath: {{ .Values.logIngest.logPath }}
+  securityContext:
+    allowPrivilegeEscalation: true
+    privileged: true
+    runAsUser: 0
+    runAsGroup: 0
+  env:
+    - name: KUBE_META_POD_NAME
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.name
+    - name: KUBE_META_NAMESPACE
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.namespace
+    - name: KUBE_META_NODE_NAME
+      valueFrom:
+        fieldRef:
+          fieldPath: spec.nodeName
+    - name: KUBE_META_POD_IP
+      valueFrom:
+        fieldRef:
+          fieldPath: status.podIP
+    - name: KUBE_META_POD_UID
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.uid
+    - name: KUBE_META_OBJECT_NAME
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.labels['app.kubernetes.io/instance']
+  config:
+    receivers:
+      filelog:
+        include:
+          - {{ .Values.logIngest.logPathPattern }}
+        start_at: end
+        include_file_path: false
+        include_file_name: false
+        operators:
+          - type: json_parser
+            parse_from: body
+            parse_to: attributes
+    processors:
+      resource:
+        attributes:
+          - action: insert
+            key: k8s.node.name
+            value: ${KUBE_META_NODE_NAME}
+          - action: insert
+            key: k8s.pod.name
+            value: ${KUBE_META_POD_NAME}
+          - action: insert
+            key: k8s.pod.ip
+            value: ${KUBE_META_POD_IP}
+          - action: insert
+            key: k8s.pod.uid
+            value: ${KUBE_META_POD_UID}
+          - action: insert
+            key: k8s.namespace.name
+            value: ${KUBE_META_NAMESPACE}
+          - action: insert
+            key: k8s.deployment.name
+            value: ${KUBE_META_OBJECT_NAME}
+      transform:
+        log_statements:
+          - context: log
+            statements:
+              - set(resource.attributes["application"], log.attributes["context"]["app"])
+              - set(resource.attributes["environment"], log.attributes["context"]["env"])
+              - set(resource.attributes["kubernetes_node_name"], resource.attributes["k8s.node.name"])
+              - set(resource.attributes["kubernetes_pod_name"], resource.attributes["k8s.pod.name"])
+              - set(resource.attributes["kubernetes_pod_ip"], resource.attributes["k8s.pod.ip"])
+              - set(resource.attributes["kubernetes_deployment_name"], resource.attributes["k8s.deployment.name"])
+              - set(resource.attributes["kubernetes_namespace"], resource.attributes["k8s.namespace.name"])
+              - set(resource.attributes["body_json"], ParseJSON(log.body))
+              - set(resource.attributes["body_json"]["kubernetes"]["pod"], resource.attributes["k8s.pod.name"])
+              - set(resource.attributes["body_json"]["kubernetes"]["namespace"], resource.attributes["k8s.namespace.name"])
+              - set(resource.attributes["body_json"]["kubernetes"]["pod_ip"], resource.attributes["k8s.pod.ip"])
+              - set(resource.attributes["body_json"]["kubernetes"]["pod_uid"], resource.attributes["k8s.pod.uid"])
+              - set(resource.attributes["body_json"]["kubernetes"]["deployment"], resource.attributes["k8s.deployment.name"])
+              - set(resource.attributes["body_json"]["kubernetes"]["node"], resource.attributes["k8s.node.name"])
+              - set(resource.attributes["body_json"]["kubernetes"]["namespace"], resource.attributes["k8s.namespace.name"])
+              - set(log.body, resource.attributes["body_json"])
+              - delete_key(resource.attributes, "body_json")
+      batch:
+        send_batch_size: 5
+        timeout: 10s  
+    exporters:
+      otlphttp/logs:
+        endpoint: {{ .Values.logIngest.lokiEndpoint }}/otlp
+        tls:
+          insecure: true
+    service:
+      telemetry:
+        logs:
+          level: info
+      pipelines:
+        logs:
+          receivers: [filelog]
+          processors: [resource, transform, batch]
+          exporters: [otlphttp/logs]
+{{- end }}

freeleaps/helm-pkg/content/values.alpha.yaml

@@ -4,6 +4,11 @@ global:
   nodeSelector: {}
 dashboard:
   enabled: false
+logIngest:
+  enabled: true
+  lokiEndpoint: http://loki-gateway.freeleaps-logging-system
+  logPathPattern: /app/log/content/*.log
+  logPath: /app/log/content
 content:
   replicas: 1
   image:

freeleaps/helm-pkg/content/values.yaml

@@ -7,6 +7,11 @@ dashboard:
   name: freeleaps-prod-content-dashboard
   title: Content Service Dashboard
   metricsPrefix: freeleaps_content
+logIngest:
+  enabled: false
+  lokiEndpoint: http://loki-gateway.freeleaps-logging-system
+  logPathPattern: /app/log/content/*.log
+  logPath: /app/log/content
 content:
   replicas: 1
   image:

freeleaps/helm-pkg/notification/templates/notification/deployment.yaml

@@ -6,6 +6,10 @@ metadata:
     app.kubernetes.io/name: "notification"
     app.kubernetes.io/managed-by: {{ .Release.Service }}
     app.kubernetes.io/instance: {{ .Release.Name }}
+{{- if .Values.logIngest.enabled }}
+  annotations:
+      opentelemetry.io/config-checksum: {{ include (print $.Template.BasePath "/notification/opentelemetry.yaml") . | sha256sum }}
+{{- end }}
   name: "notification"
   namespace: {{ .Release.Namespace | quote }}
 spec:
@@ -25,7 +29,14 @@ spec:
         app.kubernetes.io/instance: {{ .Release.Name }}
       annotations:
         app.kubernetes.io/config-checksum: {{ include (print $.Template.BasePath "/notification/notification-config.yaml") . | sha256sum }}
+{{- if .Values.logIngest.enabled }}
+        opentelemetry.io/config-checksum: {{ include (print $.Template.BasePath "/notification/opentelemetry.yaml") . | sha256sum }}
+        sidecar.opentelemetry.io/inject: "{{ .Release.Namespace}}/{{ .Release.Name }}-opentelemetry-collector"
+{{- end }}
     spec:
+{{- if .Values.logIngest.enabled }}
+      serviceAccountName: "{{ .Release.Name }}-otel-collector"
+{{- end }}
       containers:
         - name: "notification"
           image: "{{ coalesce .Values.notification.image.registry .Values.global.registry "docker.io"}}/{{ coalesce .Values.notification.image.repository .Values.global.repository }}/{{ .Values.notification.image.name }}:{{ .Values.notification.image.tag | default "latest" }}"
@@ -95,4 +106,13 @@ spec:
                   name: notification-config
                   key: {{ $key | snakecase | upper }}
             {{- end }}
-
+{{- if .Values.logIngest.enabled }}
+          volumeMounts:
+            - name: app-logs
+              mountPath: {{ .Values.logIngest.logPath }}
+{{- end }}
+{{- if .Values.logIngest.enabled }}
+      volumes:
+        - name: app-logs
+          emptyDir: {}
+{{- end }}

freeleaps/helm-pkg/notification/templates/notification/opentelemetry-rbac.yaml

@@ -0,0 +1,46 @@
+{{- if .Values.logIngest.enabled }}
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ .Release.Name }}-otel-collector
+  namespace: {{ .Release.Namespace }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ .Release.Name }}-otel-collector
+rules:
+  - apiGroups: [""]
+    resources:
+      - pods
+      - namespaces
+      - nodes
+    verbs:
+      - get
+      - watch
+      - list
+  - apiGroups: ["apps"]
+    resources:
+      - replicasets
+      - deployments
+      - statefulsets
+      - daemonsets
+    verbs:
+      - get
+      - watch
+      - list
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ .Release.Name }}-otel-collector
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ .Release.Name }}-otel-collector
+subjects:
+  - kind: ServiceAccount
+    name: {{ .Release.Name }}-otel-collector
+    namespace: {{ .Release.Namespace }}
+{{- end }}

freeleaps/helm-pkg/notification/templates/notification/opentelemetry.yaml

@@ -0,0 +1,115 @@
+{{- if .Values.logIngest.enabled }}
+apiVersion: opentelemetry.io/v1beta1
+kind: OpenTelemetryCollector
+metadata:
+  name: {{ .Release.Name }}-opentelemetry-collector
+  namespace: {{ .Release.Namespace }}
+spec:
+  mode: sidecar
+  image: ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib:latest
+  serviceAccount: "{{ .Release.Name }}-otel-collector"
+  volumeMounts:
+    - name: app-logs
+      mountPath: {{ .Values.logIngest.logPath }}
+  securityContext:
+    allowPrivilegeEscalation: true
+    privileged: true
+    runAsUser: 0
+    runAsGroup: 0
+  env:
+    - name: KUBE_META_POD_NAME
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.name
+    - name: KUBE_META_NAMESPACE
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.namespace
+    - name: KUBE_META_NODE_NAME
+      valueFrom:
+        fieldRef:
+          fieldPath: spec.nodeName
+    - name: KUBE_META_POD_IP
+      valueFrom:
+        fieldRef:
+          fieldPath: status.podIP
+    - name: KUBE_META_POD_UID
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.uid
+    - name: KUBE_META_OBJECT_NAME
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.labels['app.kubernetes.io/instance']
+  config:
+    receivers:
+      filelog:
+        include:
+          - {{ .Values.logIngest.logPathPattern }}
+        start_at: end
+        include_file_path: false
+        include_file_name: false
+        operators:
+          - type: json_parser
+            parse_from: body
+            parse_to: attributes
+    processors:
+      resource:
+        attributes:
+          - action: insert
+            key: k8s.node.name
+            value: ${KUBE_META_NODE_NAME}
+          - action: insert
+            key: k8s.pod.name
+            value: ${KUBE_META_POD_NAME}
+          - action: insert
+            key: k8s.pod.ip
+            value: ${KUBE_META_POD_IP}
+          - action: insert
+            key: k8s.pod.uid
+            value: ${KUBE_META_POD_UID}
+          - action: insert
+            key: k8s.namespace.name
+            value: ${KUBE_META_NAMESPACE}
+          - action: insert
+            key: k8s.deployment.name
+            value: ${KUBE_META_OBJECT_NAME}
+      transform:
+        log_statements:
+          - context: log
+            statements:
+              - set(resource.attributes["application"], log.attributes["context"]["app"])
+              - set(resource.attributes["environment"], log.attributes["context"]["env"])
+              - set(resource.attributes["kubernetes_node_name"], resource.attributes["k8s.node.name"])
+              - set(resource.attributes["kubernetes_pod_name"], resource.attributes["k8s.pod.name"])
+              - set(resource.attributes["kubernetes_pod_ip"], resource.attributes["k8s.pod.ip"])
+              - set(resource.attributes["kubernetes_deployment_name"], resource.attributes["k8s.deployment.name"])
+              - set(resource.attributes["kubernetes_namespace"], resource.attributes["k8s.namespace.name"])
+              - set(resource.attributes["body_json"], ParseJSON(log.body))
+              - set(resource.attributes["body_json"]["kubernetes"]["pod"], resource.attributes["k8s.pod.name"])
+              - set(resource.attributes["body_json"]["kubernetes"]["namespace"], resource.attributes["k8s.namespace.name"])
+              - set(resource.attributes["body_json"]["kubernetes"]["pod_ip"], resource.attributes["k8s.pod.ip"])
+              - set(resource.attributes["body_json"]["kubernetes"]["pod_uid"], resource.attributes["k8s.pod.uid"])
+              - set(resource.attributes["body_json"]["kubernetes"]["deployment"], resource.attributes["k8s.deployment.name"])
+              - set(resource.attributes["body_json"]["kubernetes"]["node"], resource.attributes["k8s.node.name"])
+              - set(resource.attributes["body_json"]["kubernetes"]["namespace"], resource.attributes["k8s.namespace.name"])
+              - set(log.body, resource.attributes["body_json"])
+              - delete_key(resource.attributes, "body_json")
+      batch:
+        send_batch_size: 5
+        timeout: 10s  
+    exporters:
+      otlphttp/logs:
+        endpoint: {{ .Values.logIngest.lokiEndpoint }}/otlp
+        tls:
+          insecure: true
+    service:
+      telemetry:
+        logs:
+          level: info
+      pipelines:
+        logs:
+          receivers: [filelog]
+          processors: [resource, transform, batch]
+          exporters: [otlphttp/logs]
+{{- end }}

freeleaps/helm-pkg/notification/values.alpha.yaml

@@ -4,6 +4,11 @@ global:
   nodeSelector: {}
 dashboard:
   enabled: false
+logIngest:
+  enabled: true
+  lokiEndpoint: http://loki-gateway.freeleaps-logging-system
+  logPathPattern: /app/log/notification/*.log
+  logPath: /app/log/notification
 notification:
   replicas: 1
   image:

freeleaps/helm-pkg/notification/values.yaml

@@ -7,6 +7,11 @@ dashboard:
   name: freeleaps-prod-notification-dashboard
   title: Notification Service Dashboard
   metricsPrefix: freeleaps_notification
+logIngest:
+  enabled: false
+  lokiEndpoint: http://loki-gateway.freeleaps-logging-system
+  logPathPattern: /app/log/notification/*.log
+  logPath: /app/log/notification
 notification:
   replicas: 1
   image:

freeleaps/helm-pkg/payment/templates/payment/deployment.yaml

@@ -6,6 +6,10 @@ metadata:
     app.kubernetes.io/name: "payment"
     app.kubernetes.io/managed-by: {{ .Release.Service }}
     app.kubernetes.io/instance: {{ .Release.Name }}
+{{- if .Values.logIngest.enabled }}
+  annotations:
+      opentelemetry.io/config-checksum: {{ include (print $.Template.BasePath "/payment/opentelemetry.yaml") . | sha256sum }}
+{{- end }}
   name: "payment"
   namespace: {{ .Release.Namespace | quote }}
 spec:
@@ -25,7 +29,14 @@ spec:
         app.kubernetes.io/instance: {{ .Release.Name }}
       annotations:
         app.kubernetes.io/config-checksum: {{ include (print $.Template.BasePath "/payment/payment-config.yaml") . | sha256sum }}
+{{- if .Values.logIngest.enabled }}
+        opentelemetry.io/config-checksum: {{ include (print $.Template.BasePath "/payment/opentelemetry.yaml") . | sha256sum }}
+        sidecar.opentelemetry.io/inject: "{{ .Release.Namespace}}/{{ .Release.Name }}-opentelemetry-collector"
+{{- end }}
     spec:
+{{- if .Values.logIngest.enabled }}
+      serviceAccountName: "{{ .Release.Name }}-otel-collector"
+{{- end }}
       containers:
         - name: "payment"
           image: "{{ coalesce .Values.payment.image.registry .Values.global.registry "docker.io"}}/{{ coalesce .Values.payment.image.repository .Values.global.repository }}/{{ .Values.payment.image.name }}:{{ .Values.payment.image.tag | default "latest" }}"
@@ -95,3 +106,13 @@ spec:
                   name: payment-config
                   key: {{ $key | snakecase | upper }}
             {{- end }}
+{{- if .Values.logIngest.enabled }}
+          volumeMounts:
+            - name: app-logs
+              mountPath: {{ .Values.logIngest.logPath }}
+{{- end }}
+{{- if .Values.logIngest.enabled }}
+      volumes:
+        - name: app-logs
+          emptyDir: {}
+{{- end }}

freeleaps/helm-pkg/payment/templates/payment/opentelemetry-rbac.yaml

@@ -0,0 +1,46 @@
+{{- if .Values.logIngest.enabled }}
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ .Release.Name }}-otel-collector
+  namespace: {{ .Release.Namespace }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ .Release.Name }}-otel-collector
+rules:
+  - apiGroups: [""]
+    resources:
+      - pods
+      - namespaces
+      - nodes
+    verbs:
+      - get
+      - watch
+      - list
+  - apiGroups: ["apps"]
+    resources:
+      - replicasets
+      - deployments
+      - statefulsets
+      - daemonsets
+    verbs:
+      - get
+      - watch
+      - list
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ .Release.Name }}-otel-collector
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ .Release.Name }}-otel-collector
+subjects:
+  - kind: ServiceAccount
+    name: {{ .Release.Name }}-otel-collector
+    namespace: {{ .Release.Namespace }}
+{{- end }}

freeleaps/helm-pkg/payment/templates/payment/opentelemetry.yaml

@@ -0,0 +1,115 @@
+{{- if .Values.logIngest.enabled }}
+apiVersion: opentelemetry.io/v1beta1
+kind: OpenTelemetryCollector
+metadata:
+  name: {{ .Release.Name }}-opentelemetry-collector
+  namespace: {{ .Release.Namespace }}
+spec:
+  mode: sidecar
+  image: ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib:latest
+  serviceAccount: "{{ .Release.Name }}-otel-collector"
+  volumeMounts:
+    - name: app-logs
+      mountPath: {{ .Values.logIngest.logPath }}
+  securityContext:
+    allowPrivilegeEscalation: true
+    privileged: true
+    runAsUser: 0
+    runAsGroup: 0
+  env:
+    - name: KUBE_META_POD_NAME
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.name
+    - name: KUBE_META_NAMESPACE
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.namespace
+    - name: KUBE_META_NODE_NAME
+      valueFrom:
+        fieldRef:
+          fieldPath: spec.nodeName
+    - name: KUBE_META_POD_IP
+      valueFrom:
+        fieldRef:
+          fieldPath: status.podIP
+    - name: KUBE_META_POD_UID
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.uid
+    - name: KUBE_META_OBJECT_NAME
+      valueFrom:
+        fieldRef:
+          fieldPath: metadata.labels['app.kubernetes.io/instance']
+  config:
+    receivers:
+      filelog:
+        include:
+          - {{ .Values.logIngest.logPathPattern }}
+        start_at: end
+        include_file_path: false
+        include_file_name: false
+        operators:
+          - type: json_parser
+            parse_from: body
+            parse_to: attributes
+    processors:
+      resource:
+        attributes:
+          - action: insert
+            key: k8s.node.name
+            value: ${KUBE_META_NODE_NAME}
+          - action: insert
+            key: k8s.pod.name
+            value: ${KUBE_META_POD_NAME}
+          - action: insert
+            key: k8s.pod.ip
+            value: ${KUBE_META_POD_IP}
+          - action: insert
+            key: k8s.pod.uid
+            value: ${KUBE_META_POD_UID}
+          - action: insert
+            key: k8s.namespace.name
+            value: ${KUBE_META_NAMESPACE}
+          - action: insert
+            key: k8s.deployment.name
+            value: ${KUBE_META_OBJECT_NAME}
+      transform:
+        log_statements:
+          - context: log
+            statements:
+              - set(resource.attributes["application"], log.attributes["context"]["app"])
+              - set(resource.attributes["environment"], log.attributes["context"]["env"])
+              - set(resource.attributes["kubernetes_node_name"], resource.attributes["k8s.node.name"])
+              - set(resource.attributes["kubernetes_pod_name"], resource.attributes["k8s.pod.name"])
+              - set(resource.attributes["kubernetes_pod_ip"], resource.attributes["k8s.pod.ip"])
+              - set(resource.attributes["kubernetes_deployment_name"], resource.attributes["k8s.deployment.name"])
+              - set(resource.attributes["kubernetes_namespace"], resource.attributes["k8s.namespace.name"])
+              - set(resource.attributes["body_json"], ParseJSON(log.body))
+              - set(resource.attributes["body_json"]["kubernetes"]["pod"], resource.attributes["k8s.pod.name"])
+              - set(resource.attributes["body_json"]["kubernetes"]["namespace"], resource.attributes["k8s.namespace.name"])
+              - set(resource.attributes["body_json"]["kubernetes"]["pod_ip"], resource.attributes["k8s.pod.ip"])
+              - set(resource.attributes["body_json"]["kubernetes"]["pod_uid"], resource.attributes["k8s.pod.uid"])
+              - set(resource.attributes["body_json"]["kubernetes"]["deployment"], resource.attributes["k8s.deployment.name"])
+              - set(resource.attributes["body_json"]["kubernetes"]["node"], resource.attributes["k8s.node.name"])
+              - set(resource.attributes["body_json"]["kubernetes"]["namespace"], resource.attributes["k8s.namespace.name"])
+              - set(log.body, resource.attributes["body_json"])
+              - delete_key(resource.attributes, "body_json")
+      batch:
+        send_batch_size: 5
+        timeout: 10s  
+    exporters:
+      otlphttp/logs:
+        endpoint: {{ .Values.logIngest.lokiEndpoint }}/otlp
+        tls:
+          insecure: true
+    service:
+      telemetry:
+        logs:
+          level: info
+      pipelines:
+        logs:
+          receivers: [filelog]
+          processors: [resource, transform, batch]
+          exporters: [otlphttp/logs]
+{{- end }}

freeleaps/helm-pkg/payment/values.alpha.yaml

@@ -4,6 +4,11 @@ global:
   nodeSelector: {}
 dashboard:
   enabled: false
+logIngest:
+  enabled: true
+  lokiEndpoint: http://loki-gateway.freeleaps-logging-system
+  logPathPattern: /app/log/payment/*.log
+  logPath: /app/log/payment
 payment:
   replicas: 1
   image:

freeleaps/helm-pkg/payment/values.yaml

@@ -7,6 +7,11 @@ dashboard:
   name: freeleaps-prod-payment-dashboard
   title: Payment Service Dashboard
   metricsPrefix: freeleaps_payment
+logIngest:
+  enabled: false
+  lokiEndpoint: http://loki-gateway.freeleaps-logging-system
+  logPathPattern: /app/log/payment/*.log
+  logPath: /app/log/payment
 payment:
   replicas: 1
   image: