Merge branch 'master' of https://gitea.freeleaps.mathmast.com/freeleaps/freeleaps-ops

freeleaps-devops-reconciler/helm-pkg/reconciler/values.alpha.yaml

@@ -10,7 +10,7 @@ reconciler:
     registry: docker.io
     repository: null
     name: reconciler
-    tag: 1.0.1
+    tag: snapshot-2a5bb92
     imagePullPolicy: IfNotPresent
   ports:
   - name: http

freeleaps/alpha/ci/freeleaps2-frontend/Jenkinsfile

@@ -3,7 +3,7 @@ library 'first-class-pipeline'
 executeFreeleapsPipeline {
   serviceName = 'freeleaps'
   environmentSlug = 'alpha'
-  serviceGitBranch = 'dev'
+  serviceGitBranch = 'Nicolas_log'
   serviceGitRepo = "https://gitea.freeleaps.mathmast.com/products/freeleaps.git"
   serviceGitRepoType = 'monorepo'
   serviceGitCredentialsId = 'freeleaps-repos-gitea-credentails'
@@ -17,7 +17,7 @@ executeFreeleapsPipeline {
       dependenciesManager: 'pip',
       requirementsFile: 'chat/requirements.txt',
       buildCacheEnabled: true,
-      buildAgentImage: 'python:3.10-slim-buster',
+      buildAgentImage: 'python:3.10-slim-bullseye',
       buildArtifacts: ['.'],
       lintEnabled: false,
       sastEnabled: false,
@@ -38,7 +38,7 @@ executeFreeleapsPipeline {
       dependenciesManager: 'pip',
       requirementsFile: 'freeleaps/requirements.txt',
       buildCacheEnabled: true,
-      buildAgentImage: 'python:3.10-slim-buster',
+      buildAgentImage: 'python:3.10-slim-bullseye',
       buildArtifacts: ['.'],
       lintEnabled: false,
       sastEnabled: false,

freeleaps/helm-pkg/authentication/values.alpha.yaml

@@ -15,7 +15,7 @@ authentication:
     registry: docker.io
     repository: null
     name: authentication
-    tag: snapshot-b9f275c
+    tag: snapshot-311fd21
     imagePullPolicy: IfNotPresent
   ports:
   - name: http

freeleaps/helm-pkg/centralStorage/templates/central-storage/central-storage-config.yaml

@@ -16,3 +16,7 @@ data:
   AZURE_STORAGE_DOCUMENT_API_ENDPOINT: {{ .Values.centralStorage.configs.azureStorageDocumentApiEndpoint | b64enc | quote }}
   METRICS_ENABLED: {{ .Values.centralStorage.configs.metricsEnabled | default false | toString | b64enc }}
   PROBES_ENABLED: {{ .Values.centralStorage.configs.probesEnabled | default false | toString | b64enc }}
+  DEBUG_MODE: {{ .Values.centralStorage.configs.debugMode | default false | toString | b64enc }}
+  LOG_BASE_PATH: {{ .Values.logIngest.logPath | b64enc | quote }}
+  BACKEND_LOG_FILE_NAME: {{ .Values.centralStorage.configs.appName | b64enc | quote }}
+  APPLICATION_ACTIVITY_LOG: {{ .Values.centralStorage.configs.appName | printf "%s-activity" | b64enc | quote }}

freeleaps/helm-pkg/centralStorage/templates/central-storage/deployment.yaml

@@ -114,5 +114,10 @@ spec:
 {{- if .Values.logIngest.enabled }}
       volumes:
         - name: app-logs
+          {{- if and (hasKey .Values.logIngest "persistence") (.Values.logIngest.persistence.enabled) }}
+          persistentVolumeClaim:
+            claimName: {{ .Release.Name }}-logs-pvc
+          {{- else }}
           emptyDir: {}
           {{- end }}
+{{- end }}

freeleaps/helm-pkg/centralStorage/templates/central-storage/opentelemetry.yaml

@@ -47,12 +47,23 @@ spec:
         include:
           - {{ .Values.logIngest.logPathPattern }}
         start_at: end
+        poll_interval: 1s
         include_file_path: false
         include_file_name: false
         operators:
           - type: json_parser
             parse_from: body
             parse_to: attributes
+      otlp:
+        protocols:
+          grpc:
+            endpoint: 0.0.0.0:4317
+          http:
+            endpoint: 0.0.0.0:4318
+      # Add container logs receiver
+      k8scluster:
+        collection_interval: 1s
+        auth_type: serviceAccount
     processors:
       resource:
         attributes:
@@ -96,20 +107,22 @@ spec:
               - set(log.body, resource.attributes["body_json"])
               - delete_key(resource.attributes, "body_json")
       batch:
-        send_batch_size: 5
+        send_batch_size: 1
-        timeout: 10s  
+        timeout: 1s  
     exporters:
       otlphttp/logs:
         endpoint: {{ .Values.logIngest.lokiEndpoint }}/otlp
         tls:
           insecure: true
+        headers:
+          X-Scope-OrgID: "central-storage"
     service:
       telemetry:
         logs:
           level: info
       pipelines:
         logs:
-          receivers: [filelog]
+          receivers: [filelog, otlp, k8scluster]
           processors: [resource, transform, batch]
           exporters: [otlphttp/logs]
 {{- end }}

freeleaps/helm-pkg/centralStorage/templates/central-storage/pvc.yaml

@@ -0,0 +1,19 @@
+{{- if and .Values.logIngest.enabled (hasKey .Values.logIngest "persistence") (.Values.logIngest.persistence.enabled) }}
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ .Release.Name }}-logs-pvc
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: "central-storage"
+    app.kubernetes.io/instance: {{ .Release.Name }}
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: {{ .Values.logIngest.persistence.size | default "1Gi" }}
+  {{- if .Values.logIngest.persistence.storageClass }}
+  storageClassName: {{ .Values.logIngest.persistence.storageClass }}
+  {{- end }}
+{{- end }} 

freeleaps/helm-pkg/centralStorage/values.alpha.yaml

@@ -15,7 +15,7 @@ centralStorage:
     registry: docker.io
     repository: null
     name: central_storage
-    tag: snapshot-b9f275c
+    tag: snapshot-311fd21
     imagePullPolicy: IfNotPresent
   ports:
   - name: http
@@ -87,6 +87,7 @@ centralStorage:
     azureStorageDocumentApiEndpoint: https://freeleaps1document.blob.core.windows.net/
     metricsEnabled: 'false'
     probesEnabled: 'true'
+    debugMode: 'false'
   vpa:
     minAllowed:
       enabled: false

freeleaps/helm-pkg/chat/templates/chat/opentelemetry.yaml

@@ -53,6 +53,9 @@ spec:
           - type: json_parser
             parse_from: body
             parse_to: attributes
+          - type: move
+            from: attributes.text
+            to: body
     processors:
       resource:
         attributes:
@@ -78,23 +81,13 @@ spec:
         log_statements:
           - context: log
             statements:
-              - set(resource.attributes["application"], log.attributes["context"]["app"])
+              - set(resource.attributes["application"], "chat")
-              - set(resource.attributes["environment"], log.attributes["context"]["env"])
+              - set(resource.attributes["environment"], "{{ .Values.global.environment | default .Release.Namespace }}")
               - set(resource.attributes["kubernetes_node_name"], resource.attributes["k8s.node.name"])
               - set(resource.attributes["kubernetes_pod_name"], resource.attributes["k8s.pod.name"])
               - set(resource.attributes["kubernetes_pod_ip"], resource.attributes["k8s.pod.ip"])
               - set(resource.attributes["kubernetes_deployment_name"], resource.attributes["k8s.deployment.name"])
               - set(resource.attributes["kubernetes_namespace"], resource.attributes["k8s.namespace.name"])
-              - set(resource.attributes["body_json"], ParseJSON(log.body))
-              - set(resource.attributes["body_json"]["kubernetes"]["pod"], resource.attributes["k8s.pod.name"])
-              - set(resource.attributes["body_json"]["kubernetes"]["namespace"], resource.attributes["k8s.namespace.name"])
-              - set(resource.attributes["body_json"]["kubernetes"]["pod_ip"], resource.attributes["k8s.pod.ip"])
-              - set(resource.attributes["body_json"]["kubernetes"]["pod_uid"], resource.attributes["k8s.pod.uid"])
-              - set(resource.attributes["body_json"]["kubernetes"]["deployment"], resource.attributes["k8s.deployment.name"])
-              - set(resource.attributes["body_json"]["kubernetes"]["node"], resource.attributes["k8s.node.name"])
-              - set(resource.attributes["body_json"]["kubernetes"]["namespace"], resource.attributes["k8s.namespace.name"])
-              - set(log.body, resource.attributes["body_json"])
-              - delete_key(resource.attributes, "body_json")
       batch:
         send_batch_size: 5
         timeout: 10s  

freeleaps/helm-pkg/chat/values.alpha.yaml

@@ -1,6 +1,7 @@
 global:
   registry: docker.io
   repository: freeleaps
+  environment: alpha
   nodeSelector: {}
 dashbaord:
   enabled: false
@@ -15,7 +16,7 @@ chat:
     registry: docker.io
     repository: null
     name: chat
-    tag: snapshot-97090e9
+    tag: snapshot-34206c8
     imagePullPolicy: IfNotPresent
   ports:
   - name: http
@@ -24,10 +25,10 @@ chat:
   resources:
     requests:
       cpu: 100m
-      memory: 64Mi
+      memory: 128Mi
     limits:
       cpu: 200m
-      memory: 128Mi
+      memory: 256Mi
   probes:
     readiness:
       type: httpGet

freeleaps/helm-pkg/chat/values.prod.yaml

@@ -1,6 +1,7 @@
 global:
   registry: docker.io
   repository: freeleaps
+  environment: prod
   nodeSelector: {}
 dashboard:
   enabled: true
@@ -13,7 +14,7 @@ chat:
     registry: docker.io
     repository: null
     name: chat
-    tag: snapshot-e46efad
+    tag: snapshot-e42d7f6
     imagePullPolicy: Always
   ports:
   - name: http

freeleaps/helm-pkg/chat/values.yaml

@@ -1,6 +1,7 @@
 global:
   registry: docker.io
   repository: freeleaps
+  environment: dev
   nodeSelector: {}
 dashboard:
   enabled: false
@@ -22,7 +23,7 @@ chat:
     imagePullPolicy: IfNotPresent
   ports:
     - name: http
-      containerPort: 8007
+      containerPort: 8012
       protocol: TCP
   resources:
     requests:
@@ -31,13 +32,30 @@ chat:
     limits:
       cpu: "1"
       memory: "1Gi"
-  # FIXME: Wait until the developers implements the probes APIs
+  probes:
-  probes: {}
+    liveness:
+      type: httpGet
+      config:
+        path: /api/_/livez
+        port: 8012
+        initialDelaySeconds: 30
+        periodSeconds: 10
+        timeoutSeconds: 5
+        failureThreshold: 3
+    readiness:
+      type: httpGet
+      config:
+        path: /api/_/readyz
+        port: 8012
+        initialDelaySeconds: 10
+        periodSeconds: 5
+        timeoutSeconds: 3
+        failureThreshold: 3
   services:
     - name: chat-service
       type: ClusterIP
-      port: 8007
+      port: 8012
-      targetPort: 8007
+      targetPort: 8012
       serviceMonitor:
         enabled: false
         labels:
@@ -55,7 +73,7 @@ chat:
     # SERVICE_API_ACCESS_HOST
     serviceApiAccessHost: "0.0.0.0"
     # SERVICE_API_ACCESS_PORT
-    serviceApiAccessPort: "8013"
+    serviceApiAccessPort: "8012"
     # MONGODB_NAME
     mongodbName: ""
     # MONGODB_URI
@@ -107,7 +125,7 @@ chat:
     # METRICS_ENABLED
     metricsEnabled: "false"
     # PROBES_ENABLED
-    probesEnabled: "false"
+    probesEnabled: "true"
   vpa:
     minAllowed:
       enabled: false

freeleaps/helm-pkg/content/values.alpha.yaml

@@ -15,7 +15,7 @@ content:
     registry: docker.io
     repository: null
     name: content
-    tag: snapshot-b9f275c
+    tag: snapshot-311fd21
     imagePullPolicy: IfNotPresent
   ports:
   - name: http

freeleaps/helm-pkg/devops/values.alpha.yaml

@@ -10,7 +10,7 @@ devops:
     registry: docker.io
     repository: null
     name: devops
-    tag: snapshot-b9f275c
+    tag: snapshot-311fd21
     imagePullPolicy: IfNotPresent
   ports:
   - name: http
@@ -44,15 +44,15 @@ devops:
     jwtAlgorithm: HS256
     accessTokenExpireMinutes: '3600'
     refreshTokenExpireDays: '1'
-    mongodbName: ''
+    mongodbName: freeleaps2
     mongodbPort: '27017'
-    mongodbUri: ''
+    mongodbUri: mongodb+srv://jetli:8IHKx6dZK8BfugGp@freeleaps2.hanbj.mongodb.net/
     metricsEnabled: 'false'
     probesEnabled: 'true'
     baseGiteaUrl: https://gitea.freeleaps.mathmast.com
     baseReconcileUrl: https://reconcile.freeleaps.mathmast.com
     baseLokiUrl: http://loki-gateway.freeleaps-logging-system
-    logBasePath: /app/log
+    logBasePath: /app/log/devops
     logRetention: 30 days
     logRotation: 00:00
     logBackupFiles: '5'

freeleaps/helm-pkg/devsvc/templates/devsvc/devsvc-config.yaml

@@ -19,3 +19,5 @@ data:
   CODE_DEPOT_DOMAIN_NAME: {{ .Values.devsvc.configs.codeDepotDomainName | b64enc | quote }}
   METRICS_ENABLED: {{ .Values.devsvc.configs.metricsEnabled | default false | toString | b64enc }}
   PROBES_ENABLED: {{ .Values.devsvc.configs.probesEnabled | default false | toString | b64enc }}
+  LOG_BASE_PATH: {{ .Values.devsvc.configs.logBasePath | default "/app/log/devsvc" | b64enc | quote }}
+  APPLICATION_ACTIVITY_LOG: {{ .Values.devsvc.configs.appName | default "devsvc" | printf "%s-application-activity" | b64enc | quote }}

freeleaps/helm-pkg/devsvc/templates/devsvc/opentelemetry.yaml

@@ -46,13 +46,24 @@ spec:
       filelog:
         include:
           - {{ .Values.logIngest.logPathPattern }}
+          - {{ .Values.logIngest.logPath }}/.log
         start_at: end
         include_file_path: false
         include_file_name: false
+        poll_interval: 1s
         operators:
           - type: json_parser
             parse_from: body
             parse_to: attributes
+      otlp:
+        protocols:
+          grpc:
+            endpoint: 0.0.0.0:4317
+          http:
+            endpoint: 0.0.0.0:4318
+      k8scluster:
+        collection_interval: 1s
+        auth_type: serviceAccount
     processors:
       resource:
         attributes:
@@ -96,8 +107,8 @@ spec:
               - set(log.body, resource.attributes["body_json"])
               - delete_key(resource.attributes, "body_json")
       batch:
-        send_batch_size: 5
+        send_batch_size: 1
-        timeout: 10s  
+        timeout: 1s  
     exporters:
       otlphttp/logs:
         endpoint: {{ .Values.logIngest.lokiEndpoint }}/otlp
@@ -109,7 +120,7 @@ spec:
           level: info
       pipelines:
         logs:
-          receivers: [filelog]
+          receivers: [filelog, otlp, k8scluster]
           processors: [resource, transform, batch]
           exporters: [otlphttp/logs]
 {{- end }}

freeleaps/helm-pkg/devsvc/values.alpha.yaml

@@ -90,15 +90,16 @@ devsvc:
     codeDepotDomainName: alpha.gitea.freeleaps.mathmast.com
     metricsEnabled: 'false'
     probesEnabled: 'true'
+    logBasePath: /app/log/devsvc
   vpa:
     minAllowed:
       enabled: false
-      cpu: "50m"
+      cpu: 50m
-      memory: "64Mi"
+      memory: 64Mi
     maxAllowed:
       enabled: true
-      cpu: "200m"
+      cpu: 200m
-      memory: "128Mi"
+      memory: 128Mi
     controlledResources:
     - cpu
     - memory

freeleaps/helm-pkg/freeleaps/templates/freeleaps/freeleaps-config.yaml

@@ -37,3 +37,5 @@ data:
   REDIS_IS_CLUSTER: {{ .Values.freeleaps.configs.redisIsCluster | b64enc | quote }}
   METRICS_ENABLED: {{ .Values.freeleaps.configs.metricsEnabled | default false | toString | b64enc }}
   PROBES_ENABLED: {{ .Values.freeleaps.configs.probesEnabled | default false | toString | b64enc }}
+  GITEA_API_KEY: {{ .Values.freeleaps.configs.giteaApiKey | b64enc | quote }}
+  GITEA_ENDPOINT: {{ .Values.freeleaps.configs.giteaEndpoint | b64enc | quote }}

freeleaps/helm-pkg/freeleaps/templates/freeleaps/opentelemetry.yaml

@@ -53,6 +53,9 @@ spec:
           - type: json_parser
             parse_from: body
             parse_to: attributes
+          - type: move
+            from: attributes.text
+            to: body
     processors:
       resource:
         attributes:
@@ -78,23 +81,13 @@ spec:
         log_statements:
           - context: log
             statements:
-              - set(resource.attributes["application"], log.attributes["context"]["app"])
+              - set(resource.attributes["application"], "freeleaps")
-              - set(resource.attributes["environment"], log.attributes["context"]["env"])
+              - set(resource.attributes["environment"], "{{ .Values.global.environment | default .Release.Namespace }}")
               - set(resource.attributes["kubernetes_node_name"], resource.attributes["k8s.node.name"])
               - set(resource.attributes["kubernetes_pod_name"], resource.attributes["k8s.pod.name"])
               - set(resource.attributes["kubernetes_pod_ip"], resource.attributes["k8s.pod.ip"])
               - set(resource.attributes["kubernetes_deployment_name"], resource.attributes["k8s.deployment.name"])
               - set(resource.attributes["kubernetes_namespace"], resource.attributes["k8s.namespace.name"])
-              - set(resource.attributes["body_json"], ParseJSON(log.body))
-              - set(resource.attributes["body_json"]["kubernetes"]["pod"], resource.attributes["k8s.pod.name"])
-              - set(resource.attributes["body_json"]["kubernetes"]["namespace"], resource.attributes["k8s.namespace.name"])
-              - set(resource.attributes["body_json"]["kubernetes"]["pod_ip"], resource.attributes["k8s.pod.ip"])
-              - set(resource.attributes["body_json"]["kubernetes"]["pod_uid"], resource.attributes["k8s.pod.uid"])
-              - set(resource.attributes["body_json"]["kubernetes"]["deployment"], resource.attributes["k8s.deployment.name"])
-              - set(resource.attributes["body_json"]["kubernetes"]["node"], resource.attributes["k8s.node.name"])
-              - set(resource.attributes["body_json"]["kubernetes"]["namespace"], resource.attributes["k8s.namespace.name"])
-              - set(log.body, resource.attributes["body_json"])
-              - delete_key(resource.attributes, "body_json")
       batch:
         send_batch_size: 5
         timeout: 10s  

freeleaps/helm-pkg/freeleaps/values.alpha.yaml

@@ -1,6 +1,7 @@
 global:
   registry: docker.io
   repository: freeleaps
+  environment: alpha
   nodeSelector: {}
 dashboard:
   enabled: false
@@ -15,7 +16,7 @@ freeleaps:
     registry: docker.io
     repository: null
     name: backend
-    tag: snapshot-97090e9
+    tag: snapshot-34206c8
     imagePullPolicy: IfNotPresent
   ports:
   - name: http
@@ -72,7 +73,7 @@ freeleaps:
     rabbitmqHost: freeleaps-alpha-rabbitmq.freeleaps-alpha.svc.freeleaps.cluster
     rabbitmqPort: 5672
     rabbitmqUsername: user
-    rabbitmqPassword: toKhVTIgRqHe3PLy
+    rabbitmqPassword: 4O80YlxnlhHrjzaM
     freeleapsDevsvcEndpoint: http://devsvc-service.freeleaps-alpha.svc.freeleaps.cluster:8007/api/devsvc/
     freeleapsContentEndpoint: http://content-service.freeleaps-alpha.svc.freeleaps.cluster:8013/api/content/
     freeleapsCentralStorageEndpoint: http://central-storage-service.freeleaps-alpha.svc.freeleaps.cluster:8005/api/central_storage/
@@ -87,6 +88,8 @@ freeleaps:
     redisIsCluster: 'false'
     metricsEnabled: 'false'
     probesEnabled: 'true'
+    giteaApiKey: 737645b8a52b24bf6fc5081c461255fd4c28ab0a
+    giteaEndpoint: https://alpha.gitea.freeleaps.mathmast.com/
   vpa:
     minAllowed:
       enabled: false

freeleaps/helm-pkg/freeleaps/values.prod.yaml

@@ -1,6 +1,7 @@
 global:
   registry: docker.io
   repository: freeleaps
+  environment: prod
   nodeSelector: {}
 dashboard:
   enabled: true
@@ -13,7 +14,7 @@ freeleaps:
     registry: docker.io
     repository: null
     name: backend
-    tag: snapshot-e46efad
+    tag: 1.8.0
     imagePullPolicy: Always
   ports:
   - name: http
@@ -73,6 +74,8 @@ freeleaps:
     redisIsCluster: 'true'
     metricsEnabled: 'true'
     probesEnabled: 'true'
+    giteaApiKey: a61216761e1cda4797c1bc13c4cc26472e9e1eb3
+    giteaEndpoint: https://gitea.freeleaps.mathmast.com/
   vpa:
     minAllowed:
       enabled: true

freeleaps/helm-pkg/freeleaps/values.yaml

@@ -1,6 +1,7 @@
 global:
   registry: docker.io
   repository: freeleaps
+  environment: dev
   nodeSelector: {}
 dashboard:
   enabled: false
@@ -112,6 +113,11 @@ freeleaps:
     metricsEnabled: "false"
     # PROBES_ENABLED
     probesEnabled: "false"
+    # GITEA_API_KEY
+    giteaApiKey: ""
+    # GITEA_ENDPOINT
+    giteaEndpoint: ""
+
   vpa:
     minAllowed:
       enabled: true

freeleaps/helm-pkg/notification/values.alpha.yaml

@@ -15,7 +15,7 @@ notification:
     registry: docker.io
     repository: null
     name: notification
-    tag: snapshot-b9f275c
+    tag: snapshot-311fd21
     imagePullPolicy: IfNotPresent
   ports:
   - name: http
@@ -83,13 +83,13 @@ notification:
     rabbitmqHost: freeleaps-alpha-rabbitmq.freeleaps-alpha.svc.freeleaps.cluster
     rabbitmqPort: 5672
     rabbitmqUsername: user
-    rabbitmqPassword: toKhVTIgRqHe3PLy
+    rabbitmqPassword: 4O80YlxnlhHrjzaM
     rabbitmqVritualHost: /
     systemUserId: 117f191e810c19729de860aa
     smsFrom: '+16898887156'
     emailFrom: freeleaps@freeleaps.com
     secretKey: 8f87ca8c3c9c3df09a9c78e0adb0927855568f6072d9efc892534aee35f5867b
-    sendgridApiKey: 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
+    sendgridApiKey: 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
     twilioAccountSid: ACf8c9283a6acda060258eadb29be58bc8
     twilioAuthToken: ef160748cc22c8b7195b49df4b8eca7e
     metricsEnabled: 'false'

freeleaps/helm-pkg/payment/values.alpha.yaml

@@ -15,7 +15,7 @@ payment:
     registry: docker.io
     repository: null
     name: payment
-    tag: snapshot-b9f275c
+    tag: snapshot-311fd21
     imagePullPolicy: IfNotPresent
   ports:
   - name: http