#!groovy import com.freeleaps.devops.SourceFetcher import com.freeleaps.devops.DependenciesResolver import com.freeleaps.devops.CommitMessageLinter import com.freeleaps.devops.ChangedComponentsDetector import com.freeleaps.devops.CodeLintExecutor import com.freeleaps.devops.SASTExecutor import com.freeleaps.devops.ImageBuilder import com.freeleaps.devops.SemanticReleasingExecutor import com.freeleaps.devops.enums.DependenciesManager import com.freeleaps.devops.enums.ServiceLanguage import com.freeleaps.devops.enums.CodeLinterTypes import com.freeleaps.devops.enums.ImageBuilderTypes def generateComponentStages(component, configurations) { def stages = [] stages.addAll([ // Build Agent Setup {stage("${component.name} :: Build Agent Setup") { script { if (env.executeMode == "fully" || env.changedComponents.contains(component.name)) { def buildAgentImage = component.buildAgentImage if (buildAgentImage == null || buildAgentImage.isEmpty()) { log.warn("Pipeline", "Not set buildAgentImage for ${component.name}, using default build agent image") def language = ServiceLanguage.parse(component.language) switch(language) { case ServiceLanguage.PYTHON: buildAgentImage = "docker.io/python:3.10-slim-buster" break case ServiceLanguage.JS: buildAgentImage = "docker.io/node:lts-alpine" break default: error("Unknown service language") } } log.info("Pipeline", "Using ${buildAgentImage} as build agent image for ${component.name}") env."${component.name}_buildAgentImage" = buildAgentImage } } }}, // Dependencies Resolving {stage("${component.name} :: Dependencies Resolving") { podTemplate( label: "dep-resolver-${component.name}", containers: [ containerTemplate( name: 'dep-resolver', image: env."${component.name}_buildAgentImage", ttyEnabled: true, command: 'sleep', args: 'infinity' ) ] ) { node("dep-resolver-${component.name}") { container('dep-resolver') { script { if (env.executeMode == "fully" || env.changedComponents.contains(component.name)) { def buildAgentImage = env."${component.name}_buildAgentImage" log.info("Pipeline", "Using ${buildAgentImage} as build agent image for dependencies resolving") def sourceFetcher = new SourceFetcher(this) sourceFetcher.fetch(configurations) def language = ServiceLanguage.parse(component.language) def depManager = DependenciesManager.parse(component.dependenciesManager) def dependenciesResolver = new DependenciesResolver(this, language, env.workroot + "/" + component.root + "/") dependenciesResolver.useManager(depManager) if (component.buildCacheEnabled) { dependenciesResolver.enableCachingSupport() } else { dependenciesResolver.disableCachingSupport() } dependenciesResolver.resolve(component) } } } } } }}, ]) if (component.lintEnabled != null && component.lintEnabled) { stages.addAll([ // Code Linter Environment Preparation {stage("${component.name} :: Code Linter Preparation") { script { if (env.executeMode == "fully" || env.changedComponents.contains(component.name)) { if (component.lintEnabled != null && component.lintEnabled) { log.info("Pipeline", "Code linting has enabled, preparing linter...") if (component.linter == null || component.linter.isEmpty()) { log.error("Pipeline", "Not set linter for ${component.name}, using default linter settings as fallback") } def linter = CodeLinterTypes.parse(component.linter) if (linter == null) { log.error("Pipeline", "Unknown linter for ${component.name}, skipping code linting") } if (linter.language != ServiceLanguage.parse(component.language)) { log.error("Pipeline", "Linter ${linter.linter} is not supported for ${component.language}, skipping code linting") } log.info("Pipeline", "Using ${linter.linter} with image ${linter.containerImage} as linter for ${component.name}") env."${component.name}_linterContainerImage" = linter.containerImage } else { log.info("Pipeline", "Code linting is not enabled for ${component.name}, skipping...") } } } }}, // Code Linting {stage("${component.name} :: Code Linting") { podTemplate( label: "code-linter-${component.name}", containers: [ containerTemplate( name: 'code-linter', image: env."${component.name}_linterContainerImage", ttyEnabled: true, command: 'sleep', args: 'infinity' ) ] ) { node("code-linter-${component.name}") { container('code-linter') { script { if (env.executeMode == "fully" || env.changedComponents.contains(component.name)) { if (component.lintEnabled != null && component.lintEnabled) { log.info("Pipeline", "Code linting has enabled, linting code...") def sourceFetcher = new SourceFetcher(this) sourceFetcher.fetch(configurations) def linterType = CodeLinterTypes.parse(component.linter) def language = ServiceLanguage.parse(component.language) def depManager = DependenciesManager.parse(component.dependenciesManager) // resolving deps from cache def dependenciesResolver = new DependenciesResolver(this, language, env.workroot + "/" + component.root + "/") dependenciesResolver.useManager(depManager) if (component.buildCacheEnabled) { dependenciesResolver.enableCachingSupport() } else { dependenciesResolver.disableCachingSupport() } dependenciesResolver.resolve(component) // lint codes def codeLintExecutor = new CodeLintExecutor(this, env.workroot + "/" + component.root + "/", component.linterConfig, linterType, component) codeLintExecutor.execute() } } } } } } }} ]) } if (component.sastEnabled != null && component.sastEnabled) { stages.addAll([ // SAST Scanner Environment Preparation {stage("${component.name} :: SAST Scanner Preparation") { script { if (env.executeMode == "fully" || env.changedComponents.contains(component.name)) { if (component.sastEnabled != null && component.sastEnabled) { log.info("Pipeline", "SAST scanning has enabled, preparing scanner...") if (sastScanner == null || sastScanner.isEmpty()) { log.error("Pipeline", "Not set sastScanner for ${component.name}") } def sastScannerType = SASTScannerTypes.parse(component.sastScanner) if (sastScannerType == null) { log.error("Pipeline", "Unknown SAST scanner for ${component.name}, skipping SAST scanning") } else if (sastScannerType.language != ServiceLanguage.parse(component.language)) { log.error("Pipeline", "SAST scanner ${sastScannerType.scanner} is not supported for ${component.language}, skipping SAST scanning") } else { log.info("Pipeline", "Using ${sastScanner} as SAST scanner for ${component.name}") env."${component.name}_sastScannerContainerImage" = sastScannerType.containerImage } } } } }}, // SAST Scanning {stage("${component.name} :: SAST Scanning") { when { expression { return (env.executeMode == "fully" || env.changedComponents.contains(component.name)) && env.sastScannerContainerImage != null && !env.sastScannerContainerImage.isEmpty() } } podTemplate( label: "sast-scanner-${component.name}", containers: [ containerTemplate( name: 'sast-scanner', image: env."${component.name}_sastScannerContainerImage", ttyEnabled: true, command: 'sleep', args: 'infinity' ) ] ) { node("sast-scanner-${component.name}") { container('sast-scanner') { script { if (env.executeMode == "fully" || env.changedComponents.contains(component.name)) { if (component.sastEnabled != null && component.sastEnabled) { log.info("Pipeline", "SAST scanning has enabled, scanning code...") def sourceFetcher = new SourceFetcher(this) sourceFetcher.fetch(configurations) def sastScannerType = SASTScannerTypes.parse(component.sastScanner) def sastScanner = new SASTExecutor(this, env.workroot + "/" + component.root + "/", sastScannerType) sastScanner.scan() } } } } } } }} ]) } if (component.semanticReleaseEnabled != null && component.semanticReleaseEnabled) { stages.addAll([ // Semantic Releasing {stage("${component.name} :: Semantic Releasing") { podTemplate( label: "semantic-releasing-${component.name}", containers: [ containerTemplate( name: 'semantic-releasing', image: 'node:18-bullseye-slim', ttyEnabled: true, command: 'sleep', args: 'infinity' ) ] ) { node("semantic-releasing-${component.name}") { container('semantic-releasing') { script { if (env.executeMode == "fully" || env.changedComponents.contains(component.name)) { if (component.semanticReleaseEnabled != null && component.semanticReleaseEnabled) { log.info("Pipeline", "Semantic releasing has enabled, releasing...") if (env.SEMANTIC_RELEASED != null && !env.SEMANTIC_RELEASED.isEmpty() && env.SEMANTIC_RELEASED) { log.info("Pipeline", "Semantic release has been executed, skipping...") return } def sourceFetcher = new SourceFetcher(this) sourceFetcher.fetch(configurations) def semanticReleasingExecutor = new SemanticReleasingExecutor(this, env.workroot) semanticReleasingExecutor.release(configurations.serviceGitCredentialsId) } } } } } } }} ]) } stages.addAll([ // Compilation & Packaging {stage("${component.name} :: Compilation & Packaging") { podTemplate( label: "build-agent-${component.name}", containers: [ containerTemplate( name: 'build-agent', image: env."${component.name}_buildAgentImage", ttyEnabled: true, command: 'sleep', args: 'infinity' ) ] ) { node("build-agent-${component.name}") { container('build-agent') { script { if (env.executeMode == "fully" || env.changedComponents.contains(component.name)) { def buildAgentImage = env."${component.name}_buildAgentImage" log.info("Pipeline", "Using ${buildAgentImage} as build agent image for compilation & packaging") def sourceFetcher = new SourceFetcher(this) sourceFetcher.fetch(configurations) def language = ServiceLanguage.parse(component.language) def depManager = DependenciesManager.parse(component.dependenciesManager) def dependenciesResolver = new DependenciesResolver(this, language, env.workroot + "/" + component.root + "/") dependenciesResolver.useManager(depManager) if (component.buildCacheEnabled) { dependenciesResolver.enableCachingSupport() } else { dependenciesResolver.disableCachingSupport() } dependenciesResolver.resolve(component) dir(env.workroot + "/" + component.root) { if (component.buildCommand != null && !component.buildCommand.isEmpty()) { sh component.buildCommand } component.buildArtifacts.each { artifact -> log.info("Pipeline", "Stashing artifact ${artifact} for ${component.name}...") def artifactList = sh(script: "ls ${artifact} -al", returnStdout: true) log.info("Pipeline", "Artifacts list: ${artifactList}") def targetPathType = sh( script: """ if [ -d "${artifact}" ]; then echo "dir" elif [ -f "${artifact}" ]; then echo "file" else echo "unknown" fi """, returnStdout: true ) if (artifact == '.' || artifact == './') { log.info("Pipeline", "Stashing root directory for ${component.name}...") stash includes: "", name: "${component.name}-root" } else if (targetPathType.trim() == "dir") { stash includes: "${artifact}/**", name: "${component.name}-${artifact}" } else { stash includes: artifact, name: "${component.name}-${artifact}" } } } } } } } } }}, // Image Builder Setup {stage("${component.name} :: Image Builder Setup") { script { if (env.executeMode == "fully" || env.changedComponents.contains(component.name)) { log.info("Pipeline", "Ready to setup image builder for ${component.name}") def imageBuilder if (component.imageBuilder == null || component.imageBuilder.isEmpty()) { log.info("Pipeline", "imageBuilder not set for ${component.name}, using kaniko as default image builder") imageBuilder = ImageBuilderTypes.KANIKO } else { imageBuilder = ImageBuilderTypes.parse(component.imageBuilder) if (imageBuilder == null) { log.error("Pipeline", "Unknown image builder for ${component.name}, skipping image building") } } env."${component.name}_imageBuilderImage" = imageBuilder.image log.info("Pipeline", "Using ${imageBuilder.builder} (image: ${imageBuilder.image}) as image builder for ${component.name}") } } }}, { // Image Building & Publishing stage("${component.name} :: Image Building & Publishing") { podTemplate( label: "image-builder-${component.name}", containers: [ containerTemplate( name: 'image-builder', image: env."${component.name}_imageBuilderImage", privileged: true, ttyEnabled: true, command: 'sleep', args: 'infinity' ) ], volumes: [ hostPathVolume(hostPath: '/var/run/docker.sock', mountPath: '/var/run/docker.sock') ] ) { node("image-builder-${component.name}") { container('image-builder') { script { if (env.executeMode == "fully" || env.changedComponents.contains(component.name)) { def sourceFetcher = new SourceFetcher(this) sourceFetcher.fetch(configurations) dir(env.workroot + "/" + component.root) { component.buildArtifacts.each { artifact -> if (artifact == '.' || artifact == './') { unstash "${component.name}-root" } else { unstash "${component.name}-${artifact}" } } if (component.dockerfile != null && !component.dockerfile.isEmpty()) { log.error("Pipeline", "Component ${component.name} dockerfile not set!") } def imageBuilderType = ImageBuilderTypes.parse(component.imageBuilder) if (imageBuilderType == null) { log.error("Pipeline", "Unknown image builder for ${component.name}, skipping image building") } def imageBuilder = new ImageBuilder(this, env.workroot + "/" + component.root + "/", component.imageBuildRoot, component.dockerfilePath, imageBuilderType ) log.info("Pipeline", "Retrieve version of image from pervious stage...") if (env.LATEST_VERSION == null || env.LATEST_VERSION.isEmpty()) { log.warn("Pipeline", "LATEST_VERSION environment value not set, using 'snapshot-' as default version") def commitHash = sh(script: 'git rev-parse HEAD', returnStdout: true).trim() def shortCommitHash = commitHash.take(7) env.LATEST_VERSION = "snapshot-${shortCommitHash}" } def version imageBuilder.setManifestsOfImage(component.imageRegistry, component.imageRepository, component.imageName, env.LATEST_VERSION) imageBuilder.useCredentials(component.registryCredentialsId) imageBuilder.setArchitectures(component.imageReleaseArchitectures) imageBuilder.build() } } } } } } }, { stage("${component.name} :: Argo Application Version Updating") { script { if (env.executeMode == "fully" || env.changedComponents.contains(component.name)) { def argoApplicationVersionUpdater = new ArgoApplicationVersionUpdater(this) argoApplicationVersionUpdater.update(configurations.environmentSlug, component) } } } }, } ]) return { stages.each { stageClosure -> stageClosure() } } } def call(Closure closure) { def configurations = [:] closure.resolveStrategy = Closure.DELEGATE_FIRST closure.delegate = configurations closure() pipeline { agent any options { buildDiscarder(logRotator(numToKeepStr: '25')) timeout(time: 30, unit: 'MINUTES') parallelsAlwaysFailFast() } stages { stage("Pipeline :: Commit Linting If Enabled") { when { expression { return configurations.commitMessageLintEnabled != null && configurations.commitMessageLintEnabled } } agent { kubernetes { defaultContainer 'commit-message-linter' yaml """ apiVersion: v1 kind: Pod metadata: labels: freeleaps-devops-system/milestone: commit-message-linting spec: containers: - name: commit-message-linter image: docker.io/commitlint/commitlint:master command: - cat tty: true volumeMounts: - name: workspace mountPath: /workspace volumes: - name: workspace emptyDir: {} """ } } steps { script { log.info("Pipeline","Commit message linting is enabled") def sourceFetcher = new SourceFetcher(this) sourceFetcher.fetch(configurations) def linter = new CommitMessageLinter(this) linter.lint(configurations) } } } stage("Pipeline :: Execute Mode Detection") { steps { script { def executeMode = configurations.executeMode if (executeMode == null || executeMode.isEmpty()) { log.warn("Pipeline","Not set executeMode, using fully as default execute mode") env.executeMode = "fully" } else if (executeMode == 'on-demand' && configurations.serviceGitRepoType != 'monorepo') { log.warn("Pipeline","serviceGirRepoType is not monorepo, on-demand mode is not supported, using fully mode") env.executeMode = "fully" } else { log.info("Pipeline","Using ${executeMode} as execute mode") env.executeMode = executeMode } } } } stage("Pipeline :: Code Changes Detection") { when { expression { return env.executeMode == "on-demand" } } steps { script { sourceFetcher.fetch(configurations) def changedComponentsDetector = new ChangedComponentsDetector(this) def changedComponents = changedComponentsDetector.detect(env.workroot, configurations.components) log.info("Pipeline","Changed components: ${changedComponents}") env.changedComponents = changedComponents.join(' ') } } } stage("Pipeline :: Components Build (Dynamic Generated Stages)") { when { expression { return env.executeMode == "fully" || env.changedComponents.size() > 0 } } steps { script { configurations.components.each { component -> log.info("Pipeline", "Executing generated stages for ${component.name}...") generateComponentStages(component, configurations)() } } } } } } }