CNJR-5954: Copy pipeline changes from branch/release1.21.1 for releas…

…e creation

Jenkinsfile

@@ -48,6 +48,12 @@ These are defined in runConjurTests, and also include the one-offs
 */
 @Library("product-pipelines-shared-library") _
 
+// runSecurityScans uses 2 environment variables to determine where in DefectDojo to 
+// upload the results of the scans. We have to set those variables in both the regular
+// pipeline and the promote block, so use variables for it to keep them consistent.
+def productName = 'Conjur'
+def productTypeName = 'Conjur OSS'
+
 // Automated release, promotion and dependencies
 properties([
   // Include the automated release parameters for the build
@@ -60,23 +66,76 @@ properties([
 
 // Performs release promotion.  No other stages will be run
 if (params.MODE == "PROMOTE") {
-  release.promote(params.VERSION_TO_PROMOTE) { sourceVersion, targetVersion, assetDirectory ->
-    sh "docker pull registry.tld/cyberark/conjur:${sourceVersion}-amd64"
-    sh "docker tag registry.tld/cyberark/conjur:${sourceVersion}-amd64 conjur:${sourceVersion}-amd64"
-    sh "docker pull registry.tld/cyberark/conjur:${sourceVersion}-arm64"
-    sh "docker tag registry.tld/cyberark/conjur:${sourceVersion}-arm64 conjur:${sourceVersion}-arm64"
-
-    sh "docker pull registry.tld/conjur-ubi:${sourceVersion}-amd64"
-    sh "docker tag registry.tld/conjur-ubi:${sourceVersion}-amd64 conjur-ubi:${sourceVersion}-amd64"
-    sh "docker pull registry.tld/conjur-ubi:${sourceVersion}-arm64"
-    sh "docker tag registry.tld/conjur-ubi:${sourceVersion}-arm64 conjur-ubi:${sourceVersion}-arm64"
-
-    // Promote both images for AMD64 and ARM64
-    sh "summon -f ./secrets.yml ./publish-images.sh --promote --redhat --base-version=${sourceVersion} --version=${targetVersion}"
-    sh "summon -f ./secrets.yml ./publish-images.sh --promote --base-version=${sourceVersion} --version=${targetVersion} --arch=arm64"
-    // Promote manifest that links above images
-    sh "summon -f ./secrets.yml ./publish-manifest.sh --promote --base-version=${sourceVersion} --version=${targetVersion}"
+  release.promote(params.VERSION_TO_PROMOTE) { infrapool, sourceVersion, targetVersion, assetDirectory ->
+    env.INFRAPOOL_PRODUCT_NAME = "${productName}"
+    env.INFRAPOOL_DD_PRODUCT_TYPE_NAME = "${productTypeName}"
+
+    def scans = [:]
+
+    scans["Conjur AMD64"] = {
+      stage("Conjur AMD64 scans") {
+        runSecurityScans(infrapool,
+          image: "registry.tld/cyberark/conjur:${sourceVersion}-amd64",
+          buildMode: params.MODE,
+          branch: env.BRANCH_NAME,
+          arch: 'linux/amd64')
+      }
+    }
+
+    scans["Conjur ARM64"] = {
+      stage("Conjur ARM64 scans") {
+        runSecurityScans(infrapool,
+          image: "registry.tld/cyberark/conjur:${sourceVersion}-arm64",
+          buildMode: params.MODE,
+          branch: env.BRANCH_NAME,
+          arch: 'linux/arm64')
+      }
+    }
+
+    scans["Conjur UBI AMD64"] = {
+      stage("Conjur UBI AMD64 scans") {
+        runSecurityScans(infrapool,
+          image: "registry.tld/conjur-ubi:${sourceVersion}-amd64",
+          buildMode: params.MODE,
+          branch: env.BRANCH_NAME,
+          arch: 'linux/amd64')
+      }
+    }
 
+    scans["Conjur UBI ARM64"] = {
+      stage("Conjur UBI ARM64 scans") {
+        runSecurityScans(infrapool,
+          image: "registry.tld/conjur-ubi:${sourceVersion}-arm64",
+          buildMode: params.MODE,
+          branch: env.BRANCH_NAME,
+          arch: 'linux/arm64')
+      }
+    }
+
+    parallel(scans)
+
+    infrapool.agentSh """
+      docker pull registry.tld/cyberark/conjur:${sourceVersion}-amd64
+      docker tag registry.tld/cyberark/conjur:${sourceVersion}-amd64 conjur:${sourceVersion}-amd64
+      docker pull registry.tld/cyberark/conjur:${sourceVersion}-arm64
+      docker tag registry.tld/cyberark/conjur:${sourceVersion}-arm64 conjur:${sourceVersion}-arm64
+
+      docker pull registry.tld/conjur-ubi:${sourceVersion}-amd64
+      docker tag registry.tld/conjur-ubi:${sourceVersion}-amd64 conjur-ubi:${sourceVersion}-amd64
+      docker pull registry.tld/conjur-ubi:${sourceVersion}-arm64
+      docker tag registry.tld/conjur-ubi:${sourceVersion}-arm64 conjur-ubi:${sourceVersion}-arm64
+
+      # Promote both images for AMD64 and ARM64
+      summon -f ./secrets.yml ./publish-images.sh --promote --redhat --base-version=${sourceVersion} --version=${targetVersion}
+      summon -f ./secrets.yml ./publish-images.sh --promote --base-version=${sourceVersion} --version=${targetVersion} --arch=arm64
+      
+      # Promote manifest that links above images
+      summon -f ./secrets.yml ./publish-manifest.sh --promote --dockerhub --base-version=${sourceVersion} --version=${targetVersion}
+    """
+
+    // TODO: In talking to Neil King, this likely won't work until conjurops is migrated over
+    // to github enterprise. In the absence of promoting an OSS conjur release, though, we haven't
+    // tried it since the Conjur repo migrated over.
     // Trigger Conjurops build to push newly promoted releases of conjur to ConjurOps Staging
     build(
       job:'../conjurinc--conjurops/master',
@@ -89,6 +148,7 @@ if (params.MODE == "PROMOTE") {
 
   // Copy Github Enterprise release to Github
   release.copyEnterpriseRelease(params.VERSION_TO_PROMOTE)
+
   return
 }
 
@@ -138,6 +198,9 @@ pipeline {
     MODE = release.canonicalizeMode()
     TAG_SHA = tagWithSHA()
 
+    // Values to direct scan results to the right place in DefectDojo
+    INFRAPOOL_PRODUCT_NAME = "${productName}"
+    INFRAPOOL_DD_PRODUCT_TYPE_NAME = "${productTypeName}"
   }
 
   stages {
@@ -306,49 +369,65 @@ pipeline {
           }
         }
 
-        stage('Scan Docker Image') {
-          when {
-            expression { params.RUN_ONLY == '' }
-          }
+        stage('Run security scans') {
           parallel {
-            stage("Scan Docker Image for fixable issues (AMD64)") {
+            stage('AMD64 Ubuntu-based Docker image scans') { 
               steps {
-                scanAndReport(INFRAPOOL_EXECUTORV2_AGENT_0, "conjur:${TAG_SHA}", "HIGH", false)
+                runSecurityScans(INFRAPOOL_EXECUTORV2_AGENT_0, 
+                  image: "registry.tld/conjur:${TAG_SHA}",
+                  buildMode: MODE,
+                  branch: env.BRANCH_NAME,
+                  arch: "linux/amd64")
               }
             }
-            stage("Scan Docker image for total issues (AMD64)") {
+
+            stage('ARM64 Ubuntu-based Docker image scans') { 
               steps {
-                scanAndReport(INFRAPOOL_EXECUTORV2_AGENT_0, "conjur:${TAG_SHA}", "NONE", true)
+                runSecurityScans(INFRAPOOL_EXECUTORV2ARM_AGENT_0, 
+                  image: "registry.tld/conjur:${TAG_SHA}",
+                  buildMode: MODE,
+                  branch: env.BRANCH_NAME,
+                  arch: "linux/arm64")
               }
             }
-            stage("Scan UBI-based Docker Image for fixable issues (AMD64)") {
+
+            stage('AMD64 UBI-based Docker image scans') {
               steps {
-                scanAndReport(INFRAPOOL_EXECUTORV2_AGENT_0, "conjur-ubi:${TAG_SHA}", "HIGH", false)
+                runSecurityScans(INFRAPOOL_EXECUTORV2_AGENT_1, 
+                  image: "registry.tld/conjur-ubi:${TAG_SHA}",
+                  buildMode: MODE,
+                  branch: env.BRANCH_NAME,
+                  arch: "linux/amd64")
               }
             }
-            stage("Scan UBI-based Docker image for total issues (AMD64)") {
+
+            stage('ARM64 UBI-based Docker image scans') {
               steps {
-                scanAndReport(INFRAPOOL_EXECUTORV2_AGENT_0, "conjur-ubi:${TAG_SHA}", "NONE", true)
+                runSecurityScans(INFRAPOOL_EXECUTORV2ARM_AGENT_0, 
+                  image: "registry.tld/conjur-ubi:${TAG_SHA}",
+                  buildMode: MODE,
+                  branch: env.BRANCH_NAME,
+                  arch: "linux/arm64")
               }
             }
-            stage("Scan Docker Image for fixable issues (ARM64)") {
+
+            stage('AMD64 Conjur-Test Docker image scans') {
               steps {
-                scanAndReport(INFRAPOOL_EXECUTORV2ARM_AGENT_0, "conjur:${TAG_SHA}", "HIGH", false)
+                runSecurityScans(INFRAPOOL_EXECUTORV2_AGENT_2, 
+                  image: "registry.tld/conjur-test:${TAG_SHA}",
+                  buildMode: MODE,
+                  branch: env.BRANCH_NAME,
+                  arch: "linux/amd64")
               }
             }
-            stage("Scan Docker image for total issues (ARM64)") {
+
+            stage('ARM64 Conjur-Test Docker image scans') {
               steps {
-                scanAndReport(INFRAPOOL_EXECUTORV2ARM_AGENT_0, "conjur:${TAG_SHA}", "NONE", true)
-              }
-            }
-            stage("Scan UBI-based Docker Image for fixable issues (ARM64)") {
-              steps {
-                scanAndReport(INFRAPOOL_EXECUTORV2ARM_AGENT_0, "conjur-ubi:${TAG_SHA}", "HIGH", false)
-              }
-            }
-            stage("Scan UBI-based Docker image for total issues (ARM64)") {
-              steps {
-                scanAndReport(INFRAPOOL_EXECUTORV2ARM_AGENT_0, "conjur-ubi:${TAG_SHA}", "NONE", true)
+                runSecurityScans(INFRAPOOL_EXECUTORV2ARM_AGENT_0, 
+                  image: "registry.tld/conjur-test:${TAG_SHA}",
+                  buildMode: MODE,
+                  branch: env.BRANCH_NAME,
+                  arch: "linux/arm64")              
               }
             }
           }
@@ -963,17 +1042,6 @@ pipeline {
       }
 
       post {
-        success {
-          script {
-            if (env.BRANCH_NAME == 'master') {
-              build(
-                job:'/Conjur-Enterprise/Conjur-Enterprise-secrets-provider-for-k8s/main/Conjur-Enterprise-secrets-provider-for-k8s-main-full/main',
-                wait: false
-              )
-            }
-          }
-        }
-
         always {
           script {
             if (testShouldRunOnAgent(params.RUN_ONLY, runSpecificTestOnAgent(params.RUN_ONLY, NESTED_ARRAY_OF_TESTS_TO_RUN[0]))) {
@@ -1040,15 +1108,6 @@ pipeline {
               keepAll: true
             ])
 
-            publishHTML(
-              reportName: 'Coverage Report',
-              reportDir: 'coverage',
-              reportFiles: 'index.html',
-              reportTitles: '',
-              allowMissing: false,
-              alwaysLinkToLastBuild: true,
-              keepAll: true
-            )
             junit('''
               spec/reports/*.xml,
               spec/reports-audit/*.xml,
@@ -1083,7 +1142,23 @@ pipeline {
           INFRAPOOL_EXECUTORV2_AGENT_0.agentStash name: 'coverage', includes: 'coverage/**'
           unstash 'coverage'
           archiveFiles('coverage/*.xml')
-          codacy action: 'reportCoverage', filePath: "coverage/coverage.xml"
+
+          cobertura(
+            autoUpdateHealth: false,
+            autoUpdateStability: false,
+            coberturaReportFile: 'coverage/coverage.xml',
+            conditionalCoverageTargets: '70, 0, 0',
+            failUnhealthy: false,
+            failUnstable: false,
+            maxNumberOfBuilds: 0,
+            lineCoverageTargets: '70, 0, 0',
+            methodCoverageTargets: '70, 0, 0',
+            onlyStable: false,
+            sourceEncoding: 'ASCII',
+            zoomCoverageChart: false
+          )
+
+          codacy(action: 'reportCoverage', filePath: "coverage/coverage.xml")
         }
       }
     }
@@ -1249,6 +1324,11 @@ def conjurTests(infrapool) {
         infrapool.agentSh 'ci/test proxy'
       }
     ],
+    // "ipv6": [
+    //   "IPv6 - ${env.STAGE_NAME}": {
+    //     infrapool.agentSh 'ci/test ipv6'
+    //   }
+    // ],
     "rotators": [
       "Rotators - ${env.STAGE_NAME}": {
         infrapool.agentSh 'ci/test rotators'

build_utils.sh

@@ -25,7 +25,7 @@ function tag_and_push() {
 
   for image in "$@"; do
     local target=$image:$tag
-    echo "Tagging and pushing $target..."
+    echo "Tagging $source as $target and pushing..."
     docker tag "$source" "$target"
     docker push "$target"
   done

ci/test_suites/authenticators_gcp/deploy_function.sh

@@ -68,7 +68,7 @@ deploy_function() {
   cd "$GCF_SOURCE_DIR"
 
   echo "-- Deploying function: $GCF_FUNC_NAME"
-  gcloud functions deploy "$GCF_FUNC_NAME" --runtime python37 --trigger-http --quiet
+  gcloud functions deploy "$GCF_FUNC_NAME" --no-gen2 --runtime python37 --trigger-http --quiet
 
   echo '-> deploy_function done'
 }

publish-images.sh

@@ -118,23 +118,29 @@ if [[ "${PUBLISH_EDGE}" = true ]]; then
 
   # Publish release specific and edge tags to dockerhub
   if [[ "${DOCKERHUB}" = true ]]; then
+    echo "Pushing to DockerHub"
+
     tag_and_push "${VERSION}" "${LOCAL_IMAGE}" "${IMAGE_NAME}"
     tag_and_push "edge" "${LOCAL_IMAGE}" "${IMAGE_NAME}"
   fi
 fi
 
 if [[ "${PROMOTE}" = true ]]; then
   echo "Promoting image to ${VERSION}-${ARCH}"
-
+  
   # Push latest, 1.x.y, 1.x, and 1 images
   readarray -t prefix_versions < <(gen_versions "${VERSION}")
 
   for version in latest "${prefix_versions[@]}"; do
-    tag_and_push "${version}-${ARCH}" "${LOCAL_IMAGE}" "registry.tld/${IMAGE_NAME}"
-    tag_and_push "${version}-${ARCH}" "${RH_LOCAL_IMAGE}" "registry.tld/conjur-ubi"
+    echo "Pushing images for tag: $version-${ARCH}"
+
+    tag_and_push "${version}-${ARCH}" "registry.tld/${IMAGE_NAME}:${LOCAL_TAG}-${ARCH}" "registry.tld/${IMAGE_NAME}"
+    tag_and_push "${version}-${ARCH}" "registry.tld/conjur-ubi:${LOCAL_TAG}-${ARCH}" "registry.tld/conjur-ubi"
 
     if [[ "${DOCKERHUB}" ]]; then
-      tag_and_push "${version}" "${LOCAL_IMAGE}" "${IMAGE_NAME}"
+      echo "Pushing to DockerHub"
+
+      tag_and_push "${version}-${ARCH}" "${LOCAL_IMAGE}-${ARCH}" "${IMAGE_NAME}"
     fi
   done
 fi

publish-manifest.sh

@@ -22,6 +22,7 @@ function print_help() {
 PUBLISH_EDGE=false
 PUBLISH_INTERNAL=false
 PROMOTE=false
+DOCKERHUB=false
 VERSION=$(<VERSION)
 
 LOCAL_TAG="$(version_tag)"
@@ -40,6 +41,10 @@ for arg in "$@"; do
       PROMOTE=true
       shift
       ;;
+    --dockerhub )
+      DOCKERHUB=true
+      shift
+      ;;
     --version=* )
       VERSION="${arg#*=}"
       shift
@@ -86,6 +91,14 @@ if [[ "${PUBLISH_EDGE}" = true ]]; then
   # Push manifest to internal registry
   prepare_manifest "registry.tld/${IMAGE_NAME}" "edge"
   prepare_manifest "registry.tld/conjur-ubi" "edge"
+
+  # Publish manifests to dockerhub
+  if [[ "${DOCKERHUB}" = true ]]; then
+    echo "Pushing to DockerHub"
+
+    prepare_manifest "${IMAGE_NAME}" "${VERSION}"
+    prepare_manifest "${IMAGE_NAME}" "edge"
+  fi
 fi
 
 if [[ "${PROMOTE}" = true ]]; then
@@ -95,7 +108,16 @@ if [[ "${PROMOTE}" = true ]]; then
   readarray -t prefix_versions < <(gen_versions "${VERSION}")
 
   for version in latest "${prefix_versions[@]}"; do
+    echo "Preparing manifests for tag: $version"
+
     prepare_manifest "registry.tld/${IMAGE_NAME}" "${version}"
     prepare_manifest "registry.tld/conjur-ubi" "${version}"
+
+    # Publish manifests to dockerhub
+    if [[ "${DOCKERHUB}" = true ]]; then
+      echo "Pushing to DockerHub"
+
+      prepare_manifest "${IMAGE_NAME}" "${version}"
+    fi
   done
 fi