Jenkinsfile

//
// The main Jenkinsfile for FIBO, defining the Build/Publish/Test/Deploy process that is
// executed for each push into the repository.
//
// Note that this file is in the so called "Declarative Pipeline" syntax
//
// See https://jenkins.io/doc/book/pipeline/jenkinsfile/
//
pipeline {

  agent none

  options {

    buildDiscarder(
      logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '', numToKeepStr: '5')
    )
    //
    // We let each stage running on each jenkins slave / agent decide what to check out or not
    //
    skipDefaultCheckout()
    //
    // Skip stages once the build status has gone to UNSTABLE.
    //
    skipStagesAfterUnstable()
    //
    // There must be SOME limit, if it hangs or whatever then that's a bug and therefore cancel the job.
    //
    timeout(time: 2, unit: 'HOURS')
    //
    // Prepend all console output generated by the Pipeline run with the time at which the line was emitted
    //
    //timestamps()
  }

  stages {

    stage('Check Out') {
      parallel {
        //
        // Run the prepare stage on the stardog node, check out all the repos there
        //
        // NOTE: For now it' the stardog node since we have more processing power there and there are things
        // installed there that we don't have on the jenkins master (such as "nix.sh")
        //
        stage('Stardog Node') {

          agent {
            label 'stardog'
          }

          steps {

            //setGitHubPullRequestStatus context: 'fibo-publish', message: '', state: 'PENDING'

            //
            // Clean the workspace on this node, for now. Takes more time but we need to test this with a clean slate
            // every time. Once this all works fine we can skip this step.
            //
            sh "rm -rf ${env.WORKSPACE}/*"

            //
            // Copy the rdf-toolkit.jar file artifact from the rdf-toolkit-build job (pre-requisite for running publish script)
            //
            step([$class: 'CopyArtifact', filter: '**/rdf-toolkit.jar', fingerprintArtifacts: true, flatten: true, projectName: 'rdf-toolkit-build'])
            //
            // Copy the pellet jar from the build-pellet job
            //
            step([$class: 'CopyArtifact', filter: '**/pellet-cli-*.jar', fingerprintArtifacts: true, flatten: true, projectName: 'build-pellet', target: 'pellet'])
            //
            // Check out the fibo repo' content into the ./fibo directory
            //
            dir('fibo') {
              checkout scm
              echo 'Checked out fibo repo'
            }
            //
            // Then check out the fibo-infra repo content into the ./fibo-infra directory
            //
            dir('fibo-infra') {
              //
              // If you want to check out a specific version of fibo-infra add the "branch:" parameter
              //
              //git branch: 'INFRA-161', credentialsId: 'edmcjenkins_at_edmcouncil.org', url: 'git@github.com:edmcouncil/fibo-infra.git'
              //
              git credentialsId: 'edmcjenkins_at_edmcouncil.org', url: 'git@github.com:edmcouncil/fibo-infra.git'
              echo 'Checked out fibo-infra repo'
              //
              // Stash the nomagic templates so that they can be unstashed on the nomagix box
              //
              stash includes: 'magicdraw/Concept Modeler/**', name: 'nomagic-templates'
              //
              // Stash all the tools in the bin directory
              //
              stash includes: 'bin/**', name: 'infra-bin'
              //
              // Stash all the jenkins related stuff in the jenkins directory
              //
              stash includes: 'jenkins/**', name: 'infra-jenkins'
            }
            //
            // Then run the init command on the fibo-publish script which will set all the GIT_* and JIRA_* vars
            // correctly and store their values in some .env files.
            //
            dir('fibo-infra') {
              echo 'Execute the publish-fibo.sh script:'
              sh './jenkins/bin/publish-fibo.sh init'
              echo 'Finished executing the publish-fibo script'
            }
            initEnvironment()
            //
            // Now stash all files in the env directory for reuse in other workspaces on other agents/slaves
            //
            stash includes: 'env/**', name: 'environment'
            //
            // Then check out the LCC repo into the ./LCC directory
            //
            dir('LCC') {
              git url: 'https://github.com/edmcouncil/LCC.git', credentialsId: 'edmcjenkins'
              echo 'Checked out the LCC repo'
            }
            echo 'Checked it all out'
          }
        } // end of stage 'Prepare Stardog Node'

        stage('NoMagic Node') {

          agent {
            label 'nomagic'
          }

          steps {
            //
            // Clean the workspace on this node
            //
            sh "rm -rf ${env.WORKSPACE}/*"

            //
            // Copy the rdf-toolkit.jar file artifact from the rdf-toolkit-build job (pre-requisite for running publish script)
            //
            step([$class: 'CopyArtifact', filter: '**/rdf-toolkit.jar', fingerprintArtifacts: true, flatten: true, projectName: 'rdf-toolkit-build'])

            dir('fibo') {
              checkout scm
              echo 'Checked out fibo repo'
            }
          }
        } // end of stage 'Prepare NoMagic Node'
      } // end of parallel
    } // end of stage 'Prepare'

    stage('Build') {
      parallel {

        //
        // Run on the stardog node
        //
        // NOTE: For now it' the stardog node since we have more processing power there and there are things
        // installed there that we don't have on the jenkins master (such as "nix.sh")
        //
        stage('Stardog Node') {

          agent {
            label 'stardog'
          }

          steps {
            //
            // Execute the publish script
            //
            // TODO: Split this script up in separate "stages" in this repository
            //
            dir('fibo-infra') {
              echo "Execute the publish-fibo.sh script:"
              sh "./jenkins/bin/publish-fibo.sh"
            }

            //
            // Archive the artifacts generated by the publish-fibo.sh script
            //
            dir('target') {
              stash([
                name: 'publish-script-output-stardog-node',
                includes: '**',
                excludes: '**/.git, **/.gitignore',
                useDefaultExcludes: true
              ])
            }
          }
        } // end of stage 'Build on Stardog Node'

        //
        // Build the artifacts that that need the nomagic installation
        //
        stage('NoMagic Node') {

          agent {
            label 'nomagic'
          }
          environment {
            NOMAGIC_CREDS  = credentials('50cac519-d41c-4765-8563-c43b7f55c877')
            NOMAGIC_USERID = "${env.NOMAGIC_CREDS_USR}"
            NOMAGIC_PASSWD = "${env.NOMAGIC_CREDS_PSW}"
          }

          steps {
            unstash 'environment'
            unstash 'nomagic-templates'
            unstash 'infra-bin'
            unstash 'infra-jenkins'

            initEnvironment()

            echo "NOMAGIC_USERID=${NOMAGIC_USERID}"

            //
            // Copy the nomagic templates (as they are stored in the fibo-infra repo)
            // to the right location
            //
            dir('magicdraw/Concept Modeler') {
              sh 'pwd'
              sh 'find .'
              sh 'cp -vr . /home/ec2-user/MagicDraw/data/defaults/data/reports/Concept\\ Modeler/'
            }

            /* JG>Disabling this now until nomagic process actually gets its input from this job

            withCredentials([
              usernamePassword(
                credentialsId: '50cac519-d41c-4765-8563-c43b7f55c877',
                passwordVariable: 'NOMAGIC_USERID',
                usernameVariable: 'NOMAGIC_PASSWD'
              )
            ]) {
              echo "inside withCredentials block: NOMAGIC_USERID=${NOMAGIC_USERID}"
              withEnv([
                "NOMAGIC_USERID=${env.NOMAGIC_USERID}",
                "NOMAGIC_PASSWD=${env.NOMAGIC_PASSWD}",
                "NOMAGIC_SERVER=twc184.nomagic.com:3579"
              ]) {

                echo "inside withEnv() block:"
                sh 'echo NOMAGIC_USERID=\${NOMAGIC_USERID}'
                sh 'echo NOMAGIC_SERVER=\${NOMAGIC_SERVER}'
                //
                // Execute the publish script
                //
                // TODO: Split this script up in separate "stages" in this repository
                //
                echo "Execute the publish-fibo.sh script:"
                sh "./jenkins/bin/publish-fibo.sh"
              }
            }
            */

            //
            // Instead, we're just dumping the artifacts produced by the following job here and
            // process it further just as if it has been generated here:
            // https://jenkins.edmcouncil.org/job/NoMagic/job/fibo-publish-nomagic/
            //
            step([
              $class:       'CopyArtifact',
              excludes:     '**/*.jar, **/*.csv, **/PRODIN.html, **/DEVIN.html',
              filter:       '**',
              projectName:  'NoMagic/fibo-publish-nomagic',
              target:       "target/fibo/glossary/${env.GIT_BRANCH}/${env.GIT_TAG_NAME}"
            ])
            echo "Copied nomagic output to target/fibo/glossary/${env.GIT_BRANCH}/${env.GIT_TAG_NAME}:"
            dir("target/fibo/glossary/${env.GIT_BRANCH}/${env.GIT_TAG_NAME}") {
              sh "mv -v PRODUCTION.html  production.html || true"
              sh "mv -v DEVELOPMENT.html development.html || true"
            }
            sh "find target/fibo/glossary/${env.GIT_BRANCH}/${env.GIT_TAG_NAME}"
            step([
              $class:       'CopyArtifact',
              excludes:     '*.jar, *.html, *.css, *.js',
              filter:       '*.csv',
              projectName:  'NoMagic/fibo-publish-nomagic',
              target:       "target/fibo/datadictionary/${env.GIT_BRANCH}/${env.GIT_TAG_NAME}"
            ])
            echo "Copied nomagic output to target/fibo/datadictionary/${env.GIT_BRANCH}/${env.GIT_TAG_NAME}:"
            dir("target/fibo/datadictionary/${env.GIT_BRANCH}/${env.GIT_TAG_NAME}") {
              sh "mv -v datadictionaryPROD.csv production.csv || true"
              sh "mv -v datadictionaryDEV.csv development.csv || true"
            }
            sh "find target/fibo/datadictionary/${env.GIT_BRANCH}/${env.GIT_TAG_NAME}"

            //
            // Archive the artifacts generated by the publish-fibo.sh script
            //
            dir('target') {
              stash includes: '**', name: 'publish-script-output-nomagic-node'
            }
          }
        } // end of stage 'Build on NoMagic Node'
      } // end of parallel
    } // end of stage 'Build'

    //
    // Run the publish on the master jenkins agent by just copying all the generated artifacts right into the workspace
    // on master and let NGINX just serve it from there.
    //
    // This workspace will never be "wiped" so it contains all the older versions as well, wiping this workspace
    // will be bad because we would lose all previously published versions
    //
    stage('Publish') {

      agent {
        label 'master'
      }
      environment {
        NGINX_SPEC_ROOT = '/mnt/jenkins-disk/spec.edmcouncil.org'
      }

      steps {

        sh 'test -d ${NGINX_SPEC_ROOT}'
        sh 'pwd'
        echo "Cleaning workspace before unstashing fresh content"
        sh 'rm -rf *'

        echo 'Unstashing the output of the publish-script as it ran on the stardog node'
        unstash 'publish-script-output-stardog-node'
        sh 'ls -al'
        sh 'find . -name \'.git\' -exec rm -rf {} \\; || true'
        sh 'find . -name \'.gitignore\' -exec rm -f {} \\; || true'

        echo 'Unstashing the output of the publish-script as it ran on the nomagic node'
        unstash 'publish-script-output-nomagic-node'
        sh 'ls -al'

        echo "Copy all generated content to ${env.NGINX_SPEC_ROOT}:"
        sh 'ls -al ${NGINX_SPEC_ROOT}/'
        sh 'cp -vr . ${NGINX_SPEC_ROOT}/'

        //setGitHubPullRequestStatus context: 'fibo-publish', message: '', state: 'SUCCESS'
      }
    } // end of stage "Publish"
  } // end of stages

  post {
    failure {
      mail(
        to: 'jacobus.geluk@bnymellon.com',
        subject: "Failed Pipeline: ${currentBuild.fullDisplayName}",
        body: "Something is wrong with ${env.BUILD_URL}"
      )
    }
  }
} // end of pipeline

def initEnvironment() {

  env.GIT_BRANCH    = readFile './env/GIT_BRANCH'.trim()
  env.GIT_TAG_NAME  = readFile './env/GIT_TAG_NAME'.trim()
  env.GIT_AUTHOR    = readFile './env/GIT_AUTHOR'.trim()
  env.GIT_COMMENT   = readFile './env/GIT_COMMENT'.trim()
  env.GIT_COMMIT    = readFile './env/GIT_COMMIT'.trim()
  env.JIRA_ISSUE    = readFile './env/JIRA_ISSUE'.trim()

  echo "GIT_BRANCH=${env.GIT_BRANCH}"
  echo "GIT_TAG_NAME=${env.GIT_TAG_NAME}"
  echo "GIT_AUTHOR=${env.GIT_AUTHOR}"
  echo "GIT_COMMENT=${env.GIT_COMMENT}"
  echo "GIT_COMMIT=${env.GIT_COMMIT}"
  echo "JIRA_ISSUE=${env.JIRA_ISSUE}"
}