jovian

#!/bin/bash
# shellcheck disable=SC1091
#set -euo pipefail
#set -v
#set -x

#load in functions
source bin/functions.sh
eval "$(parse_yaml profile/pipeline_parameters.yaml "params_")"
eval "$(parse_yaml profile/config.yaml "configuration_")"

### Variables
VERSION=$(git describe --tags)
INSTALL_LOG="jovian_installation_log.txt"
PROFILE="profile"

UNIQUE_ID=$(bin/generate_id.sh)
SET_HOSTNAME=$(bin/gethostname.sh)
SYSTEM_CORES=$(grep -c ^processor /proc/cpuinfo)
AVAILABLE_CORES=$(expr ${SYSTEM_CORES} - 2)

### Jovian Conda environment names
PATH_JOVIAN_MASTER_YAML="envs/Jovian_master_environment.yaml"
MASTER_NAME=$(head -n 1 ${PATH_JOVIAN_MASTER_YAML} | cut -f2 -d ' ') # Extract Conda environment name as specified in yaml file
PATH_JOVIAN_HELPER_YAML="envs/Jovian_helper_environment.yaml"
HELPER_NAME=$(head -n 1 ${PATH_JOVIAN_HELPER_YAML} | cut -f2 -d ' ') # Extract Conda environment name as specified in yaml file

### Default values for CLI parameters
INPUT_DIR="raw_data/"
JOVIAN_HELP="FALSE"
SNAKEMAKE_HELP="FALSE"
INSTALL_CONDA="FALSE"
INSTALL_DB="FALSE"
CLEAN="FALSE"
VIRUS_TYPING="FALSE"
FORCE_OVERWRITE_TT="FALSE"
ARCHIVE="FALSE"
START_NGINX="FALSE"
STOP_NGINX="FALSE"
REBUILD_ARCHIVE="FALSE"
CONFIG_JUP="FALSE"
JOVIAN_MODE="NONE"
START_JUPYTER="FALSE"
MAKE_SAMPLE_SHEET="FALSE"
SHEET_SUCCESS="FALSE" # Default value for sample_sheet.yaml
SKIP_CONFIRMATION="FALSE"
SNAKEMAKE_UNLOCK="FALSE"
JOVIAN_UPDATE="FALSE"
CHANGE_VERSION="FALSE"

### Parse the jovian commandline arguments, if they are not part of jovian, they get send to Snakemake
POSITIONAL=()
while [[ $# -gt 0 ]]
do
    key="$1"
    case $key in
        -i|--input)
        INPUT_DIR="$2"
        shift # Next
        shift # Next
        ;;
        -m|--mode)
        JOVIAN_MODE="$2"
        shift #Next
        shift #Next
        ;;
        -h|--help)
        JOVIAN_HELP="TRUE"
        shift # Next
        ;;
        -sh|--snakemake-help)
        SNAKEMAKE_HELP="TRUE"
        shift # Next
        ;;
        -ic|--install-dependencies)
        INSTALL_CONDA="TRUE"
        shift # Next
        ;;
        -id|--install-databases)
        INSTALL_DB="TRUE"
        shift # Next
        ;;
        --clean)
        CLEAN="TRUE"
        shift # Next
        ;;
        --archive)
        ARCHIVE="TRUE"
        shift # Next
        ;;
        -v|--version)
        SHOW_VERSION="TRUE"
        shift # Next
        ;;
        --start-nginx)
        START_NGINX="TRUE"
        shift # Next
        ;;
        --stop-nginx)
        STOP_NGINX="TRUE"
        shift # Next
        ;;
        --rebuild-archive)
        REBUILD_ARCHIVE="TRUE"
        shift # Next
        ;;
        --configure-jupyter)
        CONFIG_JUP="TRUE"
        shift # Next
        ;;
        --start-jupyter)
        START_JUPYTER="TRUE"
        shift # Next
        ;;
        -vt|--virus-typing)
        VIRUS_TYPING="TRUE"
        WHICH_TT="$2"
        shift # Next
        shift # Next
        ;;
        -vt-force|--virus-typing-force)
        VIRUS_TYPING="TRUE"
        FORCE_OVERWRITE_TT="TRUE"
        WHICH_TT="$2"
        shift # Next
        shift # Next
        ;;
        -vt-help|--virus-typing-help)
        VIRUS_TYPING="TRUE"
        VIRUS_TYPING_HELP="TRUE"
        shift # Next
        ;;
        --make-sample-sheet)
        MAKE_SAMPLE_SHEET="TRUE"
        shift # Next
        ;;
        -y)
        SKIP_CONFIRMATION="TRUE"
        shift # Next
        ;;
        -u|--unlock)
        SNAKEMAKE_UNLOCK="TRUE"
        shift # Next
        ;;
        --update)
        JOVIAN_UPDATE="TRUE"
        CHANGE_VERSION="$2"
        shift # Next
        shift # Next
        ;;
        *) # Any other option
        POSITIONAL+=("$1") # save in array
        shift # Next
        ;;
    esac
done
set -- "${POSITIONAL[@]:-}" # Restores the positional arguments (i.e. without the case arguments above) which then can be called via `$@` or `$[0-9]` etc. These parameters are send to Snakemake.

### Check the jovian CLI arguments and do stuff

### Show Jovian version
if [ "${SHOW_VERSION}" == "TRUE" ]; then
    echo -e "Jovian ${VERSION}"
    exit 0
fi

### Remove all Jovian output
if [ "${CLEAN:-}" == "TRUE" ]; then
    line
    spacer
    echo -e "The following files and folders will be deleted:\ndata/\nlogs/\nresults/\nprofile/variables.yaml\nsample_sheet.yaml\n\n"
    if [ "${SKIP_CONFIRMATION}" == "TRUE" ]; then
        echo -e "Removing Jovian output: data/ logs/ results/ profile/variables.yaml sample_sheet.yaml"
            rm -rf data/
            rm -rf logs/
            rm -rf results/
            rm -f sample_sheet.yaml
            rm -f profile/variables.yaml

            ## clean the yaml files
            sed -i '\|databases|d' profile/pipeline_parameters.yaml
            sed -i '\|background_ref|d' profile/pipeline_parameters.yaml
            sed -i '\|Krona_taxonomy|d' profile/pipeline_parameters.yaml
            sed -i '\|virusHostDB|d' profile/pipeline_parameters.yaml
            sed -i '\|NCBI_new_taxdump_rankedlineage|d' profile/pipeline_parameters.yaml
            sed -i '\|NCBI_new_taxdump_host|d' profile/pipeline_parameters.yaml
            sed -i '\|drmaa|d' profile/config.yaml

    else
        while read -r -p "Are you sure you wish to permanently delete these files and folders? [y/N] " response
        do
            response=${response,,}
            if [[ "${response}" =~ ^(yes|y)$ ]] || [ "${SKIP_CONFIRMATION}" == "TRUE" ]; then
                echo -e "Removing Jovian output: data/ logs/ results/ profile/variables.yaml sample_sheet.yaml"
                rm -rf data/
                rm -rf logs/
                rm -rf results/
                rm -f sample_sheet.yaml
                rm -f profile/variables.yaml

                sed -i '\|databases|d' profile/pipeline_parameters.yaml
                sed -i '\|background_ref|d' profile/pipeline_parameters.yaml
                sed -i '\|Krona_taxonomy|d' profile/pipeline_parameters.yaml
                sed -i '\|virusHostDB|d' profile/pipeline_parameters.yaml
                sed -i '\|NCBI_new_taxdump_rankedlineage|d' profile/pipeline_parameters.yaml
                sed -i '\|NCBI_new_taxdump_host|d' profile/pipeline_parameters.yaml
                sed -i '\|drmaa|d' profile/config.yaml
                break
            elif [[ "${response}" =~ ^(no|n)$ ]]; then
                echo -e "Aborting cleaning process on user request"
                exit 0
            else
                echo -e "Please answer with 'yes' or 'no'"
                minispacer
            fi
        done
    fi
    exit 0
fi

### Print Jovian help message
if [ "${JOVIAN_HELP:-}" == "TRUE" ]; then
    line
    cat <<HELP_USAGE
Jovian, version $VERSION, built with Snakemake
  Usage: bash $0 -i <INPUT_DIR> <parameters>
  N.B. it is designed for Illumina paired-end data only

Input:
  -i, --input [DIR]                 This is the folder containing your input fastq files.
                                    Default is 'raw_data/' and only relative paths are accepted.
Output (automatically generated):
  data/                             Contains detailed intermediate files.
  logs/                             Contains all log files.
  results/                          Contains all final results, these are visualized via the
                                    web-report (Notebook_report.ipynb).
Parameters:
  -m, --mode [strict|relaxed]       Work-in-progress. Automatically configures Jovian to be
                                    stringent or relaxed in taxonomic classification of
                                    scaffolds. Please see the publication for details. In short,
                                    use 'strict' for high precision at the cost of sensitivity
                                    and use 'relaxed' for high sensitivity at the cost of more
                                    spurious results (contamination).
  -vt-help, --virus-typing-help     Print additional information about the virus-typing, i.e.
                                    which family/genus is selected and which species are typed.
  -vt, --virus-typing [NoV|EV|RVA|HAV|HEV|PV|Flavi]
                                    After a Jovian analyses has completed, do viral typing for
                                    Norovirus (NoV), Enterovirus (EV), Hepatitis A (HAV), 
                                    Hepatitis E (HEV), Rotavirus A (RVA), Papillomaviruses (PV),
                                    Flaviviruses (Flavi).
                                    For additional explanation, --vt-help
  -vt-force, --virus-typing-force [NoV|EV|RVA|HAV|HEV|PV|Flavi]
                                    Same as above, but overwrites existing output. 
  -h, --help                        Print the Jovian help document.
  -sh, --snakemake-help             Print the Snakemake help document.
  --clean (-y)                      Removes Jovian output. (-y forces "Yes" on all prompts)
  -k, --keep-going                  Useful snakemake command: Go on with independent jobs if
                                    a job fails.
  -n, --dry-run                     Useful snakemake command: Do not execute anything, and
                                    display what would be done.
  -u, --unlock                      Removes the lock on the working directory. This happens when
                                    a run ends abruptly and prevents you from doing subsequent
                                    analyses.
  -q, --quiet                       Useful snakemake command: Do not output any progress or
                                    rule information.
Archiving Jovian results:
  --archive (-y)                    Archives the output of a Jovian analysis into single
                                    compressed file. (-y forces "Yes" on all prompts)
  --rebuild-archive                 Rebuilds Jovian results and logs from a previously made 
                                    archive. It is important to use the same version of Jovian
                                    as was initially used to build the archive. N.B. databases
                                    are not restored.
Jupyter Notebook (for data visualization):
  --configure-jupyter               Sets the proper Jupyter settings. You only need to do this
                                    once per user.
  --start-jupyter                   Starts a Jupyter Notebook process. You must always have this
                                    running in a separate terminal (or in the background) if you
                                    want to open the final report with the interactive graphs
                                    and tables.
Installation:
  -ic, --install-dependencies       Install the required software dependencies.
  -id, --install-databases          Install required databases.
Debug only:
  --start-nginx                     Starts the nginx process.
  --stop-nginx                      Stops the nginx process.
  --make-sample-sheet               Only make the sample sheet.
Other:
  -v, --version                     Prints the version of Jovian.
  --update                          Updates Jovian to the latest release.
                                    Some files might be reset to their default contents as shown 
                                    on Github
  --update [Version tag]            Changes Jovian to the specified release. 
                                    Only accepts major releases.
                                    Files might get lost due to a significant change in version.
HELP_USAGE
    exit 0
fi

###############################################################################################################
##### Installation block                                                                                  #####
###############################################################################################################

### Pre-flight check: Assess availability of required files, conda and Jovian_master environment
if [ ! -e "${PATH_JOVIAN_MASTER_YAML}" ]; then # If this yaml file does not exist, give error.
    line
    spacer
    echo -e "ERROR: Missing file \"${PATH_JOVIAN_MASTER_YAML}\""
    exit 1
fi
if [ ! -e "${PATH_JOVIAN_HELPER_YAML}" ]; then # If this yaml file does not exist, give error.
    line
    spacer
    echo -e "ERROR: Missing file \"${PATH_JOVIAN_HELPER_YAML}\""
    exit 1
fi
if [ ! -e "${PROFILE}/config.yaml" ]; then # If this profile folder containing config.yaml does not exist, give error.
    line
    spacer
    echo -e "ERROR: Missing file \"${PROFILE}/config.yaml\""
    exit 1
fi

if [ ! -e "${HOME}/.jovian_installchoice_compmode" ]; then
    echo -e "installeranswers:\n    compmode: U\n   queue: U" > "${HOME}"/.jovian_installchoice_compmode
fi
if [ ! -e "${HOME}/.jovian_installchoice_db" ]; then
    echo -e "installeranswers:\n    databases: U\n    backgroundref: U\n    kronataxonomy: U\n    virushostdb: U\n    newtaxdumpranked: U\n    newtaxdumphost: U" > "${HOME}"/.jovian_installchoice_db
fi

eval "$(parse_yaml "${HOME}"/.jovian_installchoice_compmode "cm_")"
eval "$(parse_yaml "${HOME}"/.jovian_installchoice_db "db_")"

if ! command -v conda > /dev/null; then # Check if conda is not installed (i.e. the `conda` command is not available), if true, install it in ~/tmp. Otherwise, proceed.
    if [ ! -e "${HOME}/tmp/Miniconda3" ]; then
        ### confirmation of conda installation
        installer_intro
        echo -e "Miniconda missing. Installing Miniconda can take up to 15 minutes..."
        if [ "${SKIP_CONFIRMATION}" == "TRUE" ]; then
            echo "Jovian ${VERSION}" > ${INSTALL_LOG}
            line | tee -a ${INSTALL_LOG}
            curl -o latest.sh -L https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
            chmod +x latest.sh
            sh latest.sh -b -p "${HOME}/tmp/Miniconda3" | tee -a ${INSTALL_LOG}
            rm latest.sh
            tput reset
            installer_intro
        else
            while read -r -p "Do you wish to install (mini)conda now? [y/N] " response
            do
                response=${response,,}
                if [[ "${response}" =~ ^(no|n) ]]; then
                    minispacer
                    echo -e "Conda is a requirement for Jovian to run."
                    echo -e "Exiting because conda isn't installed and automatic installation of conda was declined by user"
                    exit 1
                elif [[ "${response}" =~ ^(yes|y)$ ]]; then
                    echo "Jovian ${VERSION}" > ${INSTALL_LOG}
                    line | tee -a ${INSTALL_LOG}
                    curl -o latest.sh -L https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
                    chmod +x latest.sh
                    sh latest.sh -b -p "${HOME}/tmp/Miniconda3" | tee -a ${INSTALL_LOG}
                    rm latest.sh
                    tput reset
                    installer_intro
                    break
                else
                    echo -e "Please answer with 'yes' or 'no'"
                    minispacer
                fi
            done
        fi
        ### conda has been installed by previous step, confirm addition to PATH
        echo -e "Conda is installed but not yet added to PATH, do you wish to add conda to your PATH?"
        if [ "${SKIP_CONFIRMATION}" == "TRUE" ]; then
            echo "PATH=$PATH:${HOME}/tmp/Miniconda3/bin" >> "${HOME}/.bashrc" | tee -a ${INSTALL_LOG}
            source "${HOME}"/.bashrc | tee -a ${INSTALL_LOG}
            export PATH=${HOME}/tmp/Miniconda3/bin:$PATH
            tput reset
             line
             installer_intro
             echo -e "Conda succesfully added to PATH"
        else
            while read -r -p "The conda PATH will be added to your ~/.bashrc and will only affect you. [y/N] " answer
            do
                answer=${answer,,}
                if [[ "${answer}" =~ ^(yes|y)$ ]]; then
                    echo "PATH=$PATH:${HOME}/tmp/Miniconda3/bin" >> "${HOME}/.bashrc" | tee -a ${INSTALL_LOG}
                    source "${HOME}"/.bashrc | tee -a ${INSTALL_LOG}
                    export PATH=${HOME}/tmp/Miniconda3/bin:$PATH
                    tput reset
                    line
                    installer_intro
                    echo -e "Conda succesfully added to PATH"
                    break
                elif [[ "${answer}" =~ ^(no|n)$ ]]; then
                    export PATH=${HOME}/tmp/Miniconda3/bin:$PATH | tee -a ${INSTALL_LOG}
                    tput reset
                    line
                    installer_intro
                    line
                    spacer
                    echo -e "Conda has not been permanently added to PATH, this means you cannot use the conda command once this session closes."
                    break
                else
                    echo -e "Please answer with 'yes' or 'no'"
                    minispacer
                fi
            done
        fi
    
        echo -e "\n"
        echo -e "Installation of Conda is done"
        echo -e "If you wish to use conda then it might be necessary to restart your terminal session"
        source "${HOME}"/.bashrc

        echo -e "\n\nContinuing..."
        sleep 5

        installer_intro
        line
        spacer
    fi
    if [ -e "${HOME}/tmp/Miniconda3" ]; then
        export PATH=${HOME}/tmp/Miniconda3/bin:$PATH
    fi 
fi

### Install Jovian_master if it doesn't already exist, afterwards, start Jovian_master
if [[ $PATH != *${MASTER_NAME}* ]]; then # If the master environment is not in your path (i.e. it is not currently active), do...
    line
    spacer
    set +ue # Turn bash strict mode off because that breaks conda
    source activate "${MASTER_NAME}" # Try to activate this env
    if [ ! $? -eq 0 ]; then # If exit statement is not 0, i.e. master conda env hasn't been installed yet, do...
        installer_intro
        if [ "${SKIP_CONFIRMATION}" = "TRUE" ]; then
            echo -e "\tInstalling Jovian master environment..." | tee -a ${INSTALL_LOG}   
            conda env create -f ${PATH_JOVIAN_MASTER_YAML} | tee -a ${INSTALL_LOG} # Create the env from the specified yaml file
            source activate "${MASTER_NAME}"
            echo -e "DONE"
        else
            while read -r -p "The Jovian_master environment hasn't been installed yet, do you want to install this environment now? [y/N] " envanswer
            do
                envanswer=${envanswer,,}
                if [[ "${envanswer}" =~ ^(yes|y)$ ]]; then
                    echo -e "\tInstalling Jovian master environment..." | tee -a ${INSTALL_LOG}   
                    conda env create -f ${PATH_JOVIAN_MASTER_YAML} | tee -a ${INSTALL_LOG} # Create the env from the specified yaml file
                    source activate "${MASTER_NAME}"
                    echo -e "DONE"
                    break
                elif [[ "${envanswer}" =~ ^(no|n)$ ]]; then
                    echo -e "The Jovian_master environment is a requirement. Exiting because Jovian cannot continue without this environment"
                    exit 1
                else
                    echo -e "Please answer with 'yes' or 'no'"
                    minispacer
                fi
            done
        fi
    fi
    set -ue # Turn bash strict mode on again
    echo -e "Succesfully activated Jovian_master environment"
fi

        ###############################################################################################################
        ##### From this point onwards you can safely assume Conda is installed and Jovian_master is activated     #####
        ###############################################################################################################


if [ "${JOVIAN_UPDATE}" == "TRUE" ]; then
    if [ -z "${CHANGE_VERSION}" ]; then
        bin/selfupdate.sh master
        exit 0
    else
        bin/selfupdate.sh "${CHANGE_VERSION}"
        exit
    fi
fi

### Archiving of relevant data in a single tar.gz file
if [ "${ARCHIVE:-}" == "TRUE" ]; then
    line
    spacer
    echo -e "\n\nArchiving the Jovian data and results can take several hours depending on the size of your dataset(s)."
    echo -e "This archiving process will NOT include the raw dataset in the final archive"
    echo -e "\nIt is advised to run the archiving process overnight, it is strongly advised to not stop the process once you started it"
    if [ "${SKIP_CONFIRMATION}" == "TRUE" ]; then
        eval "$(parse_yaml profile/variables.yaml "config_")"
        if [ -e data/ ]; then
            echo -e "Found the 'data/' folder, appending to archive..."
            (tar --append --file=archive_"${config_Jovian_run_identifier}"_jovian-"${VERSION}".tar data/) &
            spinner
        else
            echo -e "Couldn't find the 'data/' folder in this working directory, skipping this folder in archiving"
        fi
        
        if [ -e logs/ ]; then
            echo -e "Found the 'logs/' folder, appending to archive..."
            (tar --append --file=archive_"${config_Jovian_run_identifier}"_jovian-"${VERSION}".tar logs/) &
            spinner
        else
            echo -e "Couldn't find the 'logs/' folder in this working directory, skipping this folder in archiving"
        fi
            
        if [ -e results/ ]; then
            echo -e "Found the 'results/' folder, appending to archive..."
            (tar --append --file=archive_"${config_Jovian_run_identifier}"_jovian-"${VERSION}".tar results/) &
            spinner
        else
            echo -e "Couldn't find the 'results/' folder in this working directory, skipping this folder in archiving"
        fi
            
        if [ -e sample_sheet.yaml ]; then
            echo -e "Found 'sample_sheet.yaml', appending to archive..."
            (tar --append --file=archive_"${config_Jovian_run_identifier}"_jovian-"${VERSION}".tar sample_sheet.yaml) &
            spinner
        else
            echo -e "Couldn't find the file 'sample_sheet.yaml' in this working directory, skipping this file in archiving"
        fi
            
        if [ -e profile/ ]; then
            echo -e "Found 'profile' folder, appending to archive..."
            (tar --append --file=archive_"${config_Jovian_run_identifier}"_jovian-"${VERSION}".tar profile/) &
            spinner
        else
            echo -e "Couldn't find the 'profile' folder in this working directory, skipping this file in archiving"
        fi

        if [ -e archive_"${config_Jovian_run_identifier}"_jovian-"${VERSION}".tar ]; then
            echo -e "\n\nFiles and folders have been appended to the archive."
            echo -e "Compressing archive..."
            (pigz -p "${AVAILABLE_CORES}" archive_"${config_Jovian_run_identifier}"_jovian-"${VERSION}".tar) &
            spinner
        else
            echo "No files and folders could be found for archiving."
            exit 1
        fi
    else
        while read -r -p "Do you want to start the Jovian archiving process now? [y/N] " archive_answer
        do
        archive_answer=${archive_answer,,}
            if [[ "${archive_answer}" =~ ^(yes|y)$ ]]; then
                echo -e "\n\nStarting the archiving process\n\n"
                eval "$(parse_yaml profile/variables.yaml "config_")"
                
                if [ -e data/ ]; then
                    echo -e "Found the 'data/' folder, appending to archive..."
                    (tar --append --file=archive_"${config_Jovian_run_identifier}"_jovian-"${VERSION}".tar data/) &
                    spinner
                else
                    echo -e "Couldn't find the 'data/' folder in this working directory, skipping this folder in archiving"
                fi

                if [ -e logs/ ]; then
                    echo -e "Found the 'logs/' folder, appending to archive..."
                    (tar --append --file=archive_"${config_Jovian_run_identifier}"_jovian-"${VERSION}".tar logs/) &
                    spinner
                else
                    echo -e "Couldn't find the 'logs/' folder in this working directory, skipping this folder in archiving"
                fi
                
                if [ -e results/ ]; then
                    echo -e "Found the 'results/' folder, appending to archive..."
                    (tar --append --file=archive_"${config_Jovian_run_identifier}"_jovian-"${VERSION}".tar results/) &
                    spinner
                else
                    echo -e "Couldn't find the 'results/' folder in this working directory, skipping this folder in archiving"
                fi
                
                if [ -e sample_sheet.yaml ]; then
                    echo -e "Found 'sample_sheet.yaml', appending to archive..."
                    (tar --append --file=archive_"${config_Jovian_run_identifier}"_jovian-"${VERSION}".tar sample_sheet.yaml) &
                    spinner
                else
                    echo -e "Couldn't find the file 'sample_sheet.yaml' in this working directory, skipping this file in archiving"
                fi
                
                if [ -e profile/ ]; then
                    echo -e "Found 'profile' folder, appending to archive..."
                    (tar --append --file=archive_"${config_Jovian_run_identifier}"_jovian-"${VERSION}".tar profile/) &
                    spinner
                else
                    echo -e "Couldn't find the 'profile' folder in this working directory, skipping this file in archiving"
                fi

                if [ -e archive_"${config_Jovian_run_identifier}"_jovian-"${VERSION}".tar ]; then
                    echo -e "\n\nFiles and folders have been appended to the archive."
                    echo -e "Compressing archive..."
                    (pigz -p "${AVAILABLE_CORES}" archive_"${config_Jovian_run_identifier}"_jovian-"${VERSION}".tar) &
                    spinner
                else
                    echo "No files and folders could be found for archiving."
                    exit 1
                fi
                break
            elif [[ "${archive_answer}" =~ ^(no|n)$ ]]; then
                echo -e "Aborting the archiving process on user request"
                exit 0
            else
                echo -e "Please answer with 'yes'' or 'no'"
                minispacer
            fi
        done
    fi
    exit 0
fi

### Install databases, as specified via CLI argument '-id'
if [ "${INSTALL_DB:-}" == "TRUE" ]; then
    spacer
    line
    echo -e "Installing databases, this can take hours for a fresh installation..."
    if [[ $PATH != *${HELPER_NAME}* ]]; then # If helper env is not in your path (i.e. it is not currently active), do...
        set +ue # Turn bash strict mode off because that breaks conda
        source activate "${HELPER_NAME}" # Try to activate this env
        if [ ! $? -eq 0 ]; then # If exit statement is not 0, i.e. helper conda env hasn't been installed yet, do...
            echo -e "\tInstalling Jovian helper environment..."
            conda env create -f ${PATH_JOVIAN_HELPER_YAML} # Create the env from the specified yaml file
            source activate "${HELPER_NAME}"
        fi
        set -ue # Turn bash strict mode on again
    fi
    bin/databases.sh
    # The rest of the code is performed in the helper env, not in the master env.
    conda deactivate 
    # source deactivate # if you need to go back to the master_env. N.B. Wrap this command with set +u and set -u when working in strict bash mode.
    echo -e "DONE"
    exit 0
fi

if [ "$db_installeranswers_databases" == "U" ]; then
    installer_intro
    echo "Jovian requires several databases to run."
    echo "Please specify the location of each required database, you will only have to do this once (the answers given will be stored)"
    echo "The location of each database must start with a '/'"
    thinline
    minispacer

    while
    read -r -e -p "Please specify the location where your Background Reference is installed: " bg_ref_path
    read -r -e -p "Please specify the location where the Krona Taxonomy database is installed: " krona_db_path
    read -r -e -p "Please specify the location where the MGKit Taxonomy database is installed: " mgkit_db_path
    read -r -e -p "Please specify the location of the 'virushostdb.tsv' file: "  virushost_path
    read -r -e -p "Please specify the location of the new_taxdump 'rankedlineage.dmp.delim' file: " newtaxdump_ranked_path
    read -r -e -p "Please specify the location of the new_taxdump 'host.dmp.delim' file: " newtaxdump_host_path
    do
        
        bg_ref_path=${bg_ref_path}
        krona_db_path=${krona_db_path}
        mgkit_db_path=${mgkit_db_path}
        virushost_path=${virushost_path}
        newtaxdump_ranked_path=${newtaxdump_ranked_path}
        newtaxdump_host_path=${newtaxdump_host_path}

        minispacer
        echo -e "Your provided answers are as follows:"
        echo -e "\e[1mBackground reference location: ${bg_ref_path}\e[0m"
        echo -e "\e[1mKrona taxonomy database location: ${krona_db_path}\e[0m"
        echo -e "\e[1mMGKit taxonomy database location: ${mgkit_db_path}\e[0m"
        echo -e "\e[1mVirusHost.tsv file location: ${virushost_path}\e[0m"
        echo -e "\e[1mNewtaxdump rankedlineage.dmp.delim file location: ${newtaxdump_ranked_path}\e[0m"
        echo -e "\e[1mNewtaxdump host.dmp.delim file location: ${newtaxdump_host_path}\e[0m"
        minispacer
        
        while read -r -p "Does this look correct? [yes/no] " db_paths_confirm
        do
            db_paths_confirm=${db_paths_confirm,,}
            if [[ "${db_paths_confirm}" =~ ^(yes)$ ]]; then
                echo "The locations of databases and files have been confirmed"
                break
            elif [[ "${db_paths_confirm}" =~ ^(no)$ ]]; then
                minispacer
                break
            else
                echo "Please answer with 'yes' or 'no'"
            fi
        done

        if [[  "${db_paths_confirm}" =~ ^(yes)$ ]]; then
            echo -e "installeranswers:\n    databases: SET\n    backgroundref: ${bg_ref_path}\n    kronataxonomy: ${krona_db_path}\n    virushostdb: ${virushost_path}\n    newtaxdumpranked: ${newtaxdump_ranked_path}\n    newtaxdumphost: ${newtaxdump_host_path}\n    mgkittaxonomy: ${mgkit_db_path}" > "${HOME}"/.jovian_installchoice_db
            sed -i '\|databases|d' profile/pipeline_parameters.yaml
            sed -i '\|background_ref|d' profile/pipeline_parameters.yaml
            sed -i '\|Krona_taxonomy|d' profile/pipeline_parameters.yaml
            sed -i '\|MGKit_taxonomy|d' profile/pipeline_parameters.yaml
            sed -i '\|virusHostDB|d' profile/pipeline_parameters.yaml
            sed -i '\|NCBI_new_taxdump_rankedlineage|d' profile/pipeline_parameters.yaml
            sed -i '\|NCBI_new_taxdump_host|d' profile/pipeline_parameters.yaml

            echo -e "databases:\n    background_ref: ${bg_ref_path}\n    Krona_taxonomy: ${krona_db_path}\n    virusHostDB: ${virushost_path}\n    NCBI_new_taxdump_rankedlineage: ${newtaxdump_ranked_path}\n    NCBI_new_taxdump_host: ${newtaxdump_host_path}\n    MGKit_taxonomy: ${mgkit_db_path}" >> profile/pipeline_parameters.yaml
            break
        fi
    done
elif [ "${db_installeranswers_databases}" == "SET" ]; then
    sed -i '\|databases|d' profile/pipeline_parameters.yaml
    sed -i '\|background_ref|d' profile/pipeline_parameters.yaml
    sed -i '\|Krona_taxonomy|d' profile/pipeline_parameters.yaml
    sed -i '\|MGKit_taxonomy|d' profile/pipeline_parameters.yaml
    sed -i '\|virusHostDB|d' profile/pipeline_parameters.yaml
    sed -i '\|NCBI_new_taxdump_rankedlineage|d' profile/pipeline_parameters.yaml
    sed -i '\|NCBI_new_taxdump_host|d' profile/pipeline_parameters.yaml

    echo -e "databases:\n    background_ref: ${db_installeranswers_backgroundref}\n    Krona_taxonomy: ${db_installeranswers_kronataxonomy}\n    virusHostDB: ${db_installeranswers_virushostdb}\n    NCBI_new_taxdump_rankedlineage: ${db_installeranswers_newtaxdumpranked}\n    NCBI_new_taxdump_host: ${db_installeranswers_newtaxdumphost}\n    MGKit_taxonomy: ${db_installeranswers_mgkittaxonomy}" >> profile/pipeline_parameters.yaml
fi

if [ "$cm_installeranswers_compmode" == "U" ]; then
    installer_intro
    echo -e "Jovian can run in two computing-modes. 'standalone' and 'HPC/Grid'"
    echo -e "Please specify the computing-mode that you wish to use for Jovian.\n\n"
    echo -e "\n\e[1;91;40m\tIf you don't know what this means then please contact your local system administrators before continuing.\e[0m\n\n"
    while read -r -p "Do you wish to run Jovian in 'standalone' or 'grid' mode? [standalone/grid] " cmanswer
    do
        cmanswer=${cmanswer,,}
        if [[ "${cmanswer}" =~ ^(standalone|s)$ ]]; then
            echo -e "Standalone mode has been chosen. We will remember your choice so you don't have to do this step every time"
            echo -e "installeranswers:\n    compmode: S\n   queue: N" > "${HOME}"/.jovian_installchoice_compmode
            sed -i '\|drmaa|d' profile/config.yaml
            break
        elif [[ "${cmanswer}" =~ ^(grid|g)$ ]]; then
            echo -e "Grid mode has been chosen. Please specify your grid settings so the pipeline will function properly."
            while read -r -p "Please specify the name of the Queue that your local grid/HPC cluster uses. Please enter exclusively the name of the queue. " queueanswer
            do
                queueanswer=${queueanswer,,}
                echo -e "installeranswers:\n    compmode: G\n   queue: ${queueanswer}" > "${HOME}"/.jovian_installchoice_compmode
                sed -i '\|drmaa|d' profile/config.yaml
                echo -e "drmaa: \" -q ${queueanswer} -n {threads} -R \\\"span[hosts=1]\\\"\"" >> profile/config.yaml
                echo -e "drmaa-log-dir: logs/drmaa" >> profile/config.yaml
                break
            done
            break
        elif [[ "${cmanswer}" =~ ^(exit)$ ]]; then
            break
            exit
        else
            echo -e "Please answer with 'standalone' or 'grid'."
        fi
    done
elif [ "${cm_installeranswers_compmode}" == "S" ]; then
    echo -e "Jovian is running in the standalone computing mode"
    sed -i '\|drmaa|d' profile/config.yaml
elif [ "${cm_installeranswers_compmode}" == "G" ]; then
    echo -e "Jovian is set to run in Grid-mode.\nSetting some of the variables."
    sed -i '\|drmaa|d' profile/config.yaml
    echo -e "drmaa: \" -q ${cm_installeranswers_queue} -n {threads} -R \\\"span[hosts=1]\\\"\"" >> profile/config.yaml
    echo -e "drmaa-log-dir: logs/drmaa" >> profile/config.yaml
fi


###############################################################################################################
##### End of installation block, from this point onwards you can safely assume Jovian_master is activated #####
###############################################################################################################

### Virus typing block, as specified via CLI argument '--virus-typing'
if [ "${VIRUS_TYPING:-}" == "TRUE" ]; then
    if [ "${VIRUS_TYPING_HELP:-}" == "TRUE" ]; then
        bash bin/virus_typing.sh --help
        exit 0
    elif [ "${FORCE_OVERWRITE_TT:-}" == "TRUE" ]; then
        bash bin/virus_typing.sh ${WHICH_TT} --force
        exit 0
    else
        bash bin/virus_typing.sh "${WHICH_TT}"
        exit 0
    fi
fi

#TODO the syntax below doesn't work if there are multiple archives. Dangerous.
if [ "${REBUILD_ARCHIVE}" == "TRUE" ]; then
    if [ -e archive_*.tar.gz ]; then
        bin/rebuild_archive.sh
    exit 0
    else
        echo -e "There's no Jovian-archive present."
        exit 0
    fi
fi

### Print Snakemake help
if [ "${SNAKEMAKE_HELP:-}" == "TRUE" ]; then
    line
    snakemake --help
    exit 0
fi

### Check if this required BLAST alias file is present in HOME
if [ ! -e "${HOME}/.ncbirc" ]; then # If the BLAST required ~/.ncbirc does not exist, give error.
    echo -e "ERROR: Missing file \"~/.ncbirc\". \nThis is config file is required for BLAST to function, contains the locations to the BLAST databases on your local system. It should be in your home directory."
    exit 1
fi


### Installation of Jovian specific conda environments
if [ "${INSTALL_CONDA:-}" == "TRUE" ]; then
    minispacer
    line
    touch sample_sheet.yaml
    echo -e "Jovian_run:\n    identifier: ${UNIQUE_ID}" > profile/variables.yaml
    echo -e "Server_host:\n    hostname: http://${SET_HOSTNAME}" >> profile/variables.yaml
    mkdir -p installer_files
    echo -e "@example_read1\nGATTACA\n+\nAAAAAAA\n" > installer_files/example_R1.fq
    echo -e "@example_read2\nGATTACA\n+\nAAAAAAA\n" > installer_files/example_R2.fq
    echo -e "installer:\n    R1: installer_files/example_R1.fq\n    R2: installer_files/example_R2.fq" >> sample_sheet.yaml
    echo -e "\nInstalling required conda environments, this can take up to an hour..."
    snakemake --use-conda --create-envs-only --profile ${PROFILE}
    rm sample_sheet.yaml
    rm profile/variables.yaml
    rm -rf installer_files
    echo -e "Done with creating the required conda environments, you can now start your analysis"
    exit 0
fi

### Start and stop commands for nginx
if [ "${START_NGINX}" == "TRUE" ]; then
    (bin/start_nginx.sh start)
    exit 0
fi
if [ "${STOP_NGINX}" == "TRUE" ]; then
    nginx -s quit
    echo "nginx has been stopped"
    exit 0
fi

### * Auto configure the jupyter user profile and prepare it for direct use
if [ "${CONFIG_JUP}" == "TRUE" ]; then
    jt -t grade3 -fs 95 -altp -tfs 11 -nfs 115 -cellw 88% -T
    sed -i '1704,1706d' ~/.jupyter/custom/custom.css
    sed -i '35,55d' ~/.jupyter/custom/custom.css
    cat files/Jupyter_notebook/overrides.css >> ~/.jupyter/custom/custom.css
    jupyter notebook --generate-config
    sed -i "s/#c.NotebookApp.allow_remote_access = False/c.NotebookApp.allow_remote_access = True/g" ~/.jupyter/jupyter_notebook_config.py
    sed -i "s/#c.NotebookApp.ip = 'localhost'/c.NotebookApp.ip = '${SET_HOSTNAME}'/g" ~/.jupyter/jupyter_notebook_config.py
    sed -i "s/#c.NotebookApp.port = 8888/c.NotebookApp.port = 8888/g" ~/.jupyter/jupyter_notebook_config.py
    sed -i "s/#c.NotebookApp.open_browser = True/c.NotebookApp.open_browser = False/g" ~/.jupyter/jupyter_notebook_config.py
    sed -i "s/#c.NotebookApp.iopub_data_rate_limit = 1000000/c.NotebookApp.iopub_data_rate_limit = 100000000/g" ~/.jupyter/jupyter_notebook_config.py
    cp files/Jupyter_notebook/edit.json ~/.jupyter/nbconfig
    cp files/Jupyter_notebook/notebook.json ~/.jupyter/nbconfig
    cp files/Jupyter_notebook/tree.json ~/.jupyter/nbconfig
    echo "Done - Jupyter notebooks is configured and ready for use"
    exit 0
fi

if [ "${START_JUPYTER}" == "TRUE" ]; then
    cd /
    line
    minispacer
    # ! Give notices that jupyter is scary >:(
    printf "\e[1;91;40m\tJupyter notebooks will start in a moment                                                      \e[0m\n"
    printf "\e[1;91;40m\tPlease take note that Jupyter will keep running until you \e[4mEXPLICITLY\e[24m tell it to stop.         \e[0m\n"
    printf "\e[1;91;40m\tYou can stop Jupyter by pressing 'Ctrl + C' on your keyboard \e[4mtwice\e[24m                            \e[0m\n"
    minispacer
    sleep 3
    jupyter notebook
    exit 0
fi

if [ "${JOVIAN_MODE}" == "relaxed" ]; then
    #* Check the settings for the slidingwindow and change it when the setting does not match :5:20
    if [ "$params_Trimmomatic_quality_trimming_config" != "SLIDINGWINDOW:5:20" ]; then
        sed -i "s/    quality_trimming_config: $params_Trimmomatic_quality_trimming_config/    quality_trimming_config: SLIDINGWINDOW:5:20/g" profile/pipeline_parameters.yaml
    fi
    #* Check the settings for the minlen filtering and change it when the setting does not match 250
    if [ "$params_scaffold_minLen_filter_minlen" != "250" ]; then
        sed -i "s/    minlen: $params_scaffold_minLen_filter_minlen/    minlen: 250/g" profile/pipeline_parameters.yaml
    fi
    echo -e "Relaxed mode set..."

elif [ "${JOVIAN_MODE}" == "strict" ]; then
    #* Check the settings for the slidingwindow and change it when the setting does not match :5:30
    if [ "$params_Trimmomatic_quality_trimming_config" != "SLIDINGWINDOW:5:30" ]; then
        sed -i "s/    quality_trimming_config: $params_Trimmomatic_quality_trimming_config/    quality_trimming_config: SLIDINGWINDOW:5:30/g" profile/pipeline_parameters.yaml
    fi
    #* Check the settings for the minlen filtering and change it when the setting does not match 500
    if [ "$params_scaffold_minLen_filter_minlen" != "500" ]; then
        sed -i "s/    minlen: $params_scaffold_minLen_filter_minlen/    minlen: 500/g" profile/pipeline_parameters.yaml
    fi
    echo -e "Strict mode set..."

elif [ "${JOVIAN_MODE}" == "NONE" ]; then
    #* Check if parameters match with strict mode
    if [ "$params_Trimmomatic_quality_trimming_config" == "SLIDINGWINDOW:5:30" ] && [ "$params_scaffold_minLen_filter_minlen" == "500" ]; then
        echo -e "Jovian will run in strict mode..."
    fi
    #* Check if parameters match with relaxed mode
    if [ "$params_Trimmomatic_quality_trimming_config" == "SLIDINGWINDOW:5:20" ] && [ "$params_scaffold_minLen_filter_minlen" == "250" ]; then
        echo -e "Jovian will run in relaxed mode..."
    fi
    #* Check if parameters do not match with any of the predefined modes, exit if true
    if [[ ! "$params_Trimmomatic_quality_trimming_config" =~ ^(SLIDINGWINDOW:5:20|SLIDINGWINDOW:5:30)$ ]]; [[ ! "$params_scaffold_minLen_filter_minlen" =~ ^(250|500)$ ]]; then
        echo -e "Jovian will run in a custom mode"
    fi

elif [[ ! "${JOVIAN_MODE}" =~ ^(relaxed|strict)$ ]]; then
    #* Check if parameters match with strict mode
    if [ "$params_Trimmomatic_quality_trimming_config" == "SLIDINGWINDOW:5:30" ] && [ "$params_scaffold_minLen_filter_minlen" == "500" ]; then
        echo -e "Jovian is set to run in strict mode"
        exit 0
    fi
    #* Check if parameters match with relaxed mode
    if [ "$params_Trimmomatic_quality_trimming_config" == "SLIDINGWINDOW:5:20" ] && [ "$params_scaffold_minLen_filter_minlen" == "250" ]; then
        echo -e "Jovian is set to run in relaxed mode"
        exit 0
    fi
    #* Check if parameters do not match with any of the predefined modes, exit if true
    if [[ ! "$params_Trimmomatic_quality_trimming_config" =~ ^(SLIDINGWINDOW:5:20|SLIDINGWINDOW:5:30)$ ]]; [[ ! "$params_scaffold_minLen_filter_minlen" =~ ^(250|500)$ ]]; then
        echo -e "Jovian is set to run with a custom config/mode"
        exit 0
    fi
fi

### Pass other CLI arguments along to Snakemake
if [ ! -d "${INPUT_DIR}" ]; then
    minispacer
    echo -e "The input directory specified (${INPUT_DIR}) does not exist"
    echo -e "Please specify an existing input directory"
    exit 1
fi

### Generate sample sheet
if [ -n "$(ls -A "${INPUT_DIR}")" ]; then
    minispacer
    echo -e "Files in input directory (${INPUT_DIR}) are present"
    echo -e "Generating sample sheet..."
    bin/generate_sample_sheet.py "${INPUT_DIR}" > sample_sheet.yaml
    SHEET_SUCCESS="TRUE"
else
    minispacer
    echo -e "The input directory you specified (${INPUT_DIR}) exists but is empty...\nPlease specify a directory with input-data."
    exit 0
fi

### Checker for succesfull creation of sample_sheet
if [ "${SHEET_SUCCESS}" == "TRUE" ]; then
    echo -e "Succesfully generated the sample sheet"
    ready_for_start
else
    echo -e "Couldn't find files in the input directory that ended up being in a .FASTQ, .FQ or .GZ format"
    echo -e "Please inspect the input directory (${INPUT_DIR}) and make sure the files are in one of the formats listed below"
    echo -e ".fastq.gz (Zipped Fastq)"
    echo -e ".fq.gz (Zipped Fq)"
    echo -e ".fastq (Unzipped Fastq)"
    echo -e ".fq (unzipped Fq)"
    exit 1
fi

if [ "${MAKE_SAMPLE_SHEET}" == "TRUE" ]; then
    echo -e "Jovian_run:\n    identifier: ${UNIQUE_ID}" > profile/variables.yaml
    echo -e "Server_host:\n    hostname: http://${SET_HOSTNAME}" >> profile/variables.yaml
    echo -e "The sample sheet and variables file has now been created, you can now run the Jovian snakefile manually"
    exit 0
fi

if [ "${SNAKEMAKE_UNLOCK}" == "TRUE" ]; then
    printf "\nUnlocking working directory...\n"
    snakemake -s Snakefile --profile "${PROFILE}" --unlock
    printf "\nDone.\n"
    exit 0
fi

### Actual snakemake command with checkers for required files. N.B. here the UNIQUE_ID and SET_HOSTNAME variables are set!
if [ -e sample_sheet.yaml ]; then
    echo -e "Starting snakemake"
    set +ue #turn off bash strict mode because snakemake and conda can't work with it properly
    echo -e "Jovian_run:\n    identifier: ${UNIQUE_ID}" > profile/variables.yaml
    echo -e "Server_host:\n    hostname: http://${SET_HOSTNAME}" >> profile/variables.yaml
    eval $(parse_yaml profile/variables.yaml "config_")
    snakemake -s Snakefile --profile "${PROFILE}" ${@} && echo -e "\nUnique identifier for this Jovian run is: $config_Jovian_run_identifier "
    set -ue #turn bash strict mode back on
else
    echo -e "Sample_sheet.yaml could not be found"
    echo -e "This also means that Jovian was unable to generate a new sample sheet for you"
    echo -e "Please inspect the input directory (${INPUT_DIR}) and make sure the right files are present"
    exit 1
fi

exit 0