params {
  config_profile_description = 'Biowulf nf-core config'
  config_profile_contact = 'staff@hpc.nih.gov'
  config_profile_url = 'https://hpc.nih.gov/apps/nextflow.html'
  max_memory = '224 GB'
  max_cpus = 32
  max_time = '72 h'
  clusterOptions = null
  igenomes_base = '/fdb/igenomes_nf/'
}


// use a local executor for short jobs and it has to give -c and --mem to make nextflow
// allocate the resource automatically. For this the
// settings below may have to be adapted to the allocation for
// the main nextflow job.

singularity {
    enabled = true
    autoMounts = true
    cacheDir = "/data/$USER/nxf_singularity_cache"
    envWhitelist='https_proxy,http_proxy,ftp_proxy,DISPLAY,SLURM_JOB_ID,SINGULARITY_BINDPATH,MPLCONFIGDIR'
}

env {
    SINGULARITY_CACHEDIR="/data/$USER/.singularity"
    OMP_NUM_THREADS = 1
    OPENBLAS_NUM_THREADS = 1
    PYTHONNOUSERSITE = 1
}

profiles {
    biowulflocal {
        process {
            executor = 'local'
            cache = 'lenient'
            maxRetries = 3
            queueSize = 100
            memory = "$SLURM_MEM_PER_NODE MB"
            cpus = "$SLURM_CPUS_PER_TASK" 
       } 
    }

    biowulf {
        process {
            executor = 'slurm'
            maxRetries = 1
            queue = 'norm'
            queueSize = 200
            pollInterval = '2 min'
            queueStatInterval = '5 min'
            submitRateLimit = '6/1min'
            retry.maxAttempts = 1
    
            clusterOptions = ' --gres=lscratch:200 '

            scratch = '/lscratch/$SLURM_JOB_ID'
            // with the default stageIn and stageOut settings using scratch can
            // result in humungous work folders
            // see https://github.com/nextflow-io/nextflow/issues/961 and
            //     https://www.nextflow.io/docs/latest/process.html?highlight=stageinmode
            stageInMode = 'symlink'
            stageOutMode = 'rsync'
            
            // for running pipeline on group sharing data directory, this can avoid inconsistent files timestamps
            cache = 'lenient'

        // example for setting different parameters for jobs with a 'gpu' label
        // withLabel:gpu {
        //    queue = 'gpu'
        //    time = '4h'
        //    clusterOptions = " --gres=lscratch:400,gpu:1 "
        //    clusterOptions = ' --constraint="gpua100|gpuv100|gpuv100x" '
        //    containerOptions = " --nv "
        // }

        // example for setting different parameters for a process name
        withName: 'NFCORE_RNASPLICE:RNASPLICE:BAM_SORT_STATS_SAMTOOLS:SAMTOOLS_SORT' {
          cpus = 12
          memory = '128 GB'
          time = '4h'
        }

        // example for setting different parameters for jobs with a resource label
        //  withLabel:process_low {
        //  cpus = 2
        //  memory = '12 GB'
        //  time = '4h'
        // }        
        // withLabel:process_medium {
        //  cpus = 6
        //  memory = '36 GB'
        //  time = '12h'
        // }
        // withLabel:process_high {
        //  cpus = 12
        //  memory = '72 GB'
        //  time = '16 h'
        // }
     }   
        timeline.enabled = true
        report.enabled = true
    }
}