#!/bin/bash
###############################################################################################
#
# package to perform the interpolation/remapping of CMORized data from the native grid to specified regular lon/lat grid
# consists of:
#  - steering script EUR_CORDEX_Remapping.sh
#  - script create_weights.sc to create fields of weights; has to be done only once
#  - script create_fx.sh to create files of orography and land-Sea fraction for destination grid
#  - actual interpolation script doremap.sc  ("do remapping")
#
# three input data sets must exist:
#  1.: On the native grid the already CMORized data that should be interpolated/remapped
#  2.: On the native grid the fixed (fx) fields of orog (orography in m)  and sftlf (Land_Sea_Fraction in %)
#  3.: On target grid orog_sftlf_${destDomain}.nc containing orography (orog in m) and land-sea fraction (sftlf unit=1) of the target/destination domain
#      has to be created before performing the interpolation/remapping
#      It is a good advice that different modelers who want to interpolate/remap their data to the same target/destination grid share this file orog_sftlf_${destDomain}.nc
#      ${destDomain} stands for the name of the target/destination domain, e.g ALP-3i
#
# Authors:
#  - steering script and sript "create_weights.sc" and script "create_fx.sh" written by Hans-Juergen Panitz (HJP), IMK/KIT, Sep/Oct 2019
#  - actual interpolation script written by Erik van Meijgaard, KNMI; has been modified by H.-J. Panitz
#
# Important note:
# if the execution of the package crashes for whatever reason and if you then want to repeat the excecution
# - you MUST delete all *nc3 files in the directory/directories ${BaseDirSource}/${freq}/${var} before you rerun the tool
# - shoud delete all tmp* directories in directory $SWD
#
###############################################################################################
#
###############################################################################
#
#     USER INTERFACE
#
###############################################################################
#
# assumes: UNIX/LINUX bash-shell environment
#          CDO and NCO library and NetCDF library havbe to be installed 
#          (you probably need to adjust paths)

#
# Paths for HAWK at HLRS
#
CDO_PATH=/opt/hlrs/spack/rev-004_2020-06-17/cdo/1.9.8-intel-19.1.0-6ozqp2oo/bin
NCO_PATH=/opt/hlrs/spack/rev-004_2020-06-17/nco/git-master-intel-19.1.0-2ivs7fvn/bin
NETCDF_PATH=/opt/hlrs/spack/rev-004_2020-06-17/netcdf/4.7.3-intel-19.1.0-ni7c6g4l/bin
#
# CDO, NCO, NETCDF Software on MISTRAL at  DKRZ, Hamburg
##CDO_PATH=/sw/rhel6-x64/cdo/cdo-1.9.8-magicsxx-gcc64/bin
#NCO_PATH=/sw/rhel6-x64/nco/nco-4.7.5-gcc64/bin
#NETCDF_PATH=/sw/rhel6-x64/netcdf/netcdf_c-4.7.2-gcc64/bin
#
export PATH=.:${CDO_PATH}:${NCO_PATH}:${NETCDF_PATH}:$PATH
#
# WorkDir: Baisc Work path; all other directories are relative to WOrkDir
#
WorkDir=/lustre/cray/ws9/5/ws/imkclm1-Daten_20201011
#
# BaseDir: path to the location where your data-path according to the Data Reference System (DRS) begins
#  see: fps_convection_filennaming.pdf on FPSC webpage: https://www.hymex.org/cordexfps-convection/wiki/doku.php?id=protocol
BaseDir=${WorkDir}//FPS_Convection/Mandatory_Domain_00275_23GPsponge/CEUR0275_RK25L50ERAINT_KIT/Evaluation/CMOR/OUT
#
Project="CORDEX-FPSCONV"              # DRS element <activity>=project_id
Product="output"                   # DRS element <product>
Domain="ALP-3"                    # DRS element <Domain>  (Source!!)
Institute="CLMcom-KIT"             # DRS element <Institution>
#Driving_Model="MPI-M-MPI-ESM-LR"       # DRS element <GCMModelName>
#Experiment="historical"            # DRS elelemt <CMIP5ExperimentName>
Driving_Model="ECMWF-ERAINT"       # DRS element <GCMModelName>
Experiment="evaluation"            # DRS elelemt <CMIP5ExperimentName>
Ensemble="r1i1p1"                  # DRS element <CMIP5EnsembleMember>
#RCM_Model="CLMcom-KIT-CCLM5-0-15"  # DRS element <CPRCM_Model_Id> (RCM model_id)
RCM_Model="CLMcom-KIT-CCLM5-0-14"  # DRS element <CPRCM_Model_Id> (RCM model_id)
RCM_version_id="fpsconv-x2yn2-v1"           # DRS elements <CPRCMNestingVersionID> 
#
# define the frequencies to be considered, separated by blank character; will run in a loop
#  if you leave the list blank, all frequencies that are found will be considered
frequencies="1hr"     
#frequencies=" "  # example for blank list   
#
# Target Domain
destDomain=ALP-3i
#
# List of variables,separated by blank chartacter, that have to be remapped
#  if you leave the list blank, all variables being stored for a given frequency will be considered
#  NOTE: in case that more than one frequency is considered in one job the var_list MUST be empty
#        it will be "filled" later with all variables found in the corresponding Input Directory
#
var_list="hfls uas vas"
#var_list=" "      # example for blank list
#
# specify mapping method for CDO: remap${map}; remapcon is the method recommended by the EURO-CORDEX Community
map="con"
#
# there might be some  variables with an "extreme" land-water contrast
#  "extreme" means that these variables are only defined over land but not over any kind of water body (open sea, rivers, lakes)
#  at least in CCLM these variables are represented by the missing value (=1.E20) at grid points over water
#  in CCLM these variables are: snc snd mrros mrro mrso1 mrso
#  this "extreme" land-water contrast is kept also in the target grid depending on the land-sea mask of the target domain
#  if your RCM does not have such variables, leave the following list empty: Variables_with_missing_value_over_water=" "
#  if your RCM has such variables that are different from those in the following list, edit the list
Variables_with_missing_value_over_water="snc snd mrros mrro mrso1 mrso"
#
# for some near-surface variables being defined at each native grid-point (no missing values at all), but with sharp land-sea contrast, an "extended" remapping procedure is selected
# these variables are: tas tasmin tasmax huss sfcWind sfcWindmax mfrso mrros mrro mrso snw snm uas vas snc snd sic evspsbl hfss hfls
# these variables are defined in the actual remapping tool "doremap.sc"
# for the extended remapping procedure weight files have to be created, but only once per RCM and destination domain
# this will be done before the actual remapping
# Thus, the user has to choose now one of the variables listed above and one "low" frequency (e.g. day or sem) for which CMORized data on the native grid are available 
# Attention!!!!
#  if you carried out already a remapping to some destination domain and if you now want to perform another remapping to different destination domain,
#  and if you define the same scratch directories $HWD and $SWD
#  be sure that the directories $HWD and $SWD are empty. Previously created weight files are stored in $SWD, but they do not correspond to the new destiantion domain
# Attention!!!!
#   I don't recommend to choose a variable from the list "Variables_with_missing_value_over_water" (HJP's opinion)
#  only a variable should be chosen that does not have any missing value in the native grid
weight_frequency="day"
weight_variable="tasmax"
#
# specify working directory on HOME and SCRATCH partition (SWD can be chosen the same as HWD)
HWD=${WorkDir}/Scratch/cordex_convert/CORDEX_REMAP
SWD=${WorkDir}/Scratch/cordex_convert/CORDEX_REMAP
#
###############################################################################
#
#     USER INTERFACE END
#
###############################################################################
#
# exedir: directory where this script is located
exedir=$PWD
#
# Base directory of source data and target data
#
# Source: Paths to CMORized variables and constants 
BaseDirSource=${BaseDir}/${Project}/${Product}/${Domain}/${Institute}/${Driving_Model}/${Experiment}/${Ensemble}/${RCM_Model}/${RCM_version_id}
BaseDirSourcefx=${BaseDir}/${Project}/${Product}/${Domain}/${Institute}/${Driving_Model}/${Experiment}/r0i0p0/${RCM_Model}/${RCM_version_id}/fx
#
#Target 
BaseDirTarget=${BaseDir}/${Project}/${Product}/${destDomain}/${Institute}/${Driving_Model}/${Experiment}/${Ensemble}/${RCM_Model}/${RCM_version_id}
#
# common_fx_file contains orographie (orog in m) and land-sea fraction (sftlf in %) of the target domain
#  must exist and, thus, be created in advance before performing the interpolation
common_fx_file=${exedir}/orog_sftlf_${destDomain}.nc
#
# check existence of Base directory of source data
if [ ! -d ${BaseDirSource} ]
then
 echo
 echo ${BaseDirSource}
 echo Base directory of source data does not exist
 echo check it, correct it and repeat the script
 exit
fi
#
# check existence of Base directory of fx source data
if [ ! -d ${BaseDirSourcefx} ]
then
 echo
 echo ${BaseDirSourcefx}
 echo Base directory of fx source data does not exist
 echo check it, correct it and repeat the script
 exit
fi
#
# check orog directory of source data
if [ ! -d ${BaseDirSourcefx}/orog ]
then
 echo
 echo ${BaseDirSourcefx}/orog
 echo directory of orog  source file  does not exist
 echo check it, correct it and repeat the script
 exit
fi
#
# check sftlf directory of source data
if [ ! -d ${BaseDirSourcefx}/sftlf ]
then
 echo
 echo ${BaseDirSourcefx}/sftlf
 echo directory of sftlf source file  does not exist
 echo check it, correct it and repeat the script
 exit
fi
#
# determine filenames of orog and sftlf for the source domain
cd ${BaseDirSourcefx}/orog
orog_file=`echo $( ls -f orog*)`
cd ${BaseDirSourcefx}/sftlf
sftlf_file=`echo $( ls -f sftlf* )`
#
cd ${exedir}
#
# check whether common fx file for destination domain exists
if [ ! -f ${common_fx_file} ]
then
  echo
  echo common_fx_file $common_fx_file not found ... remedy and retry
  exit
fi
#
if [ ! "${frequencies}" -o "${frequencies}" == " " ]  # check whether user defined list of frequencies is empty
then
 cd ${BaseDirSource}
 frequencies=`echo $( ls -d *)`
fi
cd ${exedir}
#
# check existence of time frequencies (whether there are frequency directories at all)
list_freq=" "
icnt=0
for freq in $frequencies
do
 if [ -d ${BaseDirSource}/${freq} ]
 then
   list_freq="${list_freq} "$freq
   (( icnt = icnt+1 ))
 else
   echo 
   echo Frequency ${freq} does not exist in ${BaseDirSource}
 fi
done
#
if [ ${icnt} -eq 0 ]
then
 echo
 echo -------------------------------------------------------
 echo no frequencies found at all
 echo check it, correct it, repeat the script
 echo -------------------------------------------------------
 exit
else
 frequencies=${list_freq}
 echo
 echo -------------------------------------------------------
 echo Final list of frequencies: $frequencies
 echo -------------------------------------------------------
 echo
fi
#
# create Base directory for output/target data if it does not exist
[ -d ${BaseDirTarget} ] || mkdir -p ${BaseDirTarget}
# create Help and Temporary directories HWD and SWD; (SWD can be chosen the same as HWD)
[ -d ${HWD} ] || mkdir -p ${HWD}
[ -d ${SWD} ] || mkdir -p ${SWD}
#
# land sea mask on native grid
if [ ! -f $SWD/sftlf_${Domain}_${RCM_Model}_fx.nc ]
then
 echo sftlf
 cd $SWD
 ln -s  ${BaseDirSourcefx}/sftlf/${sftlf_file} sftlf_${Domain}_${RCM_Model}_fx.nc
 sftlf_file_source=sftlf_${Domain}_${RCM_Model}_fx.nc
else
 sftlf_file_source=sftlf_${Domain}_${RCM_Model}_fx.nc
fi
#
# orography on native grid
if [ ! -f $SWD/orog_${Domain}_${RCM_Model}_fx.nc ]
then 
 cd $SWD
 ln -s  ${BaseDirSourcefx}/orog/${orog_file} orog_${Domain}_${RCM_Model}_fx.nc
fi
#
# for security reasons in order to avoid crashes:
#  delete old tmp-files if they still exist in $SWD
cd $SWD
if ls -d tmp* 1> /dev/null 2>&1
then
  rm -r tmp*
fi
#
cd ${exedir}
#
#  now create weight files if they do not exist
#  first, check whether Input directory really exists
if [ ! -d ${BaseDirSource}/${weight_frequency}/${weight_variable} ]
then
 echo
 echo ${BaseDirSource}/${weight_frequency}/${weight_variable}
 echo directory of source data does not exist
 echo check it, correct it and repeat the script
 exit
fi
#
#
# calculation of weight functions if necessary
# has to be done only once per RCM and target grid

for  Var in  ${Variables_with_missing_value_over_water}
do 
  if [ ${Var} == ${weight_variable} ]
  then
   echo
   echo Variable $Var chosen for the calculation contains missing values which is not recommended at all
   echo Please, change this variable
   echo
   exit
  fi
done
#
# choose the first file in ${BaseDirSource}/${weight_frequency}/${weight_variable}
cd ${BaseDirSource}/${weight_frequency}/${weight_variable}
file_list=`echo $( ls *)`
read file __ <<< "${file_list}"
#
cd ${exedir}
#
csh ${exedir}/create_weights.sc $SWD $map ${Domain} ${destDomain} ${BaseDirSource}/${weight_frequency}/${weight_variable}/${file} ${sftlf_file_source} ${common_fx_file}

#
# calculation of weight functions finished
#

cd ${exedir}
#
#  now create orog and sftlf files for destination grid if they don't already exist 
${exedir}/create_fx.sh $SWD ${Domain} ${destDomain} ${common_fx_file} ${BaseDirSourcefx} ${sftlf_file} ${orog_file} ${Ensemble} ${BaseDirTarget}
#
# Save the user defined var_list
#  this nis necessary in case that more than one frequency will be conssidered in one job
#  in this case the user defined var_list MUST be empty!!
#
var_list_inp=${var_list}
#
for freq in $frequencies            ## loop over freqeuencies
do
 cd ${BaseDirSource}/${freq}
#
# check whether the user defined var_list is empty
# if yes, then all variables found in the Input Directory are considered
#
 if [ ! "${var_list_inp}" -o "${var_list_inp}" == " " ]
 then
  var_list=`echo $( ls -d *)`
 fi
#
  echo
  echo -----------------------------------------
  echo Variables to be considered for frequency ${freq}:
  echo $var_list
  echo ----------------------------------------
  echo
#
 for var in $var_list             # loop over all variables per frequency
 do

  if [ ! -d ${BaseDirSource}/${freq}/${var} ]
  then
   echo
   echo ${BaseDirSource}/${freq}/${var}
   echo directory of source data does not exist
   echo check it, correct it and repeat the script
   exit
  fi
#
  echo
  echo -----------------------------------------
  echo Variable $var considered
  echo ----------------------------------------
  echo
#
  cd ${BaseDirSource}/${freq}/${var}
#
#
# for security reasons in order to avoid crashes
#  delete *nc3-files if they still exist
  if ls *.nc3 1> /dev/null 2>&1; then
    rm *.nc3
  fi
#
# create list of all files in ${BaseDirSource}/${freq}/${var}
#  
  file_list=`echo $( ls *)`
#
  echo
  echo -----------------------------------------
  echo files considered:
  echo ${file_list}
  echo ----------------------------------------
  echo
#
#
# create final output directory for output/target data if it does not exist
  [ -d ${BaseDirTarget}/${freq}/${var} ] || mkdir -p ${BaseDirTarget}/${freq}/${var}
#
  echo 
  echo start interpolation for Variable $var at frequency $freq  at `date`
  echo

#
# loop over all files found in ${BaseDirSource}/${freq}/${var}
# all files are treated simultaneously/in parallel achieved by the clasping "( ...)&"
# however, this might lead to memory problems, also when running a batch job, depending on the number of files and the size of each file
# I got such problems on CRAY XC40 when trying to remap hourly FPSC data (HJP)
# in such a case you have to split the list of files into a smaller number being considered
#
  for file in ${file_list}
  do
#
  (

   echo file $file
   echo

   outfile=`echo "$file" | sed -e "s/$Domain/$destDomain/"`
#

   cd ${exedir}

#
#  for the actual remapping a temporary directory is created for each file
#  these directories will be deleted after a successful remapping of each file
   tmp_file=$(mktemp -d ${SWD}/tmp_${var}_XXXXXX)
   cd ${tmp_file}
#
#  now the actual remapping  tool is called
   csh ${exedir}/doremap.sc ${BaseDirSource}/${freq}/${var}/$file ${HWD} ${SWD} ${common_fx_file} ${destDomain} ${map} "${Variables_with_missing_value_over_water[*]}"

#
   status=$?
# 
# if an error occurred in doremap.sc the execution stops
# and some "cleaning" is carried out
#
   if [ $status -ne 0 ]
   then
    echo
    echo Error in interpolation script, see message above
    echo 
    if [ -r ${HWD}/target.nc ] 
    then
     rm -f ${HWD}/target.nc
    fi
    if [ -r ${tmp_file}/target.nc ] 
    then
     rm -f ${tmp_file}/target.nc
    fi
    cd ${SWD}
    rm -r tmp_${var}*
    rm -f ${BaseDirSource}/${freq}/${var}/*nc3
    exit
   fi

   mv ${tmp_file}/target.nc ${BaseDirTarget}/${freq}/${var}/${outfile}
#
  ) &
  done   # end loop over files
#
  wait
  echo 
  echo end interpolation for Variable $var at frequency $freq  at `date`
  echo
  rm -r ${SWD}/tmp_${var}*
#
 done                            # end loop over variables per frequency

done                             # end loop over frequencies
