#!/bin/csh
alias julian /users/rsg/panorama/panorama.svn/bin/julian

# THIS SCRIPT HAS BEEN UPDATED 2023-08-02 TO REFLECT NEW PATHS FOR DATA - jpe #

echo "================================="
echo "Running get_iridium_buoy_data.csh"
date 
set execdir = /users/rsg/observatory/buoy/pml/bin
#set execdir = /users/rsg/tjsm/src/buoy/iridium
#set downloaddir = /users/rsg/observatory/buoy/pml/download/E1/ODAS51
set downloaddir = /data/datasets/Buoys/E1/download/ODAS51/
#set wavedir = /users/rsg/observatory/buoy/pml/download/E1/waves
set wavedir = /data/datasets/Buoys/E1/download/waves/
#set finaldir = /users/rsg/observatory/buoy/pml/download/E1
set finaldir = /data/datasets/Buoys/E1/download/
set wavecodedir = /users/rsg/observatory/buoy/pml/bin/AMOS2XBinaryCoder-1.0
set base = 144
set HH = `date --utc +%k`
@ base = $base + $HH

#set UnitID_atm = '300025060515020'
# 2022
set UnitID_wat = '300025010838850'
set UnitID_atm = '300025010444580'

# 2023 - two units for redundancy
set UnitID_atm1 = '300025010712790'
set UnitID_atm2 = '300025010534820'

# From 2022 the wave data are encoded in binary
set waveheader = `cat $wavecodedir/bf-128.csv | head -1`
echo $waveheader > /tmp/E1buoy_waves.csv

# Get the E1 data buoy email messages
# and download them into a safe directory
#/usr/bin/python $execdir/save_all_attachments.py --deletemsg --dir $downloaddir --user e1buoy --passwd gb76nwbk600 harris.npm.ac.uk 
#/usr/bin/python2 $execdir/save_all_attachments.py --deletemsg --dir $downloaddir --user e1buoyfwd --passwd L4E1_buoy imap.gmail.com 
#/bin/rclone sync --update --verbose --transfers 30 --checkers 8 --contimeout 60s --timeout 300s --retries 3 --low-level-retries 10 --stats 1s "google-drive:/Attachments" $downloaddir 

# Get the buoy messages from Sharepoint
# and rclone into a safe directory
/bin/rclone sync --update --verbose --transfers 30 --checkers 8 --contimeout 60s --timeout 300s --retries 3 --low-level-retries 10 --stats 1s "sharepoint-WCO:/General/Data/E1/Buoy" $downloaddir 

# Move the files into day specific directories
#foreach file($downloaddir/${UnitID_wat}*.sbd)
foreach infile ( `find $downloaddir \( -name ${UnitID_atm1}\*.sbd -o -name ${UnitID_atm2}\*.sbd \) -size +300c` )
   echo $infile
   /bin/mv $infile $wavedir
end

foreach file(`find $downloaddir -name \*.sbd -mmin -90 -size -300c`)
   echo $file 
   set stem = `echo $file:t:r | cut -c1-15` 
   echo $stem
   #set datename = `cat $file | sed -e 's/\,/\ /g' | tr '\n' '\t' | awk '{print $2}'`
   set datename = `cat $file | sed -e 's/\,/\ /g' | tr '\n' '\t' | awk '{print $4}'`
   echo $datename
   # Change needed for sbd formatted files 2021 onwards
   if ($stem == $UnitID_atm1 || $stem == $UnitID_atm2) then
      set yy = `echo $datename | cut -c5-6`
      set mm = `echo $datename | cut -c3-4`
      set dd = `echo $datename | cut -c1-2`
      #set yy = `echo $datename | cut -c1-2`
      #set mm = `echo $datename | cut -c3-4`
      #set dd = `echo $datename | cut -c5-6`
      set yyyy = `echo 20$yy`
      echo $yyyy$mm$dd
   else
      set yrcheck = `echo $datename | cut -c1-2`
      if ($yrcheck == 20) then
         set yyyy = `echo $datename | cut -c1-4`
         set mm = `echo $datename | cut -c5-6`
         set dd = `echo $datename | cut -c7-8`
      else
         set yyyy = `echo 20$yrcheck`
         set mm = `echo $datename | cut -c3-4`
         set dd = `echo $datename | cut -c5-6`      
      endif
   endif
   set nchar = `echo $yyyy$mm$dd | wc -m`
   echo $nchar
   if ($nchar == 9) then
      /bin/mkdir -p $finaldir/$yyyy$mm$dd
      echo $finaldir/$yyyy$mm$dd
      set stem = `echo $file:t:r | cut -c1-15`
      /bin/mv $file $finaldir/$yyyy$mm$dd
      #/bin/scp $file $finaldir/$yyyy$mm$dd

      # move to the created directory to find files of different types
      pushd $finaldir/$yyyy$mm$dd
      # in water files (S.txt for final version)
      # remove previous versions of the file
      /bin/rm ${yyyy}${mm}${dd}S.txt
      set fs_inwater = `grep -l 5208001e4525b0d0 *.sbd`
      foreach f_inwater($fs_inwater)
         dos2unix -q $f_inwater
         cat $f_inwater | tr '\n' ',' | tr -s "\*" "," >> ${yyyy}${mm}${dd}S.txt
         echo "" >> ${yyyy}${mm}${dd}S.txt
      end 
      # atmospheric files (A.txt for final version)
      # remove previous versions of the file
      /bin/rm ${yyyy}${mm}${dd}A.txt
      #set fs_atm = `grep -l 5208001e4525da2f *.sbd`
      set fs_atm = `ls ${UnitID_atm1}_*.sbd ${UnitID_atm2}_*.sbd`
      foreach f_atm($fs_atm)
         dos2unix -q $f_atm
         #cat $f_atm | tr '\n' ',' | tr -s "\*" "," >> ${yyyy}${mm}${dd}A.txt
         cat $f_atm | tr '\n' ',' >> ${yyyy}${mm}${dd}A.txt
         echo "" >> ${yyyy}${mm}${dd}A.txt
      end 
      # wave files (W.txt for final version)
      # remove previous versions of the file
      /bin/rm ${yyyy}${mm}${dd}W.txt
      set fs_wave = `grep -l TSPWA *.sbd`
      foreach f_wave($fs_wave)
         dos2unix -q $f_wave
         cat $f_wave | tr '\n' ',' | tr -s "\*" "," >> ${yyyy}${mm}${dd}W.txt
         echo "" >> ${yyyy}${mm}${dd}W.txt
      end 
      
      # combined wave and atmospheric files
      # remove previous versions of the file
      #/bin/rm ${yyyy}${mm}${dd}WA.txt
      echo "creating combined wave atmosphere files"
      set fs_wave_atm = `grep -l 5206001e45259026 *.sbd`
      foreach f_wave_atm($fs_wave_atm)
         dos2unix -q $f_wave_atm
         cat $f_wave_atm | grep "5206001e45259026,1"  | tr -s "\*" "," >> ${yyyy}${mm}${dd}WA.txt
      end 

      popd
   endif
end

# delete everything on the Google Drive older than 1 day
#/bin/rclone delete --min-age 1d "google-drive:/Attachments"
# delete everything on the Sharepoint Drive older than 1 day
/bin/rclone delete --min-age 7d "sharepoint-WCO:/General/Data/E1/Buoy"

# From 2022 the wave files are encoded in binary
pushd $wavecodedir
set waveheader = `cat $wavecodedir/bf-128.csv | head -1`
echo $waveheader > /tmp/E1buoy_waves.csv
#foreach infile ( `find $wavedir -name ${UnitID_atm1}\*.sbd -size +300c | sort` )
foreach infile ( `find $wavedir -name ${UnitID_atm1}\*.sbd -size +300c` )
   /bin/scp $infile $wavecodedir/dataoutput.sbd
   set outputdata = `$wavecodedir/DataformatDecoder.py`
   echo $outputdata | sed 's/\[//' | sed 's/\]//' >> /tmp/E1buoy_waves.csv
end
popd
/bin/mv /tmp/E1buoy_waves.csv $finaldir/E1buoy_waves.csv

pushd $wavecodedir
set waveheader = `cat $wavecodedir/bf-128.csv | head -1`
echo $waveheader > /tmp/E1buoy_waves.csv
#foreach infile ( `find $wavedir -name ${UnitID_atm2}\*.sbd -size +300c | sort` )
foreach infile ( `find $wavedir -name ${UnitID_atm2}\*.sbd -size +300c` )
   /bin/scp $infile $wavecodedir/dataoutput.sbd
   set outputdata = `$wavecodedir/DataformatDecoder.py`
   echo $outputdata | sed 's/\[//' | sed 's/\]//' >> /tmp/E1buoy_waves.csv
end
popd
/bin/mv /tmp/E1buoy_waves.csv $finaldir/E1buoy_waves2.csv

# make long files for all the parameters
# Met files
#echo "Creating time-series met files"
#cat $finaldir/????????/*A.txt | awk 'BEGIN {FS = ","; RS = "\n"} \
#    {if ($1 ~ /2/ && $5 <= 504600 && NF > 6) \
#    {printf "$W5M5A,%02d%02d%02d,%04d05,5206001e45259026,1,E1_Buoy,%2.4fN,%1.4fW,%3.1f,%2.2f,%2.1f,-9.99,%3.1f,%4.2f,-9.99,%2.1f,-9.99,-9.99,-9.99,-9.99,02A\n",\
#    substr($2,5,2),substr($2,3,2),substr($2,1,2),$3,$5/10000,-$6/10000,$11/100,$10/10,$12/10,$7/10,$8/10,$9/100}}' \
#    > $finaldir/buoy_met_E1_all.txt

## 2022
#echo "Creating time-series met files"
#cat $finaldir/????????/*A.txt | awk 'BEGIN {FS = ","; RS = "\n"} \
#    {if ($1 ~ /2/ && $5 <= 504600 && NF == 15) \
#    {printf "$W5M5A,%02d%02d%02d,%04d05,5206001e45259026,1,E1_Buoy,%2.4fN,%1.4fW,%3.1f,%2.2f,%2.1f,-9.99,%3.1f,%4.2f,-9.99,%2.1f,-9.99,-9.99,-9.99,-9.99,02A\n",\
#    substr($2,5,2),substr($2,3,2),substr($2,1,2),$3,$5/10000,-$6/10000,$12/100,$11/10,$13/10,$7/10,$8/10,$9/100}}' \
#    > $finaldir/buoy_met_E1_all.txt

## 2023
## Added in a search for 14320 because of SST probe from Met Office being more reliable on this side.
echo "Creating time-series met files"
cat $finaldir/????????/*A.txt | awk 'BEGIN {FS = ","; RS = "\n"} \
    {if ($2 ~ /14320/ && $3 ~ /2/ && $7 <= 504600 && NF == 17) \
    {printf "$W5M5A,%02d%02d%02d,%04d05,5206001e45259026,1,E1_Buoy,%2.4fN,%1.4fW,%3.1f,%2.2f,%2.1f,%4.1f,%3.1f,%4.2f,-9.99,%2.1f,-9.99,-9.99,-9.99,-9.99,02A\n",\
    substr($4,5,2),substr($4,3,2),substr($4,1,2),$5,$7/10000,-$8/10000,$14/100,$13/10,$15/10,$12/100,$9/10,$10/10,$11/100}}' \
    | sort -u \
    > $finaldir/buoy_met_E1_all.txt

# GPS information
echo "Creating time-series GPS files"
# find if there are new files
set newatm = `find $finaldir/ -name \*A.txt -mmin -120`
echo $#newatm

# Change in 2021 that Atmospheric files don't report N and W
#cat $finaldir/????????/*A.txt | awk 'BEGIN {FS = ","; RS = "\n"} {if ($1 ~ /2/) \
#   {printf "%02d%02d%02d %04d05 %2.4fN %1.4fW\n",substr($2,5,2),substr($2,3,2),substr($2,1,2),$3,$5/10000,-$6/10000}}' \
#    > $finaldir/buoy_GPS_E1_all.txt

# Change in 2021 that Atmospheric files don't report N and W
# 2023
cat $finaldir/????????/*A.txt | awk 'BEGIN {FS = ","; RS = "\n"} {if ($3 ~ /2/) \
   {printf "%02d%02d%02d %04d05 %2.4fN %1.4fW\n",substr($4,5,2),substr($4,3,2),substr($4,1,2),$5,$7/10000,-$8/10000}}' \
   | sort -u \
    > $finaldir/buoy_GPS_E1_all.txt

# Change in 2021 that seawater data do not contain the GPS
# In water files
echo "Creating time-series in-water files"
# find if there are new files
set newwtr = `find $finaldir/ -name \*S.txt -mmin -120`
echo $#newwtr
#cat $finaldir/????????/*S*.txt | awk 'BEGIN {FS = ","; RS = "\n"} {if (NF >= 33 && $6 == 0){printf "%06d,%06d,0.00,%f,0.00,%f,%f,%f,%f,%f\n",$2,$3,$8,$12,$21,$24,$27,$30}} \
#                                            {if (NF >= 33 && $6 == 120){printf "%06d,%06d,%f,%f,0.00,%f,%f,%f,%f,%f\n",$2,$3,$7,$10,$14,$23,$26,$29,$32}}' \
#      > $finaldir/buoy_water_E1_all.txt

# 2023 - inwater data no longer via Iridium
#cat $finaldir/????????/*S*.txt | awk 'BEGIN {FS = ","; RS = "\n"} {if ((NF == 35 || NF == 48)&& $6 == 0)\
#                                            {printf "%06d,%06d,0.00,%f,0.00,%f,%f,%f,%f,%f\n",$2,$3,$8,$12,$21,$24,$27,$30}} \
#                                            {if ((NF == 35 || NF == 48) && $6 == 120)\
#                                            {printf "%06d,%06d,%f,%f,0.00,%f,%f,%f,%f,%f\n",$2,$3,$7,$10,$14,$23,$26,$29,$32}} \
#                                            {if ((NF == 33 || NF == 46) && $9 == 0)\
#                                            {printf "%06d,%06d,%f,0.00,0.00,%f,%f,%f,%f,%f\n",$2,$3,$7,$12,$21,$24,$27,$30}}' \
#      > $finaldir/buoy_water_E1_all.txt

# Final data for table on web
echo "Creating short table file"
set short_metdata = `cat $finaldir/buoy_met_E1_all.txt | tail -1 | awk 'BEGIN {FS = ","; RS = "\n"} {print $16,$10,$9,$10,$13,substr($3,1,2)}'`
if ($#newatm == 0) then
   set short_metdata = "-9.99 -9.99 -9.99 -9.99 -9.99 -9.99"
   set m_julian = 0
endif

#set short_metdata_waves = `cat $finaldir/buoy_waves_E1_all.txt | tail -1 | awk '{print $3,$4,$5,$7}'`
set short_metdata_waves = "-9.99 -9.99 -9.99 -999"
#set short_metdata_waves = `cat $finaldir/E1buoy_waves.csv | tail -1 | awk 'BEGIN{FS=","}{print $19,$19*1.3,$25,$22}'`
set short_metdata_waves = `cat $finaldir/E1buoy_waves.csv | tail -1 | awk 'BEGIN{FS=","}{print $19,$30,$25,$22}'`

if ($#newatm != 0) then
   set last_date = `cat $finaldir/buoy_met_E1_all.txt | tail -1 | awk 'BEGIN {FS = ","; RS = "\n"} {print $2}'`
   set yy = `echo $last_date | cut -c1-2`
   set yyyy = `echo "20"$yy`
   set mm = `echo $last_date | cut -c3-4`
   set dd = `echo $last_date | cut -c5-6`
   set latest_time = `echo $short_metdata[6]`
endif

# extract the date and time information from the in-water file
## 2023 - no in-water data via Iridium
#set w_last_date = `cat $finaldir/buoy_water_E1_all.txt | tail -1 | awk 'BEGIN {FS = ","; RS = "\n"} {print $1}'`
#set w_yy = `echo $w_last_date | cut -c1-2`
#set w_yyyy = `echo 20$w_yy`
#set w_mm = `echo $w_last_date | cut -c3-4`
#set w_dd = `echo $w_last_date | cut -c5-6`
#set w_latest_time = `cat $finaldir/buoy_water_E1_all.txt | tail -1 | awk 'BEGIN {FS = ","; RS = "\n"} {print $2}' | cut -c1-2`
#echo $w_latest_time

# return only the in-water data (and time-stamp) if in-water data more recent
#set w_julian = `julian $w_dd $w_mm $w_yyyy`
if ($#newatm != 0) then
  set m_julian = `julian $dd $mm $yyyy`
endif

#echo $w_julian $m_julian

#set waterdate_epoch = `date '+%s' -d "$w_yyyy$w_mm$w_dd"`
set metdate_epoch = `date '+%s' -d "$yyyy$mm$dd"`

echo $metdate_epoch
#echo $waterdate_epoch

#if ($waterdate_epoch > $metdate_epoch) then
#if ($w_julian[1] > $m_julian[1]) then
#   echo "More recent data available from in-water observations"
#   set yyyy = $w_yyyy
#   set mm = $w_mm
#   set dd = $w_dd
#   set short_metdata = "-10.0 -10.0 -10.0 -10.0 -10.0 $w_latest_time"
#   set short_metdata_waves = "-10.0 -10.0 -10.0 -10.0"
#endif

#if ($w_julian[1] == $m_julian[1]) then
   #if ($w_latest_time > $latest_time) then
   #   echo "More recent data available from in-water observations"
   #   set yyyy = $w_yyyy
   #   set mm = $w_mm
   #   set dd = $w_dd
   #   set short_metdata = "-9.99 -9.99 -9.99 -9.99 -9.99 $w_latest_time"
   #   set short_metdata_waves = "-9.99 -9.99 -9.99 -9.99"
   #endif
#endif
   
#set short_temp_sal = `cat $finaldir/buoy_water_E1_all.txt | tail -1 | awk 'BEGIN {FS = ","; RS = "\n"} {print $7,$8}'`
#set short_oxy_chl_turb = `cat $finaldir/buoy_water_E1_all.txt | tail -1 | awk 'BEGIN {FS = ","; RS = "\n"} {print $6/100.,$10,$9}'`
set short_GPS = `cat $finaldir/buoy_GPS_E1_all.txt | tail -1 | sed 's/N//g' | sed 's/W//g' | awk '{print $3,-$4}'`
#set short_PAR = `cat $finaldir/buoy_water_E1_all.txt | tail -1 | awk 'BEGIN {FS = ","; RS = "\n"} {printf "%f",$3/2.77}'`

# cludge to sort out missing in-water data (new scaling added when WQM removed from field: 24 Jul 2018).
# calibration achieved using last 5 days of data from deployment regressing WQM SST against MO SST sensor.
#set short_sst_MO = `cat $finaldir/buoy_met_E1_all.txt | tail -1 | awk '{print ($6-4.316)/0.914}'`
set short_sst_MO = `cat $finaldir/buoy_met_E1_all.txt | tail -1 | awk 'BEGIN {FS = ","; RS = "\n"} {print ($12)}'`
echo $short_sst_MO

# Do the gnuplot routines and stage files on the web
echo "Creating last ${base} hours plot (Met and PAR and waves)"
cat $finaldir/buoy_met_E1_all.txt | awk 'BEGIN {FS=","} {print $2$3,$9,$10,$11,$12,$13,$14,$15,$16}' | sort -k 1,1 -u | tail -${base} > /tmp/buoy_met_E1.txt
#cat $finaldir/buoy_water_E1_all.txt | tail -${base} | awk '{print $1$2,$3/2.77,$4,$5,$6,$7,$8,$9,$10}' > /tmp/buoy_water_E1.txt
cat $finaldir/E1buoy_waves2.csv | tail -${base} | awk -v yyyy=$yyyy 'BEGIN {FS=","}{if($3==yyyy){printf("%d%02d%02d%02d%02d00 %f %f %f %f\n", $3,$4,$5,$6,$7,$19,$19*1.3,$25,$22)}}' > /tmp/buoy_waves_E1.txt

echo "Creating gnuplot compatible format file"
cat $finaldir/buoy_water_E1_all.txt | awk 'BEGIN {FS=","} {printf "%d,%f,%f,%f,%f,%f,%f,%f,%f\n", $1$2,$3/2.77,$4,$5,$6,$7,$8,$9,$10}' > /tmp/buoy_water_E1_all.txt
#### Temporary fix while only met office temperature probe operational
cat $finaldir/buoy_met_E1_all.txt | awk 'BEGIN {FS=","} {printf "%d,%f,%f,%f,%f,%f,%f,%f,%f\n", $2$3,$15,$15,$15,$15,$12,$15,$15,$15}' > /tmp/buoy_water_E1_all.txt 

# Do some quality control of the Met Office in-water SST based on the E1 time-series analysis
/bin/scp $execdir/E1_mean_min_max.txt /tmp/E1_mean_min_max.txt
$execdir/MO_sst_QC.py -y $yyyy
# overwrite the in-water file
/bin/mv /tmp/buoy_water_E1_all_QC.txt /tmp/buoy_water_E1_all.txt 
set short_sst_MO = `cat /tmp/buoy_water_E1_all.txt | tail -1 | awk 'BEGIN {FS = ","; RS = "\n"} {print ($6)}'`
echo $short_sst_MO

echo "Creating short table format for web display"
#if ($#newwtr == 0) then 
   set short_temp_sal = `echo "$short_sst_MO -9.999"`
   set short_oxy_chl_turb = "-9.999 -9.999 -9.999"
   set short_PAR = "-9.99"
#endif

echo $short_metdata $short_temp_sal $dd $mm $yyyy $short_oxy_chl_turb $short_GPS $short_PAR $short_metdata_waves
echo $short_metdata $short_temp_sal $dd $mm $yyyy $short_oxy_chl_turb $short_GPS $short_PAR $short_metdata_waves > $finaldir/buoy_latest.txt


echo "Creating plots for the web"
pwd
/usr/bin/gnuplot /users/rsg/observatory/buoy/pml/bin/gnuplot/plot_temp_hum_ODAS51
/usr/bin/gnuplot /users/rsg/observatory/buoy/pml/bin/gnuplot/plot_pressure_ODAS51
/usr/bin/gnuplot /users/rsg/observatory/buoy/pml/bin/gnuplot/plot_wind_ODAS51
/usr/bin/gnuplot /users/rsg/observatory/buoy/pml/bin/gnuplot/plot_temp_sal_ODAS51
/usr/bin/gnuplot /users/rsg/observatory/buoy/pml/bin/gnuplot/plot_oxygen_fluor_ODAS51
/usr/bin/gnuplot /users/rsg/observatory/buoy/pml/bin/gnuplot/plot_nitrate_ODAS51
/usr/bin/gnuplot /users/rsg/observatory/buoy/pml/bin/gnuplot/plot_waves_height_ODAS51
/usr/bin/gnuplot /users/rsg/observatory/buoy/pml/bin/gnuplot/plot_waves_period_ODAS51

set latest_files = `/bin/ls [a-z]*.png`
#/usr/bin/scp $latest_files ${webuser}@${web}:/home/rsgadmin/westernchannelobservatory.org.uk/data/buoy/download/{$buoy}
/bin/mv $latest_files /data/datasets/Buoys/E1/download/
#/bin/mv -v $latest_files $downloaddir

exit(0)
