#!/bin/bash
#################################################################
# Script to find which Hydro jobs failed                        #
#                                                               #
# Loop over all slurm-<jobID>.out and print the failed jobs     #
# into a list.                                                  #
#                                                               #
# !Make sure you set properly the path to your .out files!      #
# !Check you set the correct name for your .out files!          #
#################################################################

string="Finished."			## a succesfully completed hydro job will have this string in the last line of its output
outfile="testlist.txt"			## file where we will list all the failed jobs

if [[ -f "$outfile" ]]; then		## delete previous list of failed jobs
   rm $outfile
fi

for filename in ./slurm-*.out; do 	## check all slurm-*.out files in the folder
   line=$(sed -n '$p' $filename) 	## takes the last line of each file
   if [[ $line != *$string* ]]; then	## if line does NOT contain the specified string
      newname=$(echo $filename | sed 's#./slurm-##g' | sed 's#.out##g')	## strips the filename to the jobID
      echo "${newname}:: ${line}"	## also print jobID and the last line (reason for failure)
      echo $newname >> $outfile		## append jobID into the file
   fi
done