From cafe7eeb86d6c5eb9c67b68e857bfd112e02e084 Mon Sep 17 00:00:00 2001
From: Karthik Chandrashekara <karthik@physi.uni-heidelberg.de>
Date: Wed, 19 Jun 2024 19:45:22 +0200
Subject: [PATCH] Added lines to allow for running on GPUs in cluster.

---
 .../+Scripts/run_on_cluster.m                 |  1 +
 .../+Simulator/@DipolarGas/DipolarGas.m       |  6 ++-
 .../+Simulator/@DipolarGas/initialize.m       |  4 ++
 ...m.slurm => bwhpc_matlab_gpe_sim_cpu.slurm} |  4 +-
 .../bwhpc_matlab_gpe_sim_gpu.slurm            | 38 +++++++++++++++++++
 5 files changed, 49 insertions(+), 4 deletions(-)
 rename Dipolar-Gas-Simulator/{bwhpc_matlab_gpe_sim.slurm => bwhpc_matlab_gpe_sim_cpu.slurm} (95%)
 create mode 100644 Dipolar-Gas-Simulator/bwhpc_matlab_gpe_sim_gpu.slurm

diff --git a/Dipolar-Gas-Simulator/+Scripts/run_on_cluster.m b/Dipolar-Gas-Simulator/+Scripts/run_on_cluster.m
index 12e6dc2..0a2c04d 100644
--- a/Dipolar-Gas-Simulator/+Scripts/run_on_cluster.m
+++ b/Dipolar-Gas-Simulator/+Scripts/run_on_cluster.m
@@ -25,6 +25,7 @@ OptionsStruct.NumberOfTimeSteps       = 2E6;                      % in s
 OptionsStruct.EnergyTolerance         = 5E-10;
 
 OptionsStruct.JobNumber               = 1;
+OptionsStruct.RunOnGPU                = true;
 OptionsStruct.SaveData                = true;
 OptionsStruct.SaveDirectory           = './Data';
 options                               = Helper.convertstruct2cell(OptionsStruct);
diff --git a/Dipolar-Gas-Simulator/+Simulator/@DipolarGas/DipolarGas.m b/Dipolar-Gas-Simulator/+Simulator/@DipolarGas/DipolarGas.m
index 69b1e3a..ea531f5 100644
--- a/Dipolar-Gas-Simulator/+Simulator/@DipolarGas/DipolarGas.m
+++ b/Dipolar-Gas-Simulator/+Simulator/@DipolarGas/DipolarGas.m
@@ -20,9 +20,8 @@ classdef DipolarGas < handle & matlab.mixin.Copyable
         
         SimulationParameters;
 
-        %Flags
-        
         JobNumber;
+        RunOnGPU;
         DebugMode;
         DoSave;
         SaveDirectory;
@@ -64,6 +63,8 @@ classdef DipolarGas < handle & matlab.mixin.Copyable
                 @(x) assert(isnumeric(x) && isscalar(x) && (x > 0)));
             addParameter(p, 'JobNumber',                1,...
                 @(x) assert(isnumeric(x) && isscalar(x) && (x > 0)));
+            addParameter(p, 'RunOnGPU',        false,...
+                @islogical);
             addParameter(p, 'DebugMode',        false,...
                 @islogical);
             addParameter(p, 'SaveData',         false,...
@@ -88,6 +89,7 @@ classdef DipolarGas < handle & matlab.mixin.Copyable
             this.MinimumTimeStepSize  = p.Results.MinimumTimeStepSize;
             
             this.JobNumber            = p.Results.JobNumber;
+            this.RunOnGPU             = p.Results.RunOnGPU;
             this.DebugMode            = p.Results.DebugMode;
             this.DoSave               = p.Results.SaveData;
             this.SaveDirectory        = p.Results.SaveDirectory;
diff --git a/Dipolar-Gas-Simulator/+Simulator/@DipolarGas/initialize.m b/Dipolar-Gas-Simulator/+Simulator/@DipolarGas/initialize.m
index 1cc6a9d..20964e7 100644
--- a/Dipolar-Gas-Simulator/+Simulator/@DipolarGas/initialize.m
+++ b/Dipolar-Gas-Simulator/+Simulator/@DipolarGas/initialize.m
@@ -16,4 +16,8 @@ function [psi,V,VDk] = initialize(this,Params,Transf,TransfRad)
     
     % == Setting up the initial wavefunction == %
     psi = this.setupWavefunction(Params,Transf);
+    
+    if this.RunOnGPU
+        psi = gpuArray(psi);
+    end
 end
\ No newline at end of file
diff --git a/Dipolar-Gas-Simulator/bwhpc_matlab_gpe_sim.slurm b/Dipolar-Gas-Simulator/bwhpc_matlab_gpe_sim_cpu.slurm
similarity index 95%
rename from Dipolar-Gas-Simulator/bwhpc_matlab_gpe_sim.slurm
rename to Dipolar-Gas-Simulator/bwhpc_matlab_gpe_sim_cpu.slurm
index 34a2b9b..bad4a54 100644
--- a/Dipolar-Gas-Simulator/bwhpc_matlab_gpe_sim.slurm
+++ b/Dipolar-Gas-Simulator/bwhpc_matlab_gpe_sim_cpu.slurm
@@ -6,9 +6,9 @@
 #SBATCH --nodes=1
 #SBATCH --ntasks-per-node=1
 #SBATCH --cpus-per-task=10
-#SBATCH --mem=8G 
+#SBATCH --mem=24G 
 # Estimated wallclock time for job
-#SBATCH --time=02:00:00
+#SBATCH --time=15:00:00
 #SBATCH --job-name=simulation 
 #SBATCH --error=simulation.err
 #SBATCH --output=simulation.out 
diff --git a/Dipolar-Gas-Simulator/bwhpc_matlab_gpe_sim_gpu.slurm b/Dipolar-Gas-Simulator/bwhpc_matlab_gpe_sim_gpu.slurm
new file mode 100644
index 0000000..156502f
--- /dev/null
+++ b/Dipolar-Gas-Simulator/bwhpc_matlab_gpe_sim_gpu.slurm
@@ -0,0 +1,38 @@
+#!/bin/bash 
+########### Begin SLURM header ###########
+#Partition
+#SBATCH --partition=single
+# Request number of nodes and GPU for job
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=10
+#SBATCH --gres=gpu:4
+#SBATCH --mem=24G 
+# Estimated wallclock time for job
+#SBATCH --time=15:00:00
+#SBATCH --job-name=simulation 
+#SBATCH --error=simulation.err
+#SBATCH --output=simulation.out 
+
+########### End SLURM header ##########
+
+echo "Working Directory:                    $PWD"
+echo "Running on host                       $HOSTNAME"
+echo "Job id:                               $SLURM_JOB_ID"
+echo "Job name:                             $SLURM_JOB_NAME"
+echo "Number of nodes allocated to job:     $SLURM_JOB_NUM_NODES"
+echo "Number of GPUs allocated to job:      $SLURM_GPUS"
+
+
+# Load module 
+module load math/matlab/R2023a
+
+echo Directory is `pwd`
+echo "Initiating Job..."
+
+# Start a Matlab program
+matlab -nodisplay -nosplash -r "Scripts.run_on_cluster"
+
+# notice for tests
+echo "Job terminated successfully"
+
+exit