added datamover wrapper

Datamover starts slurm jobs to move data between the fileserver and
taurus. Unfortunately, slurm does not work from within singularity
containers. Therefore, we  remote-execute them via ssh on tauruslogin3.

Singularity

@@ -10,10 +10,9 @@ From: nvidia/cuda:11.3.1-base-ubuntu20.04
   Container for py-clesperanto.
 
 %files
-#   tests /opt
     environment.yml
     runner.sh
-#   jupyter_notebook_config.json /opt/conda/etc/jupyter/
+    datamover.sh
  
 %environment
   export DEBIAN_FRONTEND=noninteractive
@@ -71,6 +70,29 @@ From: nvidia/cuda:11.3.1-base-ubuntu20.04
   mv /runner.sh $APP
   chmod a+x $APP/runner.sh
 
+  # set up the datamover wrapper script 
+  # so that we can execute datamover commands from within the container
+
+  # We use the same path as on taurus here, because they are hard-coded 
+  # into biapol-taurus. If biapol-taurus would allow us to configure the path,
+  # could also just use /usr/bin instead.
+  # In that case, the dt* commands would work automagically without absolute paths
+  slurmtools=/sw/taurus/tools/slurmtools/default/bin/ 
+
+  mkdir -p "$slurmtools"
+
+  mv /datamover.sh "$slurmtools"
+
+  datamover="${slurmtools}datamover.sh"
+
+  ln -s "$datamover" "${slurmtools}"dtwget
+  ln -s "$datamover" "${slurmtools}"dttar
+  ln -s "$datamover" "${slurmtools}"dtrsync
+  ln -s "$datamover" "${slurmtools}"dtrm
+  ln -s "$datamover" "${slurmtools}"dtmv
+  ln -s "$datamover" "${slurmtools}"dtls
+  ln -s "$datamover" "${slurmtools}"dtcp
+
 #####################
 ## run python file ##
 #####################

datamover.sh

+#!/bin/bash
+# this is a wrapper that executes datamover commands on a login node of taurus 
+# in order to be able to start the slurm jobs that move files between cluster 
+# and fileserver. It is meant to be executed via symlinks that have the name
+# of the respective command (e.g. dtcp) and point to this file
+exe=$(basename "$0")
+ssh -i /.ssh/id_rsa -o UserKnownHostsFile=/.ssh/known_hosts tauruslogin3 /sw/taurus/tools/slurmtools/default/bin/"$exe" $@

install.sh

@@ -36,5 +36,7 @@ sed -i "s|<userhome>|$userhome|g" ~/.local/share/jupyter/kernels/"$container_nam
 sed -i "s|<image_path>|$local_image|g" ~/.local/share/jupyter/kernels/"$container_name"/kernel.json
 sed -i "s|<container_name>|$container_name|g" ~/.local/share/jupyter/kernels/"$container_name"/kernel.json
 cp files/icon-64x64.png ~/.local/share/jupyter/kernels/"$container_name"/logo-64x64.png
+echo "enabling datamover"
+grep -qi ^tauruslogin3 ~/.ssh/known_hosts || echo "tauruslogin3,10.1.130.3 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBN6NLoEO9jJk09AC0182679HlVObndGh6A070EEQVKVPDGbDTburUO1K09PxFzp8vxmtknX0El4fpaKxrISe008=" >> ~/.ssh/known_hosts
 echo "testing image $local_image..."
 singularity exec "$local_image" echo "Done, you can open a jupyter notebook with a $container_name kernel now. Enjoy :)"
\ No newline at end of file

kernel_template.json

@@ -14,6 +14,8 @@
   "-B",
   "/etc/OpenCL",
   "-B",
+  "/home/<userhome>/.ssh:/.ssh",
+  "-B",
   "/projects",
   "<image_path>",
   "python",