From 128aaa2f973123a9107f1d0528e7494fe60a54e7 Mon Sep 17 00:00:00 2001
From: Emil VATAI <emil.vatai@gmail.com>
Date: Tue, 7 Nov 2023 21:30:50 +0900
Subject: [PATCH 01/22] Start multi level start cluster stcript

---
 .../singularity/start_multi_level_spark.sh    | 43 +++++++++++++++++++
 1 file changed, 43 insertions(+)
 create mode 100755 containers/singularity/start_multi_level_spark.sh

diff --git a/containers/singularity/start_multi_level_spark.sh b/containers/singularity/start_multi_level_spark.sh
new file mode 100755
index 0000000..24b2d5a
--- /dev/null
+++ b/containers/singularity/start_multi_level_spark.sh
@@ -0,0 +1,43 @@
+#!/bin/bash
+
+# set -x
+
+# BEGIN REMOVE THIS SECTION
+PMIX_RANK=1
+# END REMOVE THIS SECTION
+
+NPART=${1:-1}
+
+start_master() {
+    master_node=$(hostname)
+
+    SINGULARITY_ARGS=(
+    --bind $(mktemp -d run/`hostname`_XXXX):/run
+    --bind log/:/opt/spark-2.2.0-bin-hadoop2.6/logs
+    --bind work/:/opt/spark-2.2.0-bin-hadoop2.6/work
+    --bind hosts:/etc/hosts
+    --bind data:/tmp/data
+    --disable-cache
+    )
+    singularity instance start ${SINGULARITY_ARGS[@]} sparkleblast_latest.sif spark-process
+
+    echo "starting master on node: ${master_node}"
+    singularity exec --env SPARK_MASTER_HOST=${master_node} instance://spark-process /opt/spark-2.2.0-bin-hadoop2.6/sbin/start-master.sh
+
+    sleep 5
+
+    echo ${master_node} > master_success
+}
+
+start_worker () {
+    echo WORKER: $NPART
+}
+
+start_cluster () {
+    if [ ${PMIX_RANK} -eq "0" ]; then
+        start_master
+    else
+        start_worker
+    fi
+}
+

From 08e323040aaa6ba9452c81b375808afdcbaf324f Mon Sep 17 00:00:00 2001
From: Emil VATAI <emil.vatai@gmail.com>
Date: Thu, 16 Nov 2023 14:54:30 +0900
Subject: [PATCH 02/22] Copy `start_spark_cluster`

---
 containers/singularity/start_multi_level_spark.sh | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/containers/singularity/start_multi_level_spark.sh b/containers/singularity/start_multi_level_spark.sh
index 24b2d5a..7839171 100755
--- a/containers/singularity/start_multi_level_spark.sh
+++ b/containers/singularity/start_multi_level_spark.sh
@@ -31,6 +31,19 @@ start_master() {
 
 start_worker () {
     echo WORKER: $NPART
+    while [ ! -e "master_success" ]; do
+        sleep 1
+    done
+    master_node=$(head -n 1 master_success)
+    singularity instance start ${SINGULARITY_ARGS[@]} sparkleblast_latest.sif spark-process
+    sleep 5
+    master_url="spark://${master_node}:7077"
+    echo "Starting worker on node: $(hostname)"
+    singularity exec instance://spark-process /opt/spark-2.2.0-bin-hadoop2.6/sbin/start-slave.sh ${master_url}
+
+    while [ -e "master_success" ]; do
+        sleep 1
+    done
 }
 
 start_cluster () {

From a268f51e71a03f736817955525481107311a13f1 Mon Sep 17 00:00:00 2001
From: Emil VATAI <emil.vatai@gmail.com>
Date: Tue, 28 Nov 2023 13:14:01 +0900
Subject: [PATCH 03/22] [WIP] Add useful variables

---
 containers/singularity/run_spark_jobs.sh      |  4 ---
 .../singularity/start_multi_level_spark.sh    | 28 ++++++++++---------
 2 files changed, 15 insertions(+), 17 deletions(-)

diff --git a/containers/singularity/run_spark_jobs.sh b/containers/singularity/run_spark_jobs.sh
index d70d43d..6750c50 100755
--- a/containers/singularity/run_spark_jobs.sh
+++ b/containers/singularity/run_spark_jobs.sh
@@ -28,10 +28,6 @@ SINGULARITY_ARGS=(
   --disable-cache
   --bind hosts:/etc/hosts
   --bind ${HOST_DATA_DIR}:${CONTAINER_DATA_DIR}
-  # --bind ../../SparkLeMakeDB.sh:/opt/sparkleblast/SparkLeMakeDB.sh
-  # --bind ../../SparkLeBLASTSearch.sh:/opt/sparkleblast/SparkLeBLASTSearch.sh
-  # --bind ../../blast_args_test.txt:/opt/sparkleblast/blast_args.txt
-  # --bind ../../blast_makedb_args.txt:/opt/sparkleblast/blast_makedb_args.txt
 )
 
 MAKEDB_ARGS=(
diff --git a/containers/singularity/start_multi_level_spark.sh b/containers/singularity/start_multi_level_spark.sh
index 7839171..ef2e6e0 100755
--- a/containers/singularity/start_multi_level_spark.sh
+++ b/containers/singularity/start_multi_level_spark.sh
@@ -2,55 +2,57 @@
 
 # set -x
 
-# BEGIN REMOVE THIS SECTION
-PMIX_RANK=1
-# END REMOVE THIS SECTION
 
 NPART=${1:-1}
+PART_SIZE=$(( $PJM_MPI_PROC / $NPART ))
+GROUP_RANK=$(( $PMIX_RANK / $PART_SIZE ))
+SUB_RANK=$(( $PMIX_RANK % $PART_SIZE ))
+HOSTS_FILE=hosts-${PJM_JOBID}-${GROUP_RANK}
+MASTER_FILE=master-${PJM_JOBID}-${GROUP_RANK}
 
 start_master() {
-    master_node=$(hostname)
 
     SINGULARITY_ARGS=(
     --bind $(mktemp -d run/`hostname`_XXXX):/run
     --bind log/:/opt/spark-2.2.0-bin-hadoop2.6/logs
     --bind work/:/opt/spark-2.2.0-bin-hadoop2.6/work
-    --bind hosts:/etc/hosts
+    --bind ${HOSTS_FILE}:/etc/hosts
     --bind data:/tmp/data
     --disable-cache
     )
     singularity instance start ${SINGULARITY_ARGS[@]} sparkleblast_latest.sif spark-process
 
-    echo "starting master on node: ${master_node}"
-    singularity exec --env SPARK_MASTER_HOST=${master_node} instance://spark-process /opt/spark-2.2.0-bin-hadoop2.6/sbin/start-master.sh
+    echo "starting master on node: $(hostname)"
+    singularity exec --env SPARK_MASTER_HOST=$(hostname) instance://spark-process /opt/spark-2.2.0-bin-hadoop2.6/sbin/start-master.sh
 
     sleep 5
 
-    echo ${master_node} > master_success
+    echo $(hostname) > ${MASTER_FILE}
 }
 
 start_worker () {
     echo WORKER: $NPART
-    while [ ! -e "master_success" ]; do
+    while [ ! -e ${MASTER_FILE} ]; do
         sleep 1
     done
-    master_node=$(head -n 1 master_success)
     singularity instance start ${SINGULARITY_ARGS[@]} sparkleblast_latest.sif spark-process
     sleep 5
-    master_url="spark://${master_node}:7077"
+    master_url="spark://$(head -n1 ${MASTER_FILE}):7077"
     echo "Starting worker on node: $(hostname)"
     singularity exec instance://spark-process /opt/spark-2.2.0-bin-hadoop2.6/sbin/start-slave.sh ${master_url}
 
-    while [ -e "master_success" ]; do
+    while [ -e ${MASTER_FILE} ]; do
         sleep 1
     done
 }
 
 start_cluster () {
-    if [ ${PMIX_RANK} -eq "0" ]; then
+    if [ ${SUB_RANK} -eq "0" ]; then
         start_master
     else
         start_worker
     fi
 }
 
+split -n$NPART -d -a1 hosts hosts-${PJM_JOBID}-
+start_cluster

From 96ac6618262ceb5dc660bde1299771c24d74729c Mon Sep 17 00:00:00 2001
From: Emil VATAI <emil.vatai@gmail.com>
Date: Tue, 28 Nov 2023 13:42:38 +0900
Subject: [PATCH 04/22] s/NPART/SGROUPS/

---
 containers/singularity/start_multi_level_spark.sh | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/containers/singularity/start_multi_level_spark.sh b/containers/singularity/start_multi_level_spark.sh
index ef2e6e0..6577e2d 100755
--- a/containers/singularity/start_multi_level_spark.sh
+++ b/containers/singularity/start_multi_level_spark.sh
@@ -3,13 +3,14 @@
 # set -x
 
 
-NPART=${1:-1}
-PART_SIZE=$(( $PJM_MPI_PROC / $NPART ))
+NGROUPS=${1:-1}
+PART_SIZE=$(( $PJM_MPI_PROC / $NGROUPS ))
 GROUP_RANK=$(( $PMIX_RANK / $PART_SIZE ))
 SUB_RANK=$(( $PMIX_RANK % $PART_SIZE ))
 HOSTS_FILE=hosts-${PJM_JOBID}-${GROUP_RANK}
 MASTER_FILE=master-${PJM_JOBID}-${GROUP_RANK}
 
+
 start_master() {
 
     SINGULARITY_ARGS=(
@@ -31,7 +32,7 @@ start_master() {
 }
 
 start_worker () {
-    echo WORKER: $NPART
+    echo WORKER: $NGROUPS
     while [ ! -e ${MASTER_FILE} ]; do
         sleep 1
     done
@@ -54,5 +55,5 @@ start_cluster () {
     fi
 }
 
-split -n$NPART -d -a1 hosts hosts-${PJM_JOBID}-
+split -n$NGROUPS -d -a1 hosts hosts-${PJM_JOBID}-
 start_cluster

From 6a7cf1262b8dc110ed9a6863e9a78f8cc6302a03 Mon Sep 17 00:00:00 2001
From: Emil VATAI <emil.vatai@gmail.com>
Date: Tue, 12 Mar 2024 11:41:38 +0900
Subject: [PATCH 05/22] Rename file

---
 .../singularity/{start_multi_level_spark.sh => multi_level.sh}    | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename containers/singularity/{start_multi_level_spark.sh => multi_level.sh} (100%)

diff --git a/containers/singularity/start_multi_level_spark.sh b/containers/singularity/multi_level.sh
similarity index 100%
rename from containers/singularity/start_multi_level_spark.sh
rename to containers/singularity/multi_level.sh

From c128cebb1c5d2556d96c41166830dc52b1ba7b7b Mon Sep 17 00:00:00 2001
From: Emil VATAI <emil.vatai@gmail.com>
Date: Tue, 12 Mar 2024 11:50:08 +0900
Subject: [PATCH 06/22] Start outputdir cleanup

---
 containers/singularity/multi_level.sh | 36 +++++++++++++++++++--------
 1 file changed, 25 insertions(+), 11 deletions(-)

diff --git a/containers/singularity/multi_level.sh b/containers/singularity/multi_level.sh
index 6577e2d..27443d4 100755
--- a/containers/singularity/multi_level.sh
+++ b/containers/singularity/multi_level.sh
@@ -7,24 +7,38 @@ NGROUPS=${1:-1}
 PART_SIZE=$(( $PJM_MPI_PROC / $NGROUPS ))
 GROUP_RANK=$(( $PMIX_RANK / $PART_SIZE ))
 SUB_RANK=$(( $PMIX_RANK % $PART_SIZE ))
-HOSTS_FILE=hosts-${PJM_JOBID}-${GROUP_RANK}
-MASTER_FILE=master-${PJM_JOBID}-${GROUP_RANK}
 
+OUTFILES=outfiles/${PJM_JOBID}
+mkdir -p ${OUTFILES}
+TMPFILES=tmpfile/${PJM_JOBID}
+mkdir -p ${TMPFILES}
+OUTLOGDIR=${OUTFILES}/log
+mkdir -p ${OUTLOGDIR}
+OUTWORKDIR=${OUTFILES}/work
+mkdir -p ${OUTWORKDIR}
+OUTRUNDIR=${OUTFILES}/run
+mkdir -p ${OUTRUNDIR}
 
-start_master() {
+HOSTS_FILE=${TMPFILES}/hosts-${PJM_JOBID}-${GROUP_RANK}
+MASTER_FILE=${TMPFILES}/master-${PJM_JOBID}-${GROUP_RANK}
 
+start_master() {
     SINGULARITY_ARGS=(
-    --bind $(mktemp -d run/`hostname`_XXXX):/run
-    --bind log/:/opt/spark-2.2.0-bin-hadoop2.6/logs
-    --bind work/:/opt/spark-2.2.0-bin-hadoop2.6/work
-    --bind ${HOSTS_FILE}:/etc/hosts
-    --bind data:/tmp/data
-    --disable-cache
+        --bind $(mktemp -d ${OUTRUNDIR}/$(hostname)_XXXX):/run
+        --bind log/:/opt/spark-2.2.0-bin-hadoop2.6/logs
+        --bind work/:/opt/spark-2.2.0-bin-hadoop2.6/work
+        --bind ${HOSTS_FILE}:/etc/hosts
+        --bind data:/tmp/data
+        --disable-cache
     )
-    singularity instance start ${SINGULARITY_ARGS[@]} sparkleblast_latest.sif spark-process
+    singularity \
+        instance start ${SINGULARITY_ARGS[@]} \
+        sparkleblast_latest.sif spark-process
 
     echo "starting master on node: $(hostname)"
-    singularity exec --env SPARK_MASTER_HOST=$(hostname) instance://spark-process /opt/spark-2.2.0-bin-hadoop2.6/sbin/start-master.sh
+    singularity exec --env SPARK_MASTER_HOST=$(hostname) \
+                instance://spark-process \
+                /opt/spark-2.2.0-bin-hadoop2.6/sbin/start-master.sh
 
     sleep 5
 

From 89e5d04b0023df15a43b74499de35870ca27cdea Mon Sep 17 00:00:00 2001
From: Emil VATAI <emil.vatai@gmail.com>
Date: Tue, 12 Mar 2024 14:20:43 +0900
Subject: [PATCH 07/22] WIP

---
 containers/singularity/multi_level.sh | 21 ++++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/containers/singularity/multi_level.sh b/containers/singularity/multi_level.sh
index 27443d4..6af5a03 100755
--- a/containers/singularity/multi_level.sh
+++ b/containers/singularity/multi_level.sh
@@ -3,8 +3,8 @@
 # set -x
 
 
-NGROUPS=${1:-1}
-PART_SIZE=$(( $PJM_MPI_PROC / $NGROUPS ))
+NGROUP=${1:-1}
+PART_SIZE=$(( $PJM_MPI_PROC / $NGROUP ))
 GROUP_RANK=$(( $PMIX_RANK / $PART_SIZE ))
 SUB_RANK=$(( $PMIX_RANK % $PART_SIZE ))
 
@@ -46,7 +46,7 @@ start_master() {
 }
 
 start_worker () {
-    echo WORKER: $NGROUPS
+    echo WORKER: $NGROUP
     while [ ! -e ${MASTER_FILE} ]; do
         sleep 1
     done
@@ -69,5 +69,16 @@ start_cluster () {
     fi
 }
 
-split -n$NGROUPS -d -a1 hosts hosts-${PJM_JOBID}-
-start_cluster
+split -n$NGROUP -d -a1 hosts hosts-${PJM_JOBID}-
+
+OF_PROC=${OUTPUT_DIR}/${PJM_JOBID}-${NAME}/mpi
+
+mkdir -p log run work $(dirname ${OF_PROC})
+
+mpiexec -of-proc ${OF_PROC} ./gatherhosts_ips hosts-${PJM_JOBID}
+mpiexec -of-proc ${OF_PROC} ./multi_level.sh &
+bash -x ./run_spark_jobs.sh ${DBFILE} ${QUERYFILE}
+# mpiexec -of-proc \${OF_PROC} ./stop_spark_cluster.sh &
+rm -rf master_success-${PJM_JOBID}
+echo FSUB IS DONE
+EOF

From 16bc6ad25a5719bf3e8679267e85bec0ce325446 Mon Sep 17 00:00:00 2001
From: Emil VATAI <emil.vatai@gmail.com>
Date: Tue, 12 Mar 2024 20:41:30 +0900
Subject: [PATCH 08/22] Double run time for test_bigrun.sh

---
 containers/singularity/test_bigrun.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/containers/singularity/test_bigrun.sh b/containers/singularity/test_bigrun.sh
index b53eb6a..7f0342c 100755
--- a/containers/singularity/test_bigrun.sh
+++ b/containers/singularity/test_bigrun.sh
@@ -1,5 +1,5 @@
-CLEARALL=no ./fsub.sh nt non-rRNA-reads.fa 16  12:00:00
-CLEARALL=no ./fsub.sh nt non-rRNA-reads.fa 32  12:00:00
-CLEARALL=no ./fsub.sh nt non-rRNA-reads.fa 64  12:00:00
-CLEARALL=no ./fsub.sh nt non-rRNA-reads.fa 128  12:00:00
+CLEARALL=no ./fsub.sh nt non-rRNA-reads.fa 16  24:00:00
+CLEARALL=no ./fsub.sh nt non-rRNA-reads.fa 32  24:00:00
+CLEARALL=no ./fsub.sh nt non-rRNA-reads.fa 64  24:00:00
+CLEARALL=no ./fsub.sh nt non-rRNA-reads.fa 128  24:00:00
 

From 24f77a78f5ccf65dc42c8313c7d29f9879b66919 Mon Sep 17 00:00:00 2001
From: Emil VATAI <emil.vatai@gmail.com>
Date: Tue, 12 Mar 2024 22:48:26 +0900
Subject: [PATCH 09/22] Add tiny.sh

---
 containers/singularity/tiny.sh | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)
 create mode 100755 containers/singularity/tiny.sh

diff --git a/containers/singularity/tiny.sh b/containers/singularity/tiny.sh
new file mode 100755
index 0000000..09ea86b
--- /dev/null
+++ b/containers/singularity/tiny.sh
@@ -0,0 +1,34 @@
+#!/bin/bash
+
+#PJM -g ra000012
+#PJM -x PJM_LLIO_GFSCACHE=/vol0004
+#PJM -L elapse=1:00
+#PJM -L node=2
+#PJM --mpi proc=8
+#PJM -o ttiny/%j.stdout
+#PJM -e ttiny/%j.stderr
+
+# echo PJM_STDOUT_PATH $PJM_STDOUT_PATH
+# echo PJM_STDERR_PATH $PJM_STDERR_PATH
+
+set -x
+PLE_MPI_STD_EMPTYFILE="off"
+
+OUT_DIR=${PJM_STDOUT_PATH%.*} # ttiny/123
+TMP_FILES="${OUT_DIR}/tmpfiles" # ttiny/123/tmpfiles
+mkdir -p ${TMP_FILES}
+
+# $PJM_NODE $PJM_MPI_PROC
+GROUP_SIZE=$(( ${PJM_MPI_PROC} / ${PJM_NODE} ))
+NUM_GROUPS=${PJM_NODE}
+for gid in $(seq 0 $(( ${NUM_GROUPS} - 1 ))); do
+  grp_tmp_files="${TMP_FILES}/grp_${gid}" # ttiny/123/tmpfiles/grp-0
+  mkdir -p ${grp_tmp_files}
+  vcord=${grp_tmp_files}/vcord
+  rm -f ${vcord}
+  yes \(${gid}\) | head -n $GROUP_SIZE > ${vcord}
+  of_proc=${OUT_DIR}/mpi/grp_${gid}/
+  # mkdir -p $(dirname ${of_proc})
+  mpiexec -of-proc ${of_proc} --vcoordfile ${vcord} hostname
+done
+

From 0ed1a847f5dc9d1ef1f469e1413c4552647dc9ba Mon Sep 17 00:00:00 2001
From: Emil VATAI <emil.vatai@gmail.com>
Date: Tue, 12 Mar 2024 23:06:58 +0900
Subject: [PATCH 10/22] Fix the output

---
 containers/singularity/tiny.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/containers/singularity/tiny.sh b/containers/singularity/tiny.sh
index 09ea86b..4832516 100755
--- a/containers/singularity/tiny.sh
+++ b/containers/singularity/tiny.sh
@@ -27,8 +27,8 @@ for gid in $(seq 0 $(( ${NUM_GROUPS} - 1 ))); do
   vcord=${grp_tmp_files}/vcord
   rm -f ${vcord}
   yes \(${gid}\) | head -n $GROUP_SIZE > ${vcord}
-  of_proc=${OUT_DIR}/mpi/grp_${gid}/
-  # mkdir -p $(dirname ${of_proc})
+  of_proc=${OUT_DIR}/mpi/grp_${gid}/out
+  mkdir -p $(dirname ${of_proc})
   mpiexec -of-proc ${of_proc} --vcoordfile ${vcord} hostname
 done
 

From 5b3156530c266d74be008993a30d41f2896801b5 Mon Sep 17 00:00:00 2001
From: Emil VATAI <emil.vatai@gmail.com>
Date: Wed, 13 Mar 2024 16:01:34 +0900
Subject: [PATCH 11/22] Start refactoring

---
 containers/singularity/tiny.sh | 36 +++++++++++++++++++++-------------
 1 file changed, 22 insertions(+), 14 deletions(-)

diff --git a/containers/singularity/tiny.sh b/containers/singularity/tiny.sh
index 4832516..4ced4fe 100755
--- a/containers/singularity/tiny.sh
+++ b/containers/singularity/tiny.sh
@@ -1,5 +1,4 @@
 #!/bin/bash
-
 #PJM -g ra000012
 #PJM -x PJM_LLIO_GFSCACHE=/vol0004
 #PJM -L elapse=1:00
@@ -8,6 +7,7 @@
 #PJM -o ttiny/%j.stdout
 #PJM -e ttiny/%j.stderr
 
+# shellcheck disable=2034
 # echo PJM_STDOUT_PATH $PJM_STDOUT_PATH
 # echo PJM_STDERR_PATH $PJM_STDERR_PATH
 
@@ -15,20 +15,28 @@ set -x
 PLE_MPI_STD_EMPTYFILE="off"
 
 OUT_DIR=${PJM_STDOUT_PATH%.*} # ttiny/123
-TMP_FILES="${OUT_DIR}/tmpfiles" # ttiny/123/tmpfiles
-mkdir -p ${TMP_FILES}
+mkdir -p "${TMP_FILES}"
 
 # $PJM_NODE $PJM_MPI_PROC
-GROUP_SIZE=$(( ${PJM_MPI_PROC} / ${PJM_NODE} ))
-NUM_GROUPS=${PJM_NODE}
-for gid in $(seq 0 $(( ${NUM_GROUPS} - 1 ))); do
-  grp_tmp_files="${TMP_FILES}/grp_${gid}" # ttiny/123/tmpfiles/grp-0
-  mkdir -p ${grp_tmp_files}
-  vcord=${grp_tmp_files}/vcord
-  rm -f ${vcord}
-  yes \(${gid}\) | head -n $GROUP_SIZE > ${vcord}
-  of_proc=${OUT_DIR}/mpi/grp_${gid}/out
-  mkdir -p $(dirname ${of_proc})
-  mpiexec -of-proc ${of_proc} --vcoordfile ${vcord} hostname
+GROUP_SIZE=$(( PJM_MPI_PROC / PJM_NODE ))
+NUM_SEGMENTS=${PJM_NODE}
+
+mkvcoord () { yes "(${seg})" | head -n $GROUP_SIZE > "${vcoord}"; }
+
+segmpiexec () { mpiexec -of-proc "${of_proc}" --vcoordfile "${vcoord}" "$@"; }
+
+each_group() {
+  vcoord=${seg_tmp_dir}/vcoord
+  mkvcoord
+  segmpiexec hostname
+}
+
+for seg in $(seq 0 $(( NUM_SEGMENTS - 1 ))); do
+  seg_dir=${OUT_DIR}/seg${seg}
+  seg_tmp_dir="${seg_dir}/tmp"
+  of_proc=${seg_dir}/out/mpi
+  mkdir -p "${seg_tmp_dir}" "$(dirname "${of_proc}")"
+  ./split_query.sh
+  each_group
 done
 

From 045ce47b458a7eed66209b6da5060f55f6c1ea13 Mon Sep 17 00:00:00 2001
From: Emil VATAI <emil.vatai@gmail.com>
Date: Wed, 13 Mar 2024 18:52:42 +0900
Subject: [PATCH 12/22] Start split

---
 containers/singularity/split.sh | 10 +++++-----
 containers/singularity/tiny.sh  |  9 +++++++++
 2 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/containers/singularity/split.sh b/containers/singularity/split.sh
index 91e4efb..b30508b 100755
--- a/containers/singularity/split.sh
+++ b/containers/singularity/split.sh
@@ -1,12 +1,12 @@
 #!/usr/bin/bash
 
-if [ -z $2 ]; then echo "Usage: $0 \$file \$num_chunks"; exit; fi
+if [ -z "$2" ]; then echo "Usage: $0 \$file \$num_chunks"; exit; fi
 file=$1
 num_chunks=$2
 
-all_lines=$(wc -l $file | cut -f1 -d \ )
+all_lines=$(wc -l "$file" | cut -f1 -d \ )
 # num_lines=CEIL(all_lines / num_chunks)
-num_lines=$(( ( ${all_lines} + ${num_chunks} - 1) / ${num_chunks} ))
-if [ $(( $num_lines % 2 )) -eq 1 ]; then num_lines=$(( ${num_lines} + 1 )); fi
-split -l $num_lines $file --additional-suffix=-${file}-numchunks${num_chunks}
+num_lines=$(( ( all_lines + num_chunks - 1) / num_chunks ))
+if [ $(( num_lines % 2 )) -eq 1 ]; then num_lines=$(( num_lines + 1 )); fi
+split -l $num_lines "$file" --additional-suffix="-$(basename "${file}")-numchunks${num_chunks}"
 
diff --git a/containers/singularity/tiny.sh b/containers/singularity/tiny.sh
index 4ced4fe..4a93f22 100755
--- a/containers/singularity/tiny.sh
+++ b/containers/singularity/tiny.sh
@@ -25,6 +25,15 @@ mkvcoord () { yes "(${seg})" | head -n $GROUP_SIZE > "${vcoord}"; }
 
 segmpiexec () { mpiexec -of-proc "${of_proc}" --vcoordfile "${vcoord}" "$@"; }
 
+split () {
+
+all_lines=$(wc -l "$file" | cut -f1 -d \ )
+  # num_lines=CEIL(all_lines / num_chunks)
+  num_lines=$(( ( all_lines + num_chunks - 1) / num_chunks ))
+  if [ $(( num_lines % 2 )) -eq 1 ]; then num_lines=$(( num_lines + 1 )); fi
+  split -l $num_lines "$file" --additional-suffix="-$(basename "${file}")-numchunks${num_chunks}"
+}
+
 each_group() {
   vcoord=${seg_tmp_dir}/vcoord
   mkvcoord

From 65dc6d51740b9ba859f75422dcefcd12b303fdde Mon Sep 17 00:00:00 2001
From: Emil VATAI <emil.vatai@gmail.com>
Date: Wed, 13 Mar 2024 18:53:06 +0900
Subject: [PATCH 13/22] Remove split.sh

---
 containers/singularity/split.sh | 12 ------------
 1 file changed, 12 deletions(-)
 delete mode 100755 containers/singularity/split.sh

diff --git a/containers/singularity/split.sh b/containers/singularity/split.sh
deleted file mode 100755
index b30508b..0000000
--- a/containers/singularity/split.sh
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/usr/bin/bash
-
-if [ -z "$2" ]; then echo "Usage: $0 \$file \$num_chunks"; exit; fi
-file=$1
-num_chunks=$2
-
-all_lines=$(wc -l "$file" | cut -f1 -d \ )
-# num_lines=CEIL(all_lines / num_chunks)
-num_lines=$(( ( all_lines + num_chunks - 1) / num_chunks ))
-if [ $(( num_lines % 2 )) -eq 1 ]; then num_lines=$(( num_lines + 1 )); fi
-split -l $num_lines "$file" --additional-suffix="-$(basename "${file}")-numchunks${num_chunks}"
-

From 292477742fa6dd8b55294305b3f01cfb43f79d66 Mon Sep 17 00:00:00 2001
From: Emil VATAI <emil.vatai@gmail.com>
Date: Wed, 13 Mar 2024 22:48:13 +0900
Subject: [PATCH 14/22] Split and move files

---
 containers/singularity/tiny.sh | 58 +++++++++++++++++++++++-----------
 1 file changed, 39 insertions(+), 19 deletions(-)

diff --git a/containers/singularity/tiny.sh b/containers/singularity/tiny.sh
index 4a93f22..d5eca40 100755
--- a/containers/singularity/tiny.sh
+++ b/containers/singularity/tiny.sh
@@ -14,38 +14,58 @@
 set -x
 PLE_MPI_STD_EMPTYFILE="off"
 
-OUT_DIR=${PJM_STDOUT_PATH%.*} # ttiny/123
-mkdir -p "${TMP_FILES}"
-
 # $PJM_NODE $PJM_MPI_PROC
-GROUP_SIZE=$(( PJM_MPI_PROC / PJM_NODE ))
 NUM_SEGMENTS=${PJM_NODE}
+QUERY_FILE=$(realpath data/g50.fasta)
+# OUT_DIR=$(realpath "${PJM_STDOUT_PATH%.*}") # ttiny/123
+OUT_DIR=$(realpath ./foobar)
+NUM_SEGMENTS=3
 
-mkvcoord () { yes "(${seg})" | head -n $GROUP_SIZE > "${vcoord}"; }
+seg_dir() { echo "$(realpath "${OUT_DIR}")/seg${1}"; }
+seg_tmp_dir() { echo "$(seg_dir "$1")/tmp"; }
+seg_mpi_dir() { echo "$(seg_dir "$1")/out"; }
+seg_of_proc() { echo "$(seg_mpi_dir "$1")/mpi"; }
 
-segmpiexec () { mpiexec -of-proc "${of_proc}" --vcoordfile "${vcoord}" "$@"; }
+mkdirs () { 
+  mkdir -p "$(seg_tmp_dir "$SEG")" "$(seg_mpi_dir "$SEG")"
+}
 
-split () {
+mkvcoord () { 
+  vcoord="$(seg_tmp_dir "$SEG")/vcoord"
+  yes "(${SEG})" | head -n $group_size > "${vcoord}"
+}
+
+segmpiexec () {
+  mpiexec -of-proc "$(seg_of_proc "$SEG")" --vcoordfile "${vcoord}" "$@"
+}
 
-all_lines=$(wc -l "$file" | cut -f1 -d \ )
-  # num_lines=CEIL(all_lines / num_chunks)
-  num_lines=$(( ( all_lines + num_chunks - 1) / num_chunks ))
+split_query () {
+  pushd "$OUT_DIR" || exit
+  num_lines=$(wc -l "$QUERY_FILE" | cut -f1 -d \ )
+  # num_lines=CEIL(num_lines / NUM_SEGMENTS)
+  num_lines=$(( ( num_lines + NUM_SEGMENTS - 1) / NUM_SEGMENTS ))
   if [ $(( num_lines % 2 )) -eq 1 ]; then num_lines=$(( num_lines + 1 )); fi
-  split -l $num_lines "$file" --additional-suffix="-$(basename "${file}")-numchunks${num_chunks}"
+  suffix="-$(basename "$QUERY_FILE")-numseg${NUM_SEGMENTS}"
+  split -l $num_lines "$QUERY_FILE" --additional-suffix="$suffix"
+  mapfile -t < <(ls ./*"${suffix}"*)
+  echo "num files: ${#MAPFILE[@]}"
+  for i in "${!MAPFILE[@]}"; do
+    echo "$i ${MAPFILE[$i]}"
+    seg_query_file[i]="$(seg_dir "$i")/$(basename "${MAPFILE[$i]}")"
+    mv "${MAPFILE[$i]}" "${seg_query_file[$i]}"
+  done
+  popd || exit
 }
 
 each_group() {
-  vcoord=${seg_tmp_dir}/vcoord
   mkvcoord
   segmpiexec hostname
 }
 
-for seg in $(seq 0 $(( NUM_SEGMENTS - 1 ))); do
-  seg_dir=${OUT_DIR}/seg${seg}
-  seg_tmp_dir="${seg_dir}/tmp"
-  of_proc=${seg_dir}/out/mpi
-  mkdir -p "${seg_tmp_dir}" "$(dirname "${of_proc}")"
-  ./split_query.sh
-  each_group
+group_size=$(( PJM_MPI_PROC / PJM_NODE ))
+for SEG in $(seq 0 $(( NUM_SEGMENTS - 1 ))); do
+  mkdirs "$SEG"
+  # each_group
 done
+split_query
 

From 9fcb3fae2cab9df7c7bbc8bbea92d721f8e96edc Mon Sep 17 00:00:00 2001
From: Emil VATAI <emil.vatai@gmail.com>
Date: Wed, 13 Mar 2024 23:15:32 +0900
Subject: [PATCH 15/22] Add mpi_args

---
 containers/singularity/tiny.sh | 23 +++++++++++++----------
 1 file changed, 13 insertions(+), 10 deletions(-)

diff --git a/containers/singularity/tiny.sh b/containers/singularity/tiny.sh
index d5eca40..fe021f0 100755
--- a/containers/singularity/tiny.sh
+++ b/containers/singularity/tiny.sh
@@ -16,7 +16,8 @@ PLE_MPI_STD_EMPTYFILE="off"
 
 # $PJM_NODE $PJM_MPI_PROC
 NUM_SEGMENTS=${PJM_NODE}
-QUERY_FILE=$(realpath data/g50.fasta)
+DB_FILE=$(realpath data/g50.fasta)
+QUERY_FILE=$(realpath data/non-RNA-reads.fa)
 # OUT_DIR=$(realpath "${PJM_STDOUT_PATH%.*}") # ttiny/123
 OUT_DIR=$(realpath ./foobar)
 NUM_SEGMENTS=3
@@ -35,8 +36,11 @@ mkvcoord () {
   yes "(${SEG})" | head -n $group_size > "${vcoord}"
 }
 
-segmpiexec () {
-  mpiexec -of-proc "$(seg_of_proc "$SEG")" --vcoordfile "${vcoord}" "$@"
+mpi_args () {
+  MPI_ARGS=(
+  -of-proc "$(seg_of_proc "$SEG")" 
+  --vcoordfile "${vcoord}" "$@"
+  )
 }
 
 split_query () {
@@ -57,15 +61,14 @@ split_query () {
   popd || exit
 }
 
-each_group() {
-  mkvcoord
-  segmpiexec hostname
-}
-
 group_size=$(( PJM_MPI_PROC / PJM_NODE ))
 for SEG in $(seq 0 $(( NUM_SEGMENTS - 1 ))); do
-  mkdirs "$SEG"
-  # each_group
+  mkdirs
+  mkvcoord
 done
 split_query
+for SEG in $(seq 0 $(( NUM_SEGMENTS - 1 ))); do
+  mpi_args
+  mpiexec "${MPI_ARGS[@]}" hostname
+done
 

From 8b914d0cc0a71d0309a7e8af3f89decec6e27ee8 Mon Sep 17 00:00:00 2001
From: Emil VATAI <emil.vatai@gmail.com>
Date: Wed, 13 Mar 2024 23:28:12 +0900
Subject: [PATCH 16/22] Start mpi part

---
 containers/singularity/tiny.sh | 24 ++++++++++++++++--------
 1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/containers/singularity/tiny.sh b/containers/singularity/tiny.sh
index fe021f0..1f78776 100755
--- a/containers/singularity/tiny.sh
+++ b/containers/singularity/tiny.sh
@@ -12,20 +12,24 @@
 # echo PJM_STDERR_PATH $PJM_STDERR_PATH
 
 set -x
-PLE_MPI_STD_EMPTYFILE="off"
 
 # $PJM_NODE $PJM_MPI_PROC
-NUM_SEGMENTS=${PJM_NODE}
+if [[ "$PJM_ENVIRONMENT" = "BATCH" ]]; then
+  PLE_MPI_STD_EMPTYFILE="off"
+  NUM_SEGMENTS=${PJM_NODE}
+  OUT_DIR=$(realpath "${PJM_STDOUT_PATH%.*}") # ttiny/123
+else
+  NUM_SEGMENTS=3
+  OUT_DIR=$(realpath ./foobar)
+fi
 DB_FILE=$(realpath data/g50.fasta)
 QUERY_FILE=$(realpath data/non-RNA-reads.fa)
-# OUT_DIR=$(realpath "${PJM_STDOUT_PATH%.*}") # ttiny/123
-OUT_DIR=$(realpath ./foobar)
-NUM_SEGMENTS=3
 
 seg_dir() { echo "$(realpath "${OUT_DIR}")/seg${1}"; }
 seg_tmp_dir() { echo "$(seg_dir "$1")/tmp"; }
 seg_mpi_dir() { echo "$(seg_dir "$1")/out"; }
 seg_of_proc() { echo "$(seg_mpi_dir "$1")/mpi"; }
+seg_hosts_file() { echo "$(seg_tmp_dir "$1")/hosts"; }
 
 mkdirs () { 
   mkdir -p "$(seg_tmp_dir "$SEG")" "$(seg_mpi_dir "$SEG")"
@@ -39,8 +43,9 @@ mkvcoord () {
 mpi_args () {
   MPI_ARGS=(
   -of-proc "$(seg_of_proc "$SEG")" 
-  --vcoordfile "${vcoord}" "$@"
+  --vcoordfile "${vcoord}"
   )
+  mpiexec "${MPI_ARGS[@]}" "$@"
 }
 
 split_query () {
@@ -68,7 +73,10 @@ for SEG in $(seq 0 $(( NUM_SEGMENTS - 1 ))); do
 done
 split_query
 for SEG in $(seq 0 $(( NUM_SEGMENTS - 1 ))); do
-  mpi_args
-  mpiexec "${MPI_ARGS[@]}" hostname
+  mpi_with_args ./gatherhosts_ips "$(seg_hosts_file "$SEG")"
+  # mpiexec -of-proc \${OF_PROC} ./start_spark_cluster.sh &
+  # bash -x ./run_spark_jobs.sh ${DBFILE} ${QUERYFILE}
+  # # mpiexec -of-proc \${OF_PROC} ./stop_spark_cluster.sh &
+  # rm -rf master_success-\${PJM_JOBID}
 done
 

From b2c32539b465d192c6b86e015ee4e4d04144cafa Mon Sep 17 00:00:00 2001
From: Emil VATAI <emil.vatai@gmail.com>
Date: Thu, 14 Mar 2024 00:27:26 +0900
Subject: [PATCH 17/22] Use gatherhosts

---
 containers/singularity/tiny.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/containers/singularity/tiny.sh b/containers/singularity/tiny.sh
index 1f78776..1dc3038 100755
--- a/containers/singularity/tiny.sh
+++ b/containers/singularity/tiny.sh
@@ -22,8 +22,8 @@ else
   NUM_SEGMENTS=3
   OUT_DIR=$(realpath ./foobar)
 fi
-DB_FILE=$(realpath data/g50.fasta)
-QUERY_FILE=$(realpath data/non-RNA-reads.fa)
+DB_FILE=$(realpath data/non-rRNA-reads.fa)
+QUERY_FILE=$(realpath data/g50.fasta)
 
 seg_dir() { echo "$(realpath "${OUT_DIR}")/seg${1}"; }
 seg_tmp_dir() { echo "$(seg_dir "$1")/tmp"; }
@@ -40,7 +40,7 @@ mkvcoord () {
   yes "(${SEG})" | head -n $group_size > "${vcoord}"
 }
 
-mpi_args () {
+mpi_with_args () {
   MPI_ARGS=(
   -of-proc "$(seg_of_proc "$SEG")" 
   --vcoordfile "${vcoord}"

From bd8a00d68320003fa0a6d35117b020c12b81a89a Mon Sep 17 00:00:00 2001
From: Emil VATAI <emil.vatai@gmail.com>
Date: Thu, 21 Mar 2024 09:40:54 +0900
Subject: [PATCH 18/22] Move start_{master,worker}.sh to start_cluster.sh

---
 containers/singularity/start_spark_cluster.sh | 45 +++++++++++++++++--
 containers/singularity/start_spark_master.sh  | 22 ---------
 containers/singularity/start_spark_workers.sh | 29 ------------
 3 files changed, 41 insertions(+), 55 deletions(-)
 delete mode 100755 containers/singularity/start_spark_master.sh
 delete mode 100755 containers/singularity/start_spark_workers.sh

diff --git a/containers/singularity/start_spark_cluster.sh b/containers/singularity/start_spark_cluster.sh
index 2b355a3..d3a040a 100755
--- a/containers/singularity/start_spark_cluster.sh
+++ b/containers/singularity/start_spark_cluster.sh
@@ -1,9 +1,46 @@
 #!/bin/bash
 
-rm -f master_success-${PJM_JOBID}
-if [ ${PMIX_RANK} -eq "0" ]; then
-    bash ./start_spark_master.sh &
+master_node=$(hostname)
+
+OUT_DIR=$(realpath "${PJM_STDOUT_PATH%.*}")
+LOG_DIR=${OUT_DIR}/$PMIX_RANK/log
+RUN_DIR=${OUT_DIR}/$PMIX_RANK/run
+WORK_DIR=${OUT_DIR}/$PMIX_RANK/work
+mkdir -p "$LOG_DIR" "$RUN_DIR" "$WORK_DIR"
+
+SINGULARITY_ARGS=(
+  --bind "$RUN_DIR:/run"
+  --bind "$LOG_DIR:/opt/spark-2.2.0-bin-hadoop2.6/logs"
+  --bind "$WORK_DIR:/opt/spark-2.2.0-bin-hadoop2.6/work"
+  --bind "hosts-${PJM_JOBID}:/etc/hosts"
+  --bind data:/tmp/data
+  --disable-cache
+  --cleanenv
+)
+
+start_master () {
+  singularity instance start "${SINGULARITY_ARGS[@]}" sparkleblast_latest.sif spark-process
+  echo "starting master on node: ${master_node}"
+  singularity exec --env SPARK_MASTER_HOST="${master_node}" instance://spark-process /opt/spark-2.2.0-bin-hadoop2.6/sbin/start-master.sh
+  sleep 5
+  echo "${master_node}" > "master_success-${PJM_JOBID}"
+}
+
+start_worker () {
+  while [ ! -e "master_success-${PJM_JOBID}" ]; do sleep 1; done
+  master_node=$(head -n 1 "master_success-${PJM_JOBID}")
+  singularity instance start "${SINGULARITY_ARGS[@]}" sparkleblast_latest.sif spark-process
+  sleep 5
+  master_url="spark://${master_node}:7077"
+  echo "Starting worker on node: $(hostname)"
+  singularity exec instance://spark-process /opt/spark-2.2.0-bin-hadoop2.6/sbin/start-slave.sh "${master_url}"
+  while [ -e "master_success-${PJM_JOBID}" ]; do sleep 1; done
+}
+
+rm -f "master_success-${PJM_JOBID}"
+if [ "${PMIX_RANK}" -eq "0" ]; then
+  start_master &
 else
-    bash ./start_spark_workers.sh &
+  start_worker &
 fi
 
diff --git a/containers/singularity/start_spark_master.sh b/containers/singularity/start_spark_master.sh
deleted file mode 100755
index 0ded0c1..0000000
--- a/containers/singularity/start_spark_master.sh
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/bin/bash
-
-
-master_node=$(hostname)
-
-SINGULARITY_ARGS=(
-  --bind $(mktemp -d run/`hostname`_XXXX):/run
-  --bind log/:/opt/spark-2.2.0-bin-hadoop2.6/logs
-  --bind work/:/opt/spark-2.2.0-bin-hadoop2.6/work
-  --bind hosts-${PJM_JOBID}:/etc/hosts
-  --bind data:/tmp/data
-  --disable-cache
-)
-singularity instance start ${SINGULARITY_ARGS[@]} sparkleblast_latest.sif spark-process
-
-echo "starting master on node: ${master_node}"
-singularity exec --env SPARK_MASTER_HOST=${master_node} instance://spark-process /opt/spark-2.2.0-bin-hadoop2.6/sbin/start-master.sh
-
-sleep 5
-
-echo ${master_node} > master_success-${PJM_JOBID}
-
diff --git a/containers/singularity/start_spark_workers.sh b/containers/singularity/start_spark_workers.sh
deleted file mode 100755
index ca3ad6a..0000000
--- a/containers/singularity/start_spark_workers.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-
-
-# echo $(hostname)
-# echo ${master_node}
-SINGULARITY_ARGS=(
-  --bind $(mktemp -d run/`hostname`_${PMIX_RANK}_XXXX):/run
-  --bind log/:/opt/spark-2.2.0-bin-hadoop2.6/logs
-  --bind work/:/opt/spark-2.2.0-bin-hadoop2.6/work
-  --bind hosts-${PJM_JOBID}:/etc/hosts
-  --bind data:/tmp/data
-  --disable-cache
-)
-
-# if [[ $(hostname) != "${master_node}" ]]; then
-    while [ ! -e "master_success-${PJM_JOBID}" ]; do
-        sleep 1
-    done
-    master_node=$(head -n 1 master_success-${PJM_JOBID})
-    singularity instance start ${SINGULARITY_ARGS[@]} sparkleblast_latest.sif spark-process
-    sleep 5
-    master_url="spark://${master_node}:7077"
-    echo "Starting worker on node: $(hostname)"
-    singularity exec instance://spark-process /opt/spark-2.2.0-bin-hadoop2.6/sbin/start-slave.sh ${master_url}
-
-    while [ -e "master_success-${PJM_JOBID}" ]; do
-        sleep 1
-    done
-# fi

From 4ca2d6d6f6d2d1ec3a050d6ca6fa3bf148bf9f58 Mon Sep 17 00:00:00 2001
From: Emil VATAI <emil.vatai@gmail.com>
Date: Thu, 16 May 2024 23:37:13 +0900
Subject: [PATCH 19/22] Update tini.sh

---
 containers/singularity/tiny.sh | 64 ++++++++++++++++++++++++++++------
 1 file changed, 54 insertions(+), 10 deletions(-)

diff --git a/containers/singularity/tiny.sh b/containers/singularity/tiny.sh
index 1dc3038..6396952 100755
--- a/containers/singularity/tiny.sh
+++ b/containers/singularity/tiny.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 #PJM -g ra000012
 #PJM -x PJM_LLIO_GFSCACHE=/vol0004
-#PJM -L elapse=1:00
+#PJM -L elapse=10:00
 #PJM -L node=2
 #PJM --mpi proc=8
 #PJM -o ttiny/%j.stdout
@@ -25,25 +25,27 @@ fi
 DB_FILE=$(realpath data/non-rRNA-reads.fa)
 QUERY_FILE=$(realpath data/g50.fasta)
 
-seg_dir() { echo "$(realpath "${OUT_DIR}")/seg${1}"; }
+seg_dir() { echo "$(realpath "${OUT_DIR}")/seg$1"; }
 seg_tmp_dir() { echo "$(seg_dir "$1")/tmp"; }
 seg_mpi_dir() { echo "$(seg_dir "$1")/out"; }
 seg_of_proc() { echo "$(seg_mpi_dir "$1")/mpi"; }
 seg_hosts_file() { echo "$(seg_tmp_dir "$1")/hosts"; }
+seg_vcoord_file() { echo "$(seg_tmp_dir "$SEG")/vcoord"; }
+seg_master_file() { echo "$(seg_tmp_dir "$SEG")/master"; }
+seg_run_dir() { echo "$(seg_tmp_dir "$SEG")/run"; }
+seg_log_dir() { echo "$(seg_tmp_dir "$SEG")/log"; }
+seg_work_dir() { echo "$(seg_tmp_dir "$SEG")/work"; }
 
-mkdirs () { 
-  mkdir -p "$(seg_tmp_dir "$SEG")" "$(seg_mpi_dir "$SEG")"
-}
+mkdirs () { mkdir -p "$(seg_tmp_dir "$SEG")" "$(seg_mpi_dir "$SEG")"; }
 
 mkvcoord () { 
-  vcoord="$(seg_tmp_dir "$SEG")/vcoord"
-  yes "(${SEG})" | head -n $group_size > "${vcoord}"
+  yes "(${SEG})" | head -n $group_size > "$(seg_vcoord_file "${SEG}")"
 }
 
 mpi_with_args () {
   MPI_ARGS=(
   -of-proc "$(seg_of_proc "$SEG")" 
-  --vcoordfile "${vcoord}"
+  --vcoordfile "$(seg_vcoord_file "${SEG}")"
   )
   mpiexec "${MPI_ARGS[@]}" "$@"
 }
@@ -66,6 +68,46 @@ split_query () {
   popd || exit
 }
 
+singularity_instance_start () {
+  SINGULARITY_ARGS=(
+    --bind "$(seg_run_dir "$1"):/run"
+    --bind "$(seg_log_dir "$1"):/opt/spark-2.2.0-bin-hadoop2.6/logs"
+    --bind "$(seg_work_dir "$1"):/opt/spark-2.2.0-bin-hadoop2.6/work"
+    --bind "$(seg_hosts_file "$1"):/etc/hosts"
+    --bind data:/tmp/data
+    --disable-cache
+    --cleanenv
+    sparkleblast_latest.sif
+    spark-process
+  )
+  singularity instance start "${SINGULARITY_ARGS[@]}"
+}
+
+start_master () {
+  singularity_instance_start "$1"
+  singularity exec --env SPARK_MASTER_HOST="$(hostname)" instance://spark-process /opt/spark-2.2.0-bin-hadoop2.6/sbin/start-master.sh
+  sleep 5
+  hostname > "$(seg_master_file "$1")"
+}
+
+start_worker () {
+  while [ ! -e "$(seg_master_file "$1")" ]; do sleep 1; done
+  master_node=$(head -n 1 "$(seg_master_file "$1")")
+  singularity_instance_start "$1"
+  sleep 5
+  master_url="spark://${master_node}:7077"
+  singularity exec instance://spark-process /opt/spark-2.2.0-bin-hadoop2.6/sbin/start-slave.sh "${master_url}"
+  while [ -e "$(seg_master_file "$1")" ]; do sleep 1; done
+}
+
+start_cluster () {
+  if [ "${PMIX_RANK}" -eq "0" ]; then
+    start_master "$1"
+  else
+    start_worker "$1"
+  fi
+}
+
 group_size=$(( PJM_MPI_PROC / PJM_NODE ))
 for SEG in $(seq 0 $(( NUM_SEGMENTS - 1 ))); do
   mkdirs
@@ -74,9 +116,11 @@ done
 split_query
 for SEG in $(seq 0 $(( NUM_SEGMENTS - 1 ))); do
   mpi_with_args ./gatherhosts_ips "$(seg_hosts_file "$SEG")"
-  # mpiexec -of-proc \${OF_PROC} ./start_spark_cluster.sh &
+done
+for SEG in $(seq 0 $(( NUM_SEGMENTS - 1 ))); do
+  mpi_with_args start_cluster "$SEG" &
   # bash -x ./run_spark_jobs.sh ${DBFILE} ${QUERYFILE}
   # # mpiexec -of-proc \${OF_PROC} ./stop_spark_cluster.sh &
   # rm -rf master_success-\${PJM_JOBID}
 done
-
+wait

From 5b1fbeadb24b9a4e04ea91efc7011895ffdf4bea Mon Sep 17 00:00:00 2001
From: Emil VATAI <emil.vatai@gmail.com>
Date: Sat, 3 Aug 2024 23:16:00 +0900
Subject: [PATCH 20/22] Clean up `get_interactive_fugaku_node.sh`

---
 containers/singularity/get_interactive_fugaku_node.sh | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/containers/singularity/get_interactive_fugaku_node.sh b/containers/singularity/get_interactive_fugaku_node.sh
index e7d17a5..9b2673b 100755
--- a/containers/singularity/get_interactive_fugaku_node.sh
+++ b/containers/singularity/get_interactive_fugaku_node.sh
@@ -3,8 +3,9 @@ rm -rf  hosts master_success
 NPROC=2
 PJSUB_ARGS=(
   -L elapse=1:30:00
-  -L node=${NPROC}
-  --mpi proc=${NPROC}
+  -L node="${NPROC}"
+  # -L freq=2200 -L throttling_state=0 -L issue_state=0 -L ex_pipe_state=0 -L eco_state=0 -L retention_state=0
+  --mpi proc="${NPROC}"
   --interact
   --sparam wait-time=60
   -x PJM_LLIO_GFSCACHE=/vol0004
@@ -13,4 +14,4 @@ PJSUB_ARGS=(
   --llio localtmp-size=20Gi
   -L jobenv=singularity
 )
-pjsub  ${PJSUB_ARGS[@]}
+pjsub  "${PJSUB_ARGS[@]}"

From fd5341d30f55e337515bf8d2b7387fa2aff7f002 Mon Sep 17 00:00:00 2001
From: Emil VATAI <emil.vatai@gmail.com>
Date: Sat, 3 Aug 2024 23:17:22 +0900
Subject: [PATCH 21/22] Update test_run.sh

---
 containers/singularity/test_run.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/containers/singularity/test_run.sh b/containers/singularity/test_run.sh
index 7c70d3f..1d8393d 100755
--- a/containers/singularity/test_run.sh
+++ b/containers/singularity/test_run.sh
@@ -1 +1 @@
-CLEARALL=yes ./fsub.sh non-rRNA-reads.fa sample_text.fa 4 
+CLEARALL=no ./fsub.sh non-rRNA-reads.fa sample_text.fa 4 

From 23e116d34bd55741b0d19a75961b0381243a117c Mon Sep 17 00:00:00 2001
From: Emil VATAI <emil.vatai@gmail.com>
Date: Sat, 3 Aug 2024 23:19:00 +0900
Subject: [PATCH 22/22] Update tiny.sh

---
 containers/singularity/tiny.sh | 22 ++++++++++++++++------
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/containers/singularity/tiny.sh b/containers/singularity/tiny.sh
index 6396952..08ae486 100755
--- a/containers/singularity/tiny.sh
+++ b/containers/singularity/tiny.sh
@@ -31,12 +31,22 @@ seg_mpi_dir() { echo "$(seg_dir "$1")/out"; }
 seg_of_proc() { echo "$(seg_mpi_dir "$1")/mpi"; }
 seg_hosts_file() { echo "$(seg_tmp_dir "$1")/hosts"; }
 seg_vcoord_file() { echo "$(seg_tmp_dir "$SEG")/vcoord"; }
-seg_master_file() { echo "$(seg_tmp_dir "$SEG")/master"; }
-seg_run_dir() { echo "$(seg_tmp_dir "$SEG")/run"; }
-seg_log_dir() { echo "$(seg_tmp_dir "$SEG")/log"; }
-seg_work_dir() { echo "$(seg_tmp_dir "$SEG")/work"; }
 
-mkdirs () { mkdir -p "$(seg_tmp_dir "$SEG")" "$(seg_mpi_dir "$SEG")"; }
+# seg_master_file() { echo "$(seg_tmp_dir "$SEG")/master"; }
+# seg_run_dir() { echo "$(seg_tmp_dir "$SEG")/run"; }
+# seg_log_dir() { echo "$(seg_tmp_dir "$SEG")/log"; }
+# seg_work_dir() { echo "$(seg_tmp_dir "$SEG")/work"; }
+
+mkdirs () { 
+  DIRS=(
+  "$(seg_tmp_dir "$SEG")"
+  "$(seg_mpi_dir "$SEG")"
+  # "$(seg_run_dir "$SEG")"
+  # "$(seg_log_dir "$SEG")"
+  # "$(seg_work_dir "$SEG")"
+  )
+  mkdir -p "${DIRS[@]}" 
+}
 
 mkvcoord () { 
   yes "(${SEG})" | head -n $group_size > "$(seg_vcoord_file "${SEG}")"
@@ -118,7 +128,7 @@ for SEG in $(seq 0 $(( NUM_SEGMENTS - 1 ))); do
   mpi_with_args ./gatherhosts_ips "$(seg_hosts_file "$SEG")"
 done
 for SEG in $(seq 0 $(( NUM_SEGMENTS - 1 ))); do
-  mpi_with_args start_cluster "$SEG" &
+  mpi_with_args multi_start.sh "$(seg_tmp_dir "$SEG")"
   # bash -x ./run_spark_jobs.sh ${DBFILE} ${QUERYFILE}
   # # mpiexec -of-proc \${OF_PROC} ./stop_spark_cluster.sh &
   # rm -rf master_success-\${PJM_JOBID}