#!/bin/bash
#$ -cwd
#$ -N DirWQ
#$ -l cpu_4=1
#$ -l h_rt=24:00:00

# DISCLAIMER:
#   This script is provided "as is", without warranty of any kind.
#   The authors assume no responsibility for any damage, data loss,
#   or unexpected behavior resulting from its use.
#   
#   Users are responsible for verifying its behavior and suitability
#   for their own workloads and environments.

# Usage:
#  - Place your tasks (either files or directories) under ready/ (or DIR_TASK_READY)
#  - Edit the "Put your task here" section (run_task) to match your workload
#  - Tasks are executed under running/***/ and moved to complete/ or failed/ accordingly
#  - You can submit multiple instances of this script to execute tasks safely in parallel (tasks are picked atomically)
#  - Set RESUBMIT=1 to automatically continue across job time limits (use carefully to avoid infinite resubmission)
#  - This script exits when no tasks remain in ready/

# Settings
DIR_TASK_READY=${PWD}/ready/
UNIQUE_POSTFIX="${JOB_ID:-$(hostname -s).$$}"
DIR_TASK_RUNNING=${PWD}/running/${UNIQUE_POSTFIX}
DIR_TASK_COMPLETE=${PWD}/complete/
DIR_TASK_FAILED=${PWD}/failed/

# Set to (scheduler walltime - 2 x expected task duration)
RUNTIME_LIMIT=82800 # in seconds, 82800 is 23h

FAIL_LIMIT=10

RESUBMIT=0 # Use with extreme care, avoid infinite resubmission
QSUB_CMD=(qsub -g tgX-XXXXX) # Replace with your group name

SECONDS=0
SUCCESS=0
FAILED=0

source /etc/profile

# Put your task here
#  - The current working directory is the task directory
#  - If the original task was a file, it is available as "./input"
#  - $1 is the task name
run_task () {
	echo "Processing Task $1" >&2
	/path/to/executable .
}

# Safe Guard
if [ ! -d "${DIR_TASK_READY}" ]; then
	echo "Task directory not found" >&2
	exit 1
fi

# Prepare Directories
mkdir -p "$DIR_TASK_RUNNING" "$DIR_TASK_COMPLETE" "$DIR_TASK_FAILED" 2> /dev/null
trap 'rmdir "${DIR_TASK_RUNNING}"' EXIT

# Main Loop
while :; do
	rtask=$(find "$DIR_TASK_READY" -mindepth 1 -maxdepth 1 -print -quit 2> /dev/null)
	[ -n "$rtask" ] || break

	if [ "$SECONDS" -ge "$RUNTIME_LIMIT" ]; then
		echo "Terminating this job to prevent timeout" >&2
		[ "$RESUBMIT" -gt 0 ] && [ "$SUCCESS" -gt 0 ] && "${QSUB_CMD[@]}" "$0" "$@"
		exit 0
	fi
	
	if [ -e "${DIR_TASK_RUNNING}/terminate" ]; then
		rm -f "${DIR_TASK_RUNNING}/terminate"
		echo "Termination requested" >&2
		exit 0
	fi

	task=$(basename -- "$rtask")

	[ ! -r  "$rtask" ] && continue
	
	if [ -f "$rtask" ]; then
		mkdir "${DIR_TASK_RUNNING}/$task" || continue
		mv "$rtask" "${DIR_TASK_RUNNING}/$task/input" || {
			rmdir "${DIR_TASK_RUNNING}/$task" 2>/dev/null
			continue
		}
	else
		mv "$rtask" "${DIR_TASK_RUNNING}/$task" || continue
	fi

	# Task execution
	pushd "${DIR_TASK_RUNNING}/$task" > /dev/null 2>&1

	exitcode=0
	(
		set -o pipefail
		run_task "$task"
	) > >(tee -a "stdout.log") 2> >(tee -a "stderr.log" >&2) || exitcode=$?

	popd > /dev/null 2>&1
	
	# Task completion
	if [ "$exitcode" -eq 0 ]; then
		echo "Task $task Complete" 1>&2
		if [ -e "${DIR_TASK_COMPLETE}/$task" ]; then
			mv "${DIR_TASK_RUNNING}/$task" "${DIR_TASK_COMPLETE}/$task.$UNIQUE_POSTFIX"
		else
			mv "${DIR_TASK_RUNNING}/$task" "${DIR_TASK_COMPLETE}/$task"
		fi
		SUCCESS=$((SUCCESS + 1))
	else
		echo "Task $task Failed: $exitcode" 1>&2
		if [ -e "${DIR_TASK_FAILED}/$task" ]; then
			mv "${DIR_TASK_RUNNING}/$task" "${DIR_TASK_FAILED}/$task.$UNIQUE_POSTFIX"
		else
			mv "${DIR_TASK_RUNNING}/$task" "${DIR_TASK_FAILED}/$task"
		fi
		FAILED=$((FAILED + 1))
		if [ "$FAILED" -ge "$FAIL_LIMIT" ]; then
			echo "Failure Limit Exceeded" 1>&2
			exit 1
		fi
	fi
done