#! /usr/bin/env perl
require 'getopts.pl'; 
use File::Basename;

#
# perl script to submit jobs to SGE
# (this version is based on a script by Willy Ruesing, MPI-MG, Berlin)
#

$proname = basename($0);

sub usage { 
	print("\nusage: $proname [options] <[-f script],[ \'command text\']>\n\n"); 
	print("  $proname submits <script,\`command text\'> as a batch job \n");
	print("  to SGE (Sun GridEngine) into a selected queue.\n\n");
	print("  -h                        -- this message\n");
	print("  -m <min MB memory>        -- minimum memory in MB (without swap) of host to use\n");
	print("  -M <max MB memory>        -- maximum memory in MB of job to use\n");
	print("  -t <min total MB memory>  -- total minimum memory in MB (virtual + swap) of host to use\n");	 
	print("  -N <jobname>              -- name for the job\n");  
	print("  -p [-100 default]         -- change the priority 0 .. 1024\n");
	print("  -f <file>                 -- argument is a command text file\n");
        print("  -V                        -- do not import environment to sge job (shell variables, etc.)\n");
	print("  \'command text\'          -- take command args as command text file\n");
	print("        You have to quote the command text. Use \' or \" to expand\n");
	print("        (or not) the command line args.\n");
	print("  -q <queue complex>        -- queue to select\n\n"); 
	print("  possible queue complexes:\n");
        print("    cluster             - CIBIV cluster\n");
        print("    cluster.low         - CIBIV cluster - lowpriority queue\n");
        print("    cluster.high        - CIBIV cluster - highpriority queue\n");
	print("    desktops            - CIBIV desktop PCs with #Cores >= 2\n");
	print("    sandy               - CIBIV desktop PCs with #Cores >= 2 and Sandy Bridge CPU or newer (AVX instruction set)\n");
        print("    dmp                 - DMP Servers\n");
        print("    dmp.low             - DMP Servers [low priority]\n");
	print("    dmpwww              - DMP Servers [high priority]\n");
   print("    compute             - Compute Servers (zuse) [default]\n");
   print("    gpu                 - GPU Servers, this will automatically reserve one GPU for you\n");
   print(" -r <resource>  -- for the clusteri[.low] queues you can select resources\n");
   print("                   to run on specific nodes (if non selected scheduler decides):\n");
   print("      zuseX                 -- zuse-like nodes (48 cores,512GB ram)\n");
   print("      old                   -- just old nodes (4 cores,16GB ram)\n");
   print("      scratch               -- nodes with scratch (either old or zuse)\n\n");
   print("      NOTE: if badly chosen these can render a job un-startable.\n\n");
   print("  For parallel jobs:\n");
   print("  -s <no>                   -- number of slots to be used (thread parallel)\n\n");

	exit(-1);
} 
 
&usage if (@ARGV == 0); 
&Getopts('hq:f:p:N:m:M:t:iVs:r:'); 
use vars qw($opt_h $opt_q $opt_L $opt_H $opt_f $opt_p $opt_N $opt_m $opt_M $opt_t $opt_s $opt_r); 
&usage if ($opt_h); 

if ($opt_q) 
{ 
	$queue = $opt_q;

	if( 	$queue !~ /desktops/ &&
		$queue !~ /dmp/ &&
		$queue !~ /dmpwww/ &&
                $queue !~ /dmp.low/ &&
		$queue !~ /compute/ && 
                $queue !~ /sandy/ &&
                $queue !~ /cluster/ &&
                $queue !~ /cluster\.low/ &&
                $queue !~ /cluster\.high/ &&
                $queue !~ /gpu/ )
	{
		print("\n$proname ERROR: .... No such queue !!! ....\n");
		&usage;
	}
} 
else 
{ 
        $queue = "compute";
} 

if ( $queue =~ /cluster.*/  && `hostname` !~ /libby/ ) {
   print("\n$proname ERROR: The cluster queues are only available on the cluster-login-node libby!\n");
   exit(1);
} 

if ($opt_r)
{
    if ($queue =~ /cluster/ || $queue =~ /cluster.low/ ) {
        $resource = " -l $opt_r ";

        if( $opt_r !~ /zuseX/ &&
            $opt_r !~ /old/ &&
            $opt_r !~ /scratch/ )
        {
                print("\n$proname ERROR: .... No such resource !!! ....\n");
                &usage;
        }
   } else {
        print("\n$proname ERROR: Resources are only availble for the queues cluster and cluster.low!\n");
   exit(1);
   }
}
else
{
        $resource = "";
}

if ( $queue =~ /desktops/ )
{
   $queue .= " -soft -l amd";
}
if ( $queue =~ /sandy/ )
{
   $queue = "desktops";
}

if ($opt_p) { $priority = " -p $opt_p "; }
else        { $priority = ""; }


# memory expected on submit host

if ($opt_m) { $minmemory = " -l mf=${opt_m}M "; }
else        { $minmemory = ""; }

if ($opt_M) { $maxmemory = " -l s_data=${opt_M}M "; }
else        { $maxmemory = ""; }

if ($opt_t) { $mintotalmemory = " -l mt=${opt_t}M "; }
else        { $mintotalmemory = ""; }

if ($opt_s ) {
if ($queue =~ /^dmp$/ || $queue =~ /^compute$/ || $queue =~ /^dmpwww$/ || $queue =~ /^gpu$/ || $queue =~ /^cluster$/ ) {
$pe = "";
      $pe = " -pe threads ${opt_s} ";
}
else {
   print("\n ERROR: Only thread parallel jobs in dmp queue!\n");
   &usage;
}
}

if (@ARGV == 0 && !$opt_f)
{
	print("\n$proname ERROR: .... No script/command to run specified !!! ....\n");
	&usage;
}

if ($opt_f)
{
        $script = $opt_f;
	if (! -f $script)
	{
		print "\n$proname ERROR: .... Can not open SGE script file [$script] !!! ....\n\n";
		exit(-1);
	}
	else
	{
		print "$proname INFO: submitting command file [$script]\n";
	}
}
else
{
	print "$proname INFO: submitting command line [@ARGV]\n";
	@append = @ARGV;
}

if (!$opt_V) {
    $env = " -V ";
} else {
    $env = "";
}

#  Options used:
#
#  -S                 /usr/bin/bash will be used
#  -cwd               start job in current working directory
#  -M email-addr      your email address
#  -m e               send mail at <b>egin, <e>nd, <a>bort,
#                     <s>uspend od <n>o mail send, e.g. "-m beas"
#  -j y               merge stdout and stderr
#  -N JOBNAME         Name of the job
#  -r y               restart on abort

$CMD = ". /software/sge/default/common/settings.sh;";

if ( defined @append )
{
	if ( $opt_N )
	{
		$jname = "$opt_N.$$";
	}
	else
	{ 
		$jname = "_SGE_.$$";
	}
	$CMD =  $CMD . "echo \"@append\" | ";
}
else
{
	if ( $opt_N )
	{
		$jname = "$opt_N.$$";
	}
	else
	{ 
		$jname = basename($script);
	}
}

# changed -cwd with $PWD Enviroment Variable
#$CMD = $CMD . "$qsub $env -S /bin/bash $memory $priority -cwd -l $queue -j y -N $jname -r y $script";
#$CWD = "-v SGE_CWD_PATH=$ENV{PWD}";
#print " $CWD \n";

$CMD = $CMD . "qsub $env -S /bin/bash $resource $priority -cwd -l $queue $pe $mintotalmemory $maxmemory $minmemory -j y -N $jname -r y $script";
# $CMD = $CMD . "qsub -V -S /bin/bash $priority -cwd -l $queue $mintotalmemory $maxmemory $minmemory -j y -N $jname -r y $script";

print "using shell commands: $CMD\n\n";

$program = "/bin/bash";
push( @options, "-c" );
push( @args, $CMD ); 

# print "$program , @options , @args \n\n";

exec($program, @options, @args)  or die "can't exec sub command: $!";
