#!/usr/bin/perl

#      amak.mpirunner FILE  MEM  NODES
#...... execute commands from FILE, one by one, spread out over cores
#      MEM:  memory required by commands in GB, because we have only 32 GB
#      and cant run too many on one node,  NODES:  number of nodes (not cores)
#      cores=NODES*int(32/MEM)
#      JR, 07/2013

$thost='trillian.sr.unh.edu'            ; $uname='jraeder' ; $thome="/home/space/jraeder";
$target="${uname}\@${thost}"; $CSH="$thome/local/bin/cshrun";
$mods="module swap PrgEnv-cray PrgEnv-gnu; module load pbs; module load  cray-mpich2; ";
if($ARGV[0] eq "compile") { make_code(); exit; }   #  only need compile code once
$x=shift(@ARGV);$mem=shift(@ARGV);$nodes=shift(@ARGV);
if(($mem<1)||($nodes<1)){print STDERR "error $0 fishy |$mem| |$nodes|\n"; exit; }

$su=shift(@ARGV);if($su eq ""){$su=`date "+%m%d-%H%M"`;chop($su);}
$wdir="/home/space/jraeder/pp/tmpdir.mpirunner-$su"; ssy("cat $x > tmp.tasks");
$ppn=int(32/$mem); $cores=$ppn*$nodes; 
#....... make the PBS batch file
$c= "#!/bin/csh\n#PBS -l nodes=${nodes}:ppn=${ppn}\n#PBS -l walltime=59:59:59\nset echo\n
     cd $wdir
     pwd
     which mpirunner
     ls -al /home/space/jraeder/local/bin/mpirunner
     aprun -n $cores /home/space/jraeder/local/bin/mpirunner >&!  log.mpirunner \n";
     dumpf($c,"tmp.submit");
ssh("mkdir -p $wdir");
scp("tmp.tasks",$wdir);
scp("tmp.submit",$wdir);
ssh("$mods ; cd $wdir ; qsub tmp.submit");

exit; 

sub scpget{ local($f)=@_; $f=basename($f); ssy("rsync -auvz --progress ${target}:pp/$f ."); }
sub scp{ local($f,$d)=@_; ssy("rsync -auvz --progress $f ${target}:$d"); }
sub scpput{ local($f)=@_; ssy("rsync -auvz --progress $f ${target}:local"); }
sub ssh{ local($f)=@_; ssy("ssh ${target}  \" $f \" "); }
sub basename{ local($k)=@_; local(@a)=split('\/',$k); return($a[$#a]); }
sub ssy{ local($c)=@_; print STDERR "ssy:|$c|\n"; system($c); return; }
sub dumpf{ local($c,$f)=@_; open(DF,">$f"); print DF $c; close(DF); }

sub make_code{
  $t="
c-----------------------------------------
      program mpirunner
c-----------------------------------------
      include \"mpif.h\"
      integer ierr,istat(MPI_STATUS_SIZE)
      character*2048 r1,r2,r3
      character*6 ci
      call mpi_init(ierr)
      call mpi_comm_rank(MPI_COMM_WORLD, mod_rank, ierr)
      call mpi_comm_size(MPI_COMM_WORLD, mod_size, ierr)
      cpus = mpi_wtime()
      write(0,*)'mpirunner:start: ierr/rank/size ',ierr,mod_rank,mod_size
      open(11,file='./tmp.tasks')
      write(0,*)'mpirunner:opened:tmp.tasks'
      do 100 i=0,99999
      read(11,'(a)',end=190)r1
c...... make sure there are no ampersands in there
c       cores should not put anything into the bg
      do j=1,2048; if(r1(j:j).eq.'&')r1(j:j)=' ';enddo
      lr1=jlen(r1,2048)
      if( mod(i,mod_size) .eq. mod_rank) then
      write(0,*)'mpirunner:next_task to execute: i/rank/task ',i,nod_rank,r1(1:160)
      write(ci,'(i6.6)') i
      r3='#!/bin/csh;set echo;'//r1;r1=r3
c      do j=1,lr1
c      if(r1(j:j).eq.';')r1(j:j)=char(10)
c      enddo
      r3='tmp_run.'//ci
      open(12,file=r3,status='unknown')
      write(0,*)'mpirunner:opened: ',i,r3(1:jlen(r3,160))
c      write(12,'(a)')r1(1:lr1)
c      write(0,*)'lr1=  ',lr1
      do j=1,2000
      if(r1(j:j).eq.';')then
      write(12,*)' ' 
      else
      write(12,'(a,\$)')r1(j:j)
      endif
      enddo
      write(12,*)' ' 
      write(12,*)' ' 
      close(12)
      r3='chmod 755 tmp_run.'//ci
      call system(r3)
      r2='/home/space/jraeder/local/bin/cshrun ./  '
     +  //'tmp_log.'//ci//' nobat tmp_run.'//ci 
      write(0,*)'mpirunner:system: ',i,r2(1:jlen(r2,160))
      call system(r2)
      endif
100   continue
190   continue
c       make sure the last guy finishes
      call mpi_barrier(MPI_COMM_WORLD,ierr)
      write(0,*)'finalizing:    ',mod_rank
      call mpi_finalize(ierr)
      stop
      end
c-------------------------------------
      integer function jlen(c,m)
c-------------------------------------
      character*2048 c
      k=1
      do 100 i=1,m
      if(c(i:i).ne.' '.and.c(i:i).ne.char(0))k=i
100   continue
      jlen=k
      return
      end
";
    open(FF,">tmp99.f"); print FF $t; close(FF);
        scpput("tmp99.f"); $EXE="bin/mpirunner";
        $c="$mods ; cd local; /bin/rm -fv $EXE; ";
        $c="$c ftn -w -o $EXE -mcmodel=medium tmp99.f ; size $EXE";
        ssh($c);
}
__END__