*! Adrian Sayers 06/10/2015
* QSUB
version 13.1
cap prog drop qsub
	prog define qsub , rclass
 
  syntax   ,	 jobdir(string) 		/// Job Directory
    [ 						///
			MAXProc(integer 2) 			/// Define Maximum No of Processores
			QUEuesheet(string) 			/// Define the name of the queuesheet
			FLAGEnd(string) 			/// Give the flags a name, flags define when a process has finished.
			FLAGStart(string) 			/// Give the starting flag a  name
			wait(real 1) 				/// Define the length of wait before initiating the next process (second). long process should have large wait times
			WAITB4append(real 1) 		/// Define the length of wait before attempting to looking for running processes.
			statadir(string)   			/// Specify the location of stata i.e. give the full path and .exe
			outputdir(string)			/// Output Directory
			OUTPUTSAVEname(string) 		/// Compiled results save name
			append 						/// Instigate the append code
			deletelogs					/// Delete logs from working in batch mode
		] 	//

capture confirm existence "`jobdir'"
	if _rc !=0 di as error "Specify {opth jobdir(string)} compulsory option"

{ // Defaults local macros
	
if `"`queuesheet'"' == "" local queuesheet "queuesheet.dta"
if `"`jobstubb'"'   == "" local jobstubb   "job"
if `"`flagend'"'    == "" local flagend    "flagend"
if `"`flagstart'"'  == "" local flagstart  "flagstart"

* Build Stata Path
local executable : dir "`c(sysdir_stata)'" files "Stata*-*.exe" , respect
	foreach exe in  `executable' {
		if strpos("`exe'","old") ==0 {
			local currentstata_exe  `exe'
		} 
	}	
if `"`statadir'"'   == "" local statadir	`"`c(sysdir_stata)'`currentstata_exe'"'
	capture confirm file `"`c(sysdir_stata)'`currentstata_exe'"'
		if _rc !=0 	di as error "Specify Stata system directory and executable using {opth statadir(string)} option, automated method failed"
}

{ // Compiles a list of jobs 
noisily display _n "Building Job File"

quietly {
preserve
	local myjobs : dir "`jobdir'" files "*.do"
	clear
	gen filename =""
	local i = 1
	foreach job in `myjobs' {
    set obs `i'
    replace filename = "`job'" in `i'
    local i = `i'+1
   }
	gen jobno = _n
	gen action = 0
	save `"`jobdir'\\`queuesheet'"' , replace
restore
}
}
{ // Set initialisation parameters
local break = 0 // Break instruction
local job   = 1 // Start the Job Counter
}
{ // Initialise the jobs in each processor 
noisily display _n "Initialising Processors"

forvalues processor = 1/`maxproc' {
  cap erase "`jobdir'\\`flagend'`processor'.dta"		

  * Open the queue sheet and tick off the job 
  quietly {
    use `"`jobdir'\\`queuesheet'"' , clear
    sum jobno if action==0 , mean
    local nextjobno = `r(min)'
    local nextjobname= filename[`nextjobno']
    replace action =1 if jobno==`nextjobno'
    save `"`jobdir'\\`queuesheet'"' , replace
  }
  * Write code to create a flag to indicate when the command has started 
  _filei - `"save "`jobdir'\\`flagstart'`processor'.dta" , emptyok replace "' "`jobdir'\\`nextjobname'"

  * Write code to create a flag to indicate when the command has finished and then exit stata
  file open  mydofile using "`jobdir'\\`nextjobname'", write  append
  file write mydofile  _n "clear  all" 
  file write mydofile  _n `"save "`jobdir'\\`flagend'`processor'.dta" , emptyok replace "'
  file write mydofile  _n `"erase "`jobdir'\\`flagstart'`processor'.dta"	"'	
  file write mydofile  _n `"exit , clear STATA "'
  file close mydofile

  * Send the command
  winexec `statadir'  /e do "`jobdir'\\`nextjobname'"
  noisily di " `nextjobname' " _c
  local job = `job' + 1 // Add 1 to the Job Counter loop
}
}
{ // Look for a spare processor 
noisily display _n "Redistribution"
quietly {
while (1) {	

  forvalues processor = 1/`maxproc' {
    * Look for flags	
    cap confirm file "`jobdir'\\`flagend'`processor'.dta"
    local shell`processor' = _rc

    * Re issue the shell depending on the flag status of the process	
    if `shell`processor'' == 0 {

      * Open the queue sheet and tick off the job 
      use `"`jobdir'\\`queuesheet'"' , clear
      sum jobno if action==0 , mean
      * Create an Exit indicator if no more jobs to do, and break out the loop.
      if r(N) == 0 {
        local break =1
        continue , break
      }
      local nextjobno = `r(min)'
      local nextjobname= filename[`nextjobno']
      replace action =1 if jobno==`nextjobno'
      save `"`jobdir'\\`queuesheet'"' , replace
              
          
      * Remove the ready flag 
      erase "`jobdir'\\`flagend'`processor'.dta"		
      * Write code to create a flag to indicate when the command has started 
      _filei - `"save "`jobdir'\\`flagstart'`processor'.dta" , emptyok replace "' "`jobdir'\\`nextjobname'"

      * Write code to create a flag to indicate when the command has finished and then exit stata
      file open  mydofile using "`jobdir'\\`nextjobname'", write  append
      file write mydofile  _n "clear  all"
      file write mydofile  _n `"save "`jobdir'\\`flagend'`processor'.dta" , emptyok replace "'
      file write mydofile  _n `"erase "`jobdir'\\`flagstart'`processor'.dta"	"'
      file write mydofile  _n "exit , clear STATA "
      file close mydofile

      winexec `statadir' /e do "`jobdir'\\`nextjobname'"
              
      noisily di " `nextjobname' " _c
              
      * Move to the next job
      local job = `job' + 1 												

    } // end if 

  } // end looking over each processor

  * Break out the while if no more jobs.
  if `break'==1 {
    continue , break
  }

  * Increase the wait length for long jobs.
  local sleep =round(1000*`wait',1)
  sleep `sleep'

} // end while
} // end quietly 
}
{ // Just pause for a seconds to allow the jobs to be allocated before checking they are finished.
local sleepb4append =round(1000*`waitb4append',1)
sleep `sleepb4append'
}
{ // Wait for all jobs to finish before compiling the results.
noisily display _n "Waiting for full finish"		
* Wait for all jobs to finsih	
while (1) {	// loop over the processors until exit condition is reached
	local running = `maxproc'
  forvalues processor = 1/`maxproc' {
    cap confirm file "`jobdir'\\`flagstart'`processor'.dta"
		if _rc !=0 { // If the running file is not present subtract 1  from the total number of processors.
      local running = `running'-1
		}   // end running criteria
	}	// end loop of processors
	if `running'==0 {
    continue , break
	}
	* Increase the wait length for long jobs.
	local sleep =round(1000*`wait',1)
	sleep `sleep'	
} // end while loop
}
{ // Append outputs
quietly {
if "`append'"!="" {	
  noisily di _n "Appending output files" _n
  preserve		
	local myoutputs : dir "`outputdir'" files "*.dta"
	local i = 1
	foreach file in `myoutputs' {
		if `i'==1 {
      use "`outputdir'\\`file'" , clear
		}
		if `i'>1 {
      append using "`outputdir'\\`file'"
		}
    local i = `i'+1
	}			
	save "`outputsavename'" , replace	
  restore		
}
}
}
{ // Clear logs
quietly {	
if "`deletelogs'"!="" {
  noisily di _n "Deleting Log Files" _n
  local logs : dir `"`c(pwd)'"' files "*.log"
	foreach i in `logs' {
		erase `i'
	}			
}
}
}

end // End the Program

{ // filei command from Nick Cox
*! NJC 1.0.0 27 July 2007 
cap prog drop _filei
program _filei    
        version 8.2 

        // syntax can be 
        // + <text> <filename>   create or append 
        // - <text> <filename>   prepend 

        tokenize `"`0'"' 
        if `"`2'"' == "" | `"`4'"' != "" error 198 

        if `"`1'"' == "+" { 
                capture confirm file "`3'"
                local exists = _rc == 0 
                local file "`3'" 
        } 
        else if `"`1'"' == "-" { 
                confirm file "`3'"  
                local file "`3'" 
                local exists 1 
        } 
        else error 198  
                
        tempname ho hi 

        if !`exists' { 
                file open `ho' using "`file'", w 
                file write `ho' `"`2'"' _n
                file close `ho' 
                exit 0                   
        } 
        else {
                if "`1'" == "+" {  
                        file open `ho' using "`file'", w append 
                        file write `ho' `"`2'"' _n 
                        exit 0     
                }                    
                else { 
                        tempfile work 
                        file open `ho' using `work', w 
                        file write `ho' `"`2'"' _n 
                        file open `hi' using "`file'", r 
                        file read `hi' line 

                        while r(eof) == 0  { 
                                file write `ho' `"`macval(line)'"' _n 
                                file read `hi' line 
                        } 
                        
                        file close _all
                        copy `work' "`file'", replace 
                }
        }
end
}

*END