*! version 0.12 September 26, 2014 @ 16:30:06

* 0.1. Initial version
* 0.2. Variable names for constants in IND-Files implemented
* 0.3. Update for Wave 2007 delivery
* 0.4. Design balanced not worked for CNEF -> fixed
*      Balanced design should operate on sequence number -> changed
*      Option cnef implied "clear" -> fixed
* 0.5. Some 2007 varnames interfere with IND varnames -> fixed
* 0.6. Adaptions to CNEF 2007
* 0.7. Bug with option cnef -> fixed
* 0.8. Update for Wave 2009 delivery
* 0.9. Option "dofile" implemented (Thanks to John Haisken DeNew)
* 0.10 Badly inserted newline character. Fixed.
* 0.11 Update for Wave 2011 delivery
* 0.12 Final delivery. End of development!

program psiduse
version 10.0

	// Parse Commandline
	// ------------------

	syntax anything ///
	using/ ///
	[, clear Design(string) cnef(numlist) dofile(string)  correct reshape lower]


	local er ER
	local v  V
	if "`lower'"!="" {
		local er er
		local v  v
	}


	// Hard coded list of identifiers
	// (Update these lists for each data delivery)

	// I try to find the users delivery by myself ...
	local stop 2011
	while `stop' > 1968 {
		capture confirm file `"`using'/ind`stop'er.dta"'
		if !_rc {
			local IND ind`stop'er
			local stop = 1968
		}
		local stop = `stop' - 1
	}

	local IDind ///
	  [68] `er'30001 [69] `er'30020 [70] `er'30043 [71] `er'30067  ///
	  [72] `er'30091 [73] `er'30117 [74] `er'30138 [75] `er'30160  ///
	  [76] `er'30188 [77] `er'30217 [78] `er'30246 [79] `er'30283  ///
	  [80] `er'30313 [81] `er'30343 [82] `er'30373 [83] `er'30399  ///
	  [84] `er'30429 [85] `er'30463 [86] `er'30498 [87] `er'30535  ///
	  [88] `er'30570 [89] `er'30606 [90] `er'30642 [91] `er'30689  ///
	  [92] `er'30733 [93] `er'30806 [94] `er'33101 [95] `er'33201  ///
	  [96] `er'33301 [97] `er'33401 [99] `er'33501 [01] `er'33601  ///
	  [03] `er'33701 [05] `er'33801 [07] `er'33901 [09] `er'34001  ///
	  [11] `er'34101

	local IDfam ///
	  [68] `v'3      [69] `v'442    [70] `v'1102   [71] `v'1802   [72] `v'2402  [73] `v'3002    ///
	  [74] `v'3402   [75] `v'3802   [76] `v'4302   [77] `v'5202   [78] `v'5702    ///
	  [79] `v'6302   [80] `v'6902   [81] `v'7502   [82] `v'8202   [83] `v'8802    ///
	  [84] `v'10002  [85] `v'11102  [86] `v'12502  [87] `v'13702  [88] `v'14802   ///
	  [89] `v'16302  [90] `v'17702  [91] `v'19002  [92] `v'20302  [93] `v'21602   ///
	  [94] `er'2002  [95] `er'5002  [96] `er'7002  [97] `er'10002 [99] `er'13002  ///
	  [01] `er'17002 [03] `er'21002 [05] `er'25002 [07] `er'36002 [09] `er'42002  ///
	  [11] `er'47302

	local SQind ///
	  [69] `er'30021 [70] `er'30044 [71] `er'30068 [72] `er'30092 [73] `er'30118 [74] `er'30139  ///
	  [75] `er'30161 [76] `er'30189 [77] `er'30218 [78] `er'30247 [79] `er'30284 [80] `er'30314  ///
	  [81] `er'30344 [82] `er'30374 [83] `er'30400 [84] `er'30430 [85] `er'30464 [86] `er'30499  ///
	  [87] `er'30536 [88] `er'30571 [89] `er'30607 [90] `er'30643 [91] `er'30690 [92] `er'30734  ///
	  [93] `er'30807 [94] `er'33102 [95] `er'33202 [96] `er'33302 [97] `er'33402 [99] `er'33502  ///
	  [01] `er'33602 [03] `er'33702 [05] `er'33802 [07] `er'33902 [09] `er'34002 [11] `er'34102


	if "`dofile'"=="" {
		local dfile psiduse.do
	}
	else {
		local dfile `dofile'
	}

	capture file close fout
	file open fout using "`dfile'", write replace text


	// Check user list
	local x: subinstr local SQind `"] "' `"]"', all
	if `"`:list anything & x'"'  != `""' {
		local anything: list anything - x
		di as error "Sequence number retained automatically by psiduse. Remove as varname identifier"
		exit 198
	}

	if "`cnef'" != "" {
		_CNEF `anything' using `"`using'"' 	///
		  , `clear' design(`design') waves(`cnef') `correct'
		exit
	}

	tokenize `anything', parse("||")
	while "`1'" != "" {
		if "`1'" == "|" mac shift
		else {
			gettoken newname pairlist :1
			local newnamelist `newnamelist' `newname'
			foreach pair of local pairlist {
				gettoken year var : pair, parse(`"]"')
				local year `:subinstr local year `"["' `""''
				local yearlist "`yearlist' `year'"
				local var  `:subinstr local var `"]"' `""''

				// Extract vars from ind files
				if inlist(substr("`var'",1,4),"`er'30","`er'31","`er'32","`er'33","`er'34","`er'35")  ///
				  & length("`var'")==7 {
					capture d `var' using `"`using'/`IND'"'
					if !_rc local keepvars `keepvars' `var'
				}
				else local `year'vars "``year'vars' `var'"
				local `newname' ``newname'' [`year']`var'
			}
			mac shift
		}
	}

	// Uniquify yearlist
	local yearlist: list uniq yearlist


	// Option clear
	// ------------
	if "`clear'" == "clear" clear

	// Prepare  family files
	// ---------------------

	file write fout _n
	file write fout "// ============================================" _n
	file write fout "// Generated by Uli Kohler's <psiduse2.ado>" _n
	file write fout "// ============================================" _n
	file write fout _n _n

	file write fout _n "// ===[ PSID Data Directory ]=====================" _n
	file write fout `"local psidpath `"`using'"'"' _n _n


	foreach wave of local yearlist {

		// Create local for Century
		if `wave' >= 68 local CC 19
		else if  `wave' > 00 & `wave' < 68 local CC 20

		// Letters "er" in filenames of years 1994 and later
		local ER = cond(`CC'`wave'>=1994,"er","")

		// Extract idenifier for wave
		local IDpos: list posof `"[`wave']"' in IDfam
		local ID: word `++IDpos' of `IDfam'

		file write fout _n "// ===[ Family file `wave' ]=====================" _n
//		file write fout _n `"use `ID' ``wave'vars' using `"`using'/fam`CC'`wave'`ER'"', clear"' _n
		file write fout `"use `ID' ``wave'vars' using "\`psidpath'/fam`CC'`wave'`ER'", clear"' _n
		use `ID' ``wave'vars' using `"`using'/fam`CC'`wave'`ER'"', clear

		// Generate CNEF Idenifiers
		if `wave' >= 68 local CC 19
		else if  `wave' > 00 & `wave' < 68 local CC 20
		file write fout `"ren `ID' x11102_`CC'`wave'"' _n
		ren `ID' x11102_`CC'`wave'

		// Save for merging
		file write fout `"sort x11102_`CC'`wave'"' _n
		sort x11102_`CC'`wave'
		tempfile f`wave'
		file write fout `"quietly save f`wave', replace"' _n
		quietly save `f`wave''
	}

	// Use Ind-File
	// ------------

	macro drop _ID

	// Produce list of idenifiers and sequences for requested waves
	foreach wave of local yearlist {

		local indpos: list posof `"[`wave']"' in IDind
		local ID `ID' `: word `++indpos' of `IDind''
		if "`wave'" != "68" {
			local sqpos: list posof `"[`wave']"' in SQind
			local SQ `SQ' `: word `++sqpos' of `SQind''
		}
	}

	file write fout _n _n "// ===[ Single Individual file `IND' ]=====================" _n
	file write fout `"use `er'30002 `er'30001 `ID' `SQ' `keepvars' using "\`psidpath'/`IND'""' _n
	use `er'30002 `er'30001 `ID' `SQ' `keepvars' using `"`using'/`IND'"'

	// Autogenerated Variables
	// -----------------------

	// CNEF Idenifier
	file write fout "gen long x11101ll = `er'30001*1000 + `er'30002" _n
	gen long x11101ll = `er'30001*1000 + `er'30002
	file write fout `"lab var x11101ll "Person identification number""' _n
	lab var x11101ll "Person identification number"

	foreach wave of local yearlist {

		// Idenify century
		if `wave' >= 68 local CC 19
		else if  `wave' > 00 & `wave' < 68 local CC 20

		// Rename family number
		local IDpos: list posof `"[`wave']"' in IDind
		local IDpos1=`IDpos'+1
		local idw: word `IDpos1' of `IDind'

		file write fout "ren `idw' x11102_`CC'`wave'" _n
		ren `: word `++IDpos' of `IDind'' x11102_`CC'`wave'


		// Rename sequence number
		if "`wave'" != "68" {
			local sqpos: list posof `"[`wave']"' in SQind
			ren `: word `++sqpos' of `SQind'' xsqnr_`CC'`wave'
		}
	}

	// Option Design
	// -------------

	file write fout _n _n "// ---[ Option Design ]---------------------" _n


	if "`design'"=="any" {
		file write fout "// All observations kept" _n
		di as text "All observations kept"
	}
	else {

		tempvar g
		file write fout "gen byte g = 0" _n
		gen byte `g' = 0
		foreach wave of local yearlist {

			// Identify century
			if `wave' >= 68 {
				local CC 19
			}
			else if  `wave' > 00 & `wave' < 68 {
				local CC 20
			}

			if "`wave'" == "68" {
				file write fout "quietly replace g = g + 1 if inrange(x11102_1968,1,2930) | inrange(x11102_1968,5001,6872)" _n
				quietly replace `g' = `g' + 1 if inrange(x11102_1968,1,2930) | inrange(x11102_1968,5001,6872)
			}
			else {
				file write fout "qui replace g = g + 1 if inrange(xsqnr_`CC'`wave',1,20) | inrange(xsqnr_`CC'`wave',81,89)" _n
				qui replace `g' = `g' + 1 if inrange(xsqnr_`CC'`wave',1,20) | inrange(xsqnr_`CC'`wave',81,89)
			}
		}

		capture confirm integer number `design'
		if !_rc {
			file write fout "// Kept households interviewed " _n
			file write fout "qui keep if g>=`design'" _n
			qui keep if `g'>=`design'
			di as text _n "Kept households interviewed " ///
			  as result `design' as text " times or more"
		}
		else if "`design'"=="balanced" | "`design'" == "" {
			file write fout "// Balanced panel design " _n
			local wavecount: word count `yearlist'
			file write fout "qui keep if g == `wavecount'" _n
			qui keep if `g' == `wavecount'
			di as text "Balanced panel design"
		}
	}


	// Merge Family Files
	// ------------------

	foreach wave of local yearlist {
		if `wave' >= 68 local CC 19
		else if  `wave' > 00 & `wave' < 68 local CC 20

		file write fout _n _n "// ---[ Merge Family `wave' ]---------------------" _n
		file write fout "sort x11102_`CC'`wave'" _n
		sort x11102_`CC'`wave'

		file write fout "quietly merge x11102_`CC'`wave' using f`wave', nokeep uniqusing" _n
		file write fout "drop _merge" _n


		quietly merge x11102_`CC'`wave' using `f`wave'' , nokeep uniqusing
		drop _merge
	}
	file write fout _n


	// Rename variables
	// ----------------

	file write fout _n _n "// ---[ Recode Varnames ]---------------------" _n
	foreach newname of local newnamelist {
		foreach pair of local `newname' {
			gettoken year var : pair, parse(`"]"')
			local year `:subinstr local year `"["' `""''
			local var  `:subinstr local var `"]"' `""''

			if "`year'"!="" {
				if `year' >= 68 local CC 19
				else if `year' > 00 & `year' < 68 local CC 20
			}
			else macro drop _CC

			file write fout "ren `var' `newname'`CC'`year'" _n
			ren `var' `newname'`CC'`year'
		}
	}

	// Clean up
	// --------

	file write fout _n _n "// ---[ Clean Up ]---------------------" _n
	foreach newname of local newnamelist {
		local orderlist `orderlist' `newname'*
	}
	file write fout "order x11101ll x11102* xsqnr_* `orderlist'" _n
	order x11101ll x11102*  xsqnr_* `orderlist'

	file write fout "drop `er'30002" _n
	file write fout "capture drop `er'30001" _n
	drop `er'30002
	capture drop `er'30001




	// Link with psidadd
	// -----------------

	// We store back some chars to be used by psidadd
	char _dta[psidusedir] `"`using'"'

	if "`reshape'"!="" {
		file write fout _n _n "// ---[ Reshape LONG ]---------------------" _n
		file write fout "reshape long x11102_ xsqnr_ `newnamelist' , i(x11101ll) j(year)" _n
		reshape long x11102_ xsqnr_ `newnamelist' , i(x11101ll) j(year)
	}

	noi di "PSID retrieval algorithm actually used:"
	noi di `"{stata view `dfile':`dfile'}"'
	noi di "Filename can be changed using the option [dofile(filename.do)]"
	file close fout
end


program _CNEF
	syntax anything using/ [, clear Design(string) waves(string) correct] ///

	// CHECK
	// (This is necessary because 1st data-delivery for 2007
	// introduced ugly inconsitancies. We correct them automatically, here

	forv i = 2005(2)2007 {
		capture d *LL using `"`using'/pequiv_`i'"'
		if !_rc & "`correct'"=="" {
			di "{err}Found known inconsistency. Consider option correct"
		}
		if !_rc & "`correct'"!="" {

			use `"`using'/pequiv_`i'.dta"', replace

			foreach var of varlist *LL {
			ren `var' `=lower("`var'")'
			}
			bys x11101ll: keep if _n==1
			save `"`using'/pequiv_`i'.dta"', replace
		}
	}

	// Create necessary Lists
	// ----------------------

	tokenize `anything', parse("||")
	while "`1'" != "" {
		if "`1'" == "|" mac shift
		else {
			gettoken newname item :1
			local item = trim("`item'")
			local newnamelist `newnamelist' `newname'
			local itemlist `itemlist' `item'

			if `"`=substr(`"`item'"',-2,.)'"' == `"ll"' |  ///
			  `"`=substr(`"`item'"',-2,.)'"' == `"LL"' {
				local consitems `consitems' `item'
			}
			else {
				foreach wave of local waves {
					local `wave'items ``wave'items' `item'_`wave'
				}
			}
			mac shift
		}
	}

	// Option clear
	// ------------
	if "`clear'" == "clear" clear

	// Prepare files for merging
	// -------------------------

	gettoken first rest:waves

	foreach wave of local rest {

		use x11101ll x11102_`wave' `consitems' ``wave'items'  ///
		  using `"`using'/pequiv_`wave'"'

		// Save for merging
		sort x11101ll x11102_`CC'`wave'
		tempfile f`wave'
		quietly save `f`wave''
	}

	// Use First File
	// ------------

	use x11101ll x11102_`first' `consitems' ``first'items' 	///
	  using `"`using'/pequiv_`first'"'
	sort x11101ll

	// Merge Other Files
	// -----------------

	foreach wave of local rest {
		sort x11101ll
		quietly merge x11101ll using `f`wave'', unique update
		drop _merge
	}

	// Option Design
	// -------------

	capture confirm integer number `design'

	if !_rc {
		tempvar g
		gen byte `g' = 0
		foreach var of varlist x11102* {
			qui replace `g' = `g' + 1 if !mi(`var')
		}
		qui keep if `g'>=`design'
		di as text _n "Kept households interviewed " ///
		  as result `design' as text " times or more"
	}

	else if "`design'"=="balanced" | "`design'" == "" {
		tempvar g
		local wavecount: word count `waves'
		gen byte `g' = 0
		foreach var of varlist x11102*  {
			qui replace `g' = `g' + 1 if !mi(`var')
		}
		qui keep if `g' == `wavecount'
		di as text "Balanced panel design"
	}

	else if "`design'"=="any"  di as text "All observations kept"


	// Rename variables
	// ----------------

	local i 1
	foreach newname of local newnamelist {
		local oldprefix: word `i++' of `itemlist'

		if `"`=substr(`"`oldprefix'"',-2,.)'"' == `"ll"' {
			ren `oldprefix' `newname'
		}
		else {

			foreach wave of local waves {
				ren `oldprefix'_`wave' `newname'`wave'
			}
		}
	}

	// Clean up
	// --------

	foreach newname of local newnamelist {
		local orderlist `orderlist' `newname'*
	}
	order x11101ll x11102*  `orderlist'

	// Link with psidadd
	// -----------------

	// We store back some chars to be used by psidadd
	char _dta[cnefusedir] `"`using'"'



end

exit

Author: Ulrich Kohler
	Tel +49 (0)30 25491 361
	Fax +49 (0)30 25491 360
	Email kohler@wzb.eu