*! version 0.7 September 20, 2011 @ 17:31:38 UK

* 0.1. Initial version
* 0.2. Variable names for constants in IND-Files implemented
* 0.3. Update for Wave 2007 delivery
* 0.4. Design balanced not worked for CNEF -> fixed
*      Balanced design should operate on sequence number -> changed
*      Option cnef implied "clear" -> fixed
* 0.5. Some 2007 varnames interfere with IND varnames -> fixed
* 0.6. Adaptions to CNEF 2007
* 0.7. Bug with option cnef -> fixed

program psiduse
version 10.0
	
	// Hard coded list of identifiers
	// (Update these lists for each data delivery)

	local IND ind2007er
	
	local IDind ///
	  [68] ER30001 [69] ER30020 [70] ER30043 [71] ER30067  /// 
	  [72] ER30091 [73] ER30117 [74] ER30138 [75] ER30160  /// 
	  [76] ER30188 [77] ER30217 [78] ER30246 [79] ER30283  /// 
	  [80] ER30313 [81] ER30343 [82] ER30373 [83] ER30399  /// 
	  [84] ER30429 [85] ER30463 [86] ER30498 [87] ER30535  /// 
	  [88] ER30570 [89] ER30606 [90] ER30642 [91] ER30689  /// 
	  [92] ER30733 [93] ER30806 [94] ER33101 [95] ER33201  /// 
	  [96] ER33301 [97] ER33401 [99] ER33501 [01] ER33601  /// 
	  [03] ER33701 [05] ER33801 [07] ER33901

	local IDfam ///
	  [68] V3 [69] V442 [70] V1102 [71] V1802 [72] V2402  [73] V3002  ///  
	  [74] V3402  [75] V3802  [76] V4302  [77] V5202  [78] V5702  ///  
	  [79] V6302  [80] V6902  [81] V7502  [82] V8202  [83] V8802  ///  
	  [84] V10002 [85] V11102 [86] V12502 [87] V13702 [88] V14802  /// 
	  [89] V16302 [90] V17702 [91] V19002 [92] V20302 [93] V21602  /// 
	  [94] ER2002 [95] ER5002 [96] ER7002 [97] ER10002 [99] ER13002  /// 
	  [01] ER17002 [03] ER21002 [05] ER25002 [07] ER36002

	local SQind ///
	  [69] ER30021 [70] ER30044 [71] ER30068 [72] ER30092 [73] ER30118 [74] ER30139  ///
	  [75] ER30161 [76] ER30189 [77] ER30218 [78] ER30247 [79] ER30284 [80] ER30314  /// 
	  [81] ER30344 [82] ER30374 [83] ER30400 [84] ER30430 [85] ER30464 [86] ER30499  /// 
	  [87] ER30536 [88] ER30571 [89] ER30607 [90] ER30643 [91] ER30690 [92] ER30734  /// 
	  [93] ER30807 [94] ER33102 [95] ER33202 [96] ER33302 [97] ER33402 [99] ER33502  /// 
	  [01] ER33602 [03] ER33702 [05] ER33802 [07] ER33902
	

	// Parse Commandline 
	// ------------------
	
	syntax anything using/ [, clear Design(string) cnef(numlist) correct]

	// Check user list
	local x: subinstr local SQind `"] "' `"]"', all
	if `"`:list anything & x'"'  != `""' {
		local anything: list anything - x
		di as error "Sequence number retained automatically by psiduse. Remove as varname identifier"
		exit 198
	}

	if "`cnef'" != "" {
		_CNEF `anything' using `using' 	/// 
		  , `clear' design(`design') waves(`cnef') `correct'
		exit
	}

	tokenize `anything', parse("||")
	while "`1'" != "" {
		if "`1'" == "|" mac shift
		else {
			gettoken newname pairlist :1
			local newnamelist `newnamelist' `newname'
			foreach pair of local pairlist {
				gettoken year var : pair, parse(`"]"') 
				local year `:subinstr local year `"["' `""''  
				local yearlist "`yearlist' `year'"
				local var  `:subinstr local var `"]"' `""''
				
				// Extract vars from ind files
				if inlist(substr("`var'",1,4),"ER30","ER31","ER32","ER33","ER34","ER35")  ///
				  & length("`var'")==7 {
					capture d `var' using "`using'/`IND'"
					if !_rc local keepvars `keepvars' `var'
				}
				else local `year'vars "``year'vars' `var'"
				local `newname' ``newname'' [`year']`var'
			}
			mac shift
		}
	}

	// Uniquify yearlist
	local yearlist: list uniq yearlist


	// Option clear
	// ------------
	if "`clear'" == "clear" clear

	// Prepare  family files
	// ---------------------
	
	foreach wave of local yearlist {
	
		// Create local for Century
		if `wave' >= 68 local CC 19
		else if  `wave' > 00 & `wave' < 68 local CC 20

		// Letters "er" in filenames of years 1994 and later
		local er = cond(`CC'`wave'>=1994,"er","")

		// Extract idenifier for wave
		local IDpos: list posof `"[`wave']"' in IDfam
		local ID: word `++IDpos' of `IDfam'

		use `ID' ``wave'vars' using "`using'/fam`CC'`wave'`er'"

		// Generate CNEF Idenifiers
		if `wave' >= 68 local CC 19
		else if  `wave' > 00 & `wave' < 68 local CC 20
		ren `ID' x11102_`CC'`wave'

		// Save for merging
		sort x11102_`CC'`wave'
		tempfile f`wave'
		quietly save `f`wave''
	}

	// Use Ind-File
	// ------------

	macro drop _ID
	
	// Produce list of idenifiers and sequences for requested waves
	foreach wave of local yearlist {
		
		local indpos: list posof `"[`wave']"' in IDind
		local ID `ID' `: word `++indpos' of `IDind''
		if "`wave'" != "68" {
			local sqpos: list posof `"[`wave']"' in SQind
			local SQ `SQ' `: word `++sqpos' of `SQind''
		}
	}

	use ER30002 ER30001 `ID' `SQ' `keepvars' using "`using'/`IND'"

	// Autogenerated Variables
	// -----------------------
	
	// CNEF Idenifier
	gen long x11101ll = ER30001*1000 + ER30002
	lab var x11101ll "Person idenification number"

	foreach wave of local yearlist {

		// Idenify century
		if `wave' >= 68 local CC 19
		else if  `wave' > 00 & `wave' < 68 local CC 20

		// Rename family number
		local IDpos: list posof `"[`wave']"' in IDind
		ren `: word `++IDpos' of `IDind'' x11102_`CC'`wave'

		// Rename sequence number
		if "`wave'" != "68" {
			local sqpos: list posof `"[`wave']"' in SQind
			ren `: word `++sqpos' of `SQind'' xsqnr_`CC'`wave'
		}
	}
	
	// Option Design
	// -------------
	

	if "`design'"=="any"  di as text "All observations kept"
	else {
		tempvar g
		gen byte `g' = 0
		foreach wave of local yearlist {
			
			// Idenify century
			if `wave' >= 68 local CC 19
			else if  `wave' > 00 & `wave' < 68 local CC 20
			
			if "`wave'" == "68" 			/// 
			  quietly replace `g' = `g' + 1 if inrange(x11102_1968,1,2930) | inrange(x11102_1968,5001,6872)
			else qui replace `g' = `g' + 1 if inrange(xsqnr_`CC'`wave',1,20) | inrange(xsqnr_`CC'`wave',81,89)
		}
		
		capture confirm integer number `design'
		if !_rc {
			qui keep if `g'>=`design'
			di as text _n "Kept households interviewed " ///
			  as result `design' as text " times or more"
		}
		else if "`design'"=="balanced" | "`design'" == "" {
			local wavecount: word count `yearlist'
			qui keep if `g' == `wavecount'
			di as text "Balanced panel design"
		}
	}
	

	// Merge Family Files
	// ------------------

	foreach wave of local yearlist {
		if `wave' >= 68 local CC 19
		else if  `wave' > 00 & `wave' < 68 local CC 20
		sort x11102_`CC'`wave'
		quietly merge x11102_`CC'`wave' using `f`wave''  ///
		  , nokeep uniqusing 
		drop _merge
	}

	// Rename variables
	// ----------------

	foreach newname of local newnamelist {
		foreach pair of local `newname' {
			gettoken year var : pair, parse(`"]"') 
			local year `:subinstr local year `"["' `""''  
			local var  `:subinstr local var `"]"' `""''
			
			if "`year'"!="" {
				if `year' >= 68 local CC 19
				else if `year' > 00 & `year' < 68 local CC 20
			}
			else macro drop _CC 
			
			ren `var' `newname'`CC'`year'
		}
	}


	// Clean up
	// --------

	foreach newname of local newnamelist {
		local orderlist `orderlist' `newname'*
	}
	order x11101ll x11102*  `orderlist'
	
	drop ER30002
	capture drop ER30001


	// Link with psidadd
	// -----------------

	// We store back some chars to be used by psidadd
	char _dta[psidusedir] `"`using'"'
end


program _CNEF
	syntax anything using/ [, clear Design(string) waves(string) correct] ///

	// CHECK 
	// (This is necessary because 1st data-delivery for 2007
	// introduced ugly inconsitancies. We correct them automatically, here

	forv i = 2005(2)2007 {
		capture d *LL using "`using'/pequiv_`i'"
		if !_rc & "`correct'"=="" {
			di "{err}Found known inconsistency. Consider option correct"
		}
		if !_rc & "`correct'"!="" {
			
			use "`using'/pequiv_`i'.dta", replace
			
			foreach var of varlist *LL {
			ren `var' `=lower("`var'")'
			}
			bys x11101ll: keep if _n==1 
			save "`using'/pequiv_`i'.dta", replace
		}
	}

	// Create necessary Lists
	// ----------------------

	tokenize `anything', parse("||")
	while "`1'" != "" {
		if "`1'" == "|" mac shift
		else {
			gettoken newname item :1
			local item = trim("`item'")
			local newnamelist `newnamelist' `newname'
			local itemlist `itemlist' `item'

			if `"`=substr(`"`item'"',-2,.)'"' == `"ll"' |  /// 
			  `"`=substr(`"`item'"',-2,.)'"' == `"LL"' {
				local consitems `consitems' `item'
			}
			else {
				foreach wave of local waves {
					local `wave'items ``wave'items' `item'_`wave' 
				}
			}
			mac shift
		}
	}

	// Option clear
	// ------------
	if "`clear'" == "clear" clear

	// Prepare files for merging
	// -------------------------
	
	gettoken first rest:waves

	foreach wave of local rest {
	
		use x11101ll x11102_`wave' `consitems' ``wave'items'  /// 
		  using "`using'/pequiv_`wave'"

		// Save for merging
		sort x11101ll x11102_`CC'`wave'
		tempfile f`wave'
		quietly save `f`wave''
	}

	// Use First File
	// ------------

	use x11101ll x11102_`first' `consitems' ``first'items' 	/// 
	  using "`using'/pequiv_`first'"
	sort x11101ll 
	
	// Merge Other Files
	// -----------------

	foreach wave of local rest {
		sort x11101ll 
		quietly merge x11101ll using `f`wave'', unique update
		drop _merge
	}

	// Option Design
	// -------------

	capture confirm integer number `design'
	
	if !_rc {
		tempvar g
		gen byte `g' = 0
		foreach var of varlist x11102* {
			qui replace `g' = `g' + 1 if !mi(`var') 
		}
		qui keep if `g'>=`design'
		di as text _n "Kept households interviewed " ///
		  as result `design' as text " times or more"
	}
	
	else if "`design'"=="balanced" | "`design'" == "" {
		tempvar g
		local wavecount: word count `waves'
		gen byte `g' = 0
		foreach var of varlist x11102*  {
			qui replace `g' = `g' + 1 if !mi(`var')
		}
		qui keep if `g' == `wavecount'
		di as text "Balanced panel design"
	}
	
	else if "`design'"=="any"  di as text "All observations kept"


	// Rename variables
	// ----------------

	local i 1
	foreach newname of local newnamelist {
		local oldprefix: word `i++' of `itemlist'

		if `"`=substr(`"`oldprefix'"',-2,.)'"' == `"ll"' {
			ren `oldprefix' `newname'
		}
		else {

			foreach wave of local waves {
				ren `oldprefix'_`wave' `newname'`wave'
			}
		}
	}

	// Clean up
	// --------

	foreach newname of local newnamelist {
		local orderlist `orderlist' `newname'*
	}
	order x11101ll x11102*  `orderlist'
	
	// Link with psidadd
	// -----------------

	// We store back some chars to be used by psidadd
	char _dta[cnefusedir] `"`using'"'
end

exit

Author: Ulrich Kohler
	Tel +49 (0)30 25491 361
	Fax +49 (0)30 25491 360
	Email kohler@wzb.eu