*! version 0.5 April 22, 2010 @ 15:29:40 UK
* 0.2 Functionality for constants in IND-Files added
* 0.3 Bug for constants vars in CNEF -> fixed
* 0.4 Some 2007 varnames interfere with IND varnames -> fixed
* 0.5 Adaptions to CNEF 2007


program psidadd
version 10.0
	
	// Hard coded list of identifiers
	// (Update these lists for each data delivery)

	local IND ind2007er
	
	local IDind ///
	  [68] ER30001 [69] ER30020 [70] ER30043 [71] ER30067  /// 
	  [72] ER30091 [73] ER30117 [74] ER30138 [75] ER30160  /// 
	  [76] ER30188 [77] ER30217 [78] ER30246 [79] ER30283  /// 
	  [80] ER30313 [81] ER30343 [82] ER30373 [83] ER30399  /// 
	  [84] ER30429 [85] ER30463 [86] ER30498 [87] ER30535  /// 
	  [88] ER30570 [89] ER30606 [90] ER30642 [91] ER30689  /// 
	  [92] ER30733 [93] ER30806 [94] ER33101 [95] ER33201  /// 
	  [96] ER33301 [97] ER33401 [99] ER33501 [01] ER33601  /// 
	  [03] ER33701 [05] ER33801 [07] ER33901

	local IDfam ///
	  [68] V3 [69] V442 [70] V1102 [71] V1802 [72] V2402  [73] V3002  ///  
	  [74] V3402  [75] V3802  [76] V4302  [77] V5202  [78] V5702  ///  
	  [79] V6302  [80] V6902  [81] V7502  [82] V8202  [83] V8802  ///  
	  [84] V10002 [85] V11102 [86] V12502 [87] V13702 [88] V14802  /// 
	  [89] V16302 [90] V17702 [91] V19002 [92] V20302 [93] V21602  /// 
	  [94] ER2002 [95] ER5002 [96] ER7002 [97] ER10002 [99] ER13002  /// 
	  [01] ER17002 [03] ER21002 [05] ER25002 [07] ER36002  


	// Parse Commandline 
	// ------------------
	
	syntax anything [, CNEFfrom(string) PSIDfrom(string) correct ]  ///

	// Catch dirname from psiduse of options 
	// -------------------------------------

	// Catch version of data generating program
	if "`_dta[psidusedir]'" != "" local usetyp psid
	else local usetyp cnef

	// Catch version of this program
	if "`_dta[psidusedir]'" != "" & "`cneffrom'" == "" local addtyp psid
	else if "`_dta[psidusedir]'" != "" & "`cneffrom'" != "" local addtyp cnef
	else if "`_dta[cnefusedir]'" != "" & "`psidfrom'" == "" local addtyp cnef
	else if "`_dta[cnefusedir]'" != "" & "`psidfrom'" != "" local addtyp psid
	
	if "`addtyp'" == "cnef" {
		local using = cond("`cneffrom'"=="","`_dta[cnefusedir]'","`cneffrom'")
		_CNEF `anything' using "`using'", `correct'
		exit
	}

	local using = cond("`psidfrom'"=="","`_dta[psidusedir]'","`psidfrom'")
	tokenize `anything', parse("||")
	while "`1'" != "" {
		if "`1'" == "|" mac shift
		else {
			gettoken newname pairlist :1
			local newnamelist `newnamelist' `newname'
			foreach pair of local pairlist {
				gettoken year var : pair, parse(`"]"') 
				local year `:subinstr local year `"["' `""''  
				local yearlist "`yearlist' `year'"
				local var  `:subinstr local var `"]"' `""''
		
				// Extract vars from ind files
				if inlist(substr("`var'",1,4),"ER30","ER31","ER32","ER33","ER34","ER35")  ///
				  & length("`var'")==7 {
					capture d `var' using "`using'/`IND'"
					if !_rc local keepvars `keepvars' `var'
				}
				else {
					local `year'vars "``year'vars' `var'"
					local famfiles 1
				}
				local `newname' ``newname'' [`year']`var'
			}
			mac shift
		}
	}

	// Uniquify yearlist
	local yearlist: list uniq yearlist

	// Prepare and merge Fam files
	// ---------------------------

	if "`famfiles'" == "1" {
		preserve
		drop _all

		foreach wave of local yearlist {
			
			// Create local for Century
			if `wave' >= 68 local CC 19
			else if  `wave' > 00 & `wave' < 68 local CC 20
			
			// Letters "er" in filenames of years 1994 and later
			local er = cond(`CC'`wave'>=1994,"er","")
			
			// Extract idenifier for wave
			local IDpos: list posof `"[`wave']"' in IDfam
			local ID: word `++IDpos' of `IDfam'
			
			use `ID' ``wave'vars' using "`using'/fam`CC'`wave'`er'"
			
			// Generate CNEF Idenifiers
			if `wave' >= 68 local CC 19
			else if  `wave' > 00 & `wave' < 68 local CC 20
			ren `ID' x11102_`CC'`wave'

			// Save for merging
			sort x11102_`CC'`wave'
			tempfile f`wave'
			quietly save `f`wave''
		}

		// Merge Family Files
		restore
		
		foreach wave of local yearlist {
			if `wave' >= 68 local CC 19
			else if  `wave' > 00 & `wave' < 68 local CC 20
			sort x11102_`CC'`wave'
			quietly merge x11102_`CC'`wave' using `f`wave''  ///
			  , nokeep uniqusing 
			drop _merge
		}

	}

	// Prepare and merge IND file
	// --------------------------

	if "`keepvars'" != "" {
		preserve
		drop _all

		use ER30002 ER30001 `keepvars' using "`using'/`IND'"
		
		// Generate CNEF Idenifier
		gen long x11101ll = ER30001*1000 + ER30002
		lab var x11101ll "Person idenification number"
		drop ER30002 ER30001
		sort x11101ll

		tempfile find
		quietly save `find'
		
		restore
		sort x11101ll
		quietly merge x11101ll using `find', nokeep unique
		drop _merge
	}

	// Rename variables
	// ----------------

	foreach newname of local newnamelist {
		foreach pair of local `newname' {
			gettoken year var : pair, parse(`"]"') 
			local year `:subinstr local year `"["' `""''  
			local var  `:subinstr local var `"]"' `""''
			
			if "`year'" != "" {
				if `year' >= 68 local CC 19
				else if  `year' > 00 & `year' < 68 local CC 20
			}
			else macro drop _CC
			
			ren `var' `newname'`CC'`year'
		}
	}


	// Clean up
	// --------

	foreach newname of local newnamelist {
		local orderlist `orderlist' `newname'*
	}
	quietly ds `orderlist', not
	order `r(varlist)' `orderlist'
	
end


program _CNEF
	syntax anything using/ [, correct] ///

	// CHECK 
	// (This is necessary because 1st data-delivery for 2007
	// introduced ugly inconsitancies. We correct them automatically, here

	forv i = 2005(2)2007 {
		capture d *LL using "`using'/pequiv_`i'"
		if !_rc & "`correct'"=="" {
			di "{err}Found known inconsistency. Consider option correct"
		}
		if !_rc & "`correct'"!="" {
			
			use "`using'/pequiv_`i'.dta", replace
			
			foreach var of varlist *LL {
				ren `var' `=lower("`var'")'
			}
			qui bys x11101ll: keep if _n==1 
			qui save "`using'/pequiv_`i'.dta", replace
		}
	}

	// Create waveslist
	// ----------------

	foreach var of varlist x11102_* {
		local waves `waves' 			/// 
		  `=cond(`=substr(`"`var'"',-4,.)'>=1980,`"`=substr("`var'",-4,.)'"',`""')'
	}

	// Create necessary Lists
	// ----------------------
	
	tokenize `anything', parse("||")
	while "`1'" != "" {
		if "`1'" == "|" mac shift
		else {
			gettoken newname item :1
			local item = trim("`item'")
			local newnamelist `newnamelist' `newname'
			local itemlist `itemlist' `item'
			if `"`=substr(`"`item'"',-2,.)'"' == `"ll"' {
				local consitems `consitems' `item'
			}
			else {
				foreach wave of local waves {
					local `wave'items ``wave'items' `item'_`wave' 
				}
			}
			mac shift
		}
	}

	preserve

	// Prepare files for merging
	// -------------------------
	
	foreach wave of local waves {
	
		use x11101ll x11102_`wave' `consitems' ``wave'items'  /// 
		  using "`using'/pequiv_`wave'", clear

		// Save for merging
		sort x11101ll x11102_`CC'`wave'
		tempfile f`wave'
		quietly save `f`wave''
	}

	// Return to master
	// ----------------

	restore

	// Merge Files
	// ------------

	foreach wave of local waves {
		sort x11101ll 
		quietly merge x11101ll using `f`wave'', unique nokeep update
		drop _merge
	}

	// Rename variables
	// ----------------

	local i 1
	foreach newname of local newnamelist {
		local oldprefix: word `i++' of `itemlist'

		if `"`=substr(`"`oldprefix'"',-2,.)'"' == `"ll"' {
			ren `oldprefix' `newname'
		}
		else {
			foreach wave of local waves {
				ren `oldprefix'_`wave' `newname'`wave'
			}
		}
	}

	// Clean up
	// --------

	foreach newname of local newnamelist {
		local orderlist `orderlist' `newname'*
	}
	quietly ds `orderlist', not
	order `r(varlist)' `orderlist'


end

exit

Author: Ulrich Kohler
Tel +49 (0)30 25491 361
Fax +49 (0)30 25491 360
Email kohler@wzb.eu