program define reclink
version 8.2
*! v 1.2 19-Sep-2007 M. Blasnik
syntax varlist using/ ,  IDMaster(str) IDUsing(str) Gen(str) [WMatch(str) WNOMatch(str) EXClude(str) /*
*/   UVarlist(str) MINScore(real 0.6) MINBigram(real 0.6) ORBLock(str) REQuired(str) /*
*/ _Merge(str) UPRefix(str) EXActstr(str) ]

if "`_merge'"=="" local _merge "_merge"
cap confirm variable `_merge'
if _rc==0 {
	di as err "`_merge' already exists"
	exit 198
}

local nvars: word count `varlist'
if "`wmatch'"=="" local wmatch: di _dup(`nvars')  " 1 "
if "`wnomatch'"=="" local wnomatch "`wmatch'"
if "`uprefix'"=="" local uprefix "U"
foreach val in `wmatch' `wmnomatch'  {
	if `val'<1 {
		di as error " weights must be >=1"
		exit 198
	}
}

local nwms:  word count `wmatch'
local nwnms: word count `wnomatch'
if `nvars'!=`nwnms' | `nvars'!=`nwms'  {
	di as error "wmatch and wnomatch must each have the same # of elements as varlist"
	exit 198
}

if "`uvarlist'"=="" local uvarlist "`varlist'"
local nuvars: word count `uvarlist'
if `nvars'!=`nuvars' {
	di as error "uvarlist must have the same # of elements as varlist"
	exit 198
}

if "`orblock'"=="" & `nvars'>3 local orblock "`varlist'"

local i=1
foreach var of local varlist {
	local v`i' "`var'"
  local wm`i': word `i' of `wmatch'
	local wnm`i': word `i' of `wnomatch'
	local uv`i': word `i' of `uvarlist'
	cap confirm string var `var'
	local string`i'=(_rc==0)
	local nobigram: list var & exactstr
	local bigram`i'=("`nobigram'"=="")
	local inreq: list var & required
	local inreq`i'=("`inreq'"!="")
	local i=`i'+1
}

* scaling factor for bigram score mapping to matchscore
local biscale=.5/(1-`minbigram')^(1/3)

preserve

quietly {
isid `idmaster'
gen `gen'=0
label var `gen' "reclink matching score"
local mobs=c(N)
tempfile m u matched results idusingmatched idmastermatched
keep `idmaster' `gen' `varlist'
sort `idmaster'
local idmtype: type `idmaster'
save `m'

if "`exclude'"!="" {
	tempfile midexclude uidexclude using2
	tempvar mrgtmp
	use `idmaster' `idusing' using `exclude'
	drop if missing(`idmaster') | missing(`idusing')
	save `midexclude'

	bysort `idusing': keep if _n==1
	keep `idusing'
	sort `idusing'
	save `uidexclude'

	use `midexclude'
	bysort `idmaster': keep if _n==1
	keep `idmaster'
	sort `idmaster'
	merge `idmaster' using `m', _merge(`mrgtmp')
	keep if `mrgtmp'==2
	drop `mrgtmp'
	sort `idmaster'
	local mobs2match=c(N)
	save `m', replace

	use `using'
	local uobs=c(N)
	sort `idusing'
	merge `idusing' using `uidexclude', _merge(`mrgtmp') nokeep
	keep if `mrgtmp'==1
	drop `mrgtmp'
	save `using2'
}
else {
	local using2 "`using'"
}

use `using2'
local uobs2match=c(N)
if "`uobs'"=="" local uobs=c(N)
isid `idusing'
keep `idusing' `uvarlist'
local i=1
foreach var of varlist `uvarlist' {
	if "`var'"!="`v`i''" {
		cap rename `var' `v`i''
	}
	local i=`i'+1
}
sort `varlist'
local idutype: type `idusing'
tempvar uidtmp
egen `uidtmp'=group(`varlist'), missing
save `u'

* create lookup table back to original id variable in using dataset
keep `uidtmp' `idusing'
tempfile uids
sort `uidtmp'
save `uids'

* now collapse the using dataset down to unique combinations of varlist
use `u'
drop `idusing'
bysort `uidtmp' : keep if _n==1
sort `varlist'
save `u', replace
local uniqusingn=c(N)

* identify perfect matches
use `m'
sort `varlist'
merge `varlist' using `u', nokeep
keep if _merge==3
local perfectn=c(N)
noi di as result _n "`perfectn' perfect matches found" _n

if `perfectn'>0 {
	replace `gen'=1
	keep `idmaster' `uidtmp' `gen'
	sort `idmaster'
	save `matched'

	* save using ids for perfect matches
	keep `uidtmp'
	sort `uidtmp'
	save `idusingmatched'

	* save master ids for perfect matches
	use `matched'
	keep `idmaster'
	sort `idmaster'
	save `idmastermatched'

	* get rid of perfect matches from using datafile
	use `u'
	sort `uidtmp'
	merge `uidtmp' using  `idusingmatched'
	keep if _merge==1
	drop _merge
	save `u', replace

	* get rid of perfect matches from master data file
	use `m'
	sort `idmaster'
	merge `idmaster' using  `idmastermatched'
	keep if _merge==1
	drop _merge
	save `m', replace

}

else {
	use `m', replace
}

local tomatch=c(N)
tempname post
postfile `post' `idmtype' `idmaster' long `uidtmp' float `gen' using `results'
if `tomatch'>1000 {
noi di as text "Going through `tomatch' observation to assess fuzzy matches, each .=5% complete"
}
forvalues obs=1(1)`tomatch' {
		use `m' in `obs', replace

  if int(20*`obs'/`tomatch')!=int(20*(`obs'-1)/`tomatch') & (`tomatch'>1000) noi di "." _c
	local ifall
	local ifany
	local ifreq

* OR blocking
	local orblock : list orblock - required
	if lower("`orblock'")!="none" & "`orblock'"!="" {
		local i=1
		foreach v of local orblock {
			local thisval=`v'[1]
			if "`thisval'"!="" {
				if `"`ifany'"'!="" {
					local ifany `" `ifany' | "'
				}
				if `string`i''==1 {
					local ifany `" `ifany' `v'=="`thisval'" "'
				}
				else {
					local ifany `" `ifany' `v'==`thisval' "'
				}
			}
			local i=`i'+1
		}
	local ifall `" if (`ifany') "'
	} /* end of OR blocking */

	* now add required matches
	if "`required'"!="" {
		local i=1
		foreach v of local required {
			local thisval=`v'[1]
			if `"`ifreq'"'!="" {
				local ifreq `" `ifreq' & "'
			}
			if `string`i''==1 {
				local ifreq `" `ifreq' `v'=="`thisval'" "'
			}
			 else {
			 	local ifreq `" `ifreq' `v'==`thisval' "'
			}
			local i=`i'+1
		}
		if `"`ifall'"'!="" local ifall `" `ifall' & (`ifreq') "'
		 else local ifall `" if (`ifreq') "'
	} /* end of required matches */

	local thisidmaster=`idmaster'[1]
	local i=1
	foreach v of local varlist {
			local thisv`i'=`v'[1]
			local i=`i'+1
	}

* still need to go thru varlist to drop single quotes that cause macro problems?

	* load using data which meets required and orblock
	use `u'  `ifall', replace
	local upossn=c(N)
	if `upossn'>0 {

* scale bigram matches from minscore to 1 into 0->1
		tempvar M NM score bi
		gen `M'=0
		gen `NM'=0
		local i=1
		foreach v of local varlist {
				if "`thisv`i''"!="" {
					if `inreq`i''!=1 {
						if `string`i''==1 & `bigram`i''==1 {
							bigram1 `v', match(`thisv`i'') gen(`bi')
						}
						else {
							cap drop `bi'
							if `string`i''==1 gen `bi'=(`v'=="`thisv`i''")
							else gen `bi'=(`v'==`thisv`i'')
						}
					replace `M'=`M'+`wm`i''*(.5*`bi'^2+cond(`bi'>`minbigram',`biscale'*(`bi'-`minbigram')^(1/3),0))
					replace `NM'=`NM'+`wnm`i''*cond(`bi'<(`minbigram'-.2),1,cond(`bi'<`minbigram',1-`bi'^2,(1-`bi')^2)) if !missing(`v')
					}
					else {
						replace `M'=`M'+`wm`i''
					}
			}
				* missing in one, not other = 30% non-match
				 replace `NM'=`NM'+`wnm`i''*0.3 if (missing("`thisv`i''") + missing(`v')) ==1
			local i=`i'+1
		}

		gen `score'=`M'/(`M'+`NM')

		replace `score'=0 if `score'==.
		sort `score'
		* now keep the cases with the highest score and post them
		keep if `score'==`score'[_N] & `score'>`minscore'
		local matches=c(N)
		if `matches'>0 {
			local scoreval=`score'[1]
			forval j=1(1)`matches' {
				local thisidusing = `uidtmp'[`j']
				post `post' (`thisidmaster') (`thisidusing') (`scoreval')
				if `score'>.89 & `score'<.895 {
					noi di "M=" `M'[`j']
					noi di "NM=" `NM'[`j']
					noi di "bigram=" `bi'[`j']
				}
			}
		}
	}
}
postclose `post'

* now merge the matching back into master
use `results', replace
local resobs=c(N)
if `perfectn'>0 {
	append using `matched'
}
tempvar tmrg
sort `uidtmp'
merge `uidtmp' using `uids', nokeep _merge(`tmrg')
assert `tmrg'==3
drop `tmrg' `uidtmp'
sort `idmaster'
save, replace

* clean up using data -- make sure no varnames in common so mismatches can be examined
use `using', replace
forvalues i=1(1)`nvars' {
	cap confirm var `v`i''
	if _rc==0 rename `v`i'' `uprefix'`v`i''
}
tempfile u2
sort `idusing'
save `u2'

* now back to original dataset and merge results into that, then using dataset into that
restore
sort `idmaster'
merge `idmaster' using `results', _merge(`tmrg')
assert `tmrg'!=2
sort `idusing'
merge `idusing' using `u2' , nokeep _merge(`_merge')
local fobs=c(N)
qui count if missing(`idusing')
local nomatch=r(N)
forvalues i=1(1)`nvars' {
	cap move  `uprefix'`v`i'' `v`i''
	cap move  `v`i'' `uprefix'`v`i''
}
}

format `gen' %5.4f
noi di _n "Added: `idusing'= identifier from `using'   `gen' = matching score"
noi di as res "Observations:  Master N = `mobs'    `using' N= `uobs' "
if "`exclude'"!="" {
	noi di as res "  # Obs after excluding matches in `exclude':  Master = `mobs2match'    `using': `uobs2match' "
}
noi di as res "  Unique Master Cases: matched = " `mobs'-`nomatch'   " (exact = `perfectn'), unmatched = `nomatch'"
end


program define bigram1
version 8.2
*! version 2.2 M Blasnik 19-Sep-2005
syntax varlist (max=1) , gen(str) match(str)
tempvar slen wink
cap confirm var `gen'
if _rc==0 replace `gen'=0
else gen `gen'=0
local mlen=length("`match'")
gen `slen'=length(`varlist')
local poss=`mlen'-1
if `mlen'>2 {
	forval i=1(1)`poss' {
			qui replace `gen'=`gen'+1 if index(`varlist',substr("`match'",`i',2))>0
	}
	qui replace `gen'=`gen'*2/(`slen'+`mlen'-2)
}

* deal with strings <3 characters
	qui replace `gen'=0 if (`mlen'<3 | `slen'<3) & !index("`match'",`varlist') & !index(`varlist',"`match'")
  qui replace `gen'=min(`mlen',`slen')/(`mlen'+`slen'-1)  if (`mlen'<3 | `slen'<3) & ( index("`match'",`varlist') | index(`varlist',"`match'") )

	* Winkler adjustment: adjusts score upward based on first 1,2,3, or 4 characters matching
	gen byte `wink'=0
	forval i=1(1)4 {
			qui replace `wink'=`wink'+1 if substr(`varlist',1,`i')==substr("`m2'",1,`i')
	}
	qui replace `gen'=`gen'+`wink'*(1-`gen')/10
  * make sure exact matches =1 and missing =0
  qui replace `gen'=1 if `varlist'=="`match'"
	qui replace `gen'=0 if `slen'==0
end