*! opencagegeo version 1.2.0 (22/02/2018)
*! Lars Zeigermannn
* opencagegeo uses the OpenCage Geocoder API. It reuses some code segments of
* geocode written by Adam Ozimek and Daniel Miles. opencagegeo requires
* insheetjson and libjson written by Erik Lindsley.
* Version 1.1.0: a bug affecting users of paid keys has been fixed and a
* and the paidkey option was added
* Version 1.2.0: opencagegeo no longer requires the paidkey option
program opencagegeo
version 12.1
syntax [in] [if], ///
[ ///
key(str) ///
NUMber(varname) STReet(varname str) POSTcode(varname) ///
city(varname str) county(varname str) state(varname str) ///
country(varname str) FULLaddress(varname str) ///
COORDinates(varname str) LATitude(varname) ///
LONgitude(varname) countrycode(str) LANGuage(str) ///
replace RESume PAIDkey ///
]
*** Mark sample
marksample touse, novarlist
qui count if `touse' == 1
local todo = `r(N)'
*** Generate tempvar sorder
tempvar sorder
gen `sorder' = _n
*** Check that _N > 0
cap assert _N > 0
if _rc!= 0 {
di as err "No observations"
exit 2000
}
*** Check for insheetjson and libsjson
cap which insheetjson
if _rc == 111 di as err "Insheetjson.ado not found, please ssc install insheetjson"
cap which libjson.mlib
if _rc == 111 di as err "Libjson.mlib not found, please ssc install libjson"
qui if _rc == 111 assert 1 == 2
*** Check that replace and resume are not combined
if ("`resume'" != "") {
if ("`replace'" != "") {
di as err "Options replace and resume may not be combined"
exit 184
}
cap confirm variable g_quality
if _rc != 0 {
di as err "Variable g_quality not found, cannot resume geocoding"
exit 499
}
* Reset local touse
qui count if `touse' == 1 & g_quality != .
local geocoded = `r(N)' + 1
* Adjust touse if resume
qui replace `touse' = 0 if g_quality != .
}
else {
local geocoded = 1
}
if ("`replace'" != "") {
cap replace g_quality = . if `touse'
}
*** Check that touse is not zero for all observations
qui sum `touse'
if `r(mean)' == 0 {
di as err "Nothing to geocode"
exit 2000
}
*** Generate language tempvar (if specified)
local langvar = 0
if ("`language'" != "") {
cap confirm variable `language'
if _rc == 0 {
tempvar languageresponse
qui gen `languageresponse' = `language'
local langvar = 1
}
}
else {
local language = "en"
}
*** Generate countrycode tempvar (if specified
local countrycodevar = 0
if ("`countrycode'" != "") {
local iso_3166 AD AE AF AG AI AL AM AO AQ AR AS AT AU AW AX AZ BA ///
BB BD BE BF BG BH BI BJ BL BM BN BO BQ BR BS BT BV BW ///
BY BZ CA CC CD CF CG CH CI CK CL CM CN CO CR CU CV CW ///
CX CY CZ DE DJ DK DM DO DZ EC EE EG EH ER ES ET FI FJ ///
FK FM FO FR GA GB GD GE GF GG GH GI GL GM GN GP GQ GR ///
GS GT GU GW GY HK HM HN HR HT HU ID IE IL IM IN IO IQ ///
IR IS IT JE JM JO JP KE KG KH KI KM KN KP KR KW KY KZ ///
LA LB LC LI LK LR LS LT LU LV LY MA MC MD ME MF MG MH ///
MK ML MM MN MO MP MQ MR MS MT MU MV MW MX MY MZ NA NC ///
NE NF NG NI NL NO NP NR NU NZ OM PA PE PF PG PH PK PL ///
PM PN PR PS PT PW PY QA RE RO RS RU RW SA SB SC SD SE ///
SG SH SI SJ SK SL SM SN SO SR SS ST SV SX SY SZ TC TD ///
TF TG TH TJ TK TL TM TN TO TR TT TV TW TZ UA UG UM US ///
UY UZ VA VC VE VG VI VN VU WF WS YE YT ZA ZM ZW
cap confirm variable `countrycode'
if _rc == 0 {
tempvar countrycodequery
qui gen `countrycodequery' = `countrycode'
local countrycodevar = 1
tempvar iso_check
qui gen `iso_check' = 0
foreach i of local iso_3166 {
qui replace `iso_check' = 1 if upper(`countrycodequery') == "`i'"
}
cap assert `iso_check' == 1 if `countrycodequery' != ""
if _rc == 9 {
qui levelsof `countrycodequery' if `iso_check' == 0, local(levels)
di as err "Following country codes are invalid: "`r(levels)'""
exit 9
}
}
else {
local continue = 0
foreach i of local iso_3166 {
cap assert upper("`countrycode'") == "`i'"
if _rc == 0{
local continue = 1
continue, break
}
}
if (`continue' == 0) {
di as err "Country code `countrycode' is not valid"
exit 9
}
}
}
*** Check that key was provided
if ("`key'" == "") {
di as err "Opencagedata API key required: Sign up for a free key at www.opencagedata.com"
exit 198
}
*** Check that no options are combined inconsistently
if ("`fulladdress'" != "") {
cap assert "`coordinates'" == ""
if _rc != 0 {
di as err "Options fulladdress and coordinates may not be combined"
exit 184
}
cap assert "`latitude'" == ""
if _rc != 0 {
di as err "Options fulladdress and latitude not be combined"
exit 184
}
cap assert "`longitude'" == ""
if _rc != 0 {
di as err "Options fulladdress and longitude not be combined"
exit 184
}
local type fulladdress
}
if ("`coordinates'" != "") {
cap assert "`latitude'" == ""
if _rc != 0 {
di as err "Options coordinates and latitude may not be combined"
exit 184
}
cap assert "`longitude'" == ""
if _rc != 0 {
di as err "Options coordinates and longitude may not be combined"
exit 184
}
cap assert regexm(`coordinates',",")==1
if _rc != 0 {
di as err "Latitudes and longitudes must be separated by a comma"
exit 499
}
cap assert length(`coordinates') == (length(subinstr(`coordinates',",","",.)) +1)
if _rc != 0{
di as err "Variable fed into coordinates option may only contain one comma"
exit 499
}
tempvar latcheck
qui gen `latcheck' = real(substr(`coordinates',1,strpos(`coordinates',",")-1))
cap assert `latcheck' != .
if _rc != 0 {
di as err "Latitudes may not contain non-numeric characters"
exit 499
}
cap assert abs(`latcheck') <= 90
if _rc != 0 {
di as err "Latitudes must take values between -90 and 90"
exit 499
}
tempvar loncheck
qui gen `loncheck' = real(substr(`coordinates',strpos(`coordinates',",")+1,.))
cap assert `loncheck' != .
if _rc != 0 {
di as err "Longitudes may not contain non-numeric characters"
exit 499
}
cap assert abs(`loncheck') <= 180
if _rc != 0 {
di as err "Longitudes must take values between -180 and 180"
exit 499
}
local type coordinates
}
if ("`latitude'" != "" | "`longitude'" != "") {
cap assert "`latitude'" != ""
if _rc != 0 {
di as err "The longitude option must be specified together with latitude"
exit 499
}
cap assert "`longitude'" != ""
if _rc != 0 {
di as err "The latitude option must be specified together with longitude"
exit 499
}
cap assert "`longitude'" == "`latitude'"
if _rc == 0 {
di "Warning: same variable specified for latitude and longitude"
}
* Check that latitude values are between -90 and 90
tempvar templat
tempvar latcheck
cap confirm numeric variable `latitude'
if _rc == 7 {
qui gen `templat' = `latitude'
qui gen `latcheck' = real(`latitude')
cap assert `latcheck' != .
if _rc != 0 {
di as err "Latitudes may not contain non-numeric characters"
exit 499
}
}
else {
qui gen `templat' = string(`latitude')
qui gen `latcheck' = `latitude'
}
cap assert abs(`latcheck') <= 90
if _rc != 0 {
di as err "Latitudes must take values between -90 and 90"
exit 499
}
* Check that longitude values are between -180 and 180
tempvar templon
tempvar loncheck
cap confirm numeric variable `longitude'
if _rc == 7 {
qui gen `templon' = `longitude'
qui gen `loncheck' = real(`longitude')
cap assert `loncheck' != .
if _rc != 0 {
di as err "Longitudes may not contain non-numeric characters"
exit 499
}
}
else {
qui gen `templon' = string(`longitude')
qui gen `loncheck' = `longitude'
}
cap assert abs(`loncheck') <= 180
if _rc != 0 {
di as err "Longitudes must take values between -180 and 180"
exit 499
}
* Insert leading zeros for values below absolute 1 (required by the Opencage Geocoder)
qui replace `templat' = "0" + `templat' if strpos(`templat',".") == 1
qui replace `templat' = subinstr(`templat',"-.","-0.",.)
qui replace `templon' = "0" + `templon' if strpos(`templon',".") == 1
qui replace `templon' = subinstr(`templon',"-.","-0.",.)
local type latlon
}
if "`type'" != "" {
local address_parts "number street postcode city county state country"
foreach i of local address_parts {
capture assert "``i''" == ""
if _rc != 0 {
di as err "Options `type' and `i' may not be combined"
exit 184
}
}
}
else {
if "`street'" == "" & "`number'" == "" & "`postcode'" == "" & ///
"`city'" == "" & "`county'" == "" & "`country'" == "" {
di as err "No location specified"
exit 498
}
local type address
}
quietly {
*** Generate tempvar work containing the location
tempvar work
if ("`type'" == "fulladdress") {
gen `work' = " " + `fulladdress' if `touse'
}
if ("`type'" == "address") {
tempvar blank
tempvar tempnumber
tempvar temppostcode
if ("`number'" != "") {
cap confirm numeric variable `number'
if _rc == 0 {
qui tostring `number', gen(`tempnumber')
qui replace `tempnumber' = "" if `tempnumber' == "."
}
else if _rc == 7 {
gen `tempnumber' = `number'
}
}
if ("`postcode'" != "") {
cap confirm numeric variable `postcode'
if _rc == 0 {
qui tostring `postcode', gen(`temppostcode')
qui replace `temppostcode' = "" if `temppostcode' == "."
}
else if _rc == 7 {
gen `temppostcode' = `postcode'
}
}
gen `blank' = ""
gen `work' = ""
if ("`number'" != "") replace `work' = `work' + `tempnumber' if `touse' & `tempnumber' != ""
if ("`street'" != "") replace `work' = `work' + " " + `street' if `touse' & `street' != ""
if ("`postcode'" != "") replace `work' = `work' + "%2C" + `temppostcode' if `touse' & `work' != "" & `temppostcode' != ""
if ("`city'" != "" & "`postcode'" != "") replace `work' = `work' +" " + `city' if `touse' & `city' != ""
if ("`city'" != "" & "`postcode'" == "") {
replace `work' = `work' + "%2C" if `touse' & `city' != "" & `work' != ""
replace `work' = `work' + `city' if `touse' & `city' != ""
}
if ("`county'" != ""){
replace `work' = `work' + "%2C" if `touse' & `work' != "" & `county' != ""
replace `work' = `work' + `county' if `touse'
}
if ("`state'" != "") {
replace `work' = `work' + "%2C" if `touse' & `work' != "" & `state' != ""
replace `work' = `work' + `state' if `touse'
}
if ("`country'" != "") {
replace `work' = `work' + "%2C" if `touse' & `work' != "" & `country' != ""
replace `work' = `work' + `country' if `touse'
}
}
if ("`type'" == "coordinates") {
gen `work' = `coordinates' if `touse'
}
if ("`type'" == "latlon") {
gen `work' = `templat' + "%2C" + `templon' if `touse'
}
*** Generate local containing all var names to be filled in
local tobefilled g_lat g_lon g_country g_state g_county g_city ///
g_postcode g_street g_number g_confidence g_formatted
*** Replace observations if replace option specified
cap confirm new var `tobefilled'
local needreplace = _rc
if ("`replace'" != "" | "`resume'" != "" | `needreplace' == 0) {
foreach var of local tobefilled{
cap gen str224 `var' = ""
if ("`replace'" != "") cap replace `var' = "" if `touse'
}
cap recast str224 `tobefilled'
}
else {
foreach var of local tobefilled{
cap confirm variable `var'
if !_rc {
noi di as err "`var' does already exist."
}
}
cap confirm variable g_quality
if !_rc {
noi di as err "g_quality does already exist."
}
noi di as err "Drop above variables or use replace option."
exit 110
}
*** Generate tempvars
tempvar g_town g_village g_hamlet g_street_name g_road g_footway ///
g_residential g_pedestrian g_code
cap gen str224 `g_town' = ""
cap gen str224 `g_village' = ""
cap gen str224 `g_hamlet' = ""
cap gen str224 `g_street_name' = ""
cap gen str224 `g_road' = ""
cap gen str224 `g_footway' = ""
cap gen str224 `g_residential' = ""
cap gen str224 `g_pedestrian' = ""
cap gen str224 `g_code' = ""
*** Clean up tempvar work to avoid problems when sending the query
* Change some common address formats causing errors
replace `work' = lower(`work')
replace `work' = subinstr(`work',"&","%26",.) if `touse'
replace `work' = subinstr(`work',"#","",.) if `touse'
replace `work' = subinstr(`work'," 01st"," 1st",.) if `touse'
replace `work' = subinstr(`work'," 02nd"," 2nd",.) if `touse'
replace `work' = subinstr(`work'," 03rd"," 3rd",.) if `touse'
replace `work' = subinstr(`work'," 04th"," 4th",.) if `touse'
replace `work' = subinstr(`work'," 05th"," 5th",.) if `touse'
replace `work' = subinstr(`work'," 06th"," 6th",.) if `touse'
replace `work' = subinstr(`work'," 07th"," 7th",.) if `touse'
replace `work' = subinstr(`work'," 08th"," 8th",.) if `touse'
replace `work' = subinstr(`work'," 09th"," 9th",.) if `touse'
replace `work' = subinstr(`work',`"""'," ",.) if `touse'
* Remove multiple blanks
replace `work' = itrim(`work')
* Remove blanks after commas
replace `work' = subinstr(strtrim(`work'), ", ", ",",.)
* Remove leading and trailing blanks and replace interior blanks with +
replace `work' = subinstr(strtrim(`work'), " ", "+",.)
*** Check tempvar work for special characters (Stata 13 or older) or encode (if Stata 14 or newer)
if c(stata_version) >= 14 {
replace `work' = ustrto(`work', "ascii", 4)
replace `work' = subinstr(`work',"\","%",.)
}
else {
foreach num of numlist 1/31 127/255 {
cap assert index(`work',char(`num')) == 0
if _rc != 0 {
di as err "Location names may not contain special characters"
exit 499
}
}
}
*** Generate local containing column selectors
local selectors results:1:geometry:lat results:1:geometry:lng ///
results:1:components:country results:1:components:state ///
results:1:components:county results:1:components:city ///
results:1:components:postcode results:1:components:street ///
results:1:components:house_number results:1:confidence ///
results:1:formatted results:1:components:town ///
results:1:components:village results:1:components:hamlet ///
results:1:components:street_name results:1:components:road ///
results:1:components:footway results:1:components:residential ///
results:1:components:pedestrian status:code
*** Order data set for geocoding
sort `touse' `sorder'
*** Generate locals for loop
local cnt = _N
count if `touse' == 0
local start = `r(N)' + 1
*** Loop over observations to be geocoded
forval i = `start'/`cnt' {
cap {
local offset = `i'-1
local query = `work'[`i']
if (`countrycodevar' == 1) local countrycode = `countrycodequery'[`i']
if (`langvar' == 1) local language = `languageresponse'[`i']
insheetjson `tobefilled' `g_town' `g_village' `g_hamlet' `g_street_name' `g_road' ///
`g_footway' `g_residential' `g_pedestrian' `g_code' ///
using "http://api.opencagedata.com/geocode/v1/json?q=`query'&key=`key'&no_annotations=1&language=`language'&countrycode=`countrycode'&limit=1", col(`selectors') flatten limit(1) offset(`offset') replace
if (`g_code'[`i'] != "200") {
if (`i' == `start') {
local error = 111
}
else {
local error = 112
}
qui replace `touse' = 0 if _n >= `i'
continue, break
}
noi di "OpenCage geocoded `geocoded' of `todo'"
local ++geocoded
}
if _rc!=0 {
local error = _rc
qui replace `touse' = 0 if _n >= `i'
continue, break
}
}
*** Generate unique street variable
cap replace g_street = g_street + `g_street_name' + `g_road' + ///
`g_pedestrian' + `g_residential' if (missing(g_street) | g_street == "[]") & `touse'
*** Generate unique city variable
cap replace g_city = g_city + `g_town' + `g_village' + `g_hamlet' ///
if (missing(g_city) | g_city == "[]") & `touse'
*** Compress variables and convert to UTF-8 if Stata 14
foreach var of local tobefilled {
if c(stata_version) >= 14 {
replace `var' = ustrunescape(`var') if `touse'
}
replace `var' = subinstr(`var',"[]","",.) if `touse'
}
cap compress `tobefilled'
sort `sorder'
*** Generate g_quality variable
cap gen g_quality = .
replace g_quality = 0 if `touse'
replace g_quality = 1 if g_country != "" & `touse'
replace g_quality = 2 if g_state != "" & `touse'
replace g_quality = 3 if g_county != "" & `touse'
replace g_quality = 4 if g_city != "" & `touse'
replace g_quality = 5 if g_postcode != "" & `touse'
replace g_quality = 6 if g_street != "" & `touse'
replace g_quality = 7 if g_number != "" & `touse'
*** Define labels and label values of g_quality
cap label define quality 0 "not found" 1 "country" 2 "state" ///
3 "county" 4 "city" 5 "postcode" 6 "street" 7 "number"
label values g_quality quality
*** Exit if error occured
if ("`error'" != "") {
if (`error' == 111) {
noi di as err "No observations geocoded: Invalid key, rate limit exceeded or no internet connection"
exit
}
else if (`error' == 112) {
noi di as err "Rate limit exceeded or internet connection failed"
exit
}
exit `error'
}
*** Display summary table of g_quality
if ("`resume'" != "") marksample touse, novarlist
noi tab g_quality if `touse'
noi di _newline "Data generated is jointly licensed under the ODbL and CC-BY-SA licenses"
}
end