/*======================================================= Program Name: povcalnet.ado Author: R.Andres Castaneda Aguilar Jorge Soler Lopez Espen Beer Prydz Christoph Lakner Ruoxuan Wu Qinghua Zhao World Bank Group project: Stata package to easily query the [PovcalNet API](http://iresearch.worldbank.org/PovcalNet/docs/PovcalNet%20API.pdf) Dependencies: The World Bank - DEC ----------------------------------------------------------------------- Creation Date: Sept 2018 References: Output: dta file =======================================================*/ program def povcalnet, rclass set checksum off //temporarily bypasses control of files taken from internet version 11.0 syntax [anything(name=subcommand)] /// [, /// COUNtry(string) /// REGion(string) /// YEAR(string) /// POVline(numlist) /// POPShare(numlist) /// PPP(numlist) /// AGGregate /// CLEAR /// INFOrmation /// coverage(string) /// ISO /// Standard ISO codes SERVER(string) /// internal use pause /// debugging FILLgaps /// N2disp(integer 15) /// noDIPSQuery /// querytimes(integer 5) /// ] if ("`pause'" == "pause") pause on else pause off qui { dis as text "{hline}" noi disp in red "WARNING:" noi dis as text `"{p 6 4 0 80}The povcalnet command and website have been replaced with the new pip command and the Poverty and Inequality Portal, {browse "https://pip.worldbank.org/"}. Povcalnet will remain functional until the end of 2022, but its data is now {bf:outdated}. Once retired, you will no longer be able to access these data. If you anticipate needing to reproduce existing analysis based on PovcalNet data, you will want to save a version of the data. Moving forward, we encourage you to use the new PIP site and its corresponding {browse "https://github.com/worldbank/pip": pip} Stata command, with the latest data and many new features.{p_end}"' dis as text "{hline}" _n //======================================================== // Conditions //======================================================== if ("`aggregate'" != "" & "`fillgaps'" != "") { noi disp in red "options {it:aggregate} and {it:fillgaps} are mutually exclusive." _n /* */ "Please select only one." error } if ("`popshare'" != "" & (lower("`subcommand'") == "wb" | "`aggregate'" != "")) { noi disp in red "option {it:popshare} can't be combined with option {it:aggregate}" _c /* */ " or with subcommand {it:wb}" _n error } // ------------------------------------------------------------------------ // New session procedure // ------------------------------------------------------------------------ if ("${pcn_cmds_ssc}" == "") { // --------------------------------------------------------------- // Update PovcalNet // --------------------------------------------------------------- * mata: povcalnet_source("povcalnet") // creates local src _pcn_find_src povcalnet local src = "`r(src)'" * If povcalnet was installed from github if (regexm("`src'", "github")) { local git_cmds povcalnet foreach cmd of local git_cmds { * Check repository of files * mata: povcalnet_source("`cmd'") _pcn_find_src `cmd' local src = "`r(src)'" if regexm("`src'", "\.io/") { // if site if regexm("`src'", "://([^ ]+)\.github") { local repo = regexs(1) + "/`cmd'" } } else { // if branch if regexm("`src'", "\.com/([^ ]+)(/`cmd')") { local repo = regexs(1) + regexs(2) } } qui github query `repo' local latestversion = "`r(latestversion)'" if regexm("`r(latestversion)'", "v([0-9]+)\.([0-9]+)\.([0-9]+)"){ local lastMajor = regexs(1) local lastMinor = regexs(2) local lastPatch = regexs(3) } qui github version `cmd' local crrtversion = "`r(version)'" if regexm("`r(version)'", "v([0-9]+)\.([0-9]+)\.([0-9]+)"){ local crrMajor = regexs(1) local crrMinor = regexs(2) local crrPatch = regexs(3) } foreach x in repo cmd { local `x' : subinstr local `x' "." "", all local `x' : subinstr local `x' "-" ".", all if regexm("``x''", "v([0-9]+)(\.)?([a-z]+)?([0-9]?)") { disp regexs(1) regexs(2) regexs(4) } } * force installation if ("`crrtversion'" == "") { github install `repo', replace cap window stopbox note "github command has been reinstalled to " /// "keep record of new updates. Please type discard and retry." global pcn_cmds_ssc = "" exit } if (`lastMajor' > `crrMajor' | `lastMinor' > `crrMinor' | `lastPatch' > `crrPatch') { * if (`lastMajor'`lastMinor'`lastPatch' > `crrMajor'`crrMinor'`crrPatch') { cap window stopbox rusure "There is a new version of `cmd' in Github (`latestversion')." /// "Would you like to install it now?" if (_rc == 0) { cap github install `repo', replace if (_rc == 0) { cap window stopbox note "Installation complete. please type" /// "discard in your command window to finish" local bye "exit" } else { noi disp as err "there was an error in the installation. " _n /// "please run the following to retry, " _n(2) /// "{stata github install `repo', replace}" local bye "error" } } else local bye "" } // end of checking github update else { noi disp as result "Github version of {cmd:`cmd'} is up to date." local bye "" } } // end of loop } // end if installed from github else if (regexm("`src'", "repec")) { // if povcalnet was installed from SSC qui adoupdate povcalnet, ssconly if ("`r(pkglist)'" == "povcalnet") { cap window stopbox rusure "There is a new version of povcalnet in SSC." /// "Would you like to install it now?" if (_rc == 0) { cap ado update povcalnet, ssconly update if (_rc == 0) { cap window stopbox note "Installation complete. please type" /// "discard in your command window to finish" local bye "exit" } else { noi disp as err "there was an error in the installation. " _n /// "please run the following to retry, " _n(2) /// "{stata ado update povcalnet, ssconly update}" local bye "error" } } else local bye "" } // end of checking SSC update else { noi disp as result "SSC version of {cmd:povcalnet} is up to date." local bye "" } } // Finish checking povcalnet update else { noi disp as result "Source of {cmd:povcalnet} package not found." _n /// "You won't be able to benefit from latest updates." local bye "" } /*================================================== Dependencies ==================================================*/ *---------- check SSC commands local ssc_cmds missings noi disp in y "Note: " in w "{cmd:povcalnet} requires the packages " /// "below from SSC: " _n in g "`ssc_cmds'" foreach cmd of local ssc_cmds { capture which `cmd' if (_rc != 0) { ssc install `cmd' noi disp in g "{cmd:`cmd'} " in w _col(15) "installed" } } adoupdate `ssc_cmds', ssconly if ("`r(pkglist)'" != "") adoupdate `r(pkglist)', update ssconly global pcn_cmds_ssc = 1 // make sure it does not execute again per session `bye' } /*================================================== Defaults ==================================================*/ *---------- API defaults if "`server'"!="" { if !inlist(lower("`server'"), "int", "testing", "ar") { noi disp in red "the server requested does not exist" error } if (lower("`server'") == "int") { local server "${pcn_svr_in}" } if (lower("`server'") == "testing") { local server "${pcn_svr_ts}" } if (upper("`server'") == "AR") { local server "${pcn_svr_ar}" } if ("`server'" == "") { noi disp in red "You don't have access to internal servers" _n /* */ "You're being redirected to public server" local server "http://iresearch.worldbank.org/PovcalNet" } } else { local server "http://iresearch.worldbank.org/PovcalNet" } local base = "`server'/PovcalNetAPI.ashx" return local server = "`server'" //------------ Check internet connection scalar tpage = fileread(`"`server'/js/common_NET.js"') if regexm(tpage, "error") { noi disp in red "You may not have Internet connections. Please verify" error } *---------- lower case subcommand local subcommand = lower("`subcommand'") *---------- Test if ("`subcommand'" == "test") { if ("${pcn_query}" == "") { noi di as err "global pcn_query does not exist. You cannot test the query." error } local fq = "`base'?${pcn_query}" noi disp in y "querying" in w _n "`fq'" noi view browse "`fq'" exit } *---------- Modify country(all) with aggregate if (lower("`country'") == "all" & "`aggregate'" != "") { local country "" local aggregate "" local subcommand "wb" local wb_change 1 noi disp as err "Warning: " as text " {cmd:povcalnet, country(all) aggregate} " /* */ "is equivalent to {cmd:povcalnet wb}. " _n /* */ " if you want to aggregate all countries by survey years, " /* */ "you need to parse the list of countries in {it:country()} option. See " /* */ "{help povcalnet##options:aggregate option description} for an example on how to do it" } else { local wb_change 0 } if ("`year'" == "") local year "all" * *---------- Coverage if ("`coverage'" == "") local coverage = "all" local coverage = lower("`coverage'") foreach c of local coverage { if !inlist(lower("`c'"), "national", "rural", "urban", "all") { noi disp in red `"option {it:coverage()} must be "national", "rural", "urban" or "all" "' error } } *---------- Poverty line/population share // Blank popshare and blank povline = default povline 1.9 if ("`popshare'" == "" & "`povline'" == "") { local povline = 1.9 local pcall = "povline" } // defined popshare and defined povline = error else if ("`popshare'" != "" & "`povline'" != "") { noi disp as err "povline and popshare cannot be used at the same time" error } // blank popshare and defined povline else if ("`popshare'" == "" & "`povline'" != "") { local pcall = "povline" } // defined popshare and blank povline else { local pcall = "popshare" } *---------- Info if regexm("`subcommand'", "^info") { local information = "information" local subcommand = "information" } *---------- Subcommand consistency if !inlist("`subcommand'", "wb", "information", "cl", "") { noi disp as err "subcommand must be either {it:wb}, {it:cl}, or {it:info}" error } *---------- One-on-one execution if ("`subcommand'" == "cl" & lower("`country'") == "all") { noi disp in red "you cannot use option {it:countr(all)} with subcommand {it:cl}" error 197 } *---------- PPP if (lower("`country'") == "all" & "`ppp'" != "") { noi disp as err "Option {it:ppp()} is not allowed with {it:country(all)}" error } *---------- WB aggregate if ("`subcommand'" == "wb") { if ("`country'" != "") { noi disp as err "option {it:country()} is not allowed with subcommand {it:wb}" error } noi disp as res "Note: " as txt "subcommand {it:wb} only accepts options " _n /* */ "{it:region()} and {it:year()}" } *---------- Country if ("`country'" == "" & "`region'" == "") local country "all" if ("`country'" != "") { if (lower("`country'") != "all") local country = upper("`country'") else local country "all" } /*================================================== Main conditions ==================================================*/ if ("`information'" == "") { if (c(N) != 0 & "`clear'" == "" & /* */ "`information'" == "") { noi di as err "You must start with an empty dataset; or enable the option {it:clear}." error 4 } drop _all } *---------- Country and region if ("`country'" != "") & ("`region'" != "") { noi disp in r "options {it:country()} and {it:region()} are mutually exclusive" error } if ("`aggregate'" != "") { if ("`ppp'" != ""){ noi di as err "Option PPP cannot be combined with aggregate." error 198 } noi disp as res "Note: " as text "Aggregation is only possible over reference years." local agg_display = "Aggregation in base year(s) `year'" } if (wordcount("`country'")>2) { if ("`ppp'" != ""){ noi di as err "Option PPP can only be used with one country." error 198 } } /*================================================== Execution ==================================================*/ pause povcalnet - before execution *---------- Information if ("`information'" != ""){ noi povcalnet_info, `clear' `pause' server(`server') return add exit } *---------- Country Level (one-on-one query) if ("`subcommand'" == "cl") { noi povcalnet_cl, country("`country'") /// year("`year'") /// povline("`povline'") /// ppp("`ppp'") /// server("`server'") /// coverage(`coverage') /// `clear' /// `iso' /// `pause' return add exit } *---------- Regular query and Aggregate Query if ("`subcommand'" == "wb") { local wb "wb" } else local wb "" tempfile povcalf save `povcalf', empty local f = 0 if ("`pcall'" == "povline") loc i_call "i_povline" else loc i_call "i_popshare" foreach `i_call' of local `pcall' { local ++f /*================================================== Create Query ==================================================*/ povcalnet_query, country("`country'") /// region("`region'") /// year("`year'") /// povline("`i_povline'") /// popshare("`i_popshare'") /// ppp("`ppp'") /// coverage(`coverage') /// server(`server') /// `clear' /// `information' /// `iso' /// `fillgaps' /// `aggregate' /// `wb' /// `pause' /// `groupedby' // local query_ys = "`r(query_ys)'" local query_ct = "`r(query_ct)'" local query_pl = "`r(query_pl)'" local query_ds = "`r(query_ds)'" local query_pp = "`r(query_pp)'" local query_ps = "`r(query_ps)'" return local query_ys_`f' = "`query_ys'" return local query_ct_`f' = "`query_ct'" return local query_pl_`f' = "`query_pl'" return local query_ds_`f' = "`query_ds'" return local query_pp_`f' = "`query_pp'" return local query_ps_`f' = "`query_ps'" return local base = "`base'" *---------- Query if ("`popshare'" == ""){ local query = "`query_ys'&`query_ct'&`query_pl'`query_pp'`query_ds'&format=csv" } else{ local query = "`query_ys'&`query_ct'&`query_ps'`query_pp'`query_ds'&format=csv" } return local query_`f' "`query'" global pcn_query = "`query'" *---------- Base + query local queryfull "`base'?`query'" return local queryfull_`f' = "`queryfull'" /*================================================== Download and clean data ==================================================*/ *---------- download data local rc = 0 local qr = 1 // query round while (`qr' <= `querytimes') { tempfile clfile cap copy "`queryfull'" `clfile' if (_rc == 0) { cap insheet using `clfile', clear name if (_rc != 0) local rc "in" continue, break } else { local rc "copy" local ++qr } } * global qr = `qr' if ("`aggregate'" == "" & "`wb'" == "") { local rtype 1 } else { local rtype 2 } pause after download *---------- Clean data povcalnet_clean `rtype', year("`year'") `iso' /* */ rc(`rc') region(`region') `pause' `wb' pause after cleaning /*================================================== Display Query ==================================================*/ if ("`dipsquery'" == "" & "`rc'" == "0") { noi di as res _n "{ul: Query at \$`i_povline' poverty line}" noi di as res "{hline}" if ("`query_ys'" != "") noi di as res "Year:" as txt "{p 4 6 2} `query_ys' {p_end}" if ("`query_ct'" != "") noi di as res "Country:" as txt "{p 4 6 2} `query_ct' {p_end}" if ("`query_pl'" != "") noi di as res "Poverty line:" as txt "{p 4 6 2} `query_pl' {p_end}" if ("`query_ps'" != "") noi di as res "Population share:" as txt "{p 4 6 2} `query_ps' {p_end}" if ("`query_ds'" != "") noi di as res "Aggregation:" as txt "{p 4 6 2} `query_ds' {p_end}" if ("`query_pp'" != "") noi di as res "PPP:" as txt "{p 4 6 2} `query_pp' {p_end}" noi di as res _dup(20) "-" noi di as res "No. Obs:" as txt _col(20) c(N) noi di as res "{hline}" } /*================================================== Append data ==================================================*/ if (`wb_change' == 1) { keep if regioncode == "WLD" } append using `povcalf' save `povcalf', replace } // end of povline loop return local npl = `f' // ------------------------------ // display results // ------------------------------ local n2disp = min(`c(N)', `n2disp') noi di as res _n "{ul: first `n2disp' observations}" if ("`subcommand'" == "wb") { sort year regioncode noi list region year povertyline headcount mean in 1/`n2disp', /* */ abbreviate(12) sepby(year) } else { if ("`aggregate'" == "") { sort countrycode year regioncode noi list countrycode year povertyline headcount mean median datatype /* */ in 1/`n2disp', abbreviate(12) sepby(countrycode) } else { sort year noi list year povertyline headcount mean , /* */ abbreviate(12) sepby(povertyline) } } //======================================================== // Create notes //======================================================== local pllabel "" foreach p of local povline { local pllabel "`pllabel' \$`p'" } local pllabel = trim("`pllabel'") local pllabel: subinstr local pllabel " " ", ", all if ("`wb'" == "") { if ("`aggregate'" == "" & "`fillgaps'" == "") { local lvlabel "country level" } else if ("`aggregate'" != "" & "`fillgaps'" == "") { local lvlabel "aggregated level" } else if ("`aggregate'" == "" & "`fillgaps'" != "") { local lvlabel "Country level (lined up)" } else { local lvlabel "" } } else { local lvlabel "regional and global level" } local datalabel "WB poverty at `lvlabel' using `pllabel'" local datalabel = substr("`datalabel'", 1, 80) label data "`datalabel' (`c(current_date)')" * citations local cite `"Please cite as: Castaneda Aguilar, R. A., C. Lakner, E. B. Prydz, J. S. Lopez, R. Wu and Q. Zhao (2019) "povcalnet: Stata module to access World Bank’s Global Poverty and Inequality data," Statistical Software Components 2019, Boston College Department of Economics."' notes: `cite' dis as text "{hline}" noi disp in red "WARNING:" noi dis as text `"{p 6 4 0 80}The povcalnet command and website have been replaced with the new pip command and the Poverty and Inequality Portal, {browse "https://pip.worldbank.org/"}. Povcalnet will remain functional until the end of 2022, but its data is now {bf:outdated}. Once retired, you will no longer be able to access these data. If you anticipate needing to reproduce existing analysis based on PovcalNet data, you will want to save a version of the data. Moving forward, we encourage you to use the new PIP site and its corresponding {browse "https://github.com/worldbank/pip": pip} Stata command, with the latest data and many new features.{p_end}"' dis as text "{hline}" _n noi disp in y _n `"`cite'"' return local cite `"`cite'"' } // end of qui end //======================================================== // Aux programs //======================================================== program define _pcn_find_src, rclass syntax anything(name=cmd id="Package name") qui { preserve drop _all // find stata.trk file findfile stata.trk local fn = "`r(fn)'" // create copy tempfile statatrk copy "`r(fn)'" "`statatrk'" // import copy into stata import delimited using `statatrk', bindquote(nobind) gen n = _n // line number // find line where the package is used levelsof n if regexm(v1, "`cmd'.pkg"), sep(,) loca(pklines) if (`"`pklines'"' == `""') local src = "NotInstalled" else { // the latest source and subtract which refers to the source local sourceline = max(0, `pklines') - 1 // get the source without the initial S if regexm(v1[`sourceline'], "S (.*)") local src = regexs(1) } // return info return local src = "`src'" } // end of qui end // ------------------------------------------------------------------------ // MATA functions // ------------------------------------------------------------------------ * findfile stata.trk * local fn = "`r(fn)'" cap mata: mata drop povcalnet_*() mata: // function to look for source of code void povcalnet_source(string scalar cmd) { cmd = cmd :+ "\.pkg" fh = _fopen("`fn'", "r") pos_a = ftell(fh) pos_b = 0 while ((line=strtrim(fget(fh)))!=J(0,0,"")) { if (regexm(strtrim(line), cmd)) { fseek(fh, pos_b, -1) break } pos_b = pos_a pos_a = ftell(fh) } src = strtrim(fget(fh)) if (rows(src) > 0) { src = substr(src, 3) st_local("src", src) } else { st_local("src", "NotFound") } fclose(fh) } end exit /* End of do-file */ ><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><>< Notes: Version Control: *! version 1.2.0 *! version 1.1.8 *! version 1.0.0