/*=======================================================
Program Name: pip.ado
Author:
R.Andres Castaneda
acastanedaa@worldbank.org

Contributor:
Tefera Bekele Degefu
tdegefu@worldbank.org

World Bank Group

project:	  Adaptation Stata package (from povcalnet) to easily query the [PIP API]
Dependencies: The World Bank - DECIS
-----------------------------------------------------------------------
References:
Output:
=======================================================*/

/*==================================================
0: Program set up
==================================================*/
program define pip, rclass
version 16.0

syntax [anything(name=subcommand)]  ///
[,                             	   ///
COUntry(string)                ///
REGion(string)                 ///
YEAR(string)                   ///
POVLine(numlist)               ///
POPShare(numlist)	   		       ///
PPP_year(numlist)              ///
AGGregate                      ///
CLEAR                          ///
INFOrmation                    ///
COVerage(string)               ///
ISO                            ///
SERver(string)                 ///
pause                          ///
FILLgaps                       ///
N2disp(integer 1)             ///
DISPQuery                      ///
querytimes(integer 5)          ///
TIMEr                          ///
POVCALNET_format               ///
noEFFICIENT                    ///
KEEPFrames                     ///
frame_prefix(string)           ///
replace                        ///
VERsion(string)                ///
IDEntity(string)               ///
RELease(numlist)               ///
TABle(string)                  ///
path(string)                   ///
]

if ("`pause'" == "pause") pause on
else                      pause off
set checksum off

	//========================================================
	// housekeeping
	//========================================================
	local curframe = c(frame)

	if regexm("`subcommand'", "^clean") {
		noi pip_cleanup
		exit
	}


	if regexm("`subcommand'", "^dropframe") {
		pip_drop frame, frame_prefix(`frame_prefix')
		exit
	}

	if regexm("`subcommand'", "^dropglobal") {
		pip_drop global
		exit
	}

	if regexm("`subcommand'", "^install") {
		local sscmd: word 2 of `subcommand'
		noi pip_install `sscmd', path(`path') `pause'
		exit
	}

	if regexm("`subcommand'", "^uninstall") {
		pip_install uninstall, path(`path') `pause'
		exit
	}

	if regexm("`subcommand'", "^update") {
		noi pip_update, path(`path') `pause'
		exit
	}

	// ------------------------------------------------------------------------
	// New session procedure
	// ------------------------------------------------------------------------

qui {

	if ("${pip_cmds_ssc}" == "") {
		pip_new_session , `pause'
	}

	//========================================================
	// setup defaults
	//========================================================

	local server     = lower("`server'")
	local identity   = upper("`identity'")
	local country    = upper("`country'")
	local coverage   = lower("`coverage'")
	local table      = lower("`table'")

	* In case global server is specified
	if ("${pip_server}" != "" & "`server'" == "") {
		noi disp in red "warning:" in y "Global {it:pip_server} (${pip_server}) is in use"
		local server = "${pip_server}"
	}


	//========================================================
	// Auxiliary tables
	//========================================================
	if regexm("`subcommand'", "^tab") {
		noi pip_tables `table', server(`server')        ///
		version(`version')                ///
		release(`release')                ///
		ppp_year(`ppp_year')              ///
		identity(`identity')              ///
		`clear'
		return add
		exit
	}


	//========================================================
	//  Timer
	//========================================================

	local i = 0
	local crlf "`=char(10)'`=char(13)'"
	scalar tt = ""

	if ("`timer'" != "") {
		timer clear
		local i = 1
	}
	//========================================================
	// Conditions (Defenses)
	//========================================================
	if ("`aggregate'" != "") {
		noi disp in red "Option {it:aggregate} is disable for now."
		exit
	}
	if ("`aggregate'" != "" & "`fillgaps'" != "") {
		noi disp in red "options {it:aggregate} and {it:fillgaps} are mutually exclusive." _n /*
		*/ "Please select only one."
		error
	}

	if ("`popshare'" != "" &  (lower("`subcommand'") == "wb" | "`aggregate'" != "")) {
		noi disp in red "option {it:popshare} can't be combined with option {it:aggregate} or with subcommand {it:wb}" _n
		error
	}

	if ("`frame_prefix'" == "") {
		local frame_prefix "pip_"
	}


	/*==================================================
	Defaults
	==================================================*/
	// --- timer
	if ("`timer'" != "") {
		local i_on = `i'
		scalar tt = tt + "`crlf' `i': Set server"
		local i_off = `i++'
	}
	// --- timer

	// --- timer
	if ("`timer'" != "") timer on `i_on'
	// --- timer

	*---------- API defaults
	pip_set_server  `server', `pause'
	*return add
	local url       = "`r(url)'"
	local server    = "`r(server)'"
	local base      = "`r(base)'"
	local base_grp  = "`r(base_grp)'"

	// --- timer
	if ("`timer'" != "") timer off `i_off'
	// --- timer

	//========================================================
	// versions
	//========================================================

	// --- timer
	if ("`timer'" != "") {
		local i_on = `i'
		scalar tt = tt + "`crlf' `i': Get version"
		local i_off = `i++'
	}
	// --- timer

	// --- timer
	if ("`timer'" != "") timer on `i_on'
	// --- timer


	if regexm("`subcommand'", "^ver") {
		noi pip_versions, server(`server') availability
		return add
		exit
	}
	noi pip_versions, server(`server') ///
	version(`version')                ///
	release(`release')               ///
	ppp_year(`ppp_year')             ///
	identity(`identity')

	local version_qr = "`r(version_qr)'"
	local version    = "`r(version)'"
	local release    = "`r(release)'"
	local ppp_year   = "`r(ppp_year)'"
	local identity   = "`r(identity)'"

	return local pip_version = "`version'"

	// --- timer
	if ("`timer'" != "") timer off `i_off'
	// --- timer


	//========================================================
	// conditions
	//========================================================
	*---------- lower case subcommand
	local subcommand = lower("`subcommand'")

	*---------- Test
	if ("`subcommand'" == "test") {
		if ("${pip_query}" == "") {
			noi di as err "global pip_query does not exist. You cannot test the query."
			error
		}
		local fq = "`base'?${pip_query}"
		noi disp in y "querying" in w _n "`fq'"
		noi view browse "`fq'"
		exit
	}

	*---------- Modify country(all) with aggregate
	if (lower("`country'") == "all" & "`aggregate'" != "") {
		local country    ""
		local aggregate  ""
		local subcommand "wb"
		local wb_change  1
		noi disp as err "Warning: " as text " {cmd:pip, country(all) aggregate} " /*
	  */	"is equivalent to {cmd:pip wb}. " _n /*
	  */  " if you want to aggregate all countries by survey years, " /*
	  */  "you need to parse the list of countries in {it:country()} option. See " /*
	  */  "{help pip##options:aggregate option description} for an example on how to do it"
	}
	else {
		local wb_change 0
	}

	if ("`year'" == "") local year "all"

	if ("`year'" != "" & "`year'" != "all") {
		local yrtemp
		foreach yr of local year {
			local tt = substr("`yr'", 1, 4)
			local yrtemp `yrtemp' `tt'
		}
		local year "`yrtemp'"
	}
	*

	*---------- Coverage
	if ("`coverage'" == "") local coverage = "all"
	local coverage = lower("`coverage'")

	foreach c of local coverage {

		if !inlist(lower("`c'"), "national", "rural", "urban", "all") {
			noi disp in red `"option {it:coverage()} must be "national", "rural",  "urban" or "all" "'
			error
		}

	}

	*---------- Poverty line/population share

	// Blank popshare and blank povline = default povline 1.9
	if ("`popshare'" == "" & "`povline'" == "")  {

		if ("`ppp_year'" == "2005") local povline = 1.25
		if ("`ppp_year'" == "2011") local povline = 1.9
		if ("`ppp_year'" == "2017") local povline = 2.15

		local pcall = "povline"
	}

	// defined popshare and defined povline = error
	else if ("`popshare'" != "" & "`povline'" != "")  {
		noi disp as err "povline and popshare cannot be used at the same time"
		error
	}

	// blank popshare and defined povline
	else if ("`popshare'" == "" & "`povline'" != "")  {
		local pcall = "povline"
	}

	// defined popshare and blank povline
	else {
		local pcall = "popshare"
	}

	*---------- Info
	if regexm("`subcommand'", "^info")	{
		local information = "information"
		local subcommand  = "information"
	}


	//------------ Region

	if ("`region'" != "") {
		local region = upper("`region'")

		if ("`country'" != "") {
			noi disp in red "You must use either {it:country()} or {it:region()}."
			error
		}

		if (regexm("`region'", "SAR")) {
			noi disp in red "Note: " in y "The official code of South Asia is" ///
			"{it: SAS}, not {it:SAR}. We'll make the change for you"
			local region: subinstr local region "SAR" "SAS", word
		}

		tokenize "`version'", parse("_")
		local _version   = "_`1'_`3'_`9'"

		frame dir
		local av_frames "`r(frames)'"
		local av_frames: subinstr local  av_frames " " "|", all
		local av_frames = "^(" + "`av_frames'" + ")"

		//------------ Regions frame
		local frpiprgn "_pip_regions`_version'"
		if (!regexm("`frpiprgn'", "`av_frames'")) {
			pip_info, clear justdata `pause' server(`server') version(`version')
		}
		frame `frpiprgn' {
			levelsof region_code, local(av_regions)  clean
		}

		// Add all to have the same functionality as in country(all)
		local av_regions = "`av_regions'" + " ALL"

		local inregion: list region in av_regions
		if (`inregion' == 0) {

			noi disp in red "region `region' is not available." _n ///
			"Only the following are available:" _n "`av_regions'"

			error
		}

	}




	*---------- WB aggregate

	if ("`subcommand'" == "wb") {
		if ("`country'" != "") {
			noi disp as err "option {it:country()} is not allowed with subcommand {it:wb}"
			noi disp as res "Note: " as txt "subcommand {it:wb} only accepts options {it:region()} and {it:year()}"
			error
		}
	}


	*---------- Country
	if ("`country'" == "" & "`region'" == "") local country "ALL" // to modify
	if ("`country'" != "") {
		if (lower("`country'") != "all") local country = upper("`country'")
		else                             local country "ALL" // to modify
	}


	/*==================================================
	Main conditions
	==================================================*/

	if ("`information'" == "") {

		if (c(N) != 0 & "`clear'" == "" & "`information'" == "") {

			noi di as err "You must start with an empty dataset; or enable the option {it:clear}."
			error 4
		}
		drop _all
	}


	if ("`aggregate'" != "") {
		noi disp as res "Note: " as text "Aggregation is only possible over reference years."
		local agg_display = "Aggregation in base year(s) `year'"
	}

	/*==================================================
	Execution
	==================================================*/
	pause pip - before execution

	*---------- Information
	// --- timer
	if ("`timer'" != "") {
		local i_on = `i'
		scalar tt = tt + "`crlf' `i': Get info"
		local i_off = `i++'
	}
	// --- timer

	// --- timer
	if ("`timer'" != "") timer on `i_on'
	// --- timer


	if ("`information'" != ""){
		noi pip_info, `clear' `pause' server(`server') version(`version')
		return add
		exit
	}

	// --- timer
	if ("`timer'" != "") timer off `i_off'
	// --- timer

	*---------- Regular query and Aggregate Query
	if ("`subcommand'" == "wb") {
		local wb "wb"
	}
	else local wb ""


	tempfile povcalf
	save `povcalf', empty

	local f = 0

	if ("`pcall'" == "povline") 	loc i_call "i_povline"
	else 							loc i_call "i_popshare"


	// --- timer
	if ("`timer'" != "") {
		local j = `i++'
		local k = `i++'
		local h = `i++'
		scalar tt = tt + "`crlf' `j': bulding query"
		scalar tt = tt + "`crlf' `k': downloading data"
		scalar tt = tt + "`crlf' `h': cleaning data"
	}
	// --- timer

	foreach `i_call' of local `pcall' {

		// --- timer
		if ("`timer'" != "") timer on `j'
		// --- timer

		local ++f

		/*==================================================
		Create Query
		==================================================*/
		pip_query,   country("`country'")       ///
		region("`region'")                      ///
		year("`year'")                          ///
		povline("`i_povline'")                  ///
		popshare("`i_popshare'")	   					  ///
		ppp("`ppp_year'")                            ///
		coverage(`coverage')                    ///
		server(`server')                        ///
		version(`version')                      ///
		`clear'                                 ///
		`information'                           ///
		`iso'                                   ///
		`fillgaps'                              ///
		`aggregate'                             ///
		`wb'                                    ///
		`pause'                                 ///
		`groupedby'                             //

		local query_ys = "`r(query_ys)'"
		local query_ct = "`r(query_ct)'"
		local query_pl = "`r(query_pl)'"
		local query_ds = "`r(query_ds)'"
		local query_pp = "`r(query_pp)'"
		local query_ps = "`r(query_ps)'"

		local query_cv = "`r(query_cv)'"

		return local query_ys_`f' = "`query_ys'"
		return local query_ct_`f' = "`query_ct'"
		return local query_pl_`f' = "`query_pl'"
		return local query_ds_`f' = "`query_ds'"
		return local query_pp_`f' = "`query_pp'"
		return local query_ps_`f' = "`query_ps'"

		return local query_cv_`f' = "`query_cv'"

		return local base      = "`base'"

		*---------- Query
		if ("`popshare'" == ""){
			local query = "`query_ys'&`query_ct'&`query_cv'&`query_pl'`query_pp'`query_ds'&`version_qr'"
		}
		else{
			local query = "`query_ys'&`query_ct'&`query_cv'&`query_ps'`query_pp'`query_ds'&`version_qr'"
		}
		return local query_`f' "`query'"
		global pip_query = "`query'&format=csv"

		*---------- Base + query
		if ("`aggregate'" != "" | "`subcommand'" == "wb"){
			local queryfull "`base_grp'?`query'"
		}
		else{
			local queryfull "`base'?`query'"
		}

		return local queryfull_`f' = "`queryfull'"

		// --- timer
		if ("`timer'" != "") timer off `j'
		// --- timer

		/*==================================================
		Download  and clean data
		==================================================*/


		// --- timer
		if ("`timer'" != "") timer on `k'
		// --- timer

		*---------- download data
		cap import delimited  "`queryfull'&format=csv", `clear' varn(1) asdouble
		if (_rc) {
			noi dis ""
			noi dis in red "It was not possible to download data from the PIP API."
			noi dis ""
			noi dis in white `"(1) Please check your Internet connection by "' _c
			noi dis in white  `"{browse "`url'/health-check" :clicking here}"'
			noi dis in white `"(2) Test that the data is retrievable. By"' _c
		  noi dis in white  `"{stata pip test, server(`server'): clicking here }"' _c
			noi dis in white  "you should be able to download the data."
			noi dis in white `"(3) Please consider adjusting your Stata timeout parameters. For more details see {help netio}"'
			noi dis in white `"(4) Please send us an email to:"'
			noi dis in white _col(8) `"email: pip@worldbank.org"'
			noi dis in white _col(8) `"subject: pip query error on `c(current_date)' `c(current_time)'"'
			noi di ""
			error 673
		}
		* noi disp "`queryfull'&format=csv"
		* exit


		// --- timer
		if ("`timer'" != "") timer off `k'
		// --- timer

		* global qr = `qr'

		if ("`aggregate'" == "" & "`wb'" == "") {
			local rtype 1
		}
		else {
			local rtype 2
		}

		pause after download

		// --- timer
		if ("`timer'" != "") timer on `h'
		// --- timer

		*---------- Clean data
		noi pip_clean `rtype', year("`year'") `iso' server(`server') /*
		*/ region(`region') `pause' `fillgaps' version(`version')

		pause after cleaning
		// --- timer
		if ("`timer'" != "") timer off `h'
		// --- timer

		/*==================================================
		Display Query
		==================================================*/

		if ("`dispquery'" != "") {
			noi di as res _n "{ul: Query at \$`i_povline' poverty line}"
			noi di as res "{hline}"


			if ("`query_ys'" != "") {
				noi di as res "Year:" as txt "{p 4 6 2} `query_ys' {p_end}"
			}

			if ("`query_ct'" != "") {
				noi di as res "Country:" as txt "{p 4 6 2} `query_ct' {p_end}"
			}

			if ("`query_pl'" != "") {
				noi di as res "Poverty line:" as txt "{p 4 6 2} `query_pl' {p_end}"
			}

			if ("`query_ps'" != "") {
				noi di as res "Population share:" as txt "{p 4 6 2} `query_ps' {p_end}"
			}

			if ("`query_ds'" != "") {
				noi di as res "Aggregation:" as txt "{p 4 6 2} `query_ds' {p_end}"
			}

			if ("`query_pp'" != "") {
				noi di as res "PPP:" as txt "{p 4 6 2} `query_pp' {p_end}"
			}

			if ("`'&`version_qr''" != "") {
				noi di as res "Version:" as txt "{p 4 6 2} `version_qr' {p_end}"
			}

			noi di as res "full query:" as txt "{p 4 6 2} `queryfull' {p_end}" _n
			noi di as res "See in browser: "  `"{browse "`queryfull'":here}"'  _n
			noi di as res "Download .csv: "  `"{browse "`queryfull'&format=csv":here}"'

			noi di as res _dup(20) "-"
			noi di as res "No. Obs:"      as txt _col(20) c(N)
			noi di as res "{hline}"
		}

		/*==================================================
		Append data
		==================================================*/
		if (`wb_change' == 1) {
			keep if regioncode == "WLD"
		}
		append using `povcalf'
		save `povcalf', replace

	} // end of povline loop

	return local npl = `f'

	// ------------------------------
	//  display results
	// ------------------------------

	local n2disp = min(`c(N)', `n2disp')

	if (`n2disp' > 1) {
		noi di as res _n "{ul: first `n2disp' observations}"
	}
	else	if (`n2disp' == 1) {
		noi di as res _n "{ul: first observation}"
	}
	else {
		noi di as res _n "{ul: No observations available}"
	}


	if ("`subcommand'" == "wb") {
		sort region_code year

		tempname tolist
		frame copy `c(frame)' `tolist'
		frame `tolist' {
			gsort region_code -year

			count if (region_code == "WLD")
			local cwld = r(N)
			if (`cwld' >= `n2disp') {
				keep if (region_code == "WLD")
			}
			noi list region_code year poverty_line headcount mean ///
			in 1/`n2disp',  abbreviate(12) noobs
		}

	}

	else {
		if ("`aggregate'" == "") {
			sort country_code year
			local varstodisp "country_code year poverty_line headcount mean median welfare_type"
			local sepby "country_code"
		}
		else {
			sort year
			local varstodisp "year poverty_line headcount mean"
			local sepby "poverty_line"
		}

		foreach v of local varstodisp {
			cap confirm var `v', exact
			if _rc continue
			local v2d "`v2d' `v'"
		}

		noi list `v2d' in 1/`n2disp',  abbreviate(12)  sepby(`sepby') noobs

	}



	//========================================================
	//  Create notes
	//========================================================

	local pllabel ""
	foreach p of local povline {
		local pllabel "`pllabel' \$`p'"
	}
	local pllabel = trim("`pllabel'")
	local pllabel: subinstr local pllabel " " ", ", all


	if ("`wb'" == "")   {
		if ("`aggregate'" == "" & "`fillgaps'" == "") {
			local lvlabel "country level"
		}
		else if ("`aggregate'" != "" & "`fillgaps'" == "") {
			local lvlabel "aggregated level"
		}
		else if ("`aggregate'" == "" & "`fillgaps'" != "") {
			local lvlabel "Country level (lined up)"
		}
		else {
			local lvlabel ""
		}
	}
	else {
		local lvlabel "regional and global level"
	}


	local datalabel "WB poverty at `lvlabel' using `pllabel'"
	local datalabel = substr("`datalabel'", 1, 80)

	label data "`datalabel' (`c(current_date)')"

	//========================================================
	// Final messages
	//========================================================

	* citations
	if ("${pip_cmds_ssc}" == "1") {
		local cnoi "noi"
		global pip_cmds_ssc = ${pip_cmds_ssc} + 1
	}
	else {
		local cnoi "qui"
		noi disp `"Click {stata "pip_cite, reg_cite":here} to display how to cite"'
	}
	`cnoi' pip_cite, reg_cite
	notes: `r(cite_data)'

	noi disp in y _n `"`cite'"'

	return local cite `"`cite'"'

	* Install alternative version
	if ("${pip_cmds_ssc}" == "") {
		noi pip_${pip_source} msg
	}

	//========================================================
	// Convert to povcalnet format
	//========================================================

	if ("`timer'" != "") {
		local i_on = `i'
		scalar tt = tt + "`crlf' `i': formating to povcalnet"
		local i_off = `i++'
	}
	// --- timer

	// --- timer
	if ("`timer'" != "") timer on `i_on'
	// --- timer

	if ("`povcalnet_format'" != "") {
		pause before povcalnet format
		pip_povcalnet_format  `rtype', `pause'
	}

	// --- timer
	if ("`timer'" != "") timer off `i_off'
	// --- timer

	//========================================================
	//  Drop frames created in the middle of the process
	//========================================================

	if ("`timer'" != "") {
		local i_on = `i'
		scalar tt = tt + "`crlf' `i': remove frames"
		local i_off = `i++'
	}
	// --- timer

	// --- timer
	if ("`timer'" != "") timer on `i_on'
	// --- timer

	frame dir
	local av_frames "`r(frames)'"

	* set trace on
	foreach fr of local av_frames {

		if (regexm("`fr'", "(^_pip_)(.+)")) {

			// If users wants to keep frames
			if ("`keepframes'" != "") {
				local frname = "`frame_prefix'" + regexs(2)
				frame copy `fr' `frname', `replace'
			}
			// if user wants to drop them
			if ("`efficient'" == "noefficient") {
				frame drop `fr'
			}
		}

	} // condition to keep frames

	// --- timer
	if ("`timer'" != "") timer off `i_on'
	// --- timer


	* set trace off



	// --- timer
	if ("`timer'" != "") {
		noi disp tt
		noi timer list
	}
	// --- timer

} // end of qui
end



exit
/* End of do-file */

><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><><

Notes:

Version Control:

*! version 0.9.5                 <2023Feb14>
*! -- fix writing error in pip.pkg file that did not allow the installation of pip_update
*! version 0.9.2                 <2023Feb14>
*! -- improve installation and update features
*! version 0.9.0                  <2023Feb09>
*! -- Update help file
*! version 0.3.9                  <2022Dec16>
*! -- BREAKING Change: Fix formating of aux tables. Rename some variables to make it consistent with other PIP outputs
*! -- Drop obs with missing values in poverty line or headcount
*! -- Fix display of citations
*! -- Improve Help file
*! -- fix bug with PPP_year  and ppp parameters
*! -- Display only one observation
*! -- Fix big with options ppp and ppp_year. Only ppp_year remained.
*! -- Change order of returning variables.
*! -- change all labels to lower cases
*! -- BREAKING Change: remove distribution estimates from line up estimates.
*! version 0.3.8             <2022Oct06>
*! -- Testing version change
*! -- Fix bugs
*! version 0.3.7        <2022Oct06>
*! -- Add new routines to install and update pip
*! -- Fix bug in `pip wb, region(WLD)`, which used to return all regions, rather than just WLD.
*! -- Labels for variables `icp` and `ppp` now depend on the PPP year of the data.
*! version 0.3.6        <2022Sep08>
*! -- make it work with new API specifications
*! -- Fix problem with variable name version
*! -- Fix problem with variable name version
*! version 0.3.5        <2022Jul06>
*! -- Add `asdouble` in all calls of `import delimited`
*! version 0.3.4        <2022Jun10>
*! version 0.3.3        <2022may25>
*! version 0.3.2        <2022apr26>
*! version 0.3.1        <2022apr08>
*! version 0.3.0        <2022apr07>
*! version 0.2.2        <2022apr06>
*! version 0.2.1        <2022apr04>
*! version 0.2.0        <2022apr01>
*! version 0.1.7        <2022mar30>
*! version 0.1.6        <2022mar28>
*! version 0.1.5        <2022mar25>
*! version 0.1.4        <2022mar18>
*! version 0.1.3        <2022mar18>
*! version 0.1.2        <2022feb07>
*! version 0.1.1        <2022feb01>
*! version 0.1.0        <2022feb01>
*! version 0.0.2        <2022jan12>
*! version 0.0.1        <2021dec01>


*##s