******************************************************************************* * _update_indicators *! v 16.1 12Apr2020 by João Pedro Azevedo * increase documentation * change the creation of the medata files for SOURCEID and TOPICID ******************************************************************************* program define _update_indicators, rclass version 9 syntax /// [ , /// FILE(string) /// NOINDLIST /// NOSTHLP1 /// NOSTHLP2 /// CHECK /// QUERY /// NOIsily /// NAME(string) /// SAVE /// REPLACE /// ] ******************************************************************************* tempfile tmp set checksum off ******************************************************************************* * download and created metadata files in case these are not available ******************************************************************************* if ("`file'" == "") { tempfile file1 file2 _api_read_indicators, update preserveout file1(`file1') file2(`file2') `check' `query' local file "`r(file2)'" } if ("`file'" != "") { local file2 "`file'" } ******************************************************************************* * Open files and prepare database to auto-update the documentation ******************************************************************************* quietly { insheet using `file2', delimiter("#") clear name * noi di "save tmptmp" * save tmptmp, replace * drop cases in which SOURCEID does not stata with a numeric CODE drop if real(substr(sourceid,1,1))==. & !missing(sourceid) gen seq = _n replace indicatorcode = indicatorcode[_n-1] if indicatorcode == "" & indicatorcode[_n-1] != "" foreach var in indicatorname sourceid sourceorganization sourcenote topicid { gsort indicatorcode -`var' bysort indicatorcode: replace `var' = `var'[1] if `var'[1] != "" & `var'== "" } bysort indicatorcode indicatorname sourceid sourceorganization sourcenote topicid : gen dups = _n keep if dups == 1 drop dups drop seq bysort indicatorcode indicatorname sourceid sourceorganization sourcenote : gen tot = _N foreach var in topicid sourceid { replace `var' = subinstr(`var',"&","and",.) replace `var' = subinstr(`var',">"," ",.) replace `var' = "0"+`var' if real(substr(`var',1,2))<=9 & real(substr(`var',1,1)) !=. } * Indentify multiple entries for the same indicator bysort indicatorcode : gen seq = _n order indicatorcode indicatorname sort sourceid indicatorcode label var sourceid "Source" label var topicid "Topics" compress * save files with all INDICATORS + SOURCEID + TOPICID save `tmp', replace ******************************************************************************** * Create Indicator list for dialogue box * if noindlist == to missing this section of the code is skipped * this is a txt file ******************************************************************************** if ("`noindlist'" == "") { noi di in smcl in g "{bf: Processing indicators list...}" local indicator indicators.txt tempfile tmp1tmp use `tmp' , clear keep indicatorcode indicatorname bysort indicatorcode : gen seq = _n keep if seq == 1 drop seq sort indicatorcode gen export = indicatorcode + " - " + indicatorname keep export sort export outsheet using `tmp1tmp', replace noquote nolabel nonames cap: findfile `indicator' , `path' if _rc == 0 { copy `tmp1tmp' `r(fn)' , replace } else { copy `tmp1tmp' `indicator' } *noi gen length = length(sourcenote) *noi sum noi di in smcl in g "{bf: Processing indicators list...COMPLETED!}" } ******************************************************************************* * create sthlp files (sourceid and topicid) * if NOSTHLP1 == to missing this part of the code is skipped * creates a single file for sourcid with all indicators by sourceid * creates a single file for topicid with all indicators by topicid ******************************************************************************* if ("`nosthlp1'" == "") { noi di in smcl in g "" noi di in smcl in g "{bf: Processing indicators by source and topic...}" noi di in smcl in g "" local date: disp %td date("`c(current_date)'", "DMY") qui foreach variable in sourceid topicid { tempfile help`variable' tempname hlp`variable' dups`variable' seq2`variable' seq3`variable' code`variable' use `tmp', clear if ("`save'" != "") { if ("`name'" == "") { local name "indicators" } save `name'.dta, `replace' noi di in y "`name'.dta saved `replace'" noi di "" } keep if `variable' != "" sort `variable' indicatorcode bysort `variable' indicatorcode : gen `dups`variable'' = _n keep if `dups`variable'' == 1 sort `dups`variable'' `variable' indicatorcode topicid bysort `dups`variable'' `variable' indicatorcode : gen `seq2`variable'' = _n keep if `seq2`variable'' == 1 sort `variable' `countryname' gen `seq3`variable'' = _n gen `code`variable'' = trim(word(`variable',1)) local title : variable label `variable' local title = subinstr("`title'","Code","",.) **************** header ******************** file open `hlp`variable'' using `help`variable'' , write text replace file write `hlp`variable'' "{smcl}" _n file write `hlp`variable'`tc0'' "{right:(as of `date')}" _n file write `hlp`variable'' "" _n file write `hlp`variable'' "{marker indicators}{...}" _n file write `hlp`variable'' "{p 40 20 2}(Go up to {it:{help wbopendata##sections:Sections Menu}}){p_end}" _n file write `hlp`variable'' "{title:`title'}" _n file write `hlp`variable'' "" _n ******************************************** levelsof `variable' file write `hlp`variable'' "{marker toc}" _n file write `hlp`variable'' "{p 40 20 2}(Go up to {it:{help wbopendata##`variable':`title'}}){p_end}" _n file write `hlp`variable'' "{synoptset 40 tabbed}{...}" _n file write `hlp`variable'' "{synopthdr:`title' Code}" _n file write `hlp`variable'' "{synoptline}" _n foreach topic in `r(levels)' { local var1code = trim(word("`topic'",1)) local var1name = trim(subinstr("`topic'","`var1code'","",.)) file write `hlp`variable'' "{synopt:{opt `var1code'}} {help wbopendata_`variable'##`variable'_`var1code':`var1name'}{p_end}" _n } file write `hlp`variable'' "{synoptline}" _n file write `hlp`variable'' "" _n file write `hlp`variable'' "" _n levelsof `variable' foreach topic in `r(levels)' { sum `seq3`variable'' if `variable' == "`topic'" local min = r(min) local max = r(max) local topicode0 = `code`variable'' in `min' local topicode1 = `variable' in `min' local topicode2 "`variable'_`topicode0'" file write `hlp`variable'' "{marker `topicode2'}" _n file write `hlp`variable'' "{p 40 20 2}(Go up to {it:{help wbopendata_`variable'##`variable'_`topicode0':`title'}} or {it:{help wbopendata_`variable'_indicators`topicode0'##`toc':TOC}}){p_end}" _n file write `hlp`variable'' "{synoptset 40 tabbed}{...}" _n file write `hlp`variable'' "{synopthdr:`topicode1'}" _n file write `hlp`variable'' "{synoptline}" _n forvalues line = `min'(1)`max' { local indicatorname = indicatorname in `line' local indicatorcode = indicatorcode in `line' file write `hlp`variable'' "" _n file write `hlp`variable'' "{synopt:{help wbopendata_`variable'_indicators`topicode0'##`variable'_`indicatorcode':`indicatorcode'{marker `indicatorcode'}}}`indicatorname'{p_end}" _n } file write `hlp`variable'' "{synoptline}" _n file write `hlp`variable'' `""' _n } file write `hlp`variable'`tc0'' "{right:(as of `date')}" _n file close `hlp`variable'' cap: findfile wbopendata_`variable'.sthlp , `path' if _rc == 0 { copy `help`variable'' `r(fn)' , replace } else { copy `help`variable'' wbopendata_`variable'.sthlp } noi di in g in smcl " See {bf:{help wbopendata_`variable'##`variable':`title'}}" } noi di in smcl in g "" noi di in smcl in g "{bf: Processing indicators by source and topic...COMPLETED!}" noi di in smcl in g "" } ******************************************************************************* * create sthlp files (sourceid_indicators and topicid_indicators) * if NOSTHLP2 == to missing this part of the code is skipped * multiple files for sourcid * multiple fiels for topicid ******************************************************************************* if ("`nosthlp2'" == "") { noi di in smcl in g "" noi di in smcl in g "{bf: Processing indicators metadata by source and topic...}" noi di in smcl in g "" local date: disp %td date("`c(current_date)'", "DMY") * loop through sourcid and topicid qui foreach variable in sourceid topicid { * created tempfiles tempfile help`variable' tmp2`variable' * create temp variable names tempname hlp`variable' dups`variable' seq2`variable' seq3`variable' code`variable' tot`variable' use `tmp', clear keep if `variable' != "" sort `type' indicatorcode topicid bysort `variable' indicatorcode : gen `dups`variable'' = _n keep if `dups`variable'' == 1 sort `dups`variable'' `variable' indicatorcode bysort `dups`variable'' `variable' indicatorcode : gen `seq2`variable'' = _n keep if `seq2`variable'' == 1 sort `variable' indicatorcode bysort `variable' : gen `seq3`variable'' = _n bysort `variable' : gen `tot`variable'' = _N gen `code`variable'' = trim(word(`variable',1)) local title : variable label `variable' local title = subinstr("`title'","Code","",.) /**************** header ********************/ levelsof `variable' local levelsof2 `"`r(levels)'"' `noi' di `"`levelsof2'"' compress save `tmp2`variable'' , replace * loop toics l qui foreach topic1 in `levelsof2' { use `tmp2`variable'', clear keep if `variable' == "`topic1'" `noi' sum `seq3`variable'' if `variable' == "`topic1'" local min = r(min) local max = r(max) local tc0 = `code`variable'' in `min' `noi' di " `variable' : `topic1' : `tc0' " tempname hlp`variable'`tc0' tempfile help`variable'`tc0' file open `hlp`variable'`tc0'' using `help`variable'`tc0'' , write text replace ******************************************** *** Header of Help file file write `hlp`variable'`tc0'' "{smcl}" _n file write `hlp`variable'`tc0'' "{right:(as of `date')}" _n file write `hlp`variable'`tc0'' "" _n file write `hlp`variable'`tc0'' "{marker indicators}{...}" _n file write `hlp`variable'`tc0'' "{p 40 20 2}(Go up to {it:{help wbopendata##sections:Sections Menu}}){p_end}" _n file write `hlp`variable'`tc0'' "{title:`title'}" _n file write `hlp`variable'`tc0'' "" _n ******************************************** *** Help file ToC file write `hlp`variable'`tc0'' "{marker toc}" _n file write `hlp`variable'`tc0'' "{p 40 20 2}(Go up to {it:{help wbopendata##`variable':`title'}}){p_end}" _n file write `hlp`variable'`tc0'' "{synoptset 25 tabbed}{...}" _n file write `hlp`variable'`tc0'' "{synopthdr:`title' Code}" _n file write `hlp`variable'`tc0'' "{synoptline}" _n ******************************************** /* begin TOC */ foreach topic2 in `levelsof2' { local var1code = trim(word("`topic2'",1)) local var1name = trim(subinstr("`topic2'","`var1code'","",.)) file write `hlp`variable'`tc0'' `"{synopt:{opt `var1code'}} {help wbopendata_`variable'_indicators`var1code'##`variable'_`var1code':`var1name'}{p_end}"' _n } /* end TOC */ ******************************************** file write `hlp`variable'`tc0'' "" _n file write `hlp`variable'`tc0'' "" _n ******************************************** *** begin Footer of ToC sum `seq3`variable'' if `variable' == "`topic1'" local min = r(min) local max = r(max) local topicode0 = `code`variable'' in `min' local topicode1 = `variable' in `min' local topicode2 "`variable'_`topicode0'" file write `hlp`variable'`tc0'' "{marker `topicode2'}" _n file write `hlp`variable'`tc0'' "{p 40 20 2}(Go up to {it:{help wbopendata##`variable':`title'}} or {it:{help wbopendata_`variable'_indicators`tc0'##`toc':TOC}}){p_end}" _n file write `hlp`variable'`tc0'' "" _n file write `hlp`variable'`tc0'' "{synoptset 25 tabbed}{...}" _n file write `hlp`variable'`tc0'' "{syntab:{title:{bf:`topicode1'}}}" _n *file write `hlp`variable'`tc0'' "{synoptline}" _n *** end Footer of ToC ******************************************** ******************************************** *** begin Indicator loop levelsof indicatorcode if `variable' == "`topic1'" foreach indicator in `r(levels)' { `noi' di "`variable' : `topic1' : `indicator'" *** Indicator metadata local indicatorcode "`indicator'" levelsof indicatorname if indicatorcode == "`indicator'" local indicatorname `r(levels)' levelsof sourceid if indicatorcode == "`indicator'" local sourceid `r(levels)' levelsof sourceorganization if indicatorcode == "`indicator'" local sourceorganization "`r(levels)'" levelsof sourcenote if indicatorcode == "`indicator'" local sourcenote "`r(levels)'" *** Adjust websites cap: _website, text(`sourceorganization') if _rc == 0 { local sourceorganization = r(text) } cap: _website, text(`sourcenote') if _rc == 0 { local sourcenote = r(text) } *** Header of indicator metadata file write `hlp`variable'`tc0'' "{synoptline}" _n file write `hlp`variable'`tc0'' `"{marker `variable'_`indicatorcode'}"' _n file write `hlp`variable'`tc0'' `"{synopt:{bf:{help wbopendata_`variable'##`indicatorcode':`indicatorcode'} - `indicatorname'}}"' _n file write `hlp`variable'`tc0'' "" _n file write `hlp`variable'`tc0'' `"{synopt:{opt Source}}`sourceid'{p_end}"' _n file write `hlp`variable'`tc0'' "" _n *** Loop to add multiple topics in single indicator documentation local ccc = 1 levelsof topicid if indicatorcode == "`indicator'" foreach topic4 in `r(levels)' { if (`ccc' == 1) { file write `hlp`variable'`tc0'' `"{synopt:{opt Topics}}`topic4'{p_end}"' _n local ccc = `ccc'+1 } else { file write `hlp`variable'`tc0'' `"{synopt: }`topic4'{p_end}"' _n } } local ccc = 0 *** Source Notes file write `hlp`variable'`tc0'' "" _n file write `hlp`variable'`tc0'' `"{synopt:{opt Source Notes}}`sourcenote'{p_end}"' _n file write `hlp`variable'`tc0'' "" _n *** Source Organization file write `hlp`variable'`tc0'' `"{synopt:{opt Source Organization}}`sourceorganization'{p_end}"' _n file write `hlp`variable'`tc0'' "" _n file write `hlp`variable'`tc0'' "" _n } *** end Indicator loop ******************************************** file write `hlp`variable'`tc0'' `""' _n file write `hlp`variable'`tc0'' "{right:(as of `date')}" _n *** generation date ******************************************** file close `hlp`variable'`tc0'' *** close file ******************************************** ******************************************** *** move file cap: findfile wbopendata_`variable'_indicators`tc0'.sthlp , `path' if _rc == 0 { copy `help`variable'`tc0'' `r(fn)' , replace } else { copy `help`variable'`tc0'' wbopendata_`variable'_indicators`tc0'.sthlp } ******************************************** *** display file noi di in g in smcl " See {bf:{help wbopendata_`variable'_indicators`tc0'##`toc':`title' `tc0'}}" } } noi di in smcl in g "" noi di in smcl in g "{bf: Processing indicators metadata by source and topic...COMPLETED!}" noi di in smcl in g "" } ******************************************** /* Create return locals */ ******************************************** use `tmp' , clear sum tot if seq == 1 return local total = r(N) * Loop to create SOURCEID locals preserve keep sourceid indicatorcode drop if sourceid == "" bysort sourceid indicatorcode : gen dups = _n keep if dups == 1 tab sourceid, m levelsof sourceid return local sourceid `"`r(levels)'"' foreach varvalue in `r(levels)' { di `"`varvalue'"' sum if sourceid == "`varvalue'" & sourceid != "" local obs = r(N) local code = word("`varvalue'",1) local name = lower("`varvalue'") local name = subinstr("`name'"," ","_",.) return local sourceid`code' `obs' local sourcereturn "`sourcereturn' sourceid`code'" } return local sourcereturn = "`sourcereturn'" restore * Loop to create TOPICID locals preserve keep topicid indicatorcode drop if topicid == "" bysort topicid indicatorcode : gen dups = _n keep if dups == 1 tab topicid, m levelsof topicid return local topicid `"`r(levels)'"' foreach varvalue in `r(levels)' { di `"`varvalue'"' sum if topicid == "`varvalue'" & topicid != "" local obs = r(N) local code = word("`varvalue'",1) local name = lower("`varvalue'") local name = subinstr("`name'"," ","_",.) return local topicid`code' `obs' local topicreturn "`topicreturn' topicid`code'" } return local topicreturn = "`topicreturn'" restore ******************************************************************************* ******************************************************************************* *restore local ctrytime = c(current_time) local ctrydatef = c(current_date) local dt_ctryupdate "`ctrydatef' `ctrytime'" return local dt_ctryupdate = "`dt_ctryupdate'" return local dt_ctrylastcheck = "`dt_ctryupdate'" *return local ctrymeta = `ctrymeta' } end ******************************************************************************* * v 16.0 27Oct2019 by João Pedro Azevedo * fix macros ******************************************************************************* * v 15.1 10Mar2019 by João Pedro Azevedo *******************************************************************************