*! webapi v0.1.0 2026-07-04 *! Fetch an authenticated JSON web API into Stata as a dataset. *! *! A small, dependency-free generalisation of the pattern documented in *! Booth (2026, googlesheets/googlechart): a request to a URL, an *! authentication step, and a JSON reply parsed back into variables and *! observations. webapi handles the HTTP + auth + JSON-to-table steps with *! the Python standard library, so no pip packages or virtualenv are needed. *! *! webapi get using "" [, options] *! webapi post using "" , body("") [, options] *! *! Options: *! records(path) dot-path to the JSON array of records (default: the *! whole reply). e.g. records("data") or records("results.items") *! params(k=v|k=v) pipe-delimited query-string parameters *! headers(H:V|H:V) pipe-delimited request headers *! bearer(token) send Authorization: Bearer *! basicauth(user:pw) send HTTP basic authentication *! apikeyheader(H:V) send an API key as a header *! apikeyparam(k=v) send an API key as a query parameter *! body(text) request body (POST); JSON content-type assumed *! timeout(#) seconds before giving up (default 30) *! tsv(path) keep the intermediate TSV at this path *! stringall import every column as a string *! clear replace data in memory (like import delimited) *! verbose echo the underlying python invocation program define webapi, rclass version 16.0 gettoken sub 0 : 0 local sub = lower(strtrim("`sub'")) if "`sub'" == "help" { help webapi exit 0 } if !inlist("`sub'", "get", "post") { display as error "webapi: first word must be {bf:get}, {bf:post}, or {bf:help}" display as error `" Syntax: {bf:webapi get using "URL", records(...) [options]}"' exit 198 } syntax using/, [ HEADers(string) PARAms(string) /// BEARER(string) BASICauth(string) /// APIKEYHeader(string) APIKEYParam(string) /// RECords(string) BODY(string) TIMEout(integer 30) /// EVERY(real 0) TIMES(integer 0) /// TSV(string) STRINGall CLEAR Verbose ] local url `"`using'"' local method = upper("`sub'") * --- discover the helper ------------------------------------------- capture findfile webapi_helper.py if _rc { display as error "webapi: webapi_helper.py not on the adopath. Reinstall the package." exit 601 } local helper "`r(fn)'" * --- output paths -------------------------------------------------- if `"`tsv'"' == "" { tempfile datatsv local outdata `"`datatsv'"' } else { local outdata `"`tsv'"' } tempfile status argjson * --- assemble the request as a tab-separated key/value file -------- * A flat keyvalue file (rather than JSON built in Stata) means URLs, * tokens, and parameter values need no escaping on the Stata side. tempname h quietly file open `h' using `"`argjson'"', write text replace file write `h' "method" _tab "`method'" _n file write `h' "url" _tab `"`url'"' _n file write `h' "records" _tab `"`records'"' _n file write `h' "timeout" _tab "`timeout'" _n if `"`headers'"' != "" file write `h' "headers_raw" _tab `"`headers'"' _n if `"`params'"' != "" file write `h' "params_raw" _tab `"`params'"' _n if `"`bearer'"' != "" file write `h' "bearer" _tab `"`bearer'"' _n if `"`basicauth'"' != "" file write `h' "basic" _tab `"`basicauth'"' _n if `"`apikeyheader'"' != "" file write `h' "apikey_header" _tab `"`apikeyheader'"' _n if `"`apikeyparam'"' != "" file write `h' "apikey_param" _tab `"`apikeyparam'"' _n if `"`body'"' != "" file write `h' "body" _tab `"`body'"' _n file write `h' "out_tsv" _tab `"`outdata'"' _n file close `h' * --- python + polling settings ------------------------------------- if lower("`c(os)'") == "windows" local PY "python" else local PY "python3" if "`verbose'" != "" { display as text `"[webapi] `PY' "`helper'" "`argjson'" "`status'""' } if `every' < 0 local every 0 * every() without times() polls a bounded number of times, so a call * always returns; times() alone (no wait) just repeats back-to-back. if `every' > 0 & `times' <= 0 local times 12 if `times' <= 0 local times 1 local polling = (`times' > 1) if "`stringall'" != "" local scols "stringcols(_all)" tempfile master forvalues _p = 1/`times' { quietly shell `PY' "`helper'" "`argjson'" "`status'" capture confirm file `"`status'"' if _rc { display as error "webapi: the helper produced no output. Is python3 on your PATH?" exit 198 } * read the key=value status local st "" local err "" local msg "" local nrows 0 local ncols 0 local http "" tempname sh file open `sh' using `"`status'"', read text file read `sh' line while r(eof) == 0 { local eq = strpos(`"`line'"', "=") if `eq' > 0 { local k = substr(`"`line'"', 1, `eq'-1) local v = substr(`"`line'"', `eq'+1, .) if "`k'" == "status" local st `"`v'"' else if "`k'" == "error" local err `"`v'"' else if "`k'" == "message" local msg `"`v'"' else if "`k'" == "nrows" local nrows `"`v'"' else if "`k'" == "ncols" local ncols `"`v'"' else if "`k'" == "http_status" local http `"`v'"' } file read `sh' line } file close `sh' if "`st'" != "ok" { display as error "webapi `sub': `err' -- `msg'" exit 198 } if `polling' { * accumulate one snapshot per poll, stamped with the poll index * and the wall-clock time, so a stream of pulls stacks into a panel. import delimited using `"`outdata'"', delimiters(tab) varnames(1) clear `scols' quietly gen long _poll = `_p' quietly gen str12 _polltime = "`c(current_time)'" if `_p' > 1 quietly append using `"`master'"' quietly save `"`master'"', replace display as text " poll `_p'/`times' at `c(current_time)': `nrows' rows (HTTP `http')" if `_p' < `times' & `every' > 0 sleep `=int(`every'*1000)' } else { import delimited using `"`outdata'"', delimiters(tab) varnames(1) `clear' `scols' } } if `polling' { quietly use `"`master'"', clear sort _poll display as result _n "[webapi `sub'] `times' polls, `=_N' total rows accumulated" return scalar polls = `times' return scalar nrows = _N } else { display as result _n "[webapi `sub'] `nrows' rows, `ncols' columns (HTTP `http')" return scalar nrows = `nrows' return scalar ncols = `ncols' } return local http `"`http'"' return local url `"`url'"' end