*! version 1.2.4 20261103 github.com/mdroste/stata-ctools program define crangestat version 14.1 * A failed or interrupted call leaves the data as they were without a * snapshot of the dataset: crangestat only adds its result variables at * the end, and the plugin writes into nothing else, so _ctools_undo drops * them and resets c(changed). Option snapshot preserves the whole dataset * instead (on disk above c(max_preservemem)); option fast undoes nothing. _ctools_mode `0' * Break is recognized only inside _crangestat_impl. nobreak { if "`mode'" == "snapshot" preserve else if "`mode'" == "" _ctools_undo record CTOOLS_u_crangestat_ capture noisily break _crangestat_impl `0' local rc = _rc if `rc' { if "`mode'" == "snapshot" restore else if "`mode'" == "" _ctools_undo rollback CTOOLS_u_crangestat_ exit `rc' } if "`mode'" == "snapshot" restore, not else if "`mode'" == "" _ctools_undo clear CTOOLS_u_crangestat_ } end program define _crangestat_impl version 14.1 * Check observation limit (Stata plugin API limitation) if _N > 2147483647 { di as error "ctools does not support datasets exceeding 2^31 (2.147 billion) observations" di as error "This is a limitation of Stata's plugin API" exit 920 } * Parse the syntax as rangestat does: * crangestat (stat) [newvar=]varname [...] [if] [in], interval(keyvar low high) [by(varlist) excludeself verbose threads(#)] syntax anything(name=stat_part id="statistics" equalok) [if] [in], /// Interval(string) [BY(varlist) EXCLudeself Verbose THReads(integer 0) SNAPshot FAST] * Parse interval specification: keyvar low high, separated by spaces or * commas as in rangestat, which refuses anything more local interval : subinstr local interval "," " ", all tokenize "`interval'" local keyvar "`1'" local int_low "`2'" local int_high "`3'" if `"`4'"' != "" { di as error "extra argument in interval() option: `4'" exit 198 } if "`keyvar'" == "" | "`int_low'" == "" | "`int_high'" == "" { di as error "crangestat: interval() requires keyvar low high" exit 198 } * Validate key variable exists and is numeric capture confirm variable `keyvar' if _rc != 0 { di as error "crangestat: variable `keyvar' not found" exit 111 } capture confirm numeric variable `keyvar' if _rc != 0 { di as error "crangestat: key variable `keyvar' must be numeric" exit 109 } * Convert interval bounds (handle . for +/- infinity) local low_is_missing = 0 local high_is_missing = 0 if "`int_low'" == "." { local low_is_missing = 1 local int_low_val = 0 } else { capture confirm number `int_low' if _rc != 0 { di as error "crangestat: invalid interval low bound '`int_low''" exit 198 } local int_low_val = `int_low' } if "`int_high'" == "." { local high_is_missing = 1 local int_high_val = 0 } else { capture confirm number `int_high' if _rc != 0 { di as error "crangestat: invalid interval high bound '`int_high''" exit 198 } local int_high_val = `int_high' } * Start timing local __do_timing = ("`verbose'" != "") if `__do_timing' { local __ct_elapsed90 = 0 local __ct_elapsed91 = 0 local __ct_elapsed92 = 0 if "`__ct_elapsed90'" == "" local __ct_elapsed90 = 0 quietly _ctools_clock __ct_start90 if "`__ct_elapsed91'" == "" local __ct_elapsed91 = 0 quietly _ctools_clock __ct_start91 } * Parse stat specifications * Format: (stat) [newvar=]varname [(stat) [newvar=]varname ...] local stat_codes "" local source_vars "" local result_vars "" local source_var_map "" local nstats = 0 local nsource = 0 local source_var_list "" local remaining "`stat_part'" * As in rangestat, a specification without a statistic means (mean). if substr(strtrim("`remaining'"), 1, 1) != "(" local remaining "(mean) `remaining'" while "`remaining'" != "" { * Skip leading whitespace local remaining = strtrim("`remaining'") if "`remaining'" == "" continue, break * Expect opening parenthesis if substr("`remaining'", 1, 1) != "(" { di as error "crangestat: expected '(' before statistic name" exit 198 } * Find closing parenthesis local paren_end = strpos("`remaining'", ")") if `paren_end' == 0 { di as error "crangestat: missing ')' after statistic name" exit 198 } * Extract stat name local stat_name = strtrim(substr("`remaining'", 2, `paren_end' - 2)) local remaining = strtrim(substr("`remaining'", `paren_end' + 1, .)) * Validate stat name and get code local stat_code = -1 if "`stat_name'" == "count" local stat_code = 0 else if "`stat_name'" == "mean" local stat_code = 1 else if "`stat_name'" == "sum" local stat_code = 2 else if "`stat_name'" == "min" local stat_code = 3 else if "`stat_name'" == "max" local stat_code = 4 else if "`stat_name'" == "sd" local stat_code = 5 else if "`stat_name'" == "variance" local stat_code = 6 else if "`stat_name'" == "median" local stat_code = 7 else if "`stat_name'" == "iqr" local stat_code = 8 else if "`stat_name'" == "first" local stat_code = 9 else if "`stat_name'" == "last" local stat_code = 10 else if "`stat_name'" == "firstnm" local stat_code = 11 else if "`stat_name'" == "lastnm" local stat_code = 12 else if "`stat_name'" == "p1" local stat_code = 13 else if "`stat_name'" == "p5" local stat_code = 14 else if "`stat_name'" == "p10" local stat_code = 15 else if "`stat_name'" == "p25" local stat_code = 16 else if "`stat_name'" == "p75" local stat_code = 17 else if "`stat_name'" == "p90" local stat_code = 18 else if "`stat_name'" == "p95" local stat_code = 19 else if "`stat_name'" == "p99" local stat_code = 20 else if "`stat_name'" == "skewness" local stat_code = 21 else if "`stat_name'" == "kurtosis" local stat_code = 22 else if "`stat_name'" == "obs" local stat_code = 23 else if "`stat_name'" == "missing" local stat_code = 24 if `stat_code' == -1 { di as error "crangestat: unknown statistic '`stat_name''" exit 3499 } * Parse variable specification: [newvar=]varname * Find next ( or end local next_paren = strpos("`remaining'", "(") if `next_paren' == 0 { local var_spec = strtrim("`remaining'") local remaining "" } else { local var_spec = strtrim(substr("`remaining'", 1, `next_paren' - 1)) local remaining = substr("`remaining'", `next_paren', .) } if "`var_spec'" == "" { di as error "crangestat: missing variable after (`stat_name')" exit 198 } * Each specification holds [newvar=]varname pairs and varlists, as in * rangestat; a varlist gives one result per variable, named * varname_statname. local var_spec : subinstr local var_spec "=" " = ", all local spec_sources "" local spec_results "" while `"`var_spec'"' != "" { gettoken word var_spec : var_spec gettoken eq : var_spec if `"`word'"' == "=" { di as error "crangestat: missing new variable name in (`stat_name')" exit 102 } if `"`eq'"' == "=" { gettoken eq var_spec : var_spec gettoken source_var var_spec : var_spec if "`source_var'" == "" | "`source_var'" == "=" { di as error "crangestat: missing variable after `word'= in (`stat_name')" exit 198 } capture unab source_var : `source_var', max(1) if _rc { di as error "crangestat: variable `source_var' not found" exit 111 } local spec_sources `spec_sources' `source_var' local spec_results `spec_results' `word' } else { capture unab expanded : `word' if _rc { di as error "crangestat: variable `word' not found" exit 111 } foreach source_var of local expanded { local spec_sources `spec_sources' `source_var' local spec_results `spec_results' `source_var'_`stat_name' } } } local nspec : word count `spec_sources' forvalues spec_j = 1/`nspec' { local source_var : word `spec_j' of `spec_sources' local result_var : word `spec_j' of `spec_results' capture confirm numeric variable `source_var' if _rc != 0 { di as error "crangestat: source variable `source_var' must be numeric" exit 109 } confirm name `result_var' * Destination creation is deferred until the entire request is valid. * Track source variable index local src_idx = 0 local found = 0 foreach sv of local source_var_list { local src_idx = `src_idx' + 1 if "`sv'" == "`source_var'" { local found = 1 continue, break } } if `found' == 0 { local source_var_list "`source_var_list' `source_var'" local nsource = `nsource' + 1 local src_idx = `nsource' } * Store mapping (0-based for C) local source_var_map "`source_var_map' `=`src_idx' - 1'" local stat_codes "`stat_codes' `stat_code'" local result_vars "`result_vars' `result_var'" local result_label_`=`nstats'+1' "`stat_name' of `source_var'" local nstats = `nstats' + 1 } } if `nstats' == 0 { di as error "crangestat: no statistics specified" exit 198 } * rangestat rejects a result name used twice, or already defined, with r(110). local seen_results "" foreach output of local result_vars { if `: list output in seen_results' { di as error "cannot create -`output'-; variable(s) already defined" exit 110 } local seen_results `seen_results' `output' } _ctools_newvars `result_vars' * The sample follows rangestat: if/in and a nonmissing key. Excluded rows * take a missing key, so they neither join a window nor get a result. if `"`if'`in'"' != "" { marksample touse, novarlist tempvar sample_key quietly generate double `sample_key' = `keyvar' if `touse' local keyvar `sample_key' } quietly count if !missing(`keyvar') if r(N) == 0 error 2000 local result_index=0 foreach output of local result_vars { local ++result_index quietly generate double `output' = . label variable `output' "`result_label_`result_index''" } * Validation hook: fail as Break would once the result variables exist if "$CTOOLS_crangestat_fault" == "outputs" error 1 * Load plugin _ctools_load * Stata scopes plugin registrations to the calling ado program. capture program ctools_plugin, plugin using("`__ctools_plugin'") if _rc != 0 & _rc != 110 exit 601 capture confirm number 0 * Get variable indices * Build list of ALL dataset variables - this determines plugin varlist order * The plugin interface uses 1-based indices relative to the varlist passed to it unab all_vars : * * Indices are dataset positions (the plugin receives every variable in * order). Look up only the variables used: a local named after each * dataset variable is an invalid name once a name has 24+ characters. _crangestat_varpos key_idx : `keyvar' _crangestat_varpos source_indices : `source_var_list' _crangestat_varpos result_indices : `result_vars' * By-variable indices local nby = 0 local by_indices "" if "`by'" != "" { local nby : word count `by' _crangestat_varpos by_indices : `by' } * Build options string local opts "" if "`excludeself'" != "" local opts "`opts' excludeself" if "`verbose'" != "" local opts "`opts' verbose" local opts "`opts' low=`int_low_val' high=`int_high_val'" if `low_is_missing' local opts "`opts' low_miss" if `high_is_missing' local opts "`opts' high_miss" local threads_opt "" if `threads' > 0 { local threads_opt "threads(`threads')" } * Build argument string for plugin * Format: nstats nsource nby key_idx source_indices... result_indices... by_indices... stat_types... source_var_map... [options] local plugin_args "`nstats' `nsource' `nby' `key_idx' `source_indices' `result_indices' `by_indices' `stat_codes' `source_var_map' `opts'" * Call plugin if `__do_timing' { quietly _ctools_clock __ct_now local __ct_elapsed91 = `__ct_elapsed91' + (`__ct_now' - `__ct_start91') if "`__ct_elapsed92'" == "" local __ct_elapsed92 = 0 quietly _ctools_clock __ct_start92 } plugin call ctools_plugin `all_vars', "crangestat `threads_opt' `plugin_args'" * Validation hook: fail after the plugin filled the result variables if "$CTOOLS_crangestat_fault" == "post" error 1 if `__do_timing' { quietly _ctools_clock __ct_now local __ct_elapsed92 = `__ct_elapsed92' + (`__ct_now' - `__ct_start92') } * Display summary or timing if `__do_timing' { quietly _ctools_clock __ct_now local __ct_elapsed90 = `__ct_elapsed90' + (`__ct_now' - `__ct_start90') * Elapsed seconds are held in caller-local timing state. local __time_total = `__ct_elapsed90' * Extract sub-timer values * Elapsed seconds are held in caller-local timing state. local __time_preplugin = `__ct_elapsed91' * Elapsed seconds are held in caller-local timing state. local __time_plugin = `__ct_elapsed92' * Calculate plugin call overhead capture local __plugin_time_total = _crangestat_time_total if _rc != 0 local __plugin_time_total = 0 local __plugin_call_overhead = `__time_plugin' - `__plugin_time_total' di as text "" di as text "{hline 55}" di as text "crangestat timing breakdown:" di as text "{hline 55}" di as text " C plugin internals:" di as text " Data load: " as result %8.4f _crangestat_time_load " sec" di as text " Sort: " as result %8.4f _crangestat_time_sort " sec" di as text " Group detection: " as result %8.4f _crangestat_time_groups " sec" di as text " Compute statistics: " as result %8.4f _crangestat_time_compute " sec" di as text " Store results: " as result %8.4f _crangestat_time_store " sec" di as text " {hline 53}" di as text " C plugin total: " as result %8.4f _crangestat_time_total " sec" di as text "" di as text " Stata overhead:" di as text " Pre-plugin setup: " as result %8.4f `__time_preplugin' " sec" di as text " Plugin call overhead: " as result %8.4f `__plugin_call_overhead' " sec" di as text " {hline 53}" local __stata_overhead = `__time_preplugin' + `__plugin_call_overhead' di as text " Stata overhead total: " as result %8.4f `__stata_overhead' " sec" di as text "{hline 55}" di as text " Wall clock total: " as result %8.4f `__time_total' " sec" di as text "{hline 55}" * Display thread diagnostics capture local __threads_max = _crangestat_threads_max if _rc == 0 { capture local __openmp_enabled = _crangestat_openmp_enabled if _rc != 0 local __openmp_enabled = 0 di as text "" di as text " Thread diagnostics:" di as text " OpenMP enabled: " as result %8.0f `__openmp_enabled' di as text " Max threads available: " as result %8.0f `__threads_max' di as text "{hline 55}" } * Clean up timing scalars capture scalar drop _crangestat_time_load _crangestat_time_sort _crangestat_time_groups capture scalar drop _crangestat_time_compute _crangestat_time_store _crangestat_time_total capture scalar drop _crangestat_threads_max _crangestat_openmp_enabled } else { di as text "(`nstats' range statistics computed)" } end * Dataset positions of the named variables (abbreviations allowed), as a list. program define _crangestat_varpos gettoken macname 0 : 0 gettoken colon 0 : 0 local pos "" if `"`0'"' != "" { mata: st_local("pos", invtokens(strofreal(st_varindex(tokens(st_local("0")), 1)))) } c_local `macname' "`pos'" end