/*
  parqit merge - Combine the view with a file or another view, or join the
  data in memory with a file

*!  VERSION 0.1.37  07sep2026

  User > parqit > Combine datasets (merge, append, joinby)...
  Launch directly with:  db parqit_combine      (menu installed by: parqit menu)

  Main tab: the lazy two-table verbs (merge, append, joinby) whose using side
  is a Parquet file, glob, Hive directory, delimited text, .dta, Excel (the
  last three bridged) or another open view typed as view:name; and the
  native in-memory routes (mergein, appendin). Options tab: the native merge
  options forwarded by mergein, appendin's force, and the code page of a
  legacy .dta/Excel using source. Populate fills the key pickers from the
  current view (lazy verbs) or from the using file (memory verbs), and the
  using-variable picker from the Parquet footer of the using file.
*/

VERSION 16.0

INCLUDE _std_wide
DEFINE _dlght 520
INCLUDE header
HELP hlp1, view("help parqit##verbs")
RESET res1

SCRIPT PREINIT
BEGIN
	create STRING dlgname
	dlgname.storeDialogClassName
	create DOUBLE pq_populate_error
	main.fi_using.formatfilter					///
		"Parquet (*.parquet)|*.parquet|"			///
		"Delimited text (*.csv *.tsv *.txt *.tab)|*.csv;*.tsv;*.txt;*.tab|" ///
		"Stata data (*.dta)|*.dta|"				///
		"Excel (*.xlsx *.xls)|*.xlsx;*.xls|"			///
		"All files (*.*)|*.*"
END

PROGRAM POSTINIT_PROGRAM
BEGIN
	call program main_mode
	call program main_context
END

DIALOG main, label("parqit merge - Combine the view with a file or another view, or join the data in memory with a file") ///
	tabtitle("Main")
BEGIN
  GROUPBOX gb_type		_lft	_top	_iwd	_ht10,		///
	label("Type of combination")
  RADIO rb_merge		_ilft	_ss	_ibwd	.,		///
	first								///
	onclickon(program main_mode)					///
	label("Merge the view with a using source on key variables (merge 1:1, m:1, 1:m; lazy)")
  RADIO rb_append		@	_ss	@	.,		///
	onclickon(program main_mode)					///
	label("Append using sources to the view (append; lazy)")
  RADIO rb_joinby		@	_ss	@	.,		///
	onclickon(program main_mode)					///
	label("Form all pairwise combinations within groups of key variables (joinby; lazy)")
  RADIO rb_mergein		@	_ss	@	.,		///
	onclickon(program main_mode)					///
	label("Merge the data in memory with a file (mergein; native merge, the data stay in memory)")
  RADIO rb_appendin		@	_ss	@	.,		///
	last								///
	onclickon(program main_mode)					///
	label("Append a file to the data in memory (appendin; native append)")

  TEXT tx_using			_lft	150	_lw80	.,		///
	label("Using source: filename, glob, Hive directory, or view:name (lazy types):")
  BUTTON bu_pop			_rj80	@	80	.,		///
	onpush(program main_populate)					///
	tooltip("Populate the key picker from the current view (or the using file for mergein/appendin) and the using-variable picker from the using file") ///
	label("Populate")
  FILE fi_using			_lft	_ss	_iwd	.,		///
	error("Using source")						///
	defext(parquet)							///
	label("Browse...")

  TEXT tx_more			_lft	_ls	_iwd	.,		///
	label("Additional using sources for append (filenames, globs, or view:name, separated by spaces):")
  EDIT ed_more			@	_ss	@	.,		///
	tooltip("Use Stata compound quotes around each additional path containing spaces or commas") ///
	label("Additional using sources")

  TEXT tx_kind			_lft	_ls	_vnwd	.,		///
	label("Merge type:")
  TEXT tx_keys			140	@	450	.,		///
	label("Key variables (match variables):")
  COMBOBOX cb_kind		_lft	_ss	_vnwd	.,		///
	dropdownlist							///
	contents(kind_list)						///
	default("m:1")							///
	label("Merge type")
  COMBOBOX cb_keys		140	@	450	.,		///
	append								///
	dropdown							///
	contents(vv_list)						///
	error("Key variables")						///
	label("Key variables")

  TEXT tx_keepus		_lft	_ls	_iwd	.,		///
	label("Variables to keep from the using source (by default all are kept): keepusing() for merge and mergein, keep() for appendin:")
  COMBOBOX cb_keepus		@	_ss	@	.,		///
	append								///
	dropdown							///
	contents(uv_list)						///
	label("Variables to keep from the using source")

  GROUPBOX gb_keep		_lft	_ls	_iwd	_ht2,		///
	label("Results to keep (merge; leave all unchecked to keep all)")
  CHECKBOX ck_master		_ilft	_ss	180	.,		///
	label("Record appeared in master only")
  CHECKBOX ck_using		210	@	180	.,		///
	label("Record appeared in using only")
  CHECKBOX ck_match		400	@	170	.,		///
	label("Record appeared in both")

  TEXT tx_gen			_lft	_xls	_iwd	.,		///
	label("Name of new variable to mark merge results (default _merge):")
  EDIT ed_gen			@	_ss	_vnwd	.,		///
	max(32)								///
	option(generate)						///
	label("Name of new variable to mark results")
  CHECKBOX ck_nogen		140	@	450	.,		///
	option(nogenerate)						///
	onclickon(main.ed_gen.disable)					///
	onclickoff(main.ed_gen.enable)					///
	label("Do not generate a variable marking merge results")
  TEXT tx_context _lft 475 470 ., label("Current view: refresh to check")
  BUTTON bu_context _rj80 @ 80 ., label("Refresh") onpush(program main_populate)
END

PROGRAM main_context
BEGIN
	put "parqit _dlgcontext "
	put dlgname
	if main.rb_mergein | main.rb_appendin {
		put ", data"
	}
	stata hidden queue
	clear
END

DIALOG opt, tabtitle("Options")
BEGIN
  GROUPBOX gb_native		_lft	_top	_iwd	_ht12,		///
	label("Native merge options forwarded by mergein")
  CHECKBOX ck_update		_ilft	_ss	_ibwd	.,		///
	option(update)							///
	onclickon(opt.ck_mreplace.enable)				///
	onclickoff(opt.ck_mreplace.disable)				///
	label("Update missing values with values from the file (update)")
  CHECKBOX ck_mreplace		@	_ss	@	.,		///
	option(replace)							///
	label("Replace nonmissing values with values from the file (replace; requires update)")
  CHECKBOX ck_nolabel		@	_ss	@	.,		///
	option(nolabel)							///
	label("Do not copy value label definitions from the file (nolabel)")
  CHECKBOX ck_nonotes		@	_ss	@	.,		///
	option(nonotes)							///
	label("Do not copy notes from the file (nonotes)")
  CHECKBOX ck_mforce		@	_ss	@	.,		///
	option(force)							///
	label("Allow string/numeric variable type mismatch without error (force)")
  CHECKBOX ck_noreport		@	_ss	@	.,		///
	option(noreport)						///
	label("Do not display the result summary table (noreport)")
  TEXT tx_assert		_lft	_xls	_iwd	.,		///
	label("Require merge results, assert() (mergein; for example: match master):")
  EDIT ed_assert		@	_ss	@	.,		///
	option(assert)							///
	label("Require merge results")

  CHECKBOX ck_aforce		_lft	_ls	_iwd	.,		///
	option(force)							///
	label("Append string variables to numeric or numeric to string without error (appendin; force)")

  TEXT tx_enc			_lft	_ls	_iwd	.,		///
	label("Code page of legacy (non-UTF-8) text in a .dta or Excel using source (lazy merge, append, joinby):")
  COMBOBOX cb_enc		@	_ss	_vnwd	.,		///
	dropdownlist							///
	contents(enc_list)						///
	default("windows-1252")						///
	option(encoding)						///
	label("Code page")
END

LIST vv_list
BEGIN
END

LIST uv_list
BEGIN
END

LIST enc_list
BEGIN
	windows-1252
	latin1
	latin9
	macroman
END

LIST kind_list
BEGIN
	"1:1"
	"m:1"
	"1:m"
	"m:m"
END

PROGRAM main_mode
BEGIN
	call main.tx_more.disable
	call main.ed_more.disable
	call main.tx_kind.disable
	call main.cb_kind.disable
	call main.tx_keys.disable
	call main.cb_keys.disable
	call main.tx_keepus.disable
	call main.cb_keepus.disable
	call main.gb_keep.disable
	call main.ck_master.disable
	call main.ck_using.disable
	call main.ck_match.disable
	call main.tx_gen.disable
	call main.ed_gen.disable
	call main.ck_nogen.disable
	call opt.gb_native.disable
	call opt.ck_update.disable
	call opt.ck_mreplace.disable
	call opt.ck_nolabel.disable
	call opt.ck_nonotes.disable
	call opt.ck_mforce.disable
	call opt.ck_noreport.disable
	call opt.tx_assert.disable
	call opt.ed_assert.disable
	call opt.ck_aforce.disable
	call opt.tx_enc.disable
	call opt.cb_enc.disable
	if main.rb_merge | main.rb_mergein {
		call main.tx_kind.enable
		call main.cb_kind.enable
		call main.tx_keys.enable
		call main.cb_keys.enable
		call main.tx_keepus.enable
		call main.cb_keepus.enable
		call main.gb_keep.enable
		call main.ck_master.enable
		call main.ck_using.enable
		call main.ck_match.enable
		call main.tx_gen.enable
		call main.ck_nogen.enable
		call main.tx_gen.setlabel "Name of new variable to mark merge results (default _merge):"
		if !main.ck_nogen {
			call main.ed_gen.enable
		}
	}
	if main.rb_append {
		call main.tx_more.enable
		call main.ed_more.enable
		call main.tx_gen.enable
		call main.ed_gen.enable
		call main.tx_gen.setlabel "Name of new variable to mark the source of each observation (optional):"
	}
	if main.rb_joinby {
		call main.tx_keys.enable
		call main.cb_keys.enable
	}
	if main.rb_merge | main.rb_append | main.rb_joinby {
		call main.tx_using.setlabel "Using source: filename, glob, Hive directory, or view:name:"
		call opt.tx_enc.enable
		call opt.cb_enc.enable
	}
	if main.rb_mergein {
		call main.tx_using.setlabel "Using file (Parquet, delimited text, .dta, or Excel):"
		call opt.gb_native.enable
		call opt.ck_update.enable
		call opt.ck_nolabel.enable
		call opt.ck_nonotes.enable
		call opt.ck_mforce.enable
		call opt.ck_noreport.enable
		call opt.tx_assert.enable
		call opt.ed_assert.enable
		if opt.ck_update {
			call opt.ck_mreplace.enable
		}
	}
	if main.rb_appendin {
		call main.tx_using.setlabel "Using file (Parquet, delimited text, .dta, or Excel):"
		call main.tx_keepus.enable
		call main.cb_keepus.enable
		call opt.ck_aforce.enable
	}
	call program main_context
END

PROGRAM main_populate
BEGIN
	call program main_context
	call pq_populate_error.setvalue 0
	if main.rb_merge | main.rb_append | main.rb_joinby {
		put "parqit _dlgvars "
		put dlgname
		put " vv_list"
		stata hidden immediate
		if pq_populate_error {
			stopbox stop "Unable to get the variable names of the current view." ///
				"Open a view first (User > parqit > Read data...)."
		}
	}
	if main.rb_mergein | main.rb_appendin {
		require main.fi_using
		put "parqit _dlgvars "
		put dlgname
		put " vv_list using "
		put /smartquote main.fi_using
		stata hidden immediate
	}
	if main.fi_using {
		call pq_populate_error.setvalue 0
		put "parqit _dlgvars "
		put dlgname
		put " uv_list using "
		put /smartquote main.fi_using
		stata hidden immediate
		if pq_populate_error {
			stopbox note "The using-variable picker was not filled: Populate reads the footer of a Parquet file, glob, or Hive directory; type the names for other sources."
		}
	}
	call main.cb_keys.repopulate
	call main.cb_keepus.repopulate
END

PROGRAM main_keep_output
BEGIN
	if main.ck_master | main.ck_using | main.ck_match {
		put "keep("
		if main.ck_master {
			put "master "
		}
		if main.ck_using {
			put "using "
		}
		if main.ck_match {
			put "match"
		}
		put ") "
	}
END

PROGRAM command
BEGIN
	if main.rb_merge {
		require main.cb_keys
		require main.fi_using
		if main.cb_kind.iseq("m:m") {
			stopbox stop "Lazy merge does not support m:m." ///
				"Use joinby for all pairwise combinations, or mergein m:m for native Stata's sequential behavior."
		}
		put "parqit merge " main.cb_kind " " main.cb_keys " using "
		put /smartquote main.fi_using
		beginoptions
			put /program main_keep_output
			if main.cb_keepus {
				put "keepusing(" main.cb_keepus ") "
			}
			option main.ck_nogen
			if !main.ck_nogen {
				optionarg main.ed_gen
			}
			optionarg /hidedefault opt.cb_enc
		endoptions
	}
	if main.rb_append {
		require main.fi_using
		put "parqit append using "
		put /smartquote main.fi_using
		if main.ed_more {
			put " " main.ed_more
		}
		beginoptions
			optionarg main.ed_gen
			optionarg /hidedefault opt.cb_enc
		endoptions
	}
	if main.rb_joinby {
		require main.cb_keys
		require main.fi_using
		put "parqit joinby " main.cb_keys " using "
		put /smartquote main.fi_using
		beginoptions
			optionarg /hidedefault opt.cb_enc
		endoptions
	}
	if main.rb_mergein {
		require main.cb_keys
		require main.fi_using
		put "parqit mergein " main.cb_kind " " main.cb_keys " using "
		put /smartquote main.fi_using
		beginoptions
			put /program main_keep_output
			if main.cb_keepus {
				put "keepusing(" main.cb_keepus ") "
			}
			option main.ck_nogen
			if !main.ck_nogen {
				optionarg main.ed_gen
			}
			optionarg opt.ed_assert
			option opt.ck_update
			if opt.ck_update {
				option opt.ck_mreplace
			}
			option opt.ck_nolabel
			option opt.ck_nonotes
			option opt.ck_mforce
			option opt.ck_noreport
		endoptions
	}
	if main.rb_appendin {
		require main.fi_using
		put "parqit appendin using "
		put /smartquote main.fi_using
		beginoptions
			if main.cb_keepus {
				put "keep(" main.cb_keepus ") "
			}
			option opt.ck_aforce
		endoptions
	}
END