Use Tk-native display selection across graphical tools

DISPLAY is an X11 connection setting, but MGED treated any inherited
value as an explicit Tk screen. Unix-like shells on Windows can export
placeholders, causing native Tk child windows and display managers to
fail even though Tk initialized successfully.

Let Tk select its platform backend and canonical screen when no -d
option is given. Preserve explicit X11 selection by applying DISPLAY
only for Tk initialization, restore the caller environment for Win32
and Aqua, and reject unsupported native screen requests clearly.

Use the initialized Tk screen for WGL and Tk software rendering, and
stop assigning X11 defaults to offscreen or Qt contexts. Remove the
same synthetic DISPLAY assumptions from MGED and Archer scripts so
Archer, RtWizard, BRLMan, and Tk ISST follow consistent startup rules.

src/libdm/qtgl/dm-qtgl.cpp

@@ -254,15 +254,9 @@ qtgl_open(void *ctx, void *UNUSED(interp), int argc, const char **argv)
     if (bu_vls_strlen(&dmp->i->dm_pathName) == 0)
 	bu_vls_printf(&dmp->i->dm_pathName, ".dm_qtgl%d", count);
     ++count;
-    if (bu_vls_strlen(&dmp->i->dm_dName) == 0) {
-	char *dp;
-
-	dp = getenv("DISPLAY");
-	if (dp)
-	    bu_vls_strcpy(&dmp->i->dm_dName, dp);
-	else
-	    bu_vls_strcpy(&dmp->i->dm_dName, ":0.0");
-    }
+    /* The supplied QOpenGLWidget determines the Qt display.  Keep dm_dName
+     * empty unless the caller supplied -d; DISPLAY is an X11 setting and
+     * cannot identify a Qt context on every supported platform. */
 
     /* initialize dm specific variables */
     pubvars->devmotionnotify = 0;

src/libdm/swrast/dm-swrast.cpp

@@ -246,15 +246,9 @@ swrast_open(void *ctx, void *UNUSED(interp), int argc, const char **argv)
     if (bu_vls_strlen(&dmp->i->dm_pathName) == 0)
 	bu_vls_printf(&dmp->i->dm_pathName, ".dm_swrast%d", count);
     ++count;
-    if (bu_vls_strlen(&dmp->i->dm_dName) == 0) {
-	char *dp;
-
-	dp = getenv("DISPLAY");
-	if (dp)
-	    bu_vls_strcpy(&dmp->i->dm_dName, dp);
-	else
-	    bu_vls_strcpy(&dmp->i->dm_dName, ":0.0");
-    }
+    /* OSMesa is offscreen and has no display to discover.  Leave dm_dName
+     * empty unless the caller supplied -d; a window-system wrapper can then
+     * select its own initialized display without inheriting X11 state. */
 
     /* initialize dm specific variables */
     pubvars->devmotionnotify = 0;

src/libdm/tkswrast/dm-tkswrast.cpp

@@ -464,6 +464,16 @@ tkswrast_open(void *ctx, void *vinterp, int argc, const char **argv)
     tv->orig_configureWin = dmp->i->dm_configureWin;
     tv->orig_close = dmp->i->dm_close;
 
+    /* The wrapped offscreen backend deliberately has no default display.
+     * For this Tk presentation layer, use the display Tk actually opened.
+     * This remains correct for X11, Win32, and Aqua even when the process
+     * inherited an unrelated DISPLAY value.  An explicit -d remains intact
+     * and is validated below by Tk_CreateWindowFromPath. */
+    if (bu_vls_strlen(&dmp->i->dm_dName) == 0) {
+	bu_vls_printf(&dmp->i->dm_dName, "%s.%d", Tk_DisplayName(tkwin),
+		Tk_ScreenNumber(tkwin));
+    }
+
     char *cp = strrchr(bu_vls_addr(&dmp->i->dm_pathName), '.');
     if (dmp->i->dm_top) {
 	tv->xtkwin = Tk_CreateWindowFromPath(interp, tkwin,

src/libdm/wgl/dm-wgl.c

@@ -1040,13 +1040,11 @@ wgl_open(void *UNUSED(ctx), void *vinterp, int argc, const char *argv[])
     ++count;
 
     if (bu_vls_strlen(&dmp->i->dm_dName) == 0) {
-	char *dp;
-
-	dp = getenv("DISPLAY");
-	if (dp)
-	    bu_vls_strcpy(&dmp->i->dm_dName, dp);
-	else
-	    bu_vls_strcpy(&dmp->i->dm_dName, ":0.0");
+	/* DISPLAY names an X server and is not meaningful to WGL.  Unix-like
+	 * shells on Windows may nevertheless export it, so use the screen of
+	 * the Tk window that supplies the native HWND. */
+	bu_vls_printf(&dmp->i->dm_dName, "%s.%d", Tk_DisplayName(tkwin),
+		Tk_ScreenNumber(tkwin));
     }
 
     /* initialize dm specific variables */

src/mged/attach.c

@@ -340,6 +340,26 @@ f_attach(ClientData clientData, Tcl_Interp *interpreter, int argc, const char *a
 }
 
 
+#ifdef HAVE_TK
+static int
+mged_restore_display_env(Tcl_Interp *interp, Tcl_Obj *saved_display)
+{
+    int status;
+
+    if (saved_display != NULL) {
+	status = (Tcl_SetVar2Ex(interp, "env", "DISPLAY", saved_display,
+		TCL_GLOBAL_ONLY | TCL_LEAVE_ERR_MSG) != NULL) ? TCL_OK : TCL_ERROR;
+	Tcl_DecrRefCount(saved_display);
+	return status;
+    }
+
+    status = Tcl_UnsetVar2(interp, "env", "DISPLAY",
+	    TCL_GLOBAL_ONLY | TCL_LEAVE_ERR_MSG);
+    return status;
+}
+#endif
+
+
 int
 gui_setup(struct mged_state *s, const char *dstr)
 {
@@ -348,6 +368,7 @@ gui_setup(struct mged_state *s, const char *dstr)
 
 #ifdef HAVE_TK
     Tk_GenericProc *handler = doEvent;
+    Tcl_Obj *saved_display = NULL;
 #endif
     /* initialize only once */
     if (tkwin != NULL)
@@ -355,24 +376,83 @@ gui_setup(struct mged_state *s, const char *dstr)
 
     Tcl_ResetResult(s->interp);
 
-    /* set DISPLAY to dstr */
-    if (dstr != (char *)NULL) {
-	Tcl_SetVar(s->interp, "env(DISPLAY)", dstr, TCL_GLOBAL_ONLY);
+#ifdef HAVE_TK
+    /* Tk has no display-name argument in its C initialization API.  For an
+     * explicit MGED -d/loadtk argument, env(DISPLAY) is therefore the only
+     * portable way to select an X11 display before Tk_Init.  Do not do this
+     * for ambient process state: X11 Tk already reads DISPLAY itself, and
+     * native Win32/Aqua Tk deliberately ignores that X11-only variable. */
+    if (dstr != NULL) {
+	saved_display = Tcl_GetVar2Ex(s->interp, "env", "DISPLAY",
+		TCL_GLOBAL_ONLY);
+	if (saved_display != NULL)
+	    Tcl_IncrRefCount(saved_display);
+	if (Tcl_SetVar2(s->interp, "env", "DISPLAY", dstr,
+		TCL_GLOBAL_ONLY | TCL_LEAVE_ERR_MSG) == NULL) {
+	    if (saved_display != NULL)
+		Tcl_DecrRefCount(saved_display);
+	    return TCL_ERROR;
+	}
     }
 
-#ifdef HAVE_TK
     /* This runs the tk.tcl script */
     if (Tk_Init(s->interp) == TCL_ERROR) {
 	const char *result = Tcl_GetStringResult(s->interp);
 	/* hack to avoid a stupid Tk error */
 	if (bu_strncmp(result, "this isn't a Tk applicationcouldn't", 35) == 0) {
-	    result = (result + 27);
-	    Tcl_ResetResult(s->interp);
-	    Tcl_AppendResult(s->interp, result, (char *)NULL);
+	    Tcl_SetObjResult(s->interp, Tcl_NewStringObj(result + 27, -1));
+	}
+	if (dstr != NULL) {
+	    Tcl_SavedResult init_result;
+
+	    /* Environment restoration is cleanup on this path; retain Tk's
+	     * diagnostic even if an unusual Tcl environment trace rejects it. */
+	    Tcl_SaveResult(s->interp, &init_result);
+	    (void)mged_restore_display_env(s->interp, saved_display);
+	    Tcl_RestoreResult(s->interp, &init_result);
 	}
 	return TCL_ERROR;
     }
 
+    if (dstr != NULL) {
+	int is_x11;
+
+	if (Tcl_Eval(s->interp, "tk windowingsystem") != TCL_OK) {
+	    Tcl_SavedResult windowing_result;
+
+	    Tcl_SaveResult(s->interp, &windowing_result);
+	    (void)mged_restore_display_env(s->interp, saved_display);
+	    Tcl_RestoreResult(s->interp, &windowing_result);
+	    return TCL_ERROR;
+	}
+	is_x11 = BU_STR_EQUAL(Tcl_GetStringResult(s->interp), "x11");
+	Tcl_ResetResult(s->interp);
+
+	if (is_x11) {
+	    /* X11 may use the explicit display for subsequently created
+	     * toplevels and display managers, so keep it in the environment. */
+	    if (saved_display != NULL)
+		Tcl_DecrRefCount(saved_display);
+	} else {
+	    Tk_Window display_probe;
+
+	    /* Win32 and Aqua cannot be redirected by DISPLAY.  Restore the
+	     * caller's environment, then let Tk validate whether the explicit
+	     * name denotes the already initialized native screen.  This accepts
+	     * native aliases such as :0/:0.0 but reports unsupported alternate
+	     * displays instead of silently opening a different screen. */
+	    if (mged_restore_display_env(s->interp, saved_display) != TCL_OK)
+		return TCL_ERROR;
+
+	    display_probe = Tk_CreateWindowFromPath(s->interp,
+		    Tk_MainWindow(s->interp), ".__mged_display_probe", dstr);
+	    if (display_probe == NULL)
+		return TCL_ERROR;
+	    Tk_DestroyWindow(display_probe);
+	    Tcl_ResetResult(s->interp);
+	}
+    }
+
     /* Initialize [incr Tk] */
     if (Tcl_Eval(s->interp, "package require Itk") != TCL_OK) {
       return TCL_ERROR;
@@ -385,6 +465,10 @@ gui_setup(struct mged_state *s, const char *dstr)
     }
 #endif
 
+#ifndef HAVE_TK
+    (void)dstr;
+#endif
+
     /* Initialize the Iwidgets package */
     if (Tcl_Eval(s->interp, "package require Iwidgets") != TCL_OK) {
 	return TCL_ERROR;

src/mged/mged.c

@@ -2446,6 +2446,29 @@ apply_cli_overrides(struct mged_state *s, struct mged_cli_overrides *cl)
     if (cli_dbl_is_set(cl->eye_sep_dist))        CLI_SETVAR_DBL("eye_sep_dist",   cl->eye_sep_dist);
 
     /* --- Tcl mged_default array entries ---------------------------------- */
+    if (cl->dpy_string) {
+	const char *display = cl->dpy_string;
+#ifdef HAVE_TK
+	struct bu_vls tk_screen = BU_VLS_INIT_ZERO;
+	Tk_Window main_window = Tk_MainWindow(s->interp);
+
+	/* Apply -d after .mgedrc just like the other command-line overrides.
+	 * Once Tk is initialized, store its canonical screen name rather than
+	 * the spelling used to select it (for example, :0 and :0.0 may denote
+	 * the same native screen).  Classic/headless mode retains the original
+	 * value for a later display-manager attach. */
+	if (main_window != NULL) {
+	    bu_vls_printf(&tk_screen, "%s.%d", Tk_DisplayName(main_window),
+		    Tk_ScreenNumber(main_window));
+	    display = bu_vls_cstr(&tk_screen);
+	}
+#endif
+	Tcl_SetVar2(s->interp, "mged_default", "display", display,
+		TCL_GLOBAL_ONLY);
+#ifdef HAVE_TK
+	bu_vls_free(&tk_screen);
+#endif
+    }
     if (cl->dm_type)
 	Tcl_SetVar2(s->interp, "mged_default", "dm_type", cl->dm_type,  TCL_GLOBAL_ONLY);
     if (cl->geom)
@@ -2694,7 +2717,7 @@ main(int argc, char *argv[])
     struct bu_opt_desc opt_defs[54];
     /* ---- Existing short options, now with long aliases ---- */
     BU_OPT(opt_defs[0],  "a", "attach",           "type",    bu_opt_str,      &cl.attach,           "display manager attach target");
-    BU_OPT(opt_defs[1],  "d", "display",           "string",  bu_opt_str,      &cl.dpy_string,       "X display string");
+    BU_OPT(opt_defs[1],  "d", "display",           "screen",  bu_opt_str,      &cl.dpy_string,       "Tk display and screen name");
     BU_OPT(opt_defs[2],  "r", "read-only",         "",        NULL,            &cl.read_only,        "open database read-only");
     BU_OPT(opt_defs[3],  "p", "pipe",              "",        NULL,            &cl.pipe_mode,        "pipe mode (emit CMD_DONE sentinels)");
     BU_OPT(opt_defs[4],  "c", "classic",           "",        NULL,            &cl.classic_mode,     "classic text-only mode");
@@ -3044,9 +3067,6 @@ main(int argc, char *argv[])
     (void)Tcl_Eval(s->interp, bu_vls_addr(&s->input_str));
     bu_vls_trunc(&s->input_str, 0);
 
-    if (!s->dpy_string)
-	s->dpy_string = getenv("DISPLAY");
-
     /* show ourselves */
     if (s->interactive) {
 	if (s->classic_mged) {
@@ -3068,23 +3088,17 @@ main(int argc, char *argv[])
 	    /* start up the gui */
 
 	    int status;
-	    struct bu_vls vls = BU_VLS_INIT_ZERO;
 	    struct bu_vls error = BU_VLS_INIT_ZERO;
 
 	    (void)Tcl_Eval(s->interp, "set mged_console_mode gui");
 
-	    bu_vls_strcpy(&vls, "loadtk");
-	    if (s->dpy_string)
-		bu_vls_printf(&vls, " %s", s->dpy_string);
-
-	    status = Tcl_Eval(s->interp, bu_vls_addr(&vls));
+	    /* An unset -d is deliberately passed as NULL.  Tk already knows how
+	     * to select its platform display: X11 consults DISPLAY, while the
+	     * Win32 and Aqua backends select their native display.  Treating an
+	     * inherited DISPLAY value as an explicit request breaks native Tk
+	     * when Unix-like shells export an X11 placeholder. */
+	    status = gui_setup(s, s->dpy_string);
 	    bu_vls_strcpy(&error, Tcl_GetStringResult(s->interp));
-	    bu_vls_free(&vls);
-
-	    if (status != TCL_OK && !s->dpy_string) {
-		/* failed to load tk, try localhost X11 if DISPLAY was not set */
-		status = Tcl_Eval(s->interp, "loadtk :0");
-	    }
 
 	    if (status != TCL_OK) {
 		if (!run_in_foreground && use_pipe) {

src/tclscripts/archer/ArcherCore.tcl

@@ -1090,9 +1090,9 @@ namespace delete ::ArcherCoreBootstrap
 	wm withdraw [namespace tail $this]
     }
 
-    if {![info exists env(DISPLAY)]} {
-	set env(DISPLAY) ":0"
-    }
+    # Tk has already selected its X11, Win32, or Aqua display.  DISPLAY is an
+    # X11 input to that initialization, not portable application state; in
+    # particular, synthesizing :0 here can misdirect native Tk child windows.
 
     set mImgDir [file join [bu_dir data] tclscripts archer images]
 

src/tclscripts/mged/combmenu.tcl

@@ -25,11 +25,7 @@
 #
 
 if ![info exists mged_default(display)] {
-    if [info exists env(DISPLAY)] {
-	set mged_default(display) $env(DISPLAY)
-    } else {
-	set mged_default(display) :0
-    }
+    set mged_default(display) [winfo screen .]
 }
 
 if ![info exists mged_gui(mged,screen)] {

src/tclscripts/mged/editmenu.tcl

@@ -24,11 +24,7 @@
 #
 
 if ![info exists mged_default(display)] {
-    if [info exists env(DISPLAY)] {
-	set mged_default(display) $env(DISPLAY)
-    } else {
-	set mged_default(display) :0
-    }
+    set mged_default(display) [winfo screen .]
 }
 
 if ![info exists mged_gui(mged,screen)] {

src/tclscripts/mged/mged.tcl

@@ -27,10 +27,6 @@ if ![info exists mged_players] {
     set mged_players {}
 }
 
-if ![info exists env(DISPLAY)] {
-    set env(DISPLAY) ":0"
-}
-
 #==============================================================================
 # Ensure that tk.tcl has been loaded already via mged command 'loadtk'.
 #==============================================================================

src/tclscripts/mged/mgedrc.tcl

@@ -154,8 +154,7 @@ proc dump_mged_state {fd} {
     puts $fd "# Determines the border width in the geometry window"
     puts $fd "set mged_default(bd) $mged_default(bd)"
     puts $fd ""
-    puts $fd "# X display string used to determine the screen"
-    puts $fd "# on which to display the GUI's windows"
+    puts $fd "# Tk screen name used for the GUI's windows"
     puts $fd "set mged_default(display) $mged_default(display)"
     puts $fd ""
     puts $fd "# Configures the GUI. Note - this is only"

src/tclscripts/mged/openw.tcl

@@ -108,13 +108,12 @@ if {![info exists mged_default(config)]} {
     set mged_default(config) b
 }
 
-if {[info exists env(DISPLAY)]} {
-    set mged_default(display) $env(DISPLAY)
-} else {
-    if {![info exists mged_default(display)]} {
-	set mged_default(display) :0
-    }
-    set env(DISPLAY) $mged_default(display)
+# DISPLAY is an X11 connection setting, not a portable Tk screen name.  Tk is
+# initialized before this script runs, so its canonical screen is authoritative
+# on X11, Win32, and Aqua alike.  Preserve a value supplied explicitly through
+# .mgedrc or the gui command for multi-display X11 use.
+if {![info exists mged_default(display)]} {
+    set mged_default(display) [winfo screen .]
 }
 
 if {![info exists mged_default(gdisplay)]} {