merge core-8-branch

Add Tcl_FreeEncoding(), in the hope of fixing testcase encoding-2.2...   doesn't seem to help

generic/tclFileName.c

@@ -2436,7 +2436,8 @@ DoGlob(
 	    joinedPtr = TclDStringToObj(&append);
 	} else if (flags) {
 	    joinedPtr = TclNewFSPathObj(pathPtr, Tcl_DStringValue(&append),
-		    Tcl_DStringLength(&append));
+		    Tcl_DStringLength(&append),
+		    TclFSPathEncoding(interp, pathPtr));
 	} else {
 	    joinedPtr = Tcl_DuplicateObj(pathPtr);
 	    if (strchr(separators, Tcl_DStringValue(&append)[0]) == NULL) {
@@ -2469,7 +2470,8 @@ DoGlob(
     if (pathPtr == NULL) {
 	joinedPtr = Tcl_NewStringObj(pattern, p-pattern);
     } else if (flags) {
-	joinedPtr = TclNewFSPathObj(pathPtr, pattern, p-pattern);
+	joinedPtr = TclNewFSPathObj(pathPtr, pattern, p-pattern,
+	    TclFSPathEncoding(interp, pathPtr));
     } else {
 	joinedPtr = Tcl_DuplicateObj(pathPtr);
 	if (strchr(separators, pattern[0]) == NULL) {

generic/tclIOUtil.c

@@ -1168,8 +1168,6 @@ FsAddMountsToGlobResult(
 		    len--;
 		}
 		len++;		/* account for '/' in the mElt [Bug 1602539] */
-
-
 		mElt = TclNewFSPathObj(pathPtr, mount + len, mlen - len);
 		Tcl_ListObjAppendElement(NULL, resultPtr, mElt);
 	    }

generic/tclInt.h

@@ -3005,6 +3005,12 @@ MODULE_SCOPE double	TclFloor(const mp_int *a);
 MODULE_SCOPE void	TclFormatNaN(double value, char *buffer);
 MODULE_SCOPE int	TclFSFileAttrIndex(Tcl_Obj *pathPtr,
 			    const char *attributeName, int *indexPtr);
+MODULE_SCOPE
+    Tcl_Encoding	TclFSPathEncoding(
+			    Tcl_Interp *interp,
+			    Tcl_Obj *pathPtr
+			);
+
 MODULE_SCOPE Tcl_Command TclNRCreateCommandInNs(Tcl_Interp *interp,
 			    const char *cmdName, Tcl_Namespace *nsPtr,
 			    Tcl_ObjCmdProc *proc, Tcl_ObjCmdProc *nreProc,
@@ -3128,7 +3134,7 @@ MODULE_SCOPE int	TclpObjLstat(Tcl_Obj *pathPtr, Tcl_StatBuf *buf);
 MODULE_SCOPE Tcl_Obj *	TclpTempFileName(void);
 MODULE_SCOPE Tcl_Obj *  TclpTempFileNameForLibrary(Tcl_Interp *interp, Tcl_Obj* pathPtr);
 MODULE_SCOPE Tcl_Obj *	TclNewFSPathObj(Tcl_Obj *dirPtr, const char *addStrRep,
-			    int len);
+			    int len, Tcl_Encoding encoding);
 MODULE_SCOPE int	TclpDeleteFile(const void *path);
 MODULE_SCOPE void	TclpFinalizeCondition(Tcl_Condition *condPtr);
 MODULE_SCOPE void	TclpFinalizeMutex(Tcl_Mutex *mutexPtr);

generic/tclPathObj.c

@@ -71,6 +71,7 @@ typedef struct FsPath {
 				 * epoch. The epoch changes when
 				 * filesystem-mounts are changed. */
     const Tcl_Filesystem *fsPtr;/* The Tcl_Filesystem that claims this path */
+    Tcl_Encoding encoding;
 } FsPath;
 
 /*
@@ -646,9 +647,18 @@ TclPathPart(
 		     * the tail.
 		     */
 
+		    /*
+		     * We don't know that the encoding of the fileName is the
+		     * same as the encoding of, but that's on the caller to
+		     * harmonize.  In any case, if the encoding doesn't match
+		     * at the time it the FsPath object is used, the object is
+		     * ignored.
+		     */
+
 		    Tcl_Obj *resultPtr =
-			    TclNewFSPathObj(fsPathPtr->cwdPtr, fileName,
-			    (int)(length - strlen(extension)));
+			TclNewFSPathObj(fsPathPtr->cwdPtr, fileName,
+			    (int)(length - strlen(extension)),
+			    TclFSPathEncoding(interp, fsPathPtr->cwdPtr));
 
 		    Tcl_IncrRefCount(resultPtr);
 		    return resultPtr;
@@ -861,6 +871,7 @@ TclJoinPath(
 		&& !((elt->bytes != NULL) && (elt->bytes[0] == '\0'))
 		&& TclGetPathType(elt, NULL, NULL, NULL) == TCL_PATH_ABSOLUTE) {
 	    Tcl_Obj *tailObj = objv[1];
+
 	    Tcl_PathType type;
 
 	    /* if forceRelative - second path is relative */
@@ -902,14 +913,17 @@ TclJoinPath(
 			    || (strchr(Tcl_GetString(elt), '\\') == NULL)) {
 
 			if (PATHFLAGS(elt)) {
-			    return TclNewFSPathObj(elt, str, len);
+			    return TclNewFSPathObj(elt, str, len,
+				TclFSPathEncoding(NULL, elt));
 			}
 			if (TCL_PATH_ABSOLUTE != Tcl_FSGetPathType(elt)) {
-			    return TclNewFSPathObj(elt, str, len);
+			    return TclNewFSPathObj(
+				elt, str, len, TclFSPathEncoding(NULL, elt));
 			}
 			(void) Tcl_FSGetNormalizedPath(NULL, elt);
 			if (elt == PATHOBJ(elt)->normPathPtr) {
-			    return TclNewFSPathObj(elt, str, len);
+			    return TclNewFSPathObj(
+				elt, str, len, TclFSPathEncoding(NULL, elt));
 			}
 		    }
 		}
@@ -1241,7 +1255,9 @@ Tcl_Obj *
 TclNewFSPathObj(
     Tcl_Obj *dirPtr,
     const char *addStrRep,
-    int len)
+    int len,
+    Tcl_Encoding encoding
+)
 {
     FsPath *fsPathPtr;
     Tcl_Obj *pathPtr;
@@ -1287,6 +1303,7 @@ TclNewFSPathObj(
     fsPathPtr->nativePathPtr = NULL;
     fsPathPtr->fsPtr = NULL;
     fsPathPtr->filesystemEpoch = 0;
+    fsPathPtr->encoding = encoding;
 
     SETPATHOBJ(pathPtr, fsPathPtr);
     PATHFLAGS(pathPtr) = TCLPATH_APPENDED;
@@ -2118,6 +2135,39 @@ TclFSSetPathDetails(
     srcFsPathPtr->filesystemEpoch = TclFSEpoch();
 }
 
+/*
+ *---------------------------------------------------------------------------
+ *
+ * TclFSPathEncoding --
+ *
+ *	Produce the encoding, if any, associated with a filesystem path.
+ *
+ * Results:
+ *	NULL or a valid Tcl_Encoding.
+ *
+ * Side effects:
+ *	None.
+ *
+ *---------------------------------------------------------------------------
+ */
+
+Tcl_Encoding
+TclFSPathEncoding(
+    Tcl_Interp *interp,
+    Tcl_Obj *pathPtr)
+{
+    FsPath *fsPathPtr;
+    Tcl_Encoding encoding = NULL;
+
+    if (Tcl_FSConvertToPathType(interp, pathPtr) != TCL_OK) {
+	return NULL;
+    }
+    if (TclFetchIntRep(pathPtr, &fsPathType)) {
+	fsPathPtr = PATHOBJ(pathPtr);
+	encoding = fsPathPtr->encoding;
+    }
+    return encoding;
+}
 /*
  *---------------------------------------------------------------------------
  *
@@ -2206,9 +2256,14 @@ SetFsPathFromAny(
     Tcl_Obj *transPtr;
     const char *name;
 
-    if (TclHasIntRep(pathPtr, &fsPathType)) {
+    Tcl_Encoding sysencoding = Tcl_GetEncoding(interp, NULL);
+
+    if (TclHasIntRep(pathPtr, &fsPathType)
+	&& TclFSPathEncoding(interp, pathPtr) == sysencoding) {
+	Tcl_FreeEncoding(sysencoding);
 	return TCL_OK;
     }
+    Tcl_FreeEncoding(sysencoding);
 
     /*
      * First step is to translate the filename. This is similar to

tests/fileSystemEncoding.test

@@ -2,10 +2,6 @@
 
 # Copyright (c) 2019 Poor Yorick
 
-if {[string equal $::tcl_platform(os) "Windows NT"]} {
-    return
-}
-
 namespace eval ::tcl::test::fileSystemEncoding {
     package require tcltest 2
     namespace import ::tcltest::*
@@ -39,7 +35,7 @@ namespace eval ::tcl::test::fileSystemEncoding {
 	set globbed [lindex [glob -directory $dir *] 0]
 	encoding system utf-8
 	lappend res [file exists $globbed]
-	encoding system iso8859-1
+	encoding system iso8859-1 
 	lappend res [file exists $globbed]
 	return $res
     } -cleanup {

unix/tclUnixFile.c

@@ -267,6 +267,7 @@ TclpMatchInDirectory(
 	Tcl_StatBuf statBuf;
 	Tcl_DString ds;		/* native encoding of dir */
 	Tcl_DString dsOrig;	/* utf-8 encoding of dir */
+	Tcl_Encoding encoding = Tcl_GetEncoding(interp ,NULL);
 
 	Tcl_DStringInit(&dsOrig);
 	dirName = TclGetString(fileNamePtr);
@@ -364,7 +365,7 @@ TclpMatchInDirectory(
 	     * and pattern. If so, add the file to the result.
 	     */
 
-	    utfname = Tcl_ExternalToUtfDString(NULL, entryPtr->d_name, -1,
+	    utfname = Tcl_ExternalToUtfDString(encoding, entryPtr->d_name, -1,
 		    &utfDs);
 	    if (Tcl_StringCaseMatch(utfname, pattern, 0)) {
 		int typeOk = 1;
@@ -379,7 +380,7 @@ TclpMatchInDirectory(
 		if (typeOk) {
 		    Tcl_ListObjAppendElement(interp, resultPtr,
 			    TclNewFSPathObj(pathPtr, utfname,
-			    Tcl_DStringLength(&utfDs)));
+			    Tcl_DStringLength(&utfDs), encoding));
 		}
 	    }
 	    Tcl_DStringFree(&utfDs);
@@ -388,6 +389,7 @@ TclpMatchInDirectory(
 	    }
 	}
 
+	Tcl_FreeEncoding(encoding);
 	TclOSclosedir(d);
 	Tcl_DStringFree(&ds);
 	Tcl_DStringFree(&dsOrig);

win/tclWinFile.c

@@ -964,6 +964,7 @@ TclpMatchInDirectory(
 	Tcl_DString dsOrig;	/* UTF-8 encoding of dir. */
 	Tcl_Obj *fileNamePtr;
 	char lastChar;
+    Tcl_Encoding encoding;
 
 	/*
 	 * Get the normalized path representation (the main thing is we dont
@@ -1085,6 +1086,7 @@ TclpMatchInDirectory(
 	} else {
 	    matchSpecialDots = 0;
 	}
+	encoding = Tcl_GetEncoding(interp ,NULL);
 
 	/*
 	 * Now iterate over all of the files in the directory, starting with
@@ -1150,10 +1152,12 @@ TclpMatchInDirectory(
 		if (NativeMatchType(isDrive, attr, native, types)) {
 		    Tcl_ListObjAppendElement(interp, resultPtr,
 			    TclNewFSPathObj(pathPtr, utfname,
-				    Tcl_DStringLength(&ds)));
+				    Tcl_DStringLength(&ds), encoding));
 		}
 	    }
 
+	    Tcl_FreeEncoding(encoding);
+
 	    /*
 	     * Free ds here to ensure that native is valid above.
 	     */
@@ -1161,6 +1165,7 @@ TclpMatchInDirectory(
 	    Tcl_DStringFree(&ds);
 	} while (FindNextFileW(handle, &data) == TRUE);
 
+	Tcl_FreeEncoding(encoding);
 	FindClose(handle);
 	Tcl_DStringFree(&dsOrig);
 	return TCL_OK;