A better fix for Valgrind "still reachable" report in TestcmdtokenCmd().

The change in line 8286 is necessary for both blocking and non-blocking mode: Whenver the encoding change we need to reset the CHANNEL_ENCODING_ERROR flag.

doc/Tcl.n

@@ -1,6 +1,7 @@
 '\"
 '\" Copyright (c) 1993 The Regents of the University of California.
 '\" Copyright (c) 1994-1996 Sun Microsystems, Inc.
+'\" Copyright (c) 2023 Nathan Coulter
 '\"
 '\" See the file "license.terms" for information on usage and redistribution
 '\" of this file, and for a DISCLAIMER OF ALL WARRANTIES.
@@ -16,176 +17,152 @@ Summary of Tcl language syntax.
 .SH DESCRIPTION
 .PP
 The following rules define the syntax and semantics of the Tcl language:
-.IP "[1] \fBCommands.\fR"
-A Tcl script is a string containing one or more commands.
-Semi-colons and newlines are command separators unless quoted as
-described below.
-Close brackets are command terminators during command substitution
-(see below) unless quoted.
-.IP "[2] \fBEvaluation.\fR"
-A command is evaluated in two steps.
-First, the Tcl interpreter breaks the command into \fIwords\fR
-and performs substitutions as described below.
-These substitutions are performed in the same way for all
-commands.
-Secondly, the first word is used to locate a routine to
-carry out the command, and the remaining words of the command are
-passed to that routine.
-The routine is free to interpret each of its words
-in any way it likes, such as an integer, variable name, list,
-or Tcl script.
-Different commands interpret their words differently.
-.IP "[3] \fBWords.\fR"
-Words of a command are separated by white space (except for
-newlines, which are command separators).
-.IP "[4] \fBDouble quotes.\fR"
-If the first character of a word is double-quote
-.PQ \N'34'
-then the word is terminated by the next double-quote character.
-If semi-colons, close brackets, or white space characters
-(including newlines) appear between the quotes then they are treated
-as ordinary characters and included in the word.
-Command substitution, variable substitution, and backslash substitution
-are performed on the characters between the quotes as described below.
-The double-quotes are not retained as part of the word.
-.IP "[5] \fBArgument expansion.\fR"
-If a word starts with the string
-.QW {*}
-followed by a non-whitespace character, then the leading
-.QW {*}
-is removed and the rest of the word is parsed and substituted as any other
-word. After substitution, the word is parsed as a list (without command or
-variable substitutions; backslash substitutions are performed as is normal for
-a list and individual internal words may be surrounded by either braces or
-double-quote characters), and its words are added to the command being
-substituted. For instance,
-.QW "cmd a {*}{b [c]} d {*}{$e f {g h}}"
-is equivalent to
-.QW "cmd a b {[c]} d {$e} f {g h}" .
-.IP "[6] \fBBraces.\fR"
-If the first character of a word is an open brace
+.
+.IP "[1] \fBScript.\fR"
+A script is composed of zero or more commands delimited by semi-colons or
+newlines.
+.IP "[2] \fBCommand.\fR"
+A command is composed of zero or more words delimited by whitespace.  The
+replacement for a substitution is included verbatim in the word. For example, a
+space in the replacement is included in the word rather than becoming a
+delimiter, and \fI\\\\\fR becomes a single backslash in the word.  Each word is
+processed from left to right and each substitution is performed as soon as it
+is complete.
+For example, the command
+.RS
+.PP
+.CS
+set y [set x 0][incr x][incr x]
+.CE
+.PP
+is composed of three words, and sets the value of \fIy\fR to \fI012\fR.
+.PP
+If hash
+.PQ #
+is the first character of what would otherwise be the first word of a command,
+all characters up to the next newline are ignored.
+.RE
+.
+.IP "[3] \fBBraced word.\fR"
+If a word is enclosed in braces
 .PQ {
-and rule [5] does not apply, then
-the word is terminated by the matching close brace
-.PQ } "" .
-Braces nest within the word: for each additional open
-brace there must be an additional close brace (however,
-if an open brace or close brace within the word is
-quoted with a backslash then it is not counted in locating the
-matching close brace).
-No substitutions are performed on the characters between the
-braces except for backslash-newline substitutions described
-below, nor do semi-colons, newlines, close brackets,
-or white space receive any special interpretation.
-The word will consist of exactly the characters between the
-outer braces, not including the braces themselves.
-.IP "[7] \fBCommand substitution.\fR"
-If a word contains an open bracket
+and
+.PQ } ""
+, the braces are removed and the enclosed characters become the word.  No
+substitutions are performed.  Nested pairs of braces may occur within the word.
+A brace preceded by an odd number of backslashes is not considered part of a
+pair, and neither brace nor the backslashes are removed from the word.
+.
+.IP "[4] \fBQuoted word.\fR"
+If a word is enclosed in double quotes
+.PQ \N'34'
+, the double quotes are removed and the enclosed characters become the word.
+Substitutions are performed.
+.
+.IP "[5] \fBList.\fR"
+A list has the form of a single command.  Newline is whitespace, and semicolon
+has no special interpretation.  There is no script evaluation so there is no
+argument expansion, variable substitution, or command substitution: Dollar-sign
+and open bracket have no special interpretation, and what would be argument
+expansion in a script is invalid in a list.
+.
+.IP "[6] \fBArgument expansion.\fR"
+If
+.QW {*}
+prefixes a word, it is removed.  After any remaining enclosing braces or quotes
+are processed and applicable substitutions performed, the word, which must
+be a list, is removed from the command, and in its place each word in the
+list becomes an additional word in the command.  For example,
+.CS
+cmd a {*}{b [c]} d {*}{$e f {g h}}
+.CE
+is equivalent to
+.CS
+cmd a b {[c]} d {$e} f {g h} .
+.CE
+.
+.IP "[7] \fBEvaluation.\fR"
+To evaluate a script, an interpreter evaluates each successive command.  The
+first word identifies a procedure, and the remaining words are passed to that
+procedure for further evaluation.  The procedure interprets each argument in
+its own way, e.g. as an integer, variable name, list, mathematical expression,
+script, or in some other arbitrary way.  The result of the last command is the
+result of the script.
+.
+.IP "[8] \fBCommand substitution.\fR"
+Each pair of brackets
 .PQ [
-then Tcl performs \fIcommand substitution\fR.
-To do this it invokes the Tcl interpreter recursively to process
-the characters following the open bracket as a Tcl script.
-The script may contain any number of commands and must be terminated
-by a close bracket
-.PQ ] "" .
-The result of the script (i.e. the result of its last command) is
-substituted into the word in place of the brackets and all of the
-characters between them.
-There may be any number of command substitutions in a single word.
-Command substitution is not performed on words enclosed in braces.
-.IP "[8] \fBVariable substitution.\fR"
-If a word contains a dollar-sign
+and
+.PQ ] ""
+encloses a script and is replaced by the result of that script.
+.IP "[9] \fBVariable substitution.\fR"
+Each of the following forms begins with dollar sign
 .PQ $
-followed by one of the forms
-described below, then Tcl performs \fIvariable
-substitution\fR:  the dollar-sign and the following characters are
-replaced in the word by the value of a variable.
-Variable substitution may take any of the following forms:
+and is replaced by the value of the identified variable.  \fIname\fR names the
+variable and is composed of ASCII letters (\fBA\fR\(en\fBZ\fR and
+\fBa\fR\(en\fBz\fR), digits (\fB0\fR\(en\fB9\fR), underscores, or namespace
+delimiters (two or more colons).  \fIindex\fR is the name of an individual
+variable within an array variable, and may be empty.
 .RS
 .TP 15
 \fB$\fIname\fR
 .
-\fIName\fR is the name of a scalar variable;  the name is a sequence
-of one or more characters that are a letter, digit, underscore,
-or namespace separators (two or more colons).
-Letters and digits are \fIonly\fR the standard ASCII ones (\fB0\fR\(en\fB9\fR,
-\fBA\fR\(en\fBZ\fR and \fBa\fR\(en\fBz\fR).
+\fIname\fR may not be empty.
+
 .TP 15
 \fB$\fIname\fB(\fIindex\fB)\fR
 .
-\fIName\fR gives the name of an array variable and \fIindex\fR gives
-the name of an element within that array.
-\fIName\fR must contain only letters, digits, underscores, and
-namespace separators, and may be an empty string.
-Letters and digits are \fIonly\fR the standard ASCII ones (\fB0\fR\(en\fB9\fR,
-\fBA\fR\(en\fBZ\fR and \fBa\fR\(en\fBz\fR).
-Command substitutions, variable substitutions, and backslash
-substitutions are performed on the characters of \fIindex\fR.
+\fIname\fR may be empty.  Substitutions are performed on \fIindex\fR.
 .TP 15
 \fB${\fIname\fB}\fR
+\fIname\fR may be empty.
+.TP 15
+\fB${\fIname(index)\fB}\fR
 .
-\fIName\fR is the name of a scalar variable or array element.  It may contain
-any characters whatsoever except for close braces.  It indicates an array
-element if \fIname\fR is in the form
-.QW \fIarrayName\fB(\fIindex\fB)\fR
-where \fIarrayName\fR does not contain any open parenthesis characters,
-.QW \fB(\fR ,
-or close brace characters,
-.QW \fB}\fR ,
-and \fIindex\fR can be any sequence of characters except for close brace
-characters.  No further
-substitutions are performed during the parsing of \fIname\fR.
-.PP
-There may be any number of variable substitutions in a single word.
-Variable substitution is not performed on words enclosed in braces.
-.PP
-Note that variables may contain character sequences other than those listed
-above, but in that case other mechanisms must be used to access them (e.g.,
-via the \fBset\fR command's single-argument form).
+\fIname\fR may be empty. No substitutions are performed.
 .RE
-.IP "[9] \fBBackslash substitution.\fR"
-If a backslash
+Variables that are not accessible through one of the forms above may be
+accessed through other mechanisms, e.g. the \fBset\fR command.
+.IP "[10] \fBBackslash substitution.\fR"
+Each backslash
 .PQ \e
-appears within a word then \fIbackslash substitution\fR occurs.
-In all cases but those described below the backslash is dropped and
-the following character is treated as an ordinary
-character and included in the word.
-This allows characters such as double quotes, close brackets,
-and dollar signs to be included in words without triggering
-special processing.
-The following table lists the backslash sequences that are
-handled specially, along with the value that replaces each sequence.
+that is not part of one of the forms listed below is removed, and the next
+character is included in the word verbatim, which allows the inclusion of
+characters that would normally be interpreted, namely whitespace, braces,
+brackets, double quote, dollar sign, and backslash.  The following sequences
+are replaced as described:
+.RS
+.RS
 .RS
 .TP 7
 \e\fBa\fR
-Audible alert (bell) (Unicode U+000007).
+Audible alert (bell) (U+7).
 .TP 7
 \e\fBb\fR
-Backspace (Unicode U+000008).
+Backspace (U+8).
 .TP 7
 \e\fBf\fR
-Form feed (Unicode U+00000C).
+Form feed (U+C).
 .TP 7
 \e\fBn\fR
-Newline (Unicode U+00000A).
+Newline (U+A).
 .TP 7
 \e\fBr\fR
-Carriage-return (Unicode U+00000D).
+Carriage-return (U+D).
 .TP 7
 \e\fBt\fR
-Tab (Unicode U+000009).
+Tab (U+9).
 .TP 7
 \e\fBv\fR
-Vertical tab (Unicode U+00000B).
+Vertical tab (U+B).
 .TP 7
 \e\fB<newline>\fIwhiteSpace\fR
 .
-A single space character replaces the backslash, newline, and all spaces
-and tabs after the newline.  This backslash sequence is unique in that it
-is replaced in a separate pre-pass before the command is actually parsed.
-This means that it will be replaced even when it occurs between braces,
-and the resulting space will be treated as a word separator if it is not
-in braces or quotes.
+Newline preceded by an odd number of backslashes, along with the consecutive
+spaces and tabs that immediately follow it, is replaced by a single space.
+Because this happens before the command is split into words, it occurs even
+within braced words, and if the resulting space may subsequently be treated as
+a word delimiter.
 .TP 7
 \e\e
 Backslash
@@ -193,83 +170,30 @@ Backslash
 .TP 7
 \e\fIooo\fR
 .
-The digits \fIooo\fR (one, two, or three of them) give a eight-bit octal
-value for the Unicode character that will be inserted, in the range
-\fI000\fR\(en\fI377\fR (i.e., the range U+000000\(enU+0000FF).
-The parser will stop just before this range overflows, or when
-the maximum of three digits is reached.  The upper bits of the Unicode
-character will be 0.
+Up to three octal digits form an eight-bit value for a Unicode character in the
+range \fI0\fR\(en\fI377\fR, i.e. U+0\(enU+FF.  Only the digits that result in a
+number in this range are consumed.
 .TP 7
 \e\fBx\fIhh\fR
 .
-The hexadecimal digits \fIhh\fR (one or two of them) give an eight-bit
-hexadecimal value for the Unicode character that will be inserted.  The upper
-bits of the Unicode character will be 0 (i.e., the character will be in the
-range U+000000\(enU+0000FF).
+Up to two hexadecimal digits form an eight-bit value for a Unicode character in
+the range \fI0\fR\(en\fIFF\fR.
 .TP 7
 \e\fBu\fIhhhh\fR
 .
-The hexadecimal digits \fIhhhh\fR (one, two, three, or four of them) give a
-sixteen-bit hexadecimal value for the Unicode character that will be
-inserted.  The upper bits of the Unicode character will be 0 (i.e., the
-character will be in the range U+000000\(enU+00FFFF).
+Up to four hexadecimal digits form a 16-bit value for a Unicode character in
+the range \fI0\fR\(en\fIFFFF\fR.
 .TP 7
 \e\fBU\fIhhhhhhhh\fR
 .
-The hexadecimal digits \fIhhhhhhhh\fR (one up to eight of them) give a
-twenty-one-bit hexadecimal value for the Unicode character that will be
-inserted, in the range U+000000\(enU+10FFFF.  The parser will stop just
-before this range overflows, or when the maximum of eight digits
-is reached.  The upper bits of the Unicode character will be 0.
-.RS
-.PP
-The range U+00D800\(enU+00DFFF is reserved for surrogates, which
-are illegal on their own. Therefore, such sequences will result in
-the replacement character U+FFFD. Surrogate pairs should be
-encoded as single \e\fBU\fIhhhhhhhh\fR character.
+Up to eight hexadecimal digits form a 21-bit value for a Unicode character in
+the range \fI0\fR\(en\fI10FFFF\fR.  Only the digits that result in a number in
+this range are consumed.
+.RE
 .RE
 .PP
-Backslash substitution is not performed on words enclosed in braces,
-except for backslash-newline as described above.
 .RE
-.IP "[10] \fBComments.\fR"
-If a hash character
-.PQ #
-appears at a point where Tcl is
-expecting the first character of the first word of a command,
-then the hash character and the characters that follow it, up
-through the next newline, are treated as a comment and ignored.
-The comment character only has significance when it appears
-at the beginning of a command.
-.IP "[11] \fBOrder of substitution.\fR"
-Each character is processed exactly once by the Tcl interpreter
-as part of creating the words of a command.
-For example, if variable substitution occurs then no further
-substitutions are performed on the value of the variable;  the
-value is inserted into the word verbatim.
-If command substitution occurs then the nested command is
-processed entirely by the recursive call to the Tcl interpreter;
-no substitutions are performed before making the recursive
-call and no additional substitutions are performed on the result
-of the nested script.
-.RS
-.PP
-Substitutions take place from left to right, and each substitution is
-evaluated completely before attempting to evaluate the next.  Thus, a
-sequence like
-.PP
-.CS
-set y [set x 0][incr x][incr x]
-.CE
-.PP
-will always set the variable \fIy\fR to the value, \fI012\fR.
-.RE
-.IP "[12] \fBSubstitution and word boundaries.\fR"
-Substitutions do not affect the word boundaries of a command,
-except for argument expansion as specified in rule [5].
-For example, during variable substitution the entire value of
-the variable becomes part of a single word, even if the variable's
-value contains spaces.
+.
 .SH KEYWORDS
 backslash, command, comment, script, substitution, variable
 '\" Local Variables:

doc/encoding.n

@@ -39,9 +39,9 @@ system encoding is used.
 .VS "TCL8.7 TIP346, TIP607, TIP601"
 .PP
 .RS
-The command does not fail on encoding errors.  Instead, any not convertable bytes
-(like incomplete UTF-8  sequences, see example below) are put as byte values into
-the output stream.
+The command does not fail on encoding errors (unless \fB-strict\fR is specified).
+Instead, any not convertable bytes (like incomplete UTF-8  sequences, see example
+below) are put as byte values into the output stream.
 .PP
 If the option \fB-failindex\fR with a variable name is given, the error reporting
 is changed in the following manner:
@@ -51,9 +51,8 @@ converted characters until the first error position.
 In case of no error, the value \fI-1\fR is written to the variable.  This option
 may not be used together with \fB-nocomplain\fR.
 .PP
-The option \fB-nocomplain\fR has no effect and is available for compatibility with
-TCL 9. In TCL 9, the encoding command fails with an error on any encoding issue.
-This switch restores the TCL8.7 behaviour.
+The option \fB-nocomplain\fR has no effect, but assures to get the same result
+in Tcl 9.
 .PP
 The \fB-strict\fR option follows more strict rules in conversion.  For the \fButf-8\fR
 encoder, it disallows invalid byte sequences and surrogates (which -
@@ -74,9 +73,9 @@ specified, the current system encoding is used.
 .VS "TCL8.7 TIP346, TIP607, TIP601"
 .PP
 .RS
-The command does not fail on encoding errors.  Instead, the replacement character
-\fB?\fR is output for any not representable character (like the dot \fB\\U2022\fR
-in \fBiso-8859-1\fR encoding, see example below).
+The command does not fail on encoding errors (unless \fB-strict\fR is specified).
+Instead, the replacement character \fB?\fR is output for any not representable
+character (like the dot \fB\\U2022\fR in \fBiso-8859-1\fR encoding, see example below).
 .PP
 If the option \fB-failindex\fR with a variable name is given, the error reporting
 is changed in the following manner:
@@ -86,9 +85,8 @@ converted bytes until the first error position. No error condition is raised.
 In case of no error, the value \fI-1\fR is written to the variable.  This option
 may not be used together with \fB-nocomplain\fR.
 .PP
-The option \fB-nocomplain\fR has no effect and is available for compatibility with
-TCL 9. In TCL 9, the encoding command fails with an error on any encoding issue.
-This switch restores the TCL8.7 behaviour.
+The option \fB-nocomplain\fR has no effect, but assures to get the same result
+in Tcl 9.
 .PP
 The \fB-strict\fR option follows more strict rules in conversion.  For the \fButf-8\fR
 encoder, it disallows surrogates (which - otherwise - are just passed through).  This
@@ -157,7 +155,7 @@ Example 4: detect the error location while transforming to ISO8859-1
 (ISO-Latin 1):
 .PP
 .CS
-% set s [\fBencoding convertto\fR -failindex i utf-8 "A\eu0141"]
+% set s [\fBencoding convertto\fR -failindex i iso8859-1 "A\eu0141"]
 A
 % set i
 1
@@ -166,11 +164,11 @@ A
 Example 5: replace a not representable character by the replacement character:
 .PP
 .CS
-% set s [\fBencoding convertto\fR -nocomplain utf-8 "A\eu0141"]
+% set s [\fBencoding convertto\fR -nocomplain iso8859-1 "A\eu0141"]
 A?
 .CE
 The option \fB-nocomplain\fR has no effect, but assures to get the same result
-with TCL9.
+in Tcl 9.
 .VE "TCL8.7 TIP346, TIP607, TIP601"
 .PP
 .SH "SEE ALSO"

doc/fcopy.n

@@ -26,8 +26,9 @@ network sockets.
 The \fBfcopy\fR
 command transfers data from \fIinchan\fR until end of file
 or \fIsize\fR bytes or characters have been
-transferred; \fIsize\fR is in bytes if the two channels are using the
-same encoding, and is in characters otherwise.
+transferred; \fIsize\fR is in bytes if the input channel is in binary mode,
+or if the two channels are using the same encoding and -strict is not specified.
+Otherwise, size is in characters.
 If no \fB\-size\fR argument is given,
 then the copy goes until end of file.
 All the data read from \fIinchan\fR is copied to \fIoutchan\fR.

generic/tclBasic.c

@@ -9716,6 +9716,7 @@ TclNRYieldToObjCmd(
      */
 
     iPtr->execEnvPtr = corPtr->callerEEPtr;
+    /* Not calling Tcl_IncrRefCount(listPtr) here because listPtr is private */
     TclSetTailcall(interp, listPtr);
     corPtr->yieldPtr = listPtr;
     iPtr->execEnvPtr = corPtr->eePtr;
@@ -9918,8 +9919,8 @@ TclNRCoroutineActivateCallback(
 	    if (corPtr->yieldPtr) {
 		for (runPtr = TOP_CB(interp); runPtr; runPtr = runPtr->nextPtr) {
 		    if (runPtr->data[1] == corPtr->yieldPtr) {
+			Tcl_DecrRefCount((Tcl_Obj *)runPtr->data[1]);
 			runPtr->data[1] = NULL;
-			Tcl_DecrRefCount(corPtr->yieldPtr);
 			corPtr->yieldPtr = NULL;
 			break;
 		    }
@@ -10520,6 +10521,7 @@ TclNRCoroutineObjCmd(
     corPtr->running.lineLABCPtr = corPtr->lineLABCPtr;
     corPtr->stackLevel = NULL;
     corPtr->auxNumLevels = 0;
+    corPtr->yieldPtr = NULL;
 
     /*
      * Create the coro's execEnv, switch to it to push the exit and coro

generic/tclCmdAH.c

@@ -732,7 +732,7 @@ EncodingConverttoObjCmd(
     if (objc == 2) {
 	encoding = Tcl_GetEncoding(interp, NULL);
 	data = objv[1];
-    } else if (objc > 2 && objc < 6) {
+    } else if (objc > 2 && objc < 7) {
 	int objcUnprocessed = objc;
 	data = objv[objc - 1];
 	stringPtr = Tcl_GetString(objv[1]);

generic/tclEncoding.c

@@ -36,7 +36,7 @@ typedef struct {
 				 * encoding is deleted. */
     void *clientData;	/* Arbitrary value associated with encoding
 				 * type. Passed to conversion functions. */
-    int nullSize;		/* Number of 0x00 bytes that signify
+    Tcl_Size nullSize;	/* Number of 0x00 bytes that signify
 				 * end-of-string in this encoding. This number
 				 * is used to determine the source string
 				 * length when the srcLen argument is
@@ -237,8 +237,13 @@ static Tcl_EncodingConvertProc	Iso88591ToUtfProc;
  */
 
 static const Tcl_ObjType encodingType = {
-    "encoding", FreeEncodingInternalRep, DupEncodingInternalRep, NULL, NULL
+    "encoding",
+    FreeEncodingInternalRep,
+    DupEncodingInternalRep,
+    NULL,
+    NULL
 };
+
 #define EncodingSetInternalRep(objPtr, encoding)				\
     do {								\
 	Tcl_ObjInternalRep ir;						\
@@ -369,7 +374,7 @@ int
 Tcl_SetEncodingSearchPath(
     Tcl_Obj *searchPath)
 {
-    int dummy;
+    Tcl_Size dummy;
 
     if (TCL_ERROR == TclListObjLengthM(NULL, searchPath, &dummy)) {
 	return TCL_ERROR;
@@ -416,7 +421,7 @@ void
 TclSetLibraryPath(
     Tcl_Obj *path)
 {
-    int dummy;
+    Tcl_Size dummy;
 
     if (TCL_ERROR == TclListObjLengthM(NULL, path, &dummy)) {
 	return;
@@ -452,7 +457,7 @@ TclSetLibraryPath(
 static void
 FillEncodingFileMap(void)
 {
-    int i, numDirs = 0;
+    Tcl_Size i, numDirs = 0;
     Tcl_Obj *map, *searchPath;
 
     searchPath = Tcl_GetEncodingSearchPath();
@@ -461,13 +466,13 @@ FillEncodingFileMap(void)
     map = Tcl_NewDictObj();
     Tcl_IncrRefCount(map);
 
-    for (i = numDirs-1; i >= 0; i--) {
+    for (i = numDirs-1; i != TCL_INDEX_NONE; i--) {
 	/*
 	 * Iterate backwards through the search path so as we overwrite
 	 * entries found, we favor files earlier on the search path.
 	 */
 
-	int j, numFiles;
+	Tcl_Size j, numFiles;
 	Tcl_Obj *directory, *matchFileList;
 	Tcl_Obj **filev;
 	Tcl_GlobTypeData readableFiles = {
@@ -519,7 +524,8 @@ FillEncodingFileMap(void)
 /* Since TCL_ENCODING_MODIFIED is only used for utf-8/cesu-8 and
  * TCL_ENCODING_LE is only used for  utf-16/utf-32/ucs-2. re-use the same value */
 #define TCL_ENCODING_LE		TCL_ENCODING_MODIFIED	/* Little-endian encoding */
-#define TCL_ENCODING_UTF	0x200	/* For UTF-8 encoding, allow 4-byte output sequences */
+#define ENCODING_UTF	0x200	/* For UTF-8 encoding, allow 4-byte output sequences */
+#define ENCODING_INPUT	0x400 /* For UTF-8/CESU-8 encoding, means external -> internal */
 
 void
 TclInitEncodingSubsystem(void)
@@ -561,9 +567,9 @@ TclInitEncodingSubsystem(void)
     type.fromUtfProc	= UtfToUtfProc;
     type.freeProc	= NULL;
     type.nullSize	= 1;
-    type.clientData	= INT2PTR(TCL_ENCODING_UTF);
+    type.clientData	= INT2PTR(ENCODING_UTF);
     Tcl_CreateEncoding(&type);
-    type.clientData	= INT2PTR(TCL_ENCODING_NOCOMPLAIN);
+    type.clientData	= INT2PTR(0);
     type.encodingName	= "cesu-8";
     Tcl_CreateEncoding(&type);
 
@@ -572,13 +578,13 @@ TclInitEncodingSubsystem(void)
     type.freeProc	= NULL;
     type.nullSize	= 2;
     type.encodingName   = "ucs-2le";
-    type.clientData	= INT2PTR(TCL_ENCODING_LE|TCL_ENCODING_NOCOMPLAIN);
+    type.clientData	= INT2PTR(TCL_ENCODING_LE);
     Tcl_CreateEncoding(&type);
     type.encodingName   = "ucs-2be";
-    type.clientData	= INT2PTR(TCL_ENCODING_NOCOMPLAIN);
+    type.clientData	= INT2PTR(0);
     Tcl_CreateEncoding(&type);
     type.encodingName   = "ucs-2";
-    type.clientData	= INT2PTR(isLe.c|TCL_ENCODING_NOCOMPLAIN);
+    type.clientData	= INT2PTR(isLe.c);
     Tcl_CreateEncoding(&type);
 
     type.toUtfProc	= Utf32ToUtfProc;
@@ -600,13 +606,13 @@ TclInitEncodingSubsystem(void)
     type.freeProc	= NULL;
     type.nullSize	= 2;
     type.encodingName   = "utf-16le";
-    type.clientData	= INT2PTR(TCL_ENCODING_LE);
+    type.clientData	= INT2PTR(TCL_ENCODING_LE|ENCODING_UTF);
     Tcl_CreateEncoding(&type);
     type.encodingName   = "utf-16be";
-    type.clientData	= INT2PTR(0);
+    type.clientData	= INT2PTR(ENCODING_UTF);
     Tcl_CreateEncoding(&type);
     type.encodingName   = "utf-16";
-    type.clientData	= INT2PTR(isLe.c);
+    type.clientData	= INT2PTR(isLe.c|ENCODING_UTF);
     Tcl_CreateEncoding(&type);
 
 #ifndef TCL_NO_DEPRECATED
@@ -999,7 +1005,7 @@ Tcl_GetEncodingNames(
  *
  *---------------------------------------------------------------------------
  */
-int
+Tcl_Size
 Tcl_GetEncodingNulLength(
     Tcl_Encoding encoding)
 {
@@ -1165,7 +1171,7 @@ Tcl_ExternalToUtfDString(
     Tcl_Encoding encoding,	/* The encoding for the source string, or NULL
 				 * for the default system encoding. */
     const char *src,		/* Source string in specified encoding. */
-    int srcLen,			/* Source string length in bytes, or < 0 for
+    Tcl_Size srcLen,		/* Source string length in bytes, or < 0 for
 				 * encoding-specific string length. */
     Tcl_DString *dstPtr)	/* Uninitialized or free DString in which the
 				 * converted string is stored. */
@@ -1181,7 +1187,7 @@ Tcl_ExternalToUtfDString(
  * Tcl_ExternalToUtfDStringEx --
  *
  *	Convert a source buffer from the specified encoding into UTF-8.
-*	The parameter flags controls the behavior, if any of the bytes in
+ *	The parameter flags controls the behavior, if any of the bytes in
  *	the source buffer are invalid or cannot be represented in utf-8.
  *	Possible flags values:
  *	TCL_ENCODING_STOPONERROR: don't replace invalid characters/bytes but
@@ -1204,12 +1210,12 @@ Tcl_ExternalToUtfDString(
  *-------------------------------------------------------------------------
  */
 
-int
+Tcl_Size
 Tcl_ExternalToUtfDStringEx(
     Tcl_Encoding encoding,	/* The encoding for the source string, or NULL
 				 * for the default system encoding. */
     const char *src,		/* Source string in specified encoding. */
-    int srcLen,			/* Source string length in bytes, or < 0 for
+    Tcl_Size srcLen,			/* Source string length in bytes, or < 0 for
 				 * encoding-specific string length. */
     int flags,			/* Conversion control flags. */
     Tcl_DString *dstPtr)	/* Uninitialized or free DString in which the
@@ -1218,7 +1224,8 @@ Tcl_ExternalToUtfDStringEx(
     char *dst;
     Tcl_EncodingState state;
     const Encoding *encodingPtr;
-    int dstLen, result, soFar, srcRead, dstWrote, dstChars;
+    int result, soFar, srcRead, dstWrote, dstChars;
+    Tcl_Size dstLen;
     const char *srcStart = src;
 
     Tcl_DStringInit(dstPtr);
@@ -1238,7 +1245,7 @@ Tcl_ExternalToUtfDStringEx(
 
     flags |= TCL_ENCODING_START | TCL_ENCODING_END;
     if (encodingPtr->toUtfProc == UtfToUtfProc) {
-	flags |= TCL_ENCODING_MODIFIED | TCL_ENCODING_UTF;
+	flags |= ENCODING_INPUT;
     }
 
     while (1) {
@@ -1249,7 +1256,7 @@ Tcl_ExternalToUtfDStringEx(
 	src += srcRead;
 	if (result != TCL_CONVERT_NOSPACE) {
 	    Tcl_DStringSetLength(dstPtr, soFar);
-	    return (result == TCL_OK) ? TCL_INDEX_NONE : (int)(src - srcStart);
+	    return (result == TCL_OK) ? TCL_INDEX_NONE : (Tcl_Size)(src - srcStart);
 	}
 	flags &= ~TCL_ENCODING_START;
 	srcLen -= srcRead;
@@ -1286,7 +1293,7 @@ Tcl_ExternalToUtf(
     Tcl_Encoding encoding,	/* The encoding for the source string, or NULL
 				 * for the default system encoding. */
     const char *src,		/* Source string in specified encoding. */
-    int srcLen,			/* Source string length in bytes, or < 0 for
+    Tcl_Size srcLen,		/* Source string length in bytes, or < 0 for
 				 * encoding-specific string length. */
     int flags,			/* Conversion control flags. */
     Tcl_EncodingState *statePtr,/* Place for conversion routine to store state
@@ -1296,7 +1303,7 @@ Tcl_ExternalToUtf(
 				 * routine under control of flags argument. */
     char *dst,			/* Output buffer in which converted string is
 				 * stored. */
-    int dstLen,			/* The maximum length of output buffer in
+    Tcl_Size dstLen,		/* The maximum length of output buffer in
 				 * bytes. */
     int *srcReadPtr,		/* Filled with the number of bytes from the
 				 * source string that were converted. This may
@@ -1355,7 +1362,7 @@ Tcl_ExternalToUtf(
 	dstLen--;
     }
     if (encodingPtr->toUtfProc == UtfToUtfProc) {
-	flags |= TCL_ENCODING_MODIFIED | TCL_ENCODING_UTF;
+	flags |= ENCODING_INPUT;
     }
     do {
 	Tcl_EncodingState savedState = *statePtr;
@@ -1403,7 +1410,7 @@ Tcl_UtfToExternalDString(
     Tcl_Encoding encoding,	/* The encoding for the converted string, or
 				 * NULL for the default system encoding. */
     const char *src,		/* Source string in UTF-8. */
-    int srcLen,			/* Source string length in bytes, or < 0 for
+    Tcl_Size srcLen,		/* Source string length in bytes, or < 0 for
 				 * strlen(). */
     Tcl_DString *dstPtr)	/* Uninitialized or free DString in which the
 				 * converted string is stored. */
@@ -1443,22 +1450,23 @@ Tcl_UtfToExternalDString(
  *-------------------------------------------------------------------------
  */
 
-int
+Tcl_Size
 Tcl_UtfToExternalDStringEx(
     Tcl_Encoding encoding,	/* The encoding for the converted string, or
 				 * NULL for the default system encoding. */
     const char *src,		/* Source string in UTF-8. */
-    int srcLen,			/* Source string length in bytes, or < 0 for
+    Tcl_Size srcLen,		/* Source string length in bytes, or < 0 for
 				 * strlen(). */
-    int flags,	/* Conversion control flags. */
+    int flags,			/* Conversion control flags. */
     Tcl_DString *dstPtr)	/* Uninitialized or free DString in which the
 				 * converted string is stored. */
 {
     char *dst;
     Tcl_EncodingState state;
     const Encoding *encodingPtr;
-    int dstLen, result, soFar, srcRead, dstWrote, dstChars;
+    int result, soFar, srcRead, dstWrote, dstChars;
     const char *srcStart = src;
+    Tcl_Size dstLen;
 
     Tcl_DStringInit(dstPtr);
     dst = Tcl_DStringValue(dstPtr);
@@ -1487,7 +1495,7 @@ Tcl_UtfToExternalDStringEx(
 	    while (i >= soFar) {
 		Tcl_DStringSetLength(dstPtr, i--);
 	    }
-	    return (result == TCL_OK) ? TCL_INDEX_NONE : (int)(src - srcStart);
+	    return (result == TCL_OK) ? TCL_INDEX_NONE : (Tcl_Size)(src - srcStart);
 	}
 
 	flags &= ~TCL_ENCODING_START;
@@ -1525,7 +1533,7 @@ Tcl_UtfToExternal(
     Tcl_Encoding encoding,	/* The encoding for the converted string, or
 				 * NULL for the default system encoding. */
     const char *src,		/* Source string in UTF-8. */
-    int srcLen,			/* Source string length in bytes, or < 0 for
+    Tcl_Size srcLen,		/* Source string length in bytes, or < 0 for
 				 * strlen(). */
     int flags,			/* Conversion control flags. */
     Tcl_EncodingState *statePtr,/* Place for conversion routine to store state
@@ -1535,7 +1543,7 @@ Tcl_UtfToExternal(
 				 * routine under control of flags argument. */
     char *dst,			/* Output buffer in which converted string
 				 * is stored. */
-    int dstLen,			/* The maximum length of output buffer in
+    Tcl_Size dstLen,		/* The maximum length of output buffer in
 				 * bytes. */
     int *srcReadPtr,		/* Filled with the number of bytes from the
 				 * source string that were converted. This may
@@ -1646,7 +1654,7 @@ OpenEncodingFileChannel(
     Tcl_Obj *map = TclGetProcessGlobalValue(&encodingFileMap);
     Tcl_Obj **dir, *path, *directory = NULL;
     Tcl_Channel chan = NULL;
-    int i, numDirs;
+    Tcl_Size i, numDirs;
 
     TclListObjGetElementsM(NULL, searchPath, &numDirs, &dir);
     Tcl_IncrRefCount(nameObj);
@@ -1911,7 +1919,7 @@ LoadTableEncoding(
     for (i = 0; i < numPages; i++) {
 	int ch;
 	const char *p;
-	int expected = 3 + 16 * (16 * 4 + 1);
+	Tcl_Size expected = 3 + 16 * (16 * 4 + 1);
 
 	if (Tcl_ReadChars(chan, objPtr, expected, 0) != expected) {
 	    return NULL;
@@ -2147,7 +2155,7 @@ LoadEscapeEncoding(
     Tcl_DStringInit(&escapeData);
 
     while (1) {
-	int argc;
+	Tcl_Size argc;
 	const char **argv;
 	char *line;
 	Tcl_DString lineString;
@@ -2302,9 +2310,9 @@ BinaryProc(
  *
  * UtfToUtfProc --
  *
- *	Convert from UTF-8 to UTF-8. Note that the UTF-8 to UTF-8 translation
- *	is not a no-op, because it will turn a stream of improperly formed
- *	UTF-8 into a properly formed stream.
+ *	Converts from UTF-8 to UTF-8. Note that the UTF-8 to UTF-8 translation
+ *	is not a no-op, because it turns a stream of improperly formed
+ *	UTF-8 into a properly-formed stream.
  *
  * Results:
  *	Returns TCL_OK if conversion was successful.
@@ -2363,7 +2371,7 @@ UtfToUtfProc(
 
     dstStart = dst;
     flags |= PTR2INT(clientData);
-    dstEnd = dst + dstLen - ((flags & TCL_ENCODING_UTF) ? TCL_UTF_MAX : 6);
+    dstEnd = dst + dstLen - ((flags & ENCODING_UTF) ? TCL_UTF_MAX : 6);
 
     for (numChars = 0; src < srcEnd && numChars <= charLimit; numChars++) {
 	if ((src > srcClose) && (!Tcl_UtfCharComplete(src, srcEnd - src))) {
@@ -2379,7 +2387,7 @@ UtfToUtfProc(
 	    result = TCL_CONVERT_NOSPACE;
 	    break;
 	}
-	if (UCHAR(*src) < 0x80 && !((UCHAR(*src) == 0) && (flags & TCL_ENCODING_MODIFIED))) {
+	if (UCHAR(*src) < 0x80 && !((UCHAR(*src) == 0) && (flags & ENCODING_INPUT))) {
 	    /*
 	     * Copy 7bit characters, but skip null-bytes when we are in input
 	     * mode, so that they get converted to 0xC080.
@@ -2387,11 +2395,13 @@ UtfToUtfProc(
 
 	    *dst++ = *src++;
 	} else if ((UCHAR(*src) == 0xC0) && (src + 1 < srcEnd)
-		&& (UCHAR(src[1]) == 0x80) && (!(flags & TCL_ENCODING_MODIFIED) || ((flags & TCL_ENCODING_STRICT) == TCL_ENCODING_STRICT) || (flags & ENCODING_FAILINDEX))) {
+		&& (UCHAR(src[1]) == 0x80) && !(flags & TCL_ENCODING_MODIFIED) && (!(flags & ENCODING_INPUT)
+			|| ((flags & TCL_ENCODING_STRICT) == TCL_ENCODING_STRICT)
+			|| (flags & ENCODING_FAILINDEX))) {
 	    /*
 	     * If in input mode, and -strict or -failindex is specified: This is an error.
 	     */
-	    if (flags & TCL_ENCODING_MODIFIED) {
+	    if ((STOPONERROR) && (flags & ENCODING_INPUT)) {
 		result = TCL_CONVERT_SYNTAX;
 		break;
 	    }
@@ -2409,33 +2419,37 @@ UtfToUtfProc(
 	     * unless the user has explicitly asked to be told.
 	     */
 
-	    if (flags & TCL_ENCODING_MODIFIED) {
+	    if (flags & ENCODING_INPUT) {
 		if ((STOPONERROR) && (flags & TCL_ENCODING_CHAR_LIMIT)) {
 		    result = TCL_CONVERT_MULTIBYTE;
 		    break;
 		}
-	    if (((flags & TCL_ENCODING_STRICT) == TCL_ENCODING_STRICT) || (flags & ENCODING_FAILINDEX)) {
-		result = TCL_CONVERT_SYNTAX;
-		break;
-	    }
-		ch = UCHAR(*src++);
-	    } else {
-		char chbuf[2];
-		chbuf[0] = UCHAR(*src++); chbuf[1] = 0;
-		TclUtfToUCS4(chbuf, &ch);
+		if (((flags & TCL_ENCODING_STRICT) == TCL_ENCODING_STRICT) || (flags & ENCODING_FAILINDEX)) {
+		    result = TCL_CONVERT_SYNTAX;
+		    break;
+		}
 	    }
+	    char chbuf[2];
+	    chbuf[0] = UCHAR(*src++); chbuf[1] = 0;
+	    TclUtfToUCS4(chbuf, &ch);
 	    dst += Tcl_UniCharToUtf(ch, dst);
 	} else {
 	    int low;
-	    const char *saveSrc = src;
 	    size_t len = TclUtfToUCS4(src, &ch);
-	    if ((len < 2) && (ch != 0) && (flags & TCL_ENCODING_MODIFIED)
-		    && (((flags & TCL_ENCODING_STRICT) == TCL_ENCODING_STRICT))) {
-		result = TCL_CONVERT_SYNTAX;
-		break;
+	    if (flags & ENCODING_INPUT) {
+		if ((len < 2) && (ch != 0)
+			&& (((flags & TCL_ENCODING_STRICT) == TCL_ENCODING_STRICT) || (flags & ENCODING_FAILINDEX))) {
+		    goto utf8Syntax;
+		} else if ((ch > 0xFFFF) && !(flags & ENCODING_UTF)
+			&& (((flags & TCL_ENCODING_STRICT) == TCL_ENCODING_STRICT) || (flags & ENCODING_FAILINDEX))) {
+		utf8Syntax:
+		    result = TCL_CONVERT_SYNTAX;
+		    break;
+		}
 	    }
+	    const char *saveSrc = src;
 	    src += len;
-	    if (!(flags & TCL_ENCODING_UTF) && (ch > 0x3FF)) {
+	    if (!(flags & ENCODING_UTF) && !(flags & ENCODING_INPUT) && (ch > 0x3FF)) {
 		if (ch > 0xFFFF) {
 		    /* CESU-8 6-byte sequence for chars > U+FFFF */
 		    ch -= 0x10000;
@@ -2450,6 +2464,11 @@ UtfToUtfProc(
 		 * A surrogate character is detected, handle especially.
 		 */
 
+		if (((flags & TCL_ENCODING_STRICT) == TCL_ENCODING_STRICT) && (flags & ENCODING_UTF)) {
+		    result = TCL_CONVERT_UNKNOWN;
+		    src = saveSrc;
+		    break;
+		}
 		low = ch;
 		len = (src <= srcEnd-3) ? TclUtfToUCS4(src, &low) : 0;
 
@@ -2469,12 +2488,12 @@ UtfToUtfProc(
 		src += len;
 		dst += Tcl_UniCharToUtf(ch, dst);
 		ch = low;
-	    } else if (STOPONERROR && !(flags & TCL_ENCODING_MODIFIED) && (((ch  & ~0x7FF) == 0xD800))) {
+	    } else if (STOPONERROR && !(flags & ENCODING_INPUT) && (((ch  & ~0x7FF) == 0xD800))) {
 		result = TCL_CONVERT_UNKNOWN;
 		src = saveSrc;
 		break;
 	    } else if (((flags & TCL_ENCODING_STRICT) == TCL_ENCODING_STRICT)
-		    && (flags & TCL_ENCODING_MODIFIED) && ((ch  & ~0x7FF) == 0xD800)) {
+		    && (flags & ENCODING_INPUT) && ((ch  & ~0x7FF) == 0xD800)) {
 		result = TCL_CONVERT_SYNTAX;
 		src = saveSrc;
 		break;
@@ -2531,7 +2550,7 @@ Utf32ToUtfProc(
     const char *srcStart, *srcEnd;
     const char *dstEnd, *dstStart;
     int result, numChars, charLimit = INT_MAX;
-    int ch;
+    int ch = 0, bytesLeft = srcLen % 4;
 
     flags |= PTR2INT(clientData);
     if (flags & TCL_ENCODING_CHAR_LIMIT) {
@@ -2543,9 +2562,22 @@ Utf32ToUtfProc(
      * Check alignment with utf-32 (4 == sizeof(UTF-32))
      */
 
-    if ((srcLen % 4) != 0) {
+    if (bytesLeft != 0) {
 	result = TCL_CONVERT_MULTIBYTE;
-	srcLen &= -4;
+	srcLen -= bytesLeft;
+    }
+
+    /*
+     * If last code point is a high surrogate, we cannot handle that yet,
+     * unless we are at the end.
+     */
+
+    if (!(flags & TCL_ENCODING_END) && (srcLen >= 4) &&
+	    ((src[srcLen - ((flags & TCL_ENCODING_LE)?3:2)] & 0xFC) == 0xD8) &&
+	    ((src[srcLen - ((flags & TCL_ENCODING_LE)?2:3)]) == 0) &&
+	    ((src[srcLen - ((flags & TCL_ENCODING_LE)?1:4)]) == 0)) {
+	result = TCL_CONVERT_MULTIBYTE;
+	srcLen-= 4;
     }
 
     srcStart = src;
@@ -2560,15 +2592,21 @@ Utf32ToUtfProc(
 	    break;
 	}
 
+	int prev = ch;
 	if (flags & TCL_ENCODING_LE) {
 	    ch = (src[3] & 0xFF) << 24 | (src[2] & 0xFF) << 16 | (src[1] & 0xFF) << 8 | (src[0] & 0xFF);
 	} else {
 	    ch = (src[0] & 0xFF) << 24 | (src[1] & 0xFF) << 16 | (src[2] & 0xFF) << 8 | (src[3] & 0xFF);
 	}
+	if (((prev  & ~0x3FF) == 0xD800) && ((ch  & ~0x3FF) != 0xDC00)) {
+	    /* Bug [10c2c17c32]. If Hi surrogate not followed by Lo surrogate, finish 3-byte UTF-8 */
+	    dst += Tcl_UniCharToUtf(-1, dst);
+	}
 	if  ((unsigned)ch > 0x10FFFF || (((flags & TCL_ENCODING_STRICT) == TCL_ENCODING_STRICT)
 		&& ((ch  & ~0x7FF) == 0xD800))) {
 	    if (STOPONERROR) {
 		result = TCL_CONVERT_SYNTAX;
+		ch = 0;
 		break;
 	    }
 	}
@@ -2586,6 +2624,26 @@ Utf32ToUtfProc(
 	src += sizeof(unsigned int);
     }
 
+    if ((ch  & ~0x3FF) == 0xD800) {
+	/* Bug [10c2c17c32]. If Hi surrogate, finish 3-byte UTF-8 */
+	dst += Tcl_UniCharToUtf(-1, dst);
+    }
+    if ((flags & TCL_ENCODING_END) && (result == TCL_CONVERT_MULTIBYTE)) {
+	/* We have a single byte left-over at the end */
+	if (dst > dstEnd) {
+	    result = TCL_CONVERT_NOSPACE;
+	} else {
+	    /* destination is not full, so we really are at the end now */
+	    if ((flags & TCL_ENCODING_STRICT) == TCL_ENCODING_STRICT) {
+		result = TCL_CONVERT_SYNTAX;
+	    } else {
+		result = TCL_OK;
+		dst += Tcl_UniCharToUtf(0xFFFD, dst);
+		numChars++;
+		src += bytesLeft;
+	    }
+	}
+    }
     *srcReadPtr = src - srcStart;
     *dstWrotePtr = dst - dstStart;
     *dstCharsPtr = numChars;
@@ -2730,7 +2788,7 @@ Utf16ToUtfProc(
     const char *srcStart, *srcEnd;
     const char *dstEnd, *dstStart;
     int result, numChars, charLimit = INT_MAX;
-    unsigned short ch;
+    unsigned short ch = 0;
 
     flags |= PTR2INT(clientData);
     if (flags & TCL_ENCODING_CHAR_LIMIT) {
@@ -2748,10 +2806,11 @@ Utf16ToUtfProc(
     }
 
     /*
-     * If last code point is a high surrogate, we cannot handle that yet.
+     * If last code point is a high surrogate, we cannot handle that yet,
+     * unless we are at the end.
      */
 
-    if ((srcLen >= 2) &&
+    if (!(flags & TCL_ENCODING_END) && (srcLen >= 2) &&
 	    ((src[srcLen - ((flags & TCL_ENCODING_LE)?1:2)] & 0xFC) == 0xD8)) {
 	result = TCL_CONVERT_MULTIBYTE;
 	srcLen-= 2;
@@ -2769,11 +2828,16 @@ Utf16ToUtfProc(
 	    break;
 	}
 
+	unsigned short prev = ch;
 	if (flags & TCL_ENCODING_LE) {
 	    ch = (src[1] & 0xFF) << 8 | (src[0] & 0xFF);
 	} else {
 	    ch = (src[0] & 0xFF) << 8 | (src[1] & 0xFF);
 	}
+	if (((prev  & ~0x3FF) == 0xD800) && ((ch  & ~0x3FF) != 0xDC00)) {
+	    /* Bug [10c2c17c32]. If Hi surrogate not followed by Lo surrogate, finish 3-byte UTF-8 */
+	    dst += Tcl_UniCharToUtf(-1, dst);
+	}
 
 	/*
 	 * Special case for 1-byte utf chars for speed. Make sure we work with
@@ -2788,6 +2852,26 @@ Utf16ToUtfProc(
 	src += sizeof(unsigned short);
     }
 
+    if ((ch  & ~0x3FF) == 0xD800) {
+	/* Bug [10c2c17c32]. If Hi surrogate, finish 3-byte UTF-8 */
+	dst += Tcl_UniCharToUtf(-1, dst);
+    }
+    if ((flags & TCL_ENCODING_END) && (result == TCL_CONVERT_MULTIBYTE)) {
+	/* We have a single byte left-over at the end */
+	if (dst > dstEnd) {
+	    result = TCL_CONVERT_NOSPACE;
+	} else {
+	    /* destination is not full, so we really are at the end now */
+	    if (((flags & TCL_ENCODING_STRICT) == TCL_ENCODING_STRICT)) {
+		result = TCL_CONVERT_SYNTAX;
+	    } else {
+		result = TCL_OK;
+		dst += Tcl_UniCharToUtf(0xFFFD, dst);
+		numChars++;
+		src++;
+	    }
+	}
+    }
     *srcReadPtr = src - srcStart;
     *dstWrotePtr = dst - dstStart;
     *dstCharsPtr = numChars;
@@ -2939,10 +3023,7 @@ UtfToUcs2Proc(
 				 * output buffer. */
 {
     const char *srcStart, *srcEnd, *srcClose, *dstStart, *dstEnd;
-    int result, numChars;
-#if TCL_UTF_MAX < 4
-    int len;
-#endif
+    int result, numChars, len;
     Tcl_UniChar ch = 0;
 
     flags |= PTR2INT(clientData);
@@ -2972,17 +3053,32 @@ UtfToUcs2Proc(
 	    break;
 	}
 #if TCL_UTF_MAX < 4
-	src += (len = TclUtfToUniChar(src, &ch));
+	len = TclUtfToUniChar(src, &ch);
 	if ((ch >= 0xD800) && (len < 3)) {
+	    if (STOPONERROR) {
+		result = TCL_CONVERT_UNKNOWN;
+		break;
+	    }
+	    src += len;
 	    src += TclUtfToUniChar(src, &ch);
 	    ch = 0xFFFD;
 	}
 #else
-	src += TclUtfToUniChar(src, &ch);
+	len = TclUtfToUniChar(src, &ch);
 	if (ch > 0xFFFF) {
+	    if (STOPONERROR) {
+		result = TCL_CONVERT_UNKNOWN;
+		break;
+	    }
 	    ch = 0xFFFD;
 	}
 #endif
+	if (STOPONERROR && ((ch & ~0x7FF) == 0xD800)) {
+	    result = TCL_CONVERT_SYNTAX;
+	    break;
+	}
+
+	src += len;
 
 	/*
 	 * Need to handle this in a way that won't cause misalignment by
@@ -3084,7 +3180,8 @@ TableToUtfProc(
 	    ch = pageZero[byte];
 	}
 	if ((ch == 0) && (byte != 0)) {
-	    if (STOPONERROR) {
+	    if ((flags & ENCODING_FAILINDEX)
+		    || ((flags & TCL_ENCODING_STRICT) == TCL_ENCODING_STRICT)) {
 		result = TCL_CONVERT_SYNTAX;
 		break;
 	    }
@@ -3823,8 +3920,7 @@ EscapeFromUtfProc(
 		    result = TCL_CONVERT_NOSPACE;
 		    break;
 		}
-		memcpy(dst, subTablePtr->sequence,
-			subTablePtr->sequenceLen);
+		memcpy(dst, subTablePtr->sequence, subTablePtr->sequenceLen);
 		dst += subTablePtr->sequenceLen;
 	    }
 	}
@@ -4042,11 +4138,11 @@ unilen4(
 static void
 InitializeEncodingSearchPath(
     char **valuePtr,
-    unsigned int *lengthPtr,
+    TCL_HASH_TYPE *lengthPtr,
     Tcl_Encoding *encodingPtr)
 {
     const char *bytes;
-    int i, numDirs, numBytes;
+    Tcl_Size i, numDirs, numBytes;
     Tcl_Obj *libPathObj, *encodingObj, *searchPathObj;
 
     TclNewLiteralStringObj(encodingObj, "encoding");

generic/tclExecute.c

@@ -2503,8 +2503,8 @@ TEBCresume(
 	 * 'yieldParameter').
 	 */
 
-	Tcl_IncrRefCount(valuePtr);
 	iPtr->execEnvPtr = corPtr->callerEEPtr;
+	Tcl_IncrRefCount(valuePtr);
 	TclSetTailcall(interp, valuePtr);
 	corPtr->yieldPtr = valuePtr;
 	iPtr->execEnvPtr = corPtr->eePtr;

generic/tclIO.c

@@ -7588,7 +7588,11 @@ Tcl_Eof(
     ChannelState *statePtr = ((Channel *) chan)->state;
 				/* State of real channel structure. */
 
-    return (GotFlag(statePtr, CHANNEL_EOF) && !GotFlag(statePtr, CHANNEL_ENCODING_ERROR)) ? 1 : 0;
+    if (GotFlag(statePtr, CHANNEL_NONBLOCKING|CHANNEL_FCOPY)
+	    && GotFlag(statePtr, CHANNEL_ENCODING_ERROR)) {
+	return 0;
+    }
+    return GotFlag(statePtr, CHANNEL_EOF) ? 1 : 0;
 }
 
 /*
@@ -8354,7 +8358,7 @@ Tcl_SetChannelOption(
 #ifdef TCL_NO_DEPRECATED
 	    ResetFlag(statePtr, CHANNEL_ENCODING_NOCOMPLAIN);
 #else
-	    if (SetFlag(statePtr, CHANNEL_ENCODING_STRICT)) {
+	    if (GotFlag(statePtr, CHANNEL_ENCODING_STRICT) != CHANNEL_ENCODING_STRICT) {
 		if (interp) {
 		    Tcl_SetObjResult(interp, Tcl_NewStringObj(
 			    "bad value for -nocomplainencoding: only true allowed",
@@ -9799,6 +9803,7 @@ CopyData(
      * the bottom of the stack.
      */
 
+    SetFlag(inStatePtr, CHANNEL_FCOPY);
     inBinary = (inStatePtr->encoding == NULL);
     outBinary = (outStatePtr->encoding == NULL);
     sameEncoding = inStatePtr->encoding == outStatePtr->encoding
@@ -9820,12 +9825,20 @@ CopyData(
 	    Tcl_SetErrno(inStatePtr->unreportedError);
 	    inStatePtr->unreportedError = 0;
 	    goto readError;
+	} else if (inStatePtr->flags & CHANNEL_ENCODING_ERROR) {
+	    Tcl_SetErrno(EILSEQ);
+	    inStatePtr->flags &= ~CHANNEL_ENCODING_ERROR;
+	    goto readError;
 	}
 	Tcl_GetChannelError(outChan, &msg);
 	if ((outStatePtr->unreportedError != 0) || (msg != NULL)) {
 	    Tcl_SetErrno(outStatePtr->unreportedError);
 	    outStatePtr->unreportedError = 0;
 	    goto writeError;
+	} else if (outStatePtr->flags & CHANNEL_ENCODING_ERROR) {
+	    Tcl_SetErrno(EILSEQ);
+	    outStatePtr->flags &= ~CHANNEL_ENCODING_ERROR;
+	    goto writeError;
 	}
 
 	if (cmdPtr && (mask == 0)) {
@@ -9906,6 +9919,7 @@ CopyData(
 		    TclDecrRefCount(bufObj);
 		    bufObj = NULL;
 		}
+		ResetFlag(inStatePtr, CHANNEL_FCOPY);
 		return TCL_OK;
 	    }
 	}
@@ -9997,6 +10011,7 @@ CopyData(
 		TclDecrRefCount(bufObj);
 		bufObj = NULL;
 	    }
+	    ResetFlag(inStatePtr, CHANNEL_FCOPY);
 	    return TCL_OK;
 	}
 
@@ -10019,6 +10034,7 @@ CopyData(
 		TclDecrRefCount(bufObj);
 		bufObj = NULL;
 	    }
+	    ResetFlag(inStatePtr, CHANNEL_FCOPY);
 	    return TCL_OK;
 	}
     } /* while */
@@ -10071,6 +10087,7 @@ CopyData(
 	    }
 	}
     }
+    ResetFlag(inStatePtr, CHANNEL_FCOPY);
     return result;
 }
 

generic/tclIO.h

@@ -233,6 +233,8 @@ typedef struct ChannelState {
 					 * flushed after every newline. */
 #define CHANNEL_UNBUFFERED	(1<<5)	/* Output to the channel must always
 					 * be flushed immediately. */
+#define CHANNEL_FCOPY	(1<<6)	/* Channel is currently doing an fcopy
+					 * mode. */
 #define BG_FLUSH_SCHEDULED	(1<<7)	/* A background flush of the queued
 					 * output buffers has been
 					 * scheduled. */

generic/tclIORChan.c

@@ -2830,6 +2830,7 @@ DeleteThreadReflectedChannelMap(
     Tcl_ThreadId self = Tcl_GetCurrentThread();
     ReflectedChannelMap *rcmPtr; /* The map */
     ForwardingResult *resultPtr;
+    ThreadSpecificData *tsdPtr = TCL_TSD_INIT(&dataKey);
 
     /*
      * The origin thread for one or more reflected channels is gone.
@@ -2909,6 +2910,7 @@ DeleteThreadReflectedChannelMap(
      */
 
     rcmPtr = GetThreadReflectedChannelMap();
+    tsdPtr->rcmPtr = NULL;
     for (hPtr = Tcl_FirstHashEntry(&rcmPtr->map, &hSearch);
 	    hPtr != NULL;
 	    hPtr = Tcl_FirstHashEntry(&rcmPtr->map, &hSearch)) {

generic/tclNamesp.c

@@ -417,6 +417,8 @@ Tcl_PopCallFrame(
     framePtr->nsPtr = NULL;
 
     if (framePtr->tailcallPtr) {
+	/* Reusing the existing reference count from framePtr->tailcallPtr, so
+	 * no need to Tcl_IncrRefCount(framePtr->tailcallPtr)*/
 	TclSetTailcall(interp, framePtr->tailcallPtr);
     }
 }

generic/tclTest.c

@@ -70,6 +70,7 @@ static Tcl_Interp *delInterp;
 typedef struct TestCommandTokenRef {
     int id;			/* Identifier for this reference. */
     Tcl_Command token;		/* Tcl's token for the command. */
+    const char *value;
     struct TestCommandTokenRef *nextPtr;
 				/* Next in list of references. */
 } TestCommandTokenRef;
@@ -1178,6 +1179,18 @@ TestcmdinfoCmd(
     return TCL_OK;
 }
 
+static int
+CmdProc0(
+    void *clientData,	/* String to return. */
+    Tcl_Interp *interp,		/* Current interpreter. */
+    TCL_UNUSED(int) /*argc*/,
+    TCL_UNUSED(const char **) /*argv*/)
+{
+    TestCommandTokenRef *refPtr = (TestCommandTokenRef *) clientData;
+    Tcl_AppendResult(interp, "CmdProc1 ", refPtr->value, NULL);
+    return TCL_OK;
+}
+
 static int
 CmdProc1(
     void *clientData,	/* String to return. */
@@ -1189,6 +1202,7 @@ CmdProc1(
     return TCL_OK;
 }
 
+
 static int
 CmdProc2(
     void *clientData,	/* String to return. */
@@ -1200,6 +1214,28 @@ CmdProc2(
     return TCL_OK;
 }
 
+static void
+CmdDelProc0(
+    void *clientData)	/* String to save. */
+{
+    TestCommandTokenRef *thisRefPtr, *prevRefPtr = NULL;
+    TestCommandTokenRef *refPtr = (TestCommandTokenRef *) clientData;
+    int id = refPtr->id;
+    for (thisRefPtr = firstCommandTokenRef; refPtr != NULL;
+	thisRefPtr = thisRefPtr->nextPtr) {
+	if (thisRefPtr->id == id) {
+	    if (prevRefPtr != NULL) {
+		prevRefPtr->nextPtr = thisRefPtr->nextPtr;
+	    } else {
+		firstCommandTokenRef = thisRefPtr->nextPtr; 
+	    }
+	    break;
+	}
+	prevRefPtr = thisRefPtr;
+    }
+    ckfree(refPtr);
+}
+
 static void
 CmdDelProc1(
     void *clientData)	/* String to save. */
@@ -1253,17 +1289,16 @@ TestcmdtokenCmd(
     }
     if (strcmp(argv[1], "create") == 0) {
 	refPtr = (TestCommandTokenRef *)Tcl_Alloc(sizeof(TestCommandTokenRef));
-	refPtr->token = Tcl_CreateCommand(interp, argv[2], CmdProc1,
-		(void *) "original", NULL);
+	refPtr->token = Tcl_CreateCommand(interp, argv[2], CmdProc0,
+		refPtr, CmdDelProc0);
 	refPtr->id = nextCommandTokenRefId;
+	refPtr->value = "original";
 	nextCommandTokenRefId++;
 	refPtr->nextPtr = firstCommandTokenRef;
 	firstCommandTokenRef = refPtr;
 	sprintf(buf, "%d", refPtr->id);
 	Tcl_AppendResult(interp, buf, NULL);
-    } else if (strcmp(argv[1], "name") == 0) {
-	Tcl_Obj *objPtr;
-
+    } else {
 	if (sscanf(argv[2], "%d", &id) != 1) {
 	    Tcl_AppendResult(interp, "bad command token \"", argv[2],
 		    "\"", NULL);
@@ -1283,18 +1318,23 @@ TestcmdtokenCmd(
 	    return TCL_ERROR;
 	}
 
-	objPtr = Tcl_NewObj();
-	Tcl_GetCommandFullName(interp, refPtr->token, objPtr);
+	if (strcmp(argv[1], "name") == 0) {
+	    Tcl_Obj *objPtr;
 
-	Tcl_AppendElement(interp,
-		Tcl_GetCommandName(interp, refPtr->token));
-	Tcl_AppendElement(interp, Tcl_GetString(objPtr));
-	Tcl_DecrRefCount(objPtr);
-    } else {
-	Tcl_AppendResult(interp, "bad option \"", argv[1],
-		"\": must be create or name", NULL);
-	return TCL_ERROR;
+	    objPtr = Tcl_NewObj();
+	    Tcl_GetCommandFullName(interp, refPtr->token, objPtr);
+
+	    Tcl_AppendElement(interp,
+		    Tcl_GetCommandName(interp, refPtr->token));
+	    Tcl_AppendElement(interp, Tcl_GetString(objPtr));
+	    Tcl_DecrRefCount(objPtr);
+	} else {
+	    Tcl_AppendResult(interp, "bad option \"", argv[1],
+		    "\": must be create, name, or free", NULL);
+	    return TCL_ERROR;
+	}
     }
+
     return TCL_OK;
 }
 

generic/tclUtf.c

@@ -182,25 +182,22 @@ Invalid(
  *
  * Tcl_UniCharToUtf --
  *
- *	Store the given Tcl_UniChar as a sequence of UTF-8 bytes in the
+ *	Stores the given Tcl_UniChar as a sequence of UTF-8 bytes in the
  *	provided buffer. Equivalent to Plan 9 runetochar().
  *
- *	Special handling of Surrogate pairs is handled as follows:
- *	When this function is called for ch being a high surrogate,
- *	the first byte of the 4-byte UTF-8 sequence is produced and
- *	the function returns 1. Calling the function again with a
- *	low surrogate, the remaining 3 bytes of the 4-byte UTF-8
- *	sequence is produced, and the function returns 3. The buffer
- *	is used to remember the high surrogate between the two calls.
+ *	Surrogate pairs are handled as follows: When ch is a high surrogate,
+ *	the first byte of the 4-byte UTF-8 sequence is stored in the buffer and
+ *	the function returns 1. If the function is called again with a low
+ *	surrogate and the same buffer, the remaining 3 bytes of the 4-byte
+ *	UTF-8 sequence are produced.
  *
  *	If no low surrogate follows the high surrogate (which is actually
  *	illegal), this can be handled reasonably by calling Tcl_UniCharToUtf
- *	again with ch = -1. This will produce a 3-byte UTF-8 sequence
+ *	again with ch = -1. This produces a 3-byte UTF-8 sequence
  *	representing the high surrogate.
  *
  * Results:
- *	The return values is the number of bytes in the buffer that were
- *	consumed.
+ *	Returns the number of bytes stored into the buffer.
  *
  * Side effects:
  *	None.
@@ -223,18 +220,19 @@ Tcl_UniCharToUtf(
     }
     if (ch >= 0) {
 	if (ch <= 0x7FF) {
-	    buf[1] = (char) ((ch | 0x80) & 0xBF);
-	    buf[0] = (char) ((ch >> 6) | 0xC0);
+	    buf[1] = (char) (0x80 | (0x3F & ch));
+	    buf[0] = (char) (0xC0 | (ch >> 6));
 	    return 2;
 	}
 	if (ch <= 0xFFFF) {
 	    if ((ch & 0xF800) == 0xD800) {
 		if (ch & 0x0400) {
 		    /* Low surrogate */
-		    if (((buf[0] & 0xC0) == 0x80) && ((buf[1] & 0xCF) == 0)) {
+		    if (   (0x80 == (0xC0 & buf[0]))
+			&& (0    == (0xCF & buf[1]))) {
 			/* Previous Tcl_UniChar was a high surrogate, so combine */
-			buf[2] = (char) ((ch & 0x3F) | 0x80);
-			buf[1] |= (char) (((ch >> 6) & 0x0F) | 0x80);
+			buf[2]  = (char) (0x80 | (0x3F & ch));
+			buf[1] |= (char) (0x80 | (0x0F & (ch >> 6)));
 			return 3;
 		    }
 		    /* Previous Tcl_UniChar was not a high surrogate, so just output */
@@ -244,37 +242,40 @@ Tcl_UniCharToUtf(
 		    /* Fill buffer with specific 3-byte (invalid) byte combination,
 		       so following low surrogate can recognize it and combine */
 		    buf[2] = (char) ((ch << 4) & 0x30);
-		    buf[1] = (char) (((ch >> 2) & 0x3F) | 0x80);
-		    buf[0] = (char) (((ch >> 8) & 0x07) | 0xF0);
+		    buf[1] = (char) (0x80 | (0x3F & (ch >> 2)));
+		    buf[0] = (char) (0xF0 | (0x07 & (ch >> 8)));
 		    return 1;
 		}
 	    }
 	    goto three;
 	}
 	if (ch <= 0x10FFFF) {
-	    buf[3] = (char) ((ch | 0x80) & 0xBF);
-	    buf[2] = (char) (((ch >> 6) | 0x80) & 0xBF);
-	    buf[1] = (char) (((ch >> 12) | 0x80) & 0xBF);
-	    buf[0] = (char) ((ch >> 18) | 0xF0);
+	    buf[3] = (char) (0x80 | (0x3F & ch));
+	    buf[2] = (char) (0x80 | (0x3F & (ch >> 6)));
+	    buf[1] = (char) (0x80 | (0x3F & (ch >> 12)));
+	    buf[0] = (char) (0xF0 |         (ch >> 18));
 	    return 4;
 	}
     } else if (ch == -1) {
-	if (((buf[0] & 0xC0) == 0x80) && ((buf[1] & 0xCF) == 0)
-		&& ((buf[-1] & 0xF8) == 0xF0)) {
-	    ch = 0xD7C0 + ((buf[-1] & 0x07) << 8) + ((buf[0] & 0x3F) << 2)
-		    + ((buf[1] & 0x30) >> 4);
-	    buf[1] = (char) ((ch | 0x80) & 0xBF);
-	    buf[0] = (char) (((ch >> 6) | 0x80) & 0xBF);
-	    buf[-1] = (char) ((ch >> 12) | 0xE0);
+	if (   (0x80 == (0xC0 & buf[0]))
+	    && (0    == (0xCF & buf[1]))
+	    && (0xF0 == (0xF8 & buf[-1]))) {
+	    ch = 0xD7C0
+		+ ((0x07 & buf[-1]) << 8)
+		+ ((0x3F & buf[0])  << 2)
+		+ ((0x30 & buf[1])  >> 4);
+	    buf[1]  = (char) (0x80 | (0x3F & ch));
+	    buf[0]  = (char) (0x80 | (0x3F & (ch >> 6)));
+	    buf[-1] = (char) (0xE0 | (ch >> 12));
 	    return 2;
 	}
     }
 
     ch = 0xFFFD;
 three:
-    buf[2] = (char) ((ch | 0x80) & 0xBF);
-    buf[1] = (char) (((ch >> 6) | 0x80) & 0xBF);
-    buf[0] = (char) ((ch >> 12) | 0xE0);
+    buf[2] = (char) (0x80 | (0x3F & ch));
+    buf[1] = (char) (0x80 | (0x3F & (ch >> 6)));
+    buf[0] = (char) (0xE0 |         (ch >> 12));
     return 3;
 }
 
@@ -2948,7 +2949,7 @@ TclUniCharMatch(
  *
  * TclUtfToUCS4 --
  *
- *	Extract the 4-byte codepoint from the leading bytes of the
+ *	Extracts the 4-byte codepoint from the leading bytes of the
  *	Modified UTF-8 string "src".  This is a utility routine to
  *	contain the surrogate gymnastics in one place.
  *
@@ -2960,8 +2961,8 @@ TclUniCharMatch(
  *	enough bytes remain in the string.
  *
  * Results:
- *	*usc4Ptr is filled with the UCS4 code point, and the return value is
- *	the number of bytes from the UTF-8 string that were consumed.
+ *	Fills *usc4Ptr with the UCS4 code point and returns the number of bytes
+ *	consumed from the source string.
  *
  * Side effects:
  *	None.

generic/tclZlib.c

@@ -442,10 +442,21 @@ GenerateHeader(
     if (GetValue(interp, dictObj, "comment", &value) != TCL_OK) {
 	goto error;
     } else if (value != NULL) {
+	Tcl_EncodingState state;
 	valueStr = TclGetStringFromObj(value, &len);
-	Tcl_UtfToExternal(NULL, latin1enc, valueStr, len, 0, NULL,
+	result = Tcl_UtfToExternal(NULL, latin1enc, valueStr, len,
+		TCL_ENCODING_START|TCL_ENCODING_END|TCL_ENCODING_STOPONERROR, &state,
 		headerPtr->nativeCommentBuf, MAX_COMMENT_LEN-1, NULL, &len,
 		NULL);
+	if (result != TCL_OK) {
+	    if (result == TCL_CONVERT_UNKNOWN) {
+		Tcl_AppendResult(interp, "Comment contains characters > 0xFF", NULL);
+	    } else {
+		Tcl_AppendResult(interp, "Comment too large for zip", NULL);
+	    }
+	    result = TCL_ERROR;
+	    goto error;
+	}
 	headerPtr->nativeCommentBuf[len] = '\0';
 	headerPtr->header.comment = (Bytef *) headerPtr->nativeCommentBuf;
 	if (extraSizePtr != NULL) {
@@ -463,9 +474,21 @@ GenerateHeader(
     if (GetValue(interp, dictObj, "filename", &value) != TCL_OK) {
 	goto error;
     } else if (value != NULL) {
+	Tcl_EncodingState state;
 	valueStr = TclGetStringFromObj(value, &len);
-	Tcl_UtfToExternal(NULL, latin1enc, valueStr, len, 0, NULL,
-		headerPtr->nativeFilenameBuf, MAXPATHLEN-1, NULL, &len, NULL);
+	result = Tcl_UtfToExternal(NULL, latin1enc, valueStr, len,
+		TCL_ENCODING_START|TCL_ENCODING_END|TCL_ENCODING_STOPONERROR, &state,
+		headerPtr->nativeFilenameBuf, MAXPATHLEN-1, NULL, &len,
+		NULL);
+	if (result != TCL_OK) {
+	    if (result == TCL_CONVERT_UNKNOWN) {
+		Tcl_AppendResult(interp, "Filename contains characters > 0xFF", NULL);
+	    } else {
+		Tcl_AppendResult(interp, "Filename too large for zip", NULL);
+	    }
+	    result = TCL_ERROR;
+	    goto error;
+	}
 	headerPtr->nativeFilenameBuf[len] = '\0';
 	headerPtr->header.name = (Bytef *) headerPtr->nativeFilenameBuf;
 	if (extraSizePtr != NULL) {
@@ -1196,7 +1219,8 @@ Tcl_ZlibStreamPut(
 {
     ZlibStreamHandle *zshPtr = (ZlibStreamHandle *) zshandle;
     char *dataTmp = NULL;
-    int e, size, outSize, toStore;
+    int e;
+    int size, outSize, toStore;
     unsigned char *bytes;
 
     if (zshPtr->streamEnd) {
@@ -1325,7 +1349,8 @@ Tcl_ZlibStreamGet(
 				 * may get less! */
 {
     ZlibStreamHandle *zshPtr = (ZlibStreamHandle *) zshandle;
-    int e, i, listLen, itemLen, dataPos = 0;
+    int e;
+    int i, listLen, itemLen, dataPos = 0;
     Tcl_Obj *itemObj;
     unsigned char *dataPtr, *itemPtr;
     int existing;
@@ -1576,7 +1601,8 @@ Tcl_ZlibDeflate(
     int level,
     Tcl_Obj *gzipHeaderDictObj)
 {
-    int wbits = 0, inLen = 0, e = 0, extraSize = 0;
+    int wbits = 0, e = 0, extraSize = 0;
+    int inLen = 0;
     Byte *inData = NULL;
     z_stream stream;
     GzipHeader header;
@@ -1730,7 +1756,8 @@ Tcl_ZlibInflate(
     int bufferSize,
     Tcl_Obj *gzipHeaderDictObj)
 {
-    int wbits = 0, inLen = 0, e = 0, newBufferSize;
+    int wbits = 0, e = 0;
+    int inLen = 0, newBufferSize;
     Byte *inData = NULL, *outData = NULL, *newOutData = NULL;
     z_stream stream;
     gz_header header, *headerPtr = NULL;
@@ -2400,7 +2427,8 @@ ZlibPushSubcmd(
     const char *const *pushOptions = pushDecompressOptions;
     enum pushOptionsEnum {poDictionary, poHeader, poLevel, poLimit};
     Tcl_Obj *headerObj = NULL, *compDictObj = NULL;
-    int limit = DEFAULT_BUFFER_SIZE, dummy;
+    int limit = DEFAULT_BUFFER_SIZE;
+    int dummy;
 
     if (objc < 4) {
 	Tcl_WrongNumArgs(interp, 2, objv, "mode channel ?options...?");
@@ -2942,7 +2970,8 @@ ZlibTransformClose(
 	int flags)
 {
     ZlibChannelData *cd = (ZlibChannelData *)instanceData;
-    int e, written, result = TCL_OK;
+    int e, result = TCL_OK;
+    int written;
 
     if ((flags & (TCL_CLOSE_READ | TCL_CLOSE_WRITE)) != 0) {
 	return EINVAL;

library/encoding/ascii.enc

@@ -9,7 +9,7 @@ S
 0040004100420043004400450046004700480049004A004B004C004D004E004F
 0050005100520053005400550056005700580059005A005B005C005D005E005F
 0060006100620063006400650066006700680069006A006B006C006D006E006F
-0070007100720073007400750076007700780079007A007B007C007D007E0000
+0070007100720073007400750076007700780079007A007B007C007D007E007F
 0000000000000000000000000000000000000000000000000000000000000000
 0000000000000000000000000000000000000000000000000000000000000000
 0000000000000000000000000000000000000000000000000000000000000000

library/encoding/big5.enc

@@ -10,8 +10,8 @@ M
 0050005100520053005400550056005700580059005A005B005C005D005E005F
 0060006100620063006400650066006700680069006A006B006C006D006E006F
 0070007100720073007400750076007700780079007A007B007C007D007E007F
-0080008100820083008400850086008700880089008A008B008C008D008E008F
-0090009100920093009400950096009700980099009A009B009C009D009E009F
+0000000000000000000000000000000000000000000000000000000000000000
+0000000000000000000000000000000000000000000000000000000000000000
 0000000000000000000000000000000000000000000000000000000000000000
 0000000000000000000000000000000000000000000000000000000000000000
 0000000000000000000000000000000000000000000000000000000000000000

library/encoding/cp1250.enc

@@ -10,8 +10,8 @@ S
 0050005100520053005400550056005700580059005A005B005C005D005E005F
 0060006100620063006400650066006700680069006A006B006C006D006E006F
 0070007100720073007400750076007700780079007A007B007C007D007E007F
-20AC0081201A0083201E2026202020210088203001602039015A0164017D0179
-009020182019201C201D202220132014009821220161203A015B0165017E017A
+20AC0000201A0000201E2026202020210000203001602039015A0164017D0179
+000020182019201C201D202220132014000021220161203A015B0165017E017A
 00A002C702D8014100A4010400A600A700A800A9015E00AB00AC00AD00AE017B
 00B000B102DB014200B400B500B600B700B80105015F00BB013D02DD013E017C
 015400C100C2010200C40139010600C7010C00C9011800CB011A00CD00CE010E

library/encoding/cp1251.enc

@@ -11,7 +11,7 @@ S
 0060006100620063006400650066006700680069006A006B006C006D006E006F
 0070007100720073007400750076007700780079007A007B007C007D007E007F
 04020403201A0453201E20262020202120AC203004092039040A040C040B040F
-045220182019201C201D202220132014009821220459203A045A045C045B045F
+045220182019201C201D202220132014000021220459203A045A045C045B045F
 00A0040E045E040800A4049000A600A7040100A9040400AB00AC00AD00AE0407
 00B000B104060456049100B500B600B704512116045400BB0458040504550457
 0410041104120413041404150416041704180419041A041B041C041D041E041F

library/encoding/cp1252.enc

@@ -10,8 +10,8 @@ S
 0050005100520053005400550056005700580059005A005B005C005D005E005F
 0060006100620063006400650066006700680069006A006B006C006D006E006F
 0070007100720073007400750076007700780079007A007B007C007D007E007F
-20AC0081201A0192201E20262020202102C62030016020390152008D017D008F
-009020182019201C201D20222013201402DC21220161203A0153009D017E0178
+20AC0000201A0192201E20262020202102C620300160203901520000017D0000
+000020182019201C201D20222013201402DC21220161203A01530000017E0178
 00A000A100A200A300A400A500A600A700A800A900AA00AB00AC00AD00AE00AF
 00B000B100B200B300B400B500B600B700B800B900BA00BB00BC00BD00BE00BF
 00C000C100C200C300C400C500C600C700C800C900CA00CB00CC00CD00CE00CF

library/encoding/cp1253.enc

@@ -10,8 +10,8 @@ S
 0050005100520053005400550056005700580059005A005B005C005D005E005F
 0060006100620063006400650066006700680069006A006B006C006D006E006F
 0070007100720073007400750076007700780079007A007B007C007D007E007F
-20AC0081201A0192201E20262020202100882030008A2039008C008D008E008F
-009020182019201C201D20222013201400982122009A203A009C009D009E009F
+20AC0000201A0192201E20262020202100002030000020390000000000000000
+000020182019201C201D202220132014000021220000203A0000000000000000
 00A00385038600A300A400A500A600A700A800A9000000AB00AC00AD00AE2015
 00B000B100B200B3038400B500B600B703880389038A00BB038C00BD038E038F
 0390039103920393039403950396039703980399039A039B039C039D039E039F

library/encoding/cp1254.enc

@@ -10,8 +10,8 @@ S
 0050005100520053005400550056005700580059005A005B005C005D005E005F
 0060006100620063006400650066006700680069006A006B006C006D006E006F
 0070007100720073007400750076007700780079007A007B007C007D007E007F
-20AC0081201A0192201E20262020202102C62030016020390152008D008E008F
-009020182019201C201D20222013201402DC21220161203A0153009D009E0178
+20AC0000201A0192201E20262020202102C62030016020390152000000000000
+000020182019201C201D20222013201402DC21220161203A0153000000000178
 00A000A100A200A300A400A500A600A700A800A900AA00AB00AC00AD00AE00AF
 00B000B100B200B300B400B500B600B700B800B900BA00BB00BC00BD00BE00BF
 00C000C100C200C300C400C500C600C700C800C900CA00CB00CC00CD00CE00CF

library/encoding/cp1255.enc

@@ -10,8 +10,8 @@ S
 0050005100520053005400550056005700580059005A005B005C005D005E005F
 0060006100620063006400650066006700680069006A006B006C006D006E006F
 0070007100720073007400750076007700780079007A007B007C007D007E007F
-20AC0081201A0192201E20262020202102C62030008A2039008C008D008E008F
-009020182019201C201D20222013201402DC2122009A203A009C009D009E009F
+20AC0000201A0192201E20262020202102C62030000020390000000000000000
+000020182019201C201D20222013201402DC21220000203A0000000000000000
 00A000A100A200A320AA00A500A600A700A800A900D700AB00AC00AD00AE00AF
 00B000B100B200B300B400B500B600B700B800B900F700BB00BC00BD00BE00BF
 05B005B105B205B305B405B505B605B705B805B9000005BB05BC05BD05BE05BF

library/encoding/cp1257.enc

@@ -10,8 +10,8 @@ S
 0050005100520053005400550056005700580059005A005B005C005D005E005F
 0060006100620063006400650066006700680069006A006B006C006D006E006F
 0070007100720073007400750076007700780079007A007B007C007D007E007F
-20AC0081201A0083201E20262020202100882030008A2039008C00A802C700B8
-009020182019201C201D20222013201400982122009A203A009C00AF02DB009F
+20AC0000201A0000201E2026202020210000203000002039000000A802C700B8
+000020182019201C201D202220132014000021220000203A000000AF02DB0000
 00A0000000A200A300A4000000A600A700D800A9015600AB00AC00AD00AE00C6
 00B000B100B200B300B400B500B600B700F800B9015700BB00BC00BD00BE00E6
 0104012E0100010600C400C501180112010C00C90179011601220136012A013B

library/encoding/cp1258.enc

@@ -10,8 +10,8 @@ S
 0050005100520053005400550056005700580059005A005B005C005D005E005F
 0060006100620063006400650066006700680069006A006B006C006D006E006F
 0070007100720073007400750076007700780079007A007B007C007D007E007F
-20AC0081201A0192201E20262020202102C62030008A20390152008D008E008F
-009020182019201C201D20222013201402DC2122009A203A0153009D009E0178
+20AC0000201A0192201E20262020202102C62030000020390152000000000000
+000020182019201C201D20222013201402DC21220000203A0153000000000178
 00A000A100A200A300A400A500A600A700A800A900AA00AB00AC00AD00AE00AF
 00B000B100B200B300B400B500B600B700B800B900BA00BB00BC00BD00BE00BF
 00C000C100C2010200C400C500C600C700C800C900CA00CB030000CD00CE00CF

library/encoding/cp864.enc

@@ -11,7 +11,7 @@ S
 0060006100620063006400650066006700680069006A006B006C006D006E006F
 0070007100720073007400750076007700780079007A007B007C007D007E007F
 00B000B72219221A259225002502253C2524252C251C25342510250C25142518
-03B2221E03C600B100BD00BC224800AB00BBFEF7FEF8009B009CFEFBFEFC009F
+03B2221E03C600B100BD00BC224800AB00BBFEF7FEF800000000FEFBFEFC0000
 00A000ADFE8200A300A4FE8400000000FE8EFE8FFE95FE99060CFE9DFEA1FEA5
 0660066106620663066406650666066706680669FED1061BFEB1FEB5FEB9061F
 00A2FE80FE81FE83FE85FECAFE8BFE8DFE91FE93FE97FE9BFE9FFEA3FEA7FEA9

library/encoding/cp869.enc

@@ -10,8 +10,8 @@ S
 0050005100520053005400550056005700580059005A005B005C005D005E005F
 0060006100620063006400650066006700680069006A006B006C006D006E006F
 0070007100720073007400750076007700780079007A007B007C007D007E007F
-0080008100820083008400850386008700B700AC00A620182019038820150389
-038A03AA038C00930094038E03AB00A9038F00B200B303AC00A303AD03AE03AF
+0000000000000000000000000386000000B700AC00A620182019038820150389
+038A03AA038C00000000038E03AB00A9038F00B200B303AC00A303AD03AE03AF
 03CA039003CC03CD039103920393039403950396039700BD0398039900AB00BB
 25912592259325022524039A039B039C039D256325512557255D039E039F2510
 25142534252C251C2500253C03A003A1255A25542569256625602550256C03A3

library/encoding/cp874.enc

@@ -10,8 +10,8 @@ S
 0050005100520053005400550056005700580059005A005B005C005D005E005F
 0060006100620063006400650066006700680069006A006B006C006D006E006F
 0070007100720073007400750076007700780079007A007B007C007D007E007F
-20AC008100820083008420260086008700880089008A008B008C008D008E008F
-009020182019201C201D20222013201400980099009A009B009C009D009E009F
+20AC000000000000000020260000000000000000000000000000000000000000
+000020182019201C201D20222013201400000000000000000000000000000000
 00A00E010E020E030E040E050E060E070E080E090E0A0E0B0E0C0E0D0E0E0E0F
 0E100E110E120E130E140E150E160E170E180E190E1A0E1B0E1C0E1D0E1E0E1F
 0E200E210E220E230E240E250E260E270E280E290E2A0E2B0E2C0E2D0E2E0E2F

library/encoding/cp932.enc

@@ -10,7 +10,7 @@ M
 0050005100520053005400550056005700580059005A005B005C005D005E005F
 0060006100620063006400650066006700680069006A006B006C006D006E006F
 0070007100720073007400750076007700780079007A007B007C007D007E007F
-0080000000000000000000850086000000000000000000000000000000000000
+0000000000000000000000000000000000000000000000000000000000000000
 0000000000000000000000000000000000000000000000000000000000000000
 0000FF61FF62FF63FF64FF65FF66FF67FF68FF69FF6AFF6BFF6CFF6DFF6EFF6F
 FF70FF71FF72FF73FF74FF75FF76FF77FF78FF79FF7AFF7BFF7CFF7DFF7EFF7F

library/encoding/cp949.enc

@@ -10,7 +10,7 @@ M
 0050005100520053005400550056005700580059005A005B005C005D005E005F
 0060006100620063006400650066006700680069006A006B006C006D006E006F
 0070007100720073007400750076007700780079007A007B007C007D007E007F
-0080000000000000000000000000000000000000000000000000000000000000
+0000000000000000000000000000000000000000000000000000000000000000
 0000000000000000000000000000000000000000000000000000000000000000
 0000000000000000000000000000000000000000000000000000000000000000
 0000000000000000000000000000000000000000000000000000000000000000

library/encoding/cp950.enc

@@ -10,8 +10,8 @@ M
 0050005100520053005400550056005700580059005A005B005C005D005E005F
 0060006100620063006400650066006700680069006A006B006C006D006E006F
 0070007100720073007400750076007700780079007A007B007C007D007E007F
-0080008100820083008400850086008700880089008A008B008C008D008E008F
-0090009100920093009400950096009700980099009A009B009C009D009E009F
+0000000000000000000000000000000000000000000000000000000000000000
+0000000000000000000000000000000000000000000000000000000000000000
 0000000000000000000000000000000000000000000000000000000000000000
 0000000000000000000000000000000000000000000000000000000000000000
 0000000000000000000000000000000000000000000000000000000000000000

library/encoding/dingbats.enc

@@ -10,8 +10,8 @@ S
 2730273127322733273427352736273727382739273A273B273C273D273E273F
 2740274127422743274427452746274727482749274A274B25CF274D25A0274F
 27502751275225B225BC25C6275625D727582759275A275B275C275D275E007F
-0080008100820083008400850086008700880089008A008B008C008D008E008F
-0090009100920093009400950096009700980099009A009B009C009D009E009F
+0000000000000000000000000000000000000000000000000000000000000000
+0000000000000000000000000000000000000000000000000000000000000000
 0000276127622763276427652766276726632666266526602460246124622463
 2464246524662467246824692776277727782779277A277B277C277D277E277F
 2780278127822783278427852786278727882789278A278B278C278D278E278F

library/encoding/ebcdic.enc

@@ -1,3 +1,4 @@
+# Encoding file: ebcdic, single-byte
 S
 006F 0 1
 00

library/encoding/euc-cn.enc

@@ -10,8 +10,8 @@ M
 0050005100520053005400550056005700580059005A005B005C005D005E005F
 0060006100620063006400650066006700680069006A006B006C006D006E006F
 0070007100720073007400750076007700780079007A007B007C007D007E007F
-0080008100820083008400850086008700880089008A008B008C008D008E008F
-0090009100920093009400950096009700980099009A009B009C009D009E009F
+0000000000000000000000000000000000000000000000000000000000000000
+0000000000000000000000000000000000000000000000000000000000000000
 0000000000000000000000000000000000000000000000000000000000000000
 0000000000000000000000000000000000000000000000000000000000000000
 0000000000000000000000000000000000000000000000000000000000000000

library/encoding/euc-jp.enc

@@ -10,8 +10,8 @@ M
 0050005100520053005400550056005700580059005A005B005C005D005E005F
 0060006100620063006400650066006700680069006A006B006C006D006E006F
 0070007100720073007400750076007700780079007A007B007C007D007E007F
-0080008100820083008400850086008700880089008A008B008C008D0000008F
-0090009100920093009400950096009700980099009A009B009C009D009E009F
+0000000000000000000000000000000000000000000000000000000000000000
+0000000000000000000000000000000000000000000000000000000000000000
 0000000000000000000000000000000000000000000000000000000000000000
 0000000000000000000000000000000000000000000000000000000000000000
 0000000000000000000000000000000000000000000000000000000000000000

library/encoding/euc-kr.enc

@@ -10,8 +10,8 @@ M
 0050005100520053005400550056005700580059005A005B005C005D005E005F
 0060006100620063006400650066006700680069006A006B006C006D006E006F
 0070007100720073007400750076007700780079007A007B007C007D007E007F
-0080008100820083008400850086008700880089008A008B008C008D008E008F
-0090009100920093009400950096009700980099009A009B009C009D009E009F
+0000000000000000000000000000000000000000000000000000000000000000
+0000000000000000000000000000000000000000000000000000000000000000
 0000000000000000000000000000000000000000000000000000000000000000
 0000000000000000000000000000000000000000000000000000000000000000
 0000000000000000000000000000000000000000000000000000000000000000

library/encoding/gb1988.enc

@@ -10,8 +10,8 @@ S
 0050005100520053005400550056005700580059005A005B005C005D005E005F
 0060006100620063006400650066006700680069006A006B006C006D006E006F
 0070007100720073007400750076007700780079007A007B007C007D203E007F
-0080008100820083008400850086008700880089008A008B008C008D008E008F
-0090009100920093009400950096009700980099009A009B009C009D009E009F
+0000000000000000000000000000000000000000000000000000000000000000
+0000000000000000000000000000000000000000000000000000000000000000
 0000FF61FF62FF63FF64FF65FF66FF67FF68FF69FF6AFF6BFF6CFF6DFF6EFF6F
 FF70FF71FF72FF73FF74FF75FF76FF77FF78FF79FF7AFF7BFF7CFF7DFF7EFF7F
 FF80FF81FF82FF83FF84FF85FF86FF87FF88FF89FF8AFF8BFF8CFF8DFF8EFF8F

library/encoding/jis0201.enc

@@ -10,8 +10,8 @@ S
 0050005100520053005400550056005700580059005A005B005C005D005E005F
 0060006100620063006400650066006700680069006A006B006C006D006E006F
 0070007100720073007400750076007700780079007A007B007C007D203E007F
-0080008100820083008400850086008700880089008A008B008C008D008E008F
-0090009100920093009400950096009700980099009A009B009C009D009E009F
+0000000000000000000000000000000000000000000000000000000000000000
+0000000000000000000000000000000000000000000000000000000000000000
 0000FF61FF62FF63FF64FF65FF66FF67FF68FF69FF6AFF6BFF6CFF6DFF6EFF6F
 FF70FF71FF72FF73FF74FF75FF76FF77FF78FF79FF7AFF7BFF7CFF7DFF7EFF7F
 FF80FF81FF82FF83FF84FF85FF86FF87FF88FF89FF8AFF8BFF8CFF8DFF8EFF8F

library/encoding/macDingbats.enc

@@ -10,8 +10,8 @@ S
 2730273127322733273427352736273727382739273A273B273C273D273E273F
 2740274127422743274427452746274727482749274A274B25CF274D25A0274F
 27502751275225B225BC25C6275625D727582759275A275B275C275D275E007F
-F8D7F8D8F8D9F8DAF8DBF8DCF8DDF8DEF8DFF8E0F8E1F8E2F8E3F8E4008E008F
-0090009100920093009400950096009700980099009A009B009C009D009E009F
+F8D7F8D8F8D9F8DAF8DBF8DCF8DDF8DEF8DFF8E0F8E1F8E2F8E3F8E400000000
+0000000000000000000000000000000000000000000000000000000000000000
 0000276127622763276427652766276726632666266526602460246124622463
 2464246524662467246824692776277727782779277A277B277C277D277E277F
 2780278127822783278427852786278727882789278A278B278C278D278E278F

library/encoding/macJapan.enc

@@ -10,7 +10,7 @@ M
 0050005100520053005400550056005700580059005A005B005C005D005E005F
 0060006100620063006400650066006700680069006A006B006C006D006E006F
 0070007100720073007400750076007700780079007A007B007C007D007E007F
-0080000000000000000000000000000000000000000000000000000000000000
+0000000000000000000000000000000000000000000000000000000000000000
 0000000000000000000000000000000000000000000000000000000000000000
 00A0FF61FF62FF63FF64FF65FF66FF67FF68FF69FF6AFF6BFF6CFF6DFF6EFF6F
 FF70FF71FF72FF73FF74FF75FF76FF77FF78FF79FF7AFF7BFF7CFF7DFF7EFF7F

library/encoding/shiftjis.enc

@@ -10,7 +10,7 @@ M
 0050005100520053005400550056005700580059005A005B005C005D005E005F
 0060006100620063006400650066006700680069006A006B006C006D006E006F
 0070007100720073007400750076007700780079007A007B007C007D007E007F
-0080000000000000000000850086008700000000000000000000000000000000
+0000000000000000000000000000000000000000000000000000000000000000
 0000000000000000000000000000000000000000000000000000000000000000
 0000FF61FF62FF63FF64FF65FF66FF67FF68FF69FF6AFF6BFF6CFF6DFF6EFF6F
 FF70FF71FF72FF73FF74FF75FF76FF77FF78FF79FF7AFF7BFF7CFF7DFF7EFF7F

library/encoding/symbol.enc

@@ -10,8 +10,8 @@ S
 03A0039803A103A303A403A503C203A9039E03A80396005B2234005D22A5005F
 F8E503B103B203C703B403B503C603B303B703B903D503BA03BB03BC03BD03BF
 03C003B803C103C303C403C503D603C903BE03C803B6007B007C007D223C007F
-0080008100820083008400850086008700880089008A008B008C008D008E008F
-0090009100920093009400950096009700980099009A009B009C009D009E009F
+0000000000000000000000000000000000000000000000000000000000000000
+0000000000000000000000000000000000000000000000000000000000000000
 000003D2203222642044221E0192266326662665266021942190219121922193
 00B000B12033226500D7221D2202202200F72260226122482026F8E6F8E721B5
 21352111211C21182297229522052229222A2283228722842282228622082209

library/encoding/tis-620.enc

@@ -9,7 +9,7 @@ S
 0040004100420043004400450046004700480049004A004B004C004D004E004F
 0050005100520053005400550056005700580059005A005B005C005D005E005F
 0060006100620063006400650066006700680069006A006B006C006D006E006F
-0070007100720073007400750076007700780079007A007B007C007D007E0000
+0070007100720073007400750076007700780079007A007B007C007D007E007F
 0000000000000000000000000000000000000000000000000000000000000000
 0000000000000000000000000000000000000000000000000000000000000000
 00000E010E020E030E040E050E060E070E080E090E0A0E0B0E0C0E0D0E0E0E0F

tests/basic.test

@@ -348,7 +348,7 @@ test basic-20.2 {Tcl_GetCommandInfo, names for commands created outside namespac
 test basic-20.3 {Tcl_GetCommandInfo, #-quoting} testcmdtoken {
     catch {rename \# ""}
     set x [testcmdtoken create \#]
-    testcmdtoken name $x
+    return [testcmdtoken name $x]
 } {{#} ::#}
 
 test basic-21.1 {Tcl_GetCommandName} {emptyTest} {

tests/chanio.test

@@ -50,6 +50,8 @@ namespace eval ::tcl::test::io {
     testConstraint notWinCI       [expr {
 	$::tcl_platform(platform) ne "windows" || ![info exists ::env(CI)]}]
     testConstraint notOSX [expr {$::tcl_platform(os) ne "Darwin"}]
+    # File permissions broken on wsl without some "exotic" wsl configuration
+    testConstraint notWsl [expr {[llength [array names ::env *WSL*]] == 0}]
     testConstraint specialfiles  [expr {[file exists /dev/zero] || [file exists NUL]}]
 
     # You need a *very* special environment to do some tests.  In particular,
@@ -5348,7 +5350,7 @@ test chan-io-40.1 {POSIX open access modes: RDWR} -setup {
 } -result {zzy abzzy}
 test chan-io-40.2 {POSIX open access modes: CREAT} -setup {
     file delete $path(test3)
-} -constraints {unix} -body {
+} -constraints {unix notWsl} -body {
     set f [open $path(test3) {WRONLY CREAT} 0o600]
     file stat $path(test3) stats
     set x [format 0o%03o [expr {$stats(mode) & 0o777}]]
@@ -5361,7 +5363,7 @@ test chan-io-40.2 {POSIX open access modes: CREAT} -setup {
 } -result {0o600 {line 1}}
 test chan-io-40.3 {POSIX open access modes: CREAT} -setup {
     file delete $path(test3)
-} -constraints {unix umask} -body {
+} -constraints {unix umask notWsl} -body {
     # This test only works if your umask is 2, like ouster's.
     chan close [open $path(test3) {WRONLY CREAT}]
     file stat $path(test3) stats
@@ -6736,8 +6738,8 @@ test chan-io-52.5 {TclCopyChannel, all} -constraints {fcopy} -setup {
 } -body {
     set f1 [open $thisScript]
     set f2 [open $path(test1) w]
-    chan configure $f1 -translation lf -blocking 0
-    chan configure $f2 -translation lf -blocking 0
+    chan configure $f1 -translation lf -encoding iso8859-1 -blocking 0
+    chan configure $f2 -translation lf -encoding iso8859-1 -blocking 0
     chan copy $f1 $f2 -size -1 ;# -1 means 'copy all', same as if no -size specified.
     set result [list [chan configure $f1 -blocking] [chan configure $f2 -blocking]]
     chan close $f1
@@ -6752,8 +6754,8 @@ test chan-io-52.5a {TclCopyChannel, all, other negative value} -setup {
 } -constraints {fcopy} -body {
     set f1 [open $thisScript]
     set f2 [open $path(test1) w]
-    chan configure $f1 -translation lf -blocking 0
-    chan configure $f2 -translation lf -blocking 0
+    chan configure $f1 -translation lf -encoding iso8859-1 -blocking 0
+    chan configure $f2 -translation lf -encoding iso8859-1 -blocking 0
     chan copy $f1 $f2 -size -2 ;# < 0 behaves like -1, copy all
     set result [list [chan configure $f1 -blocking] [chan configure $f2 -blocking]]
     chan close $f1
@@ -6768,8 +6770,8 @@ test chan-io-52.5b {TclCopyChannel, all, wrap to negative value} -setup {
 } -constraints {fcopy} -body {
     set f1 [open $thisScript]
     set f2 [open $path(test1) w]
-    chan configure $f1 -translation lf -blocking 0
-    chan configure $f2 -translation lf -blocking 0
+    chan configure $f1 -translation lf -encoding iso8859-1 -blocking 0
+    chan configure $f2 -translation lf -encoding iso8859-1 -blocking 0
     chan copy $f1 $f2 -size 3221176172 ;# Wrapped to < 0, behaves like -1, copy all
     set result [list [chan configure $f1 -blocking] [chan configure $f2 -blocking]]
     chan close $f1
@@ -6802,8 +6804,8 @@ test chan-io-52.7 {TclCopyChannel} -constraints {fcopy} -setup {
 } -body {
     set f1 [open $thisScript]
     set f2 [open $path(test1) w]
-    chan configure $f1 -translation lf -blocking 0
-    chan configure $f2 -translation lf -blocking 0
+    chan configure $f1 -translation lf -encoding iso8859-1 -blocking 0
+    chan configure $f2 -translation lf -encoding iso8859-1 -blocking 0
     chan copy $f1 $f2
     set result [list [chan configure $f1 -blocking] [chan configure $f2 -blocking]]
     if {[file size $thisScript] == [file size $path(test1)]} {

tests/cmdAH.test

@@ -32,6 +32,8 @@ testConstraint linkDirectory [expr {
      && [lindex [file system [temporaryDirectory]] 1] eq "NTFS")
 }]
 testConstraint notWine [expr {![info exists ::env(CI_USING_WINE)]}]
+# File permissions broken on wsl without some "exotic" wsl configuration
+testConstraint notWsl [expr {[llength [array names ::env *WSL*]] == 0}]
 
 global env
 set cmdAHwd [pwd]
@@ -1019,7 +1021,7 @@ test cmdAH-16.2 {Tcl_FileObjCmd: readable} {
     -result 	 1
 }
 test cmdAH-16.3 {Tcl_FileObjCmd: readable} {
-    -constraints {unix notRoot testchmod}
+    -constraints {unix notRoot testchmod notWsl}
     -setup  	 {testchmod 0o333 $gorpfile}
     -body   	 {file readable $gorpfile}
     -result 	 0
@@ -1052,7 +1054,7 @@ set gorpfile [makeFile abcde gorp.file]
 test cmdAH-18.1 {Tcl_FileObjCmd: executable} -returnCodes error -body {
     file executable a b
 } -result {wrong # args: should be "file executable name"}
-test cmdAH-18.2 {Tcl_FileObjCmd: executable} {notRoot} {
+test cmdAH-18.2 {Tcl_FileObjCmd: executable} {notRoot notWsl} {
     file executable $gorpfile
 } 0
 test cmdAH-18.3 {Tcl_FileObjCmd: executable} {unix testchmod} {
@@ -1600,7 +1602,7 @@ test cmdAH-28.4 {Tcl_FileObjCmd: stat} -setup {
     file stat $gorpfile stat
     list $stat(nlink) $stat(size) $stat(type)
 } -result {1 12 file}
-test cmdAH-28.5 {Tcl_FileObjCmd: stat} -constraints {unix} -setup {
+test cmdAH-28.5 {Tcl_FileObjCmd: stat} -constraints {unix notWsl} -setup {
     unset -nocomplain stat
 } -body {
     file stat $gorpfile stat

tests/dstring.test

@@ -211,6 +211,38 @@ test dstring-2.15 {appending list elements} -constraints testdstring -setup {
 } -cleanup {
     testdstring free
 } -result {x  #}
+test dstring-2.16 {appending list elements - bug [46dda6fc29] segfault} -constraints testdstring -setup {
+    testdstring free
+} -body {
+    testdstring element "\\\n"; # Will setfault
+    testdstring get
+} -cleanup {
+    testdstring free
+} -result \\\\\\n
+test dstring-2.17 {appending list elements - bug [46dda6fc29] segfault} -constraints testdstring -setup {
+    testdstring free
+} -body {
+    testdstring element "\\\{"; # Will setfault
+    testdstring get
+} -cleanup {
+    testdstring free
+} -result [list [list \{]]
+test dstring-2.18 {appending list elements - bug [46dda6fc29] segfault} -constraints testdstring -setup {
+    testdstring free
+} -body {
+    testdstring element "\\\}"; # Will setfault
+    testdstring get
+} -cleanup {
+    testdstring free
+} -result [list [list \}]]
+test dstring-2.19 {appending list elements - bug [46dda6fc29] segfault} -constraints testdstring -setup {
+    testdstring free
+} -body {
+    testdstring element "\\\\"; # Will setfault
+    testdstring get
+} -cleanup {
+    testdstring free
+} -result [list [list \\]]
 
 test dstring-3.1 {nested sublists} -constraints testdstring -setup {
     testdstring free

tests/encoding.test

@@ -171,7 +171,7 @@ test encoding-6.2 {Tcl_CreateEncoding: replace encoding} {testencoding} {
 
 test encoding-7.1 {Tcl_ExternalToUtfDString: small buffer} {
     encoding convertfrom jis0208 8c8c8c8c
-} "吾吾吾吾"
+} 吾吾吾吾
 test encoding-7.2 {Tcl_UtfToExternalDString: big buffer} {
     set a 8C8C8C8C8C8C8C8C8C8C8C8C8C8C8C8C8C8C8C8C8C8C8C8C8C8C8C8C8C8C8C8C
     append a $a
@@ -193,7 +193,7 @@ test encoding-8.1 {Tcl_ExternalToUtf} {
     close $f
     file delete [file join [temporaryDirectory] dummy]
     return $x
-} "ab乎g"
+} ab乎g
 
 test encoding-9.1 {Tcl_UtfToExternalDString: small buffer} {
     encoding convertto jis0208 "吾吾吾吾"
@@ -213,7 +213,7 @@ test encoding-9.2 {Tcl_UtfToExternalDString: big buffer} {
 test encoding-10.1 {Tcl_UtfToExternal} {
     set f [open [file join [temporaryDirectory] dummy] w]
     fconfigure $f -translation binary -encoding shiftjis
-    puts -nonewline $f "ab乎g"
+    puts -nonewline $f ab乎g
     close $f
     set f [open [file join [temporaryDirectory] dummy] r]
     fconfigure $f -translation binary -encoding iso8859-1
@@ -248,7 +248,7 @@ test encoding-11.1 {LoadEncodingFile: unknown encoding} {testencoding} {
 } {1 {unknown encoding "jis0208"} 8C}
 test encoding-11.2 {LoadEncodingFile: single-byte} {
     encoding convertfrom jis0201 \xA1
-} "｡"
+} ｡
 test encoding-11.3 {LoadEncodingFile: double-byte} {
     encoding convertfrom jis0208 8C
 } 乎
@@ -452,6 +452,27 @@ test encoding-15.24 {UtfToUtfProc CESU-8 bug [048dd20b4171c8da]} {
     binary scan $y H* z
     list [string length $y] $z
 } {2 cfbf}
+test encoding-15.25 {UtfToUtfProc CESU-8} {
+    encoding convertfrom cesu-8 \x00
+} \x00
+test encoding-15.26 {UtfToUtfProc CESU-8} {
+    encoding convertfrom cesu-8 \xC0\x80
+} \x00
+test encoding-15.27 {UtfToUtfProc -strict CESU-8} {
+    encoding convertfrom -strict cesu-8 \x00
+} \x00
+test encoding-15.28 {UtfToUtfProc -strict CESU-8} -body {
+    encoding convertfrom -strict cesu-8 \xC0\x80
+} -returnCodes 1 -result {unexpected byte sequence starting at index 0: '\xC0'}
+test encoding-15.29 {UtfToUtfProc CESU-8} {
+    encoding convertto cesu-8 \x00
+} \x00
+test encoding-15.30 {UtfToUtfProc -strict CESU-8} {
+    encoding convertto -strict cesu-8 \x00
+} \x00
+test encoding-15.31 {UtfToUtfProc -strict CESU-8 (bytes F0-F4 are invalid)} -body {
+    encoding convertfrom -strict cesu-8 \xF1\x86\x83\x9C
+} -returnCodes 1 -result {unexpected byte sequence starting at index 0: '\xF1'}
 
 test encoding-16.1 {Utf16ToUtfProc} -body {
     set val [encoding convertfrom utf-16 NN]
@@ -481,11 +502,71 @@ test encoding-16.7 {Utf32ToUtfProc} -body {
     set val [encoding convertfrom utf-32be \0\0NN]
     list $val [format %x [scan $val %c]]
 } -result "乎 4e4e"
+test encoding-16.8 {Utf32ToUtfProc} -body {
+    set val [encoding convertfrom -nocomplain utf-32 \x41\x00\x00\x41]
+    list $val [format %x [scan $val %c]]
+} -result "\uFFFD fffd"
+test encoding-16.9 {Utf32ToUtfProc} -body {
+    encoding convertfrom utf-32le \x00\xD8\x00\x00
+} -result \uD800
+test encoding-16.10 {Utf32ToUtfProc} -body {
+    encoding convertfrom utf-32le \x00\xDC\x00\x00
+} -result \uDC00
+test encoding-16.11 {Utf32ToUtfProc} -body {
+    encoding convertfrom utf-32le \x00\xD8\x00\x00\x00\xDC\x00\x00
+} -result \uD800\uDC00
+test encoding-16.12 {Utf32ToUtfProc} -body {
+    encoding convertfrom utf-32le \x00\xDC\x00\x00\x00\xD8\x00\x00
+} -result \uDC00\uD800
+test encoding-16.13 {Utf16ToUtfProc} -body {
+    encoding convertfrom utf-16le \x00\xD8
+} -result \uD800
+test encoding-16.14 {Utf16ToUtfProc} -body {
+    encoding convertfrom utf-16le \x00\xDC
+} -result \uDC00
+test encoding-16.15 {Utf16ToUtfProc} -body {
+    encoding convertfrom utf-16le \x00\xD8\x00\xDC
+} -result \uD800\uDC00
+test encoding-16.16 {Utf16ToUtfProc} -body {
+    encoding convertfrom utf-16le \x00\xDC\x00\xD8
+} -result \uDC00\uD800
+test encoding-16.17 {Utf32ToUtfProc} -body {
+    list [encoding convertfrom -strict -failindex  idx utf-32le \x41\x00\x00\x00\x00\xD8\x00\x00\x42\x00\x00\x00] [set idx]
+} -result {A 4}
+
+test encoding-16.18 {
+    Utf16ToUtfProc, Tcl_UniCharToUtf, surrogate pairs in utf-16
+} -body {
+    apply [list {} {
+	for {set i 0xD800} {$i < 0xDBFF} {incr i} {
+	    for {set j 0xDC00} {$j < 0xDFFF} {incr j} {
+		set string [binary format S2 [list $i $j]]
+		set status [catch {
+		    set decoded [encoding convertfrom utf-16be $string]
+		    set encoded [encoding convertto utf-16be $decoded]
+		}]
+		if {$status || ( $encoded ne $string )} {
+		    return [list [format %x $i] [format %x $j]]
+		}
+	    }
+	}
+	return done
+    } [namespace current]]
+} -result done
+test encoding-16.19 {UnicodeToUtfProc, bug [d19fe0a5b]} -body {
+    encoding convertfrom utf-16 "\x41\x41\x41"
+} -result \u4141\uFFFD
+test encoding-16.20 {UnicodeToUtfProc, bug [d19fe0a5b]} -constraints deprecated -body {
+    encoding convertfrom utf-16 "\xD8\xD8"
+} -result \uD8D8
+test encoding-16.21 {UnicodeToUtfProc, bug [d19fe0a5b]} -body {
+    encoding convertfrom utf-32 "\x00\x00\x00\x00\x41\x41"
+} -result \x00\uFFFD
 
 test encoding-17.1 {UtfToUtf16Proc} -body {
     encoding convertto utf-16 "\U460DC"
 } -result "\xD8\xD8\xDC\xDC"
-test encoding-17.2 {UtfToUcs2Proc} -body {
+test encoding-17.2 {UtfToUcs2Proc, invalid testcase, see [5607d6482c]} -constraints deprecated -body {
     encoding convertfrom utf-16 [encoding convertto ucs-2 "\U460DC"]
 } -result "\uFFFD"
 test encoding-17.3 {UtfToUtf16Proc} -body {
@@ -513,14 +594,43 @@ test encoding-17.10 {Utf32ToUtfProc} -body {
     encoding convertfrom -nocomplain utf-32 "\xFF\xFF\xFF\xFF"
 } -result \uFFFD
 
-
-test encoding-18.1 {TableToUtfProc error on invalid input with -strict} -body {
+test encoding-18.1 {TableToUtfProc on invalid input} -constraints deprecated -body {
+	list [catch {encoding convertto jis0208 \\} res] $res
+} -result {0 !)}
+test encoding-18.2 {TableToUtfProc on invalid input with -strict} -body {
 	list [catch {encoding convertto -strict jis0208 \\} res] $res
 } -result {1 {unexpected character at index 0: 'U+00005C'}}
+test encoding-18.3 {TableToUtfProc on invalid input with -strict -failindex} -body {
+	list [catch {encoding convertto -strict -failindex pos jis0208 \\} res] $res $pos
+} -result {0 {} 0}
+test encoding-18.4 {TableToUtfProc on invalid input with -failindex -strict} -body {
+	list [catch {encoding convertto -failindex pos -strict jis0208 \\} res] $res $pos
+} -result {0 {} 0}
+test encoding-18.5 {TableToUtfProc on invalid input with -failindex} -body {
+	list [catch {encoding convertto -failindex pos jis0208 \\} res] $res $pos
+} -result {0 {} 0}
+test encoding-18.6 {TableToUtfProc on invalid input with -nocomplain} -body {
+	list [catch {encoding convertto -nocomplain jis0208 \\} res] $res
+} -result {0 !)}
 
-
-test encoding-19.1 {TableFromUtfProc} {
-} {}
+test encoding-19.1 {TableFromUtfProc} -body {
+    encoding convertfrom ascii AÁ
+} -result AÁ
+test encoding-19.2 {TableFromUtfProc} -body {
+    encoding convertfrom -nocomplain ascii AÁ
+} -result AÁ
+test encoding-19.3 {TableFromUtfProc} -body {
+    encoding convertfrom -strict ascii AÁ
+} -returnCodes 1 -result {unexpected byte sequence starting at index 1: '\xC1'}
+test encoding-19.4 {TableFromUtfProc} -body {
+    list [encoding convertfrom -failindex idx ascii AÁ] [set idx]
+} -result {A 1}
+test encoding-19.5 {TableFromUtfProc} -body {
+    list [encoding convertfrom -failindex idx -strict ascii AÁ] [set idx]
+} -result {A 1}
+test encoding-19.6 {TableFromUtfProc} -body {
+    list [encoding convertfrom -failindex idx -strict ascii AÁB] [set idx]
+} -result {A 1}
 
 test encoding-20.1 {TableFreefProc} {
 } {}
@@ -663,7 +773,7 @@ test encoding-24.14 {Parse valid or invalid utf-8} {
 } 1
 test encoding-24.15 {Parse valid or invalid utf-8} -constraints deprecated -body {
     encoding convertfrom utf-8 "Z\xE0\x80"
-} -result Z\xE0\x80
+} -result Z\xE0\u20AC
 test encoding-24.16 {Parse valid or invalid utf-8} -constraints testbytestring -body {
     encoding convertto utf-8 [testbytestring "Z\u4343\x80"]
 } -returnCodes 1 -result {expected byte sequence but character 1 was '䍃€' (U+004343)}
@@ -739,6 +849,21 @@ test encoding-24.39 {Try to generate invalid utf-8 with -strict} -body {
 test encoding-24.40 {Try to generate invalid utf-8 with -nocomplain} -body {
     encoding convertto -nocomplain utf-8 \uD800
 } -result \xED\xA0\x80
+test encoding-24.41 {Parse invalid utf-8 with -strict} -body {
+    encoding convertfrom -strict utf-8 \xED\xA0\x80\xED\xB0\x80
+} -returnCodes 1 -result {unexpected byte sequence starting at index 0: '\xED'}
+test encoding-24.42 {Parse invalid utf-8, fallback to cp1252 [885c86a9a0]} -body {
+    encoding convertfrom -nocomplain utf-8 \xF0\x80\x80\x80
+} -result \xF0\u20AC\u20AC\u20AC
+test encoding-24.43 {Parse invalid utf-8, fallback to cp1252 [885c86a9a0]} -body {
+    encoding convertfrom -nocomplain utf-8 \x80
+} -result \u20AC
+test encoding-24.44 {Try to generate invalid ucs-2 with -strict} -body {
+    encoding convertto -strict ucs-2 \uD800
+} -returnCodes 1 -result {unexpected character at index 0: 'U+00D800'}
+test encoding-24.45 {Try to generate invalid ucs-2 with -strict} -body {
+    encoding convertto -strict ucs-2 \U10000
+} -returnCodes 1 -result {unexpected character at index 0: 'U+010000'}
 
 file delete [file join [temporaryDirectory] iso2022.txt]
 
@@ -893,7 +1018,9 @@ test encoding-27.2 {encoding dirs basic behavior} -returnCodes error -body {
 test encoding-28.0 {all encodings load} -body {
 	set string hello
 	foreach name [encoding names] {
-		incr count
+		if {$name ne "unicode"} {
+		    incr count
+		}
 		encoding convertto -nocomplain $name $string
 
 		# discard the cached internal representation of Tcl_Encoding
@@ -901,7 +1028,7 @@ test encoding-28.0 {all encodings load} -body {
 		llength $name
 	}
 	return $count
-} -result [expr {[info exists ::tcl_precision] ? 92 : 91}]
+} -result 91
 
 runtests
 

tests/fCmd.test

@@ -42,6 +42,9 @@ if {[testConstraint win]} {
 }
 testConstraint notInCIenv [expr {![info exists ::env(CI)] || !$::env(CI)}]
 
+# File permissions broken on wsl without some "exotic" wsl configuration
+testConstraint notWsl [expr {[llength [array names ::env *WSL*]] == 0}]
+
 set tmpspace /tmp;# default value
 # Find a group that exists on this Unix system, or else skip tests that
 # require Unix groups.
@@ -354,7 +357,7 @@ test fCmd-4.10 {TclFileMakeDirsCmd: exists, is dir} -setup {
 } -result {1 1}
 test fCmd-4.11 {TclFileMakeDirsCmd: doesn't exist: errno != ENOENT} -setup {
     cleanup
-} -constraints {unix notRoot testchmod} -returnCodes error -body {
+} -constraints {unix notRoot testchmod notWsl} -returnCodes error -body {
     file mkdir td1/td2/td3
     testchmod 0 td1/td2
     file mkdir td1/td2/td3/td4
@@ -372,7 +375,7 @@ test fCmd-4.13 {TclFileMakeDirsCmd: doesn't exist: errno == ENOENT} -setup {
 test fCmd-4.14 {TclFileMakeDirsCmd: TclpCreateDirectory fails} -setup {
     cleanup
     file delete -force foo
-} -constraints {unix notRoot} -body {
+} -constraints {unix notRoot notWsl} -body {
     file mkdir foo
     file attr foo -perm 0o40000
     file mkdir foo/tf1
@@ -497,7 +500,7 @@ test fCmd-6.5 {CopyRenameOneFile: lstat(target) != 0} -setup {
 } -result {tf2}
 test fCmd-6.6 {CopyRenameOneFile: errno != ENOENT} -setup {
     cleanup
-} -constraints {unix notRoot testchmod} -body {
+} -constraints {unix notRoot testchmod notWsl} -body {
     file mkdir td1
     testchmod 0 td1
     createfile tf1
@@ -616,7 +619,7 @@ test fCmd-6.22 {CopyRenameOneFile: copy/rename: !S_ISDIR(source)} -setup {
 } -result [file join $tmpspace tf1]
 test fCmd-6.23 {CopyRenameOneFile: TclpCopyDirectory failed} -setup {
     cleanup $tmpspace
-} -constraints {xdev notRoot} -body {
+} -constraints {xdev notRoot notWsl} -body {
     file mkdir td1/td2/td3
     file attributes td1 -permissions 0
     file rename td1 $tmpspace
@@ -668,7 +671,7 @@ test fCmd-6.27 {CopyRenameOneFile: TclpCopyDirectory failed} -setup {
 } -match glob -result {error renaming "td1" to "/tmp/tcl*/td1": file already exists}
 test fCmd-6.28 {CopyRenameOneFile: TclpCopyDirectory failed} -setup {
     cleanup $tmpspace
-} -constraints {notRoot xdev} -body {
+} -constraints {notRoot xdev notWsl} -body {
     file mkdir td1/td2/td3
     file attributes td1/td2/td3 -permissions 0
     file rename td1 $tmpspace
@@ -685,7 +688,7 @@ test fCmd-6.29 {CopyRenameOneFile: TclpCopyDirectory passed} -setup {
 } -result [file join $tmpspace td1 td2]
 test fCmd-6.30 {CopyRenameOneFile: TclpRemoveDirectory failed} -setup {
     cleanup $tmpspace
-} -constraints {unix notRoot} -body {
+} -constraints {unix notRoot notWsl} -body {
     file mkdir foo/bar
     file attr foo -perm 0o40555
     file rename foo/bar $tmpspace
@@ -760,7 +763,7 @@ test fCmd-8.3 {file copy and path translation: ensure correct error} -body {
 
 test fCmd-9.1 {file rename: comprehensive: EACCES} -setup {
     cleanup
-} -constraints {unix notRoot} -body {
+} -constraints {unix notRoot notWsl} -body {
     file mkdir td1
     file mkdir td2
     file attr td2 -perm 0o40000
@@ -786,7 +789,7 @@ test fCmd-9.3 {file rename: comprehensive: file to new name} -setup {
 } -result {{tf3 tf4} 1 0}
 test fCmd-9.4 {file rename: comprehensive: dir to new name} -setup {
     cleanup
-} -constraints {unix notRoot testchmod notDarwin9} -body {
+} -constraints {unix notRoot testchmod notDarwin9 notWsl} -body {
     file mkdir td1 td2
     testchmod 0o555 td2
     file rename td1 td3
@@ -807,7 +810,7 @@ test fCmd-9.5 {file rename: comprehensive: file to self} -setup {
 } -result {tf1 tf2 1 0}
 test fCmd-9.6 {file rename: comprehensive: dir to self} -setup {
     cleanup
-} -constraints {unix notRoot testchmod} -body {
+} -constraints {unix notRoot testchmod notWsl} -body {
     file mkdir td1
     file mkdir td2
     testchmod 0o555 td2
@@ -1001,7 +1004,7 @@ test fCmd-10.2 {file copy: comprehensive: file to new name} -setup {
 } -result {{tf1 tf2 tf3 tf4} tf1 tf2 1 0}
 test fCmd-10.3 {file copy: comprehensive: dir to new name} -setup {
     cleanup
-} -constraints {unix notRoot testchmod} -body {
+} -constraints {unix notRoot testchmod notWsl} -body {
     file mkdir [file join td1 tdx]
     file mkdir [file join td2 tdy]
     testchmod 0o555 td2
@@ -1078,7 +1081,7 @@ test fCmd-10.5 {file copy: comprehensive: dir to empty dir} -setup {
 } -result [subst {{td1 td2 tdd1 tdd2 tdd3 tdd4 tds1 tds2 tds3 tds4} {1 {error copying "td1" to "[file join td2 td1]": file already exists}} {1 {error copying "tds1" to "[file join tdd1 tds1]": file already exists}} 1 1 1}]
 test fCmd-10.6 {file copy: comprehensive: dir to non-empty dir} -setup {
     cleanup
-} -constraints {notRoot unixOrWin testchmod} -body {
+} -constraints {notRoot unixOrWin testchmod notWsl} -body {
     file mkdir tds1
     file mkdir tds2
     file mkdir [file join tdd1 tds1 xxx]
@@ -1102,7 +1105,7 @@ test fCmd-10.7 {file rename: comprehensive: file to new name and dir} -setup {
 } -result [subst {{tf1 tf2} {[file join td1 tf3] [file join td1 tf4]} 1 0}]
 test fCmd-10.8 {file rename: comprehensive: dir to new name and dir} -setup {
     cleanup
-} -constraints {unix notRoot testchmod} -body {
+} -constraints {unix notRoot testchmod notWsl} -body {
     file mkdir td1
     file mkdir td2
     file mkdir td3
@@ -1299,7 +1302,7 @@ test fCmd-12.7 {renamefile: renaming directory into offspring} -setup {
 } -result {1}
 test fCmd-12.8 {renamefile: generic error} -setup {
     catch {file delete -force -- tfa}
-} -constraints {unix notRoot} -body {
+} -constraints {unix notRoot notWsl} -body {
     file mkdir tfa
     file mkdir tfa/dir
     file attributes tfa -permissions 0o555
@@ -1486,7 +1489,7 @@ test fCmd-14.7 {copyfile: copy directory succeeding} -setup {
 } -result {1 1}
 test fCmd-14.8 {copyfile: copy directory failing} -setup {
     catch {file delete -force -- tfa}
-} -constraints {unix notRoot} -body {
+} -constraints {unix notRoot notWsl} -body {
     file mkdir tfa/dir/a/b/c
     file attributes tfa/dir -permissions 0
     catch {file copy tfa tfa2}
@@ -1628,7 +1631,7 @@ test fCmd-16.8 {remove a normal file} -constraints {notRoot} -setup {
 } -result {1}
 test fCmd-16.9 {error while deleting file} -setup {
     catch {file delete -force -- tfa}
-} -constraints {unix notRoot} -body {
+} -constraints {unix notRoot notWsl} -body {
     file mkdir tfa
     createfile tfa/a
     file attributes tfa -permissions 0o555
@@ -1659,7 +1662,7 @@ test fCmd-16.11 {TclFileDeleteCmd: removing a nonexistant file} -setup {
 # More coverage tests for mkpath()
 test fCmd-17.1 {mkdir stat failing on target but not ENOENT} -setup {
      catch {file delete -force -- tfa1}
-} -constraints {unix notRoot} -body {
+} -constraints {unix notRoot notWsl} -body {
      file mkdir tfa1
      file attributes tfa1 -permissions 0o555
      catch {file mkdir tfa1/tfa2}
@@ -1869,7 +1872,7 @@ test fCmd-19.1 {remove empty directory} -constraints {notRoot} -setup {
 } -result {0}
 test fCmd-19.2 {rmdir error besides EEXIST} -setup {
     catch {file delete -force -- tfa}
-} -constraints {unix notRoot} -body {
+} -constraints {unix notRoot notWsl} -body {
     file mkdir tfa
     file mkdir tfa/a
     file attributes tfa -permissions 0o555
@@ -1897,7 +1900,7 @@ test fCmd-19.3 {recursive remove} -constraints {notRoot} -setup {
 #
 test fCmd-20.1 {TraverseUnixTree : failure opening a subdirectory directory} -setup {
     catch {file delete -force -- tfa}
-} -constraints {unix notRoot} -body {
+} -constraints {unix notRoot notWsl} -body {
     file mkdir tfa
     file mkdir tfa/a
     file attributes tfa/a -permissions 00000

tests/io.test

@@ -48,6 +48,8 @@ testConstraint testservicemode  [llength [info commands testservicemode]]
 testConstraint notWinCI [expr {
     $::tcl_platform(platform) ne "windows" || ![info exists ::env(CI)]}]
 testConstraint notOSX [expr {$::tcl_platform(os) ne "Darwin"}]
+# File permissions broken on wsl without some "exotic" wsl configuration
+testConstraint notWsl [expr {[llength [array names ::env *WSL*]] == 0}]
 
 # You need a *very* special environment to do some tests.  In
 # particular, many file systems do not support large-files...
@@ -5813,7 +5815,7 @@ test io-40.1 {POSIX open access modes: RDWR} {
     close $f
     set x
 } {zzy abzzy}
-test io-40.2 {POSIX open access modes: CREAT} {unix} {
+test io-40.2 {POSIX open access modes: CREAT} {unix notWsl} {
     file delete $path(test3)
     set f [open $path(test3) {WRONLY CREAT} 0o600]
     file stat $path(test3) stats
@@ -5825,7 +5827,7 @@ test io-40.2 {POSIX open access modes: CREAT} {unix} {
     close $f
     set x
 } {0o600 {line 1}}
-test io-40.3 {POSIX open access modes: CREAT} {unix umask} {
+test io-40.3 {POSIX open access modes: CREAT} {unix umask notWsl} {
     # This test only works if your umask is 2, like ouster's.
     file delete $path(test3)
     set f [open $path(test3) {WRONLY CREAT}]
@@ -7280,8 +7282,8 @@ test io-52.5 {TclCopyChannel, all} {fcopy} {
     file delete $path(test1)
     set f1 [open $thisScript]
     set f2 [open $path(test1) w]
-    fconfigure $f1 -translation lf -blocking 0
-    fconfigure $f2 -translation lf -blocking 0
+    fconfigure $f1 -translation lf -encoding iso8859-1 -blocking 0
+    fconfigure $f2 -translation lf -encoding iso8859-1 -blocking 0
     fcopy $f1 $f2 -size -1 ;# -1 means 'copy all', same as if no -size specified.
     set result [list [fconfigure $f1 -blocking] [fconfigure $f2 -blocking]]
     close $f1
@@ -7297,8 +7299,8 @@ test io-52.5a {TclCopyChannel, all, other negative value} {fcopy} {
     file delete $path(test1)
     set f1 [open $thisScript]
     set f2 [open $path(test1) w]
-    fconfigure $f1 -translation lf -blocking 0
-    fconfigure $f2 -translation lf -blocking 0
+    fconfigure $f1 -translation lf -encoding iso8859-1 -blocking 0
+    fconfigure $f2 -translation lf -encoding iso8859-1 -blocking 0
     fcopy $f1 $f2 -size -2 ;# < 0 behaves like -1, copy all
     set result [list [fconfigure $f1 -blocking] [fconfigure $f2 -blocking]]
     close $f1
@@ -7314,8 +7316,8 @@ test io-52.5b {TclCopyChannel, all, wrap to negative value} {fcopy} {
     file delete $path(test1)
     set f1 [open $thisScript]
     set f2 [open $path(test1) w]
-    fconfigure $f1 -translation lf -blocking 0
-    fconfigure $f2 -translation lf -blocking 0
+    fconfigure $f1 -translation lf -encoding iso8859-1 -blocking 0
+    fconfigure $f2 -translation lf -encoding iso8859-1 -blocking 0
     fcopy $f1 $f2 -size 3221176172 ;# Wrapped to < 0, behaves like -1, copy all
     set result [list [fconfigure $f1 -blocking] [fconfigure $f2 -blocking]]
     close $f1
@@ -7348,8 +7350,8 @@ test io-52.7 {TclCopyChannel} {fcopy} {
     file delete $path(test1)
     set f1 [open $thisScript]
     set f2 [open $path(test1) w]
-    fconfigure $f1 -translation lf -blocking 0
-    fconfigure $f2 -translation lf -blocking 0
+    fconfigure $f1 -translation lf -encoding iso8859-1 -blocking 0
+    fconfigure $f2 -translation lf -encoding iso8859-1 -blocking 0
     fcopy $f1 $f2
     set result [list [fconfigure $f1 -blocking] [fconfigure $f2 -blocking]]
     set s1 [file size $thisScript]
@@ -7609,6 +7611,103 @@ test io-52.19 {coverage of eofChar handling} {
     close $out
     file size $path(test2)
 } 8
+test io-52.20 {TclCopyChannel & encodings} -setup {
+    set out [open $path(utf8-fcopy.txt) w]
+    fconfigure $out -encoding utf-8 -translation lf
+    puts $out "Á"
+    close $out
+} -constraints {fcopy} -body {
+    # binary to encoding => the input has to be
+    # in utf-8 to make sense to the encoder
+
+    set in  [open $path(utf8-fcopy.txt) r]
+    set out [open $path(kyrillic.txt) w]
+
+    # Using "-encoding ascii" means reading the "Á" gives an error
+    fconfigure $in  -encoding ascii -strictencoding 1
+    fconfigure $out -encoding koi8-r -translation lf
+
+    fcopy $in $out
+} -cleanup {
+    close $in
+    close $out
+} -returnCodes 1 -match glob -result {error reading "file*": illegal byte sequence}
+test io-52.21 {TclCopyChannel & encodings} -setup {
+    set out [open $path(utf8-fcopy.txt) w]
+    fconfigure $out -encoding utf-8 -translation lf
+    puts $out "Á"
+    close $out
+} -constraints {fcopy} -body {
+    # binary to encoding => the input has to be
+    # in utf-8 to make sense to the encoder
+
+    set in  [open $path(utf8-fcopy.txt) r]
+    set out [open $path(kyrillic.txt) w]
+
+    # Using "-encoding ascii" means writing the "Á" gives an error
+    fconfigure $in  -encoding utf-8
+    fconfigure $out -encoding ascii -translation lf -strictencoding 1
+
+    fcopy $in $out
+} -cleanup {
+    close $in
+    close $out
+} -returnCodes 1 -match glob -result {error writing "file*": illegal byte sequence}
+test io-52.22 {TclCopyChannel & encodings} -setup {
+    set out [open $path(utf8-fcopy.txt) w]
+    fconfigure $out -encoding utf-8 -translation lf
+    puts $out "Á"
+    close $out
+} -constraints {fcopy} -body {
+    # binary to encoding => the input has to be
+    # in utf-8 to make sense to the encoder
+
+    set in  [open $path(utf8-fcopy.txt) r]
+    set out [open $path(kyrillic.txt) w]
+
+    # Using "-encoding ascii" means reading the "Á" gives an error
+    fconfigure $in  -encoding ascii -strictencoding 1
+    fconfigure $out -encoding koi8-r -translation lf
+    proc ::xxx args {
+        set ::s0 $args
+    }
+
+    fcopy $in $out -command ::xxx
+    vwait ::s0
+    set ::s0
+} -cleanup {
+    close $in
+    close $out
+    unset ::s0
+} -match glob -result {0 {error reading "file*": illegal byte sequence}}
+test io-52.23 {TclCopyChannel & encodings} -setup {
+    set out [open $path(utf8-fcopy.txt) w]
+    fconfigure $out -encoding utf-8 -translation lf
+    puts $out "Á"
+    close $out
+} -constraints {fcopy} -body {
+    # binary to encoding => the input has to be
+    # in utf-8 to make sense to the encoder
+
+    set in  [open $path(utf8-fcopy.txt) r]
+    set out [open $path(kyrillic.txt) w]
+
+    # Using "-encoding ascii" means writing the "Á" gives an error
+    fconfigure $in  -encoding utf-8
+    fconfigure $out -encoding ascii -translation lf -strictencoding 1
+    proc ::xxx args {
+        set ::s0 $args
+    }
+
+    fcopy $in $out -command ::xxx
+    vwait ::s0
+    set ::s0
+} -cleanup {
+    close $in
+    close $out
+    unset ::s0
+} -match glob -result {0 {error writing "file*": illegal byte sequence}}
+
 
 test io-53.1 {CopyData} {fcopy} {
     file delete $path(test1)
@@ -9046,48 +9145,6 @@ test io-75.5 {invalid utf-8 encoding read is ignored (-nocomplainencoding 1)} -s
     removeFile io-75.5
 } -result 4181
 
-test io-75.6 {invalid utf-8 encoding read is not ignored (-strictencoding 1)} -setup {
-    set fn [makeFile {} io-75.6]
-    set f [open $fn w+]
-    fconfigure $f -encoding binary
-    # \x81 is invalid in utf-8
-    puts -nonewline $f A\x81
-    flush $f
-    seek $f 0
-    fconfigure $f -encoding utf-8 -buffering none -eofchar "" -translation lf -strictencoding 1
-} -body {
-    set d [read $f]
-    binary scan $d H* hd
-    lappend hd [catch {read $f} msg]
-    close $f
-    lappend hd $msg
-} -cleanup {
-    removeFile io-75.6
-} -match glob -result {41 1 {error reading "*": illegal byte sequence}}
-
-test io-75.7 {invalid utf-8 encoding eof handling (-strictencoding 1)} -setup {
-    set fn [makeFile {} io-75.7]
-    set f [open $fn w+]
-    fconfigure $f -encoding binary
-    # \xA1 is invalid in utf-8. -eofchar is not detected, because it comes later.
-    puts -nonewline $f A\xA1\x1A
-    flush $f
-    seek $f 0
-    fconfigure $f -encoding utf-8 -buffering none -eofchar \x1A -translation lf -strictencoding 1
-} -body {
-    set d [read $f]
-    binary scan $d H* hd
-    lappend hd [eof $f]
-    lappend hd [catch {read $f} msg]
-    lappend hd $msg
-    fconfigure $f -encoding iso8859-1
-    lappend hd [read $f];# We changed encoding, so now we can read the \xA1
-    close $f
-    set hd
-} -cleanup {
-    removeFile io-75.7
-} -match glob -result {41 0 1 {error reading "*": illegal byte sequence} ¡}
-
 test io-75.8 {invalid utf-8 encoding eof handling (-strictencoding 1)} -setup {
     set fn [makeFile {} io-75.8]
     set f [open $fn w+]

tests/ioCmd.test

@@ -376,6 +376,16 @@ test iocmd-8.21 {fconfigure command / -nocomplainencoding 0 error} -constraints
 } -body {
     fconfigure $console -nocomplainencoding 0
 } -returnCodes error -result "bad value for -nocomplainencoding: only true allowed"
+test iocmd-8.22 {fconfigure command / -nocomplainencoding 0, no error if -strictencoding already defined} -setup {
+    set console stdin
+    set oldmode [fconfigure $console -strictencoding]
+} -body {
+    fconfigure $console -strictencoding 1
+    fconfigure $console -nocomplainencoding 0
+    fconfigure $console -nocomplainencoding
+} -cleanup {
+    fconfigure $console -strictencoding $oldmode
+} -result 0
 
 
 test iocmd-9.1 {eof command} {

tests/tcltest.test

@@ -22,6 +22,9 @@ if {"::tcltest" ni [namespace children]} {
     namespace import -force ::tcltest::*
 }
 
+# File permissions broken on wsl without some "exotic" wsl configuration
+testConstraint notWsl [expr {[llength [array names ::env *WSL*]] == 0}]
+
 namespace eval ::tcltest::test {
 
 namespace import ::tcltest::*
@@ -306,7 +309,7 @@ test tcltest-5.3 {testConstraint - constraint empty (tcltest::safeFetch)}  {
 #}
 
 test tcltest-5.5 {InitConstraints: list of built-in constraints} \
-	-constraints {!singleTestInterp} \
+	-constraints {!singleTestInterp notWsl} \
 	-setup {tcltest::InitConstraints} \
 	-body { lsort [array names ::tcltest::testConstraints] } \
 	-result [lsort {
@@ -556,7 +559,7 @@ switch -- $::tcl_platform(platform) {
     }
 }
 test tcltest-8.3 {tcltest a.tcl -tmpdir notReadableDir} {
-    -constraints {unix notRoot}
+    -constraints {unix notRoot notWsl}
     -body {
 	child msg $a -tmpdir $notReadableDir
 	return $msg
@@ -572,7 +575,7 @@ testConstraint notFAT [expr {
 }]
 # FAT/NTFS permissions are fairly hopeless; ignore this test if that FS is used
 test tcltest-8.4 {tcltest a.tcl -tmpdir notWriteableDir} {
-    -constraints {unixOrWin notRoot notFAT}
+    -constraints {unixOrWin notRoot notFAT notWsl}
     -body {
 	child msg $a -tmpdir $notWriteableDir
 	return $msg
@@ -645,7 +648,7 @@ test tcltest-8.11 {tcltest a.tcl -testdir thisdirectoryisafile} {
     -result {*not a directory*}
 }
 test tcltest-8.12 {tcltest a.tcl -testdir notReadableDir} {
-    -constraints {unix notRoot}
+    -constraints {unix notRoot notWsl}
     -body {
 	child msg $a -testdir $notReadableDir
 	return $msg

tests/unixFCmd.test

@@ -18,6 +18,8 @@ if {"::tcltest" ni [namespace children]} {
 catch [list package require -exact tcl::test [info patchlevel]]
 
 testConstraint testchmod [llength [info commands testchmod]]
+# File permissions broken on wsl without some "exotic" wsl configuration
+testConstraint notWsl [expr {[llength [array names ::env *WSL*]] == 0}]
 
 # These tests really need to be run from a writable directory, which
 # it is assumed [temporaryDirectory] is.
@@ -94,7 +96,7 @@ if {[testConstraint unix] && [testConstraint notRoot]} {
 
 test unixFCmd-1.1 {TclpRenameFile: EACCES} -setup {
     cleanup
-} -constraints {unix notRoot} -body {
+} -constraints {unix notRoot notWsl} -body {
     file mkdir td1/td2/td3
     file attributes td1/td2 -permissions 0
     file rename td1/td2/td3 td2
@@ -135,7 +137,7 @@ test unixFCmd-1.6 {TclpRenameFile: ENOTDIR} {emptyTest unix notRoot} {
 } {}
 test unixFCmd-1.7 {TclpRenameFile: EXDEV} -setup {
     cleanup
-} -constraints {unix notRoot} -body {
+} -constraints {unix notRoot notWsl} -body {
     file mkdir foo/bar
     file attr foo -perm 0o40555
     file rename foo/bar /tmp
@@ -219,7 +221,7 @@ test unixFCmd-2.4 {TclpCopyFile: src is fifo} -setup {
 } -result {fifo fifo}
 test unixFCmd-2.5 {TclpCopyFile: copy attributes} -setup {
     cleanup
-} -constraints {unix notRoot} -body {
+} -constraints {unix notRoot notWsl} -body {
     close [open tf1 a]
     file attributes tf1 -permissions 0o472
     file copy tf1 tf2
@@ -334,7 +336,7 @@ test unixFCmd-16.3 {SetOwnerAttribute - invalid owner} -setup {
 
 test unixFCmd-17.1 {SetPermissionsAttribute} -setup {
     catch {file delete -force -- foo.test}
-} -constraints {unix notRoot} -body {
+} -constraints {unix notRoot notWsl} -body {
     close [open foo.test w]
     list [file attributes foo.test -permissions 0] \
 	[file attributes foo.test -permissions]
@@ -366,7 +368,7 @@ test unixFCmd-17.4 {SetPermissionsAttribute} -setup {
 close [open foo.test w]
 set ::i 4
 proc permcheck {testnum permList expected} {
-    test $testnum {SetPermissionsAttribute} {unix notRoot} {
+    test $testnum {SetPermissionsAttribute} {unix notRoot notWsl} {
       set result {}
       foreach permstr $permList {
 	file attributes foo.test -permissions $permstr

tests/zlib.test

@@ -486,6 +486,38 @@ test zlib-8.18 {Bug dd260aaf: fconfigure} -setup {
     catch {close $inSide}
     catch {close $outSide}
 } -result {{one two} {one two}}
+test zlib-8.19 {zlib transformation, bug f9eafc3886} -constraints zlib -setup {
+    set file [makeFile {} test.gz]
+} -body {
+    set f [zlib push gzip [open $file w] -header [list comment [string repeat A 500]]]
+} -cleanup {
+    catch {close $f}
+    removeFile $file
+} -returnCodes 1 -result {Comment too large for zip}
+test zlib-8.20 {zlib transformation, bug f9eafc3886} -constraints zlib -setup {
+    set file [makeFile {} test.gz]
+} -body {
+    set f [zlib push gzip [open $file w] -header [list filename [string repeat A 5000]]]
+} -cleanup {
+    catch {close $f}
+    removeFile $file
+} -returnCodes 1 -result {Filename too large for zip}
+test zlib-8.21 {zlib transformation, bug f9eafc3886} -constraints zlib -setup {
+    set file [makeFile {} test.gz]
+} -body {
+    set f [zlib push gzip [open $file w] -header [list comment \u100]]
+} -cleanup {
+    catch {close $f}
+    removeFile $file
+} -returnCodes 1 -result {Comment contains characters > 0xFF}
+test zlib-8.22 {zlib transformation, bug f9eafc3886} -constraints zlib -setup {
+    set file [makeFile {} test.gz]
+} -body {
+    set f [zlib push gzip [open $file w] -header [list filename \u100]]
+} -cleanup {
+    catch {close $f}
+    removeFile $file
+} -returnCodes 1 -result {Filename contains characters > 0xFF}
 
 test zlib-9.1 "check fcopy with push" -constraints zlib -setup {
     set sfile [makeFile {} testsrc.gz]

tools/encoding/Makefile

@@ -71,11 +71,6 @@ encodings: clean txt2enc $(EUC_ENCODINGS)
 	done
 	@echo
 	@echo Compiling special versions of encoding files.
-	@for p in ascii.txt; do \
-	    enc=`echo $$p | sed 's/\..*$$/\.enc/'`; \
-	    echo $$enc; \
-	    ./txt2enc -m $$p > $$enc; \
-	done
 	@for p in jis0208.txt; do \
 	    enc=`echo $$p | sed 's/\..*$$/\.enc/'`; \
 	    echo $$enc; \

tools/encoding/txt2enc.c

@@ -106,7 +106,7 @@ main(int argc, char **argv)
 	fputs("    -f\tfallback character (default: QUESTION MARK)\n", stderr);
 	fputs("    -t\toverride implicit type with single, double, or multi\n", stderr);
 	fputs("    -s\tsymbol+ascii encoding\n", stderr);
-	fputs("    -m\tdon't implicitly include range 0080 to 00FF\n", stderr);
+	fputs("    -m\tdon't implicitly include 007F\n", stderr);
 	return 1;
     }
 
@@ -208,10 +208,8 @@ main(int argc, char **argv)
 	    toUnicode[0][i] = i;
 	}
 	if (fixmissing) {
-	    for (i = 0x7F; i < 0xA0; i++) {
-		if (toUnicode[i] == NULL && toUnicode[0][i] == 0) {
-		    toUnicode[0][i] = i;
-		}
+	    if (toUnicode[0x7F] == NULL && toUnicode[0][0x7F] == 0) {
+		toUnicode[0][0x7F] = 0x7F;
 	    }
 	}
     }