Rebase to tip-657

Merge tip-657
2023-09-12 14:23:43 +00:00 · 2023-08-30 13:58:17 +00:00 · 2023-07-19 14:59:27 +00:00 · 2023-07-19 09:03:02 +00:00 · 2023-07-16 21:29:41 +00:00 · 2023-07-14 14:51:27 +00:00
18 changed files with 720 additions and 140 deletions
@@ -2038,6 +2038,7 @@ typedef struct Tcl_EncodingType {
 #define TCL_ENCODING_PROFILE_STRICT   TCL_ENCODING_STOPONERROR
 #define TCL_ENCODING_PROFILE_TCL8     0x01000000
 #define TCL_ENCODING_PROFILE_REPLACE  0x02000000
+#define TCL_ENCODING_PROFILE_LOSSLESS 0x03000000

 /*
 * The following definitions are the error codes returned by the conversion
@@ -83,6 +83,8 @@ typedef struct {
 				 * is no corresponding character the encoding,
 				 * the value in the matrix is 0x0000.
 				 * malloc'd. */
+    int flags;			/* Miscellaneous flags */
+#define ENCODING_ASCII_COMPATIBLE 0x1
 } TableEncodingData;

 /*
@@ -196,6 +198,7 @@ static struct TclEncodingProfiles {
    const char *name;
    int value;
 } encodingProfiles[] = {
+    {"lossless", TCL_ENCODING_PROFILE_LOSSLESS},
    {"replace", TCL_ENCODING_PROFILE_REPLACE},
    {"strict", TCL_ENCODING_PROFILE_STRICT},
    {"tcl8", TCL_ENCODING_PROFILE_TCL8},
@@ -206,8 +209,11 @@ static struct TclEncodingProfiles {
 #define PROFILE_REPLACE(flags_)                                        \
    (ENCODING_PROFILE_GET(flags_) == TCL_ENCODING_PROFILE_REPLACE)

+#define PROFILE_LOSSLESS(flags_)                                       \
+    (ENCODING_PROFILE_GET(flags_) == TCL_ENCODING_PROFILE_LOSSLESS)
+
 #define PROFILE_STRICT(flags_)                                         \
-    (!PROFILE_TCL8(flags_) && !PROFILE_REPLACE(flags_))
+    (!PROFILE_TCL8(flags_) && !PROFILE_REPLACE(flags_) && !PROFILE_LOSSLESS(flags_))

 #define UNICODE_REPLACE_CHAR ((Tcl_UniChar)0xFFFD)
 #define SURROGATE(c_)      (((c_) & ~0x7FF) == 0xD800)
@@ -257,6 +263,20 @@ static Tcl_EncodingConvertProc	UtfToUtfProc;
 static Tcl_EncodingConvertProc	Iso88591FromUtfProc;
 static Tcl_EncodingConvertProc	Iso88591ToUtfProc;

+/* Return 1/0 if unich is a lossless wrapper */
+static inline int IsLosslessWrapper(Tcl_UniChar unich) {
+    return (unich >= 0xDC00 && unich <= 0xDCFF);
+}
+/* Convert a byte to internal lossless representation */
+static inline Tcl_UniChar ToLossless(char ch) {
+    /* Only encode if non-ASCII for security reasons. See TIP */
+    return 0x80 & ch ? 0xDC00 + UCHAR(ch) : UNICODE_REPLACE_CHAR;
+}
+/* Convert an internal lossless representation to raw byte */
+static inline unsigned char FromLossless(Tcl_UniChar unich) {
+    assert(IsLosslessWrapper(unich));
+    return (unsigned char)(unich - 0xDC00);
+}

 /*
 * A Tcl_ObjType for holding a cached Tcl_Encoding in the twoPtrValue.ptr1 field
@@ -289,6 +309,42 @@ static const Tcl_ObjType encodingType = {
    } while (0)


+/*
+ *------------------------------------------------------------------------
+ *
+ * ToLosslessUtf8 --
+ *
+ *    Converts an entire string of bytes to their lossless utf-8 representation.
+ *    Caller has to ensure the entire string is to be treated as invalid encoding.
+ *
+ * Results:
+ *    Number of bytes in converted utf-8 output or a negative value if
+ *    insufficient space.
+ *
+ * Side effects:
+ *    The dst buffer is filled with the utf-8 lossless representation.
+ *
+ *------------------------------------------------------------------------
+ */
+static Tcl_Size
+ToLosslessUtf8(
+	const char *src, /* Source bytes */
+	Tcl_Size srcLen, /* Number of source bytes */
+	char *dst,       /* Destination buffer */
+	Tcl_Size dstLen) /* Size of destination buffer */
+{
+    if ((dstLen / 3) < srcLen) {
+	return -1; /* No space */
+    }
+    const char *srcEnd = src + srcLen;
+    char *dstStart = dst;
+    while (src < srcEnd) {
+	dst += Tcl_UniCharToUtf(ToLossless(UCHAR(*src)), dst);
+	++src;
+    }
+    return (dst - dstStart);
+}
+
 /*
 *----------------------------------------------------------------------
 *
@@ -2108,11 +2164,29 @@ LoadTableEncoding(
    TclDecrRefCount(objPtr);

    if (type == ENCODING_DOUBLEBYTE) {
+	/* DBCS never ascii compatible so no need to set dataPtr->flags */
 	memset(dataPtr->prefixBytes, 1, sizeof(dataPtr->prefixBytes));
    } else {
+	int asciiCompatible = 1;
 	for (hi = 1; hi < 256; hi++) {
 	    if (dataPtr->toUnicode[hi] != NULL) {
 		dataPtr->prefixBytes[hi] = 1;
+		if (hi < 128) {
+		    /* any byte < 128 is a prefix => not ASCII compatible */
+		    asciiCompatible = 0;
+		}
+	    }
+	}
+	if (asciiCompatible) {
+	    for (lo = 1; lo < 128; ++lo) {
+		if (dataPtr->toUnicode[0][lo] != lo) {
+		    /* any byte < 128 does not map to itself => not ASCII compatible */
+		    asciiCompatible = 0;
+		    break;
+		}
+	    }
+	    if (asciiCompatible) {
+		dataPtr->flags |= ENCODING_ASCII_COMPATIBLE;
 	    }
 	}
    }
@@ -2556,54 +2630,77 @@ UtfToUtfProc(
 	    *dst++ = *src++;
 	} else if ((UCHAR(*src) == 0xC0) && (src + 1 < srcEnd) &&
 		 (UCHAR(src[1]) == 0x80) &&
-		 (!(flags & ENCODING_INPUT) || PROFILE_STRICT(profile) ||
-		  PROFILE_REPLACE(profile))) {
-	    /* Special sequence \xC0\x80 */
-	    if ((PROFILE_STRICT(profile) || PROFILE_REPLACE(profile)) && (flags & ENCODING_INPUT)) {
-		if (PROFILE_REPLACE(profile)) {
+		 (!(flags & ENCODING_INPUT) || ! PROFILE_TCL8(profile))) {
+	    /*
+	     * \xC0\x80 is handled specially for either of the following:
+	     *   1. We are doing (internal) modified utf-8 to (external)
+	     *   conformant utf-8. C080 is valid internal utf-8 so we
+	     *   simply output a \0. Note this overrides case 2.
+	     *   2. The profile in use is not TCL8, in which case we have to
+	     *   to take a profile-dependent action.
+	     * Note the remaining case of external->internal with a TCL8
+	     * profile is handled in the default if clause later. (TODO - why not here?)
+	     */
+	    if (flags & ENCODING_INPUT) {
+		assert(!PROFILE_TCL8(profile));
+		if (PROFILE_STRICT(profile)) {
+		   result = TCL_CONVERT_SYNTAX;
+		   break;
+		} else if (PROFILE_REPLACE(profile)) {
 		   dst += Tcl_UniCharToUtf(UNICODE_REPLACE_CHAR, dst);
 		   src += 2;
 		} else {
-		   /* PROFILE_STRICT */
-		   result = TCL_CONVERT_SYNTAX;
-		   break;
+		   assert(PROFILE_LOSSLESS(profile));
+		   Tcl_Size len =
+		       ToLosslessUtf8(src, 2, dst, dstLen - (dst - dstStart));
+		   if (len < 0) {
+		       result = TCL_CONVERT_NOSPACE;
+		       break;
+		   }
+		   dst += len;
+		   src += 2;
 		}
 	    } else {
-		/*
-		 * For output convert 0xC080 to a real null.
-		 */
+		/* For output convert 0xC080 to a real null. */
 		*dst++ = 0;
 		src += 2;
 	    }
-
 	} else if (!Tcl_UtfCharComplete(src, srcEnd - src)) {
 	    /*
-	     * Incomplete byte sequence.
-		 * Always check before using Tcl_UtfToUniChar. Not doing so can cause it
-		 * run beyond the end of the buffer! If we happen on such an incomplete
-		 * char its bytes are made to represent themselves unless the user has
-		 * explicitly asked to be told.
+	     * Incomplete byte sequence. We need to do this check before the
+	     * Tcl_UtfToUniChar checks in the next sibling if clause. Not doing
+	     * so can cause it run beyond the end of the buffer! If we
+	     * happen on such an incomplete char its bytes are made to
+	     * represent themselves unless the user has explicitly asked to
+	     * be told.
 	     */
-
+	    assert(flags & TCL_ENCODING_END); /* Else break earlier would
+	    					 trigger (srcClose compare) */
 	    if (flags & ENCODING_INPUT) {
+		/* TODO - why is this inside a ENCODING_INPUT check? */
 		/* Incomplete bytes for modified UTF-8 target */
 		if (PROFILE_STRICT(profile)) {
-		    result = (flags & TCL_ENCODING_CHAR_LIMIT)
-			       ? TCL_CONVERT_MULTIBYTE
-			       : TCL_CONVERT_SYNTAX;
-		    break;
+		   result = (flags & TCL_ENCODING_CHAR_LIMIT)
+			      ? TCL_CONVERT_MULTIBYTE
+			      : TCL_CONVERT_SYNTAX;
+		   break;
 		}
 	    }
 	    if (PROFILE_REPLACE(profile)) {
 		ch = UNICODE_REPLACE_CHAR;
 		++src;
-	    } else {
-		/* TCL_ENCODING_PROFILE_TCL8 */
+		dst += Tcl_UniCharToUtf(ch, dst);
+	    } else if (PROFILE_TCL8(profile)) {
 		char chbuf[2];
 		chbuf[0] = UCHAR(*src++); chbuf[1] = 0;
 		Tcl_UtfToUniChar(chbuf, &ch);
+		dst += Tcl_UniCharToUtf(ch, dst);
+	    } else {
+		assert(PROFILE_LOSSLESS(profile));
+                ch = ToLossless(UCHAR(*src));
+		dst += Tcl_UniCharToUtf(ch, dst);
+		src += 1;
 	    }
-	    dst += Tcl_UniCharToUtf(ch, dst);
 	} else {
 	    int isInvalid = 0;
 	    size_t len = Tcl_UtfToUniChar(src, &ch);
@@ -2619,7 +2716,18 @@ UtfToUtfProc(
 			break;
 		    } else if (PROFILE_REPLACE(profile)) {
 			ch = UNICODE_REPLACE_CHAR;
+		    } else if (PROFILE_LOSSLESS(profile)) {
+			Tcl_Size n = ToLosslessUtf8(
+			    src, len, dst, dstLen - (dst - dstStart));
+			if (n < 0) {
+			    result = TCL_CONVERT_NOSPACE;
+			    break;
+			}
+			dst += n;
+			src += len;
+			continue;
 		    }
+		    /* else PROFILE_TCL8 - treat as normal char below */
 		}
 	    }

@@ -2638,18 +2746,18 @@ UtfToUtfProc(
 		*dst++ = (char) (((ch >> 6) | 0x80) & 0xBF);
 		*dst++ = (char) ((ch | 0x80) & 0xBF);
 		continue;
-	    } else if (PROFILE_STRICT(profile) &&
-		       (!(flags & ENCODING_INPUT)) &&
-		       SURROGATE(ch)) {
-		result = TCL_CONVERT_UNKNOWN;
-		src = saveSrc;
-		break;
-	    } else if (PROFILE_STRICT(profile) &&
-		       (flags & ENCODING_INPUT) &&
-		       SURROGATE(ch)) {
-		result = TCL_CONVERT_SYNTAX;
-		src = saveSrc;
-		break;
+	    } else if (SURROGATE(ch)) {
+		if (PROFILE_STRICT(profile)) {
+		    result = (flags & ENCODING_INPUT) ? TCL_CONVERT_SYNTAX
+						      : TCL_CONVERT_UNKNOWN;
+		    src = saveSrc;
+		    break;
+		} else if (PROFILE_LOSSLESS(profile)) {
+		    if (IsLosslessWrapper(ch)) {
+			*dst++ = FromLossless(ch); /* Invalid UTF8 by design! */
+			continue;
+		    }
+		}
 	    }
 	    dst += Tcl_UniCharToUtf(ch, dst);
 	}
@@ -2746,7 +2854,8 @@ Utf32ToUtfProc(
 	} else if (PROFILE_STRICT(flags) && SURROGATE(ch)) {
 	    result = TCL_CONVERT_SYNTAX;
 	    break;
-	} else if (PROFILE_REPLACE(flags) && SURROGATE(ch)) {
+	} else if ((! PROFILE_TCL8(flags)) && SURROGATE(ch)) {
+            /* PROFILE_REPLACE | PROFILE_LOSSLESS */
 	    ch = UNICODE_REPLACE_CHAR;
 	}

@@ -2774,7 +2883,7 @@ Utf32ToUtfProc(
 	    if (PROFILE_STRICT(flags)) {
 		result = TCL_CONVERT_SYNTAX;
 	    } else {
-		/* PROFILE_REPLACE or PROFILE_TCL8 */
+		/* PROFILE_REPLACE | PROFILE_LOSSLESS | PROFILE_TCL8 */
 		result = TCL_OK;
 		dst += Tcl_UniCharToUtf(UNICODE_REPLACE_CHAR, dst);
 		numChars++;
@@ -2863,8 +2972,8 @@ UtfToUtf32Proc(
 	    if (PROFILE_STRICT(flags)) {
 		result = TCL_CONVERT_UNKNOWN;
 		break;
-	    }
-	    if (PROFILE_REPLACE(flags)) {
+	    } else if (! PROFILE_TCL8(flags)) {
+                /* PROFILE_REPLACE | PROFILE_LOSSLESS */
 		ch = UNICODE_REPLACE_CHAR;
 	    }
 	}
@@ -2983,7 +3092,10 @@ Utf16ToUtfProc(
 		dst--; /* Also undo writing a single byte too much */
 		numChars--;
 		break;
-	    } else if (PROFILE_REPLACE(flags)) {
+	    } else if (PROFILE_TCL8(flags)) {
+                /* Bug [10c2c17c32]. Hi surrogate not followed by Lo: finish 3-byte UTF-8 */
+		dst += Tcl_UniCharToUtf(-1, dst);
+	    } else {
 		/*
 		 * Previous loop wrote a single byte to mark the high surrogate.
 		 * Replace it with the replacement character. Further, restart
@@ -2996,9 +3108,6 @@ Utf16ToUtfProc(
 		src -= 2;
 		numChars--;
 		continue;
-	    } else {
-	    /* Bug [10c2c17c32]. If Hi surrogate not followed by Lo surrogate, finish 3-byte UTF-8 */
-		dst += Tcl_UniCharToUtf(-1, dst);
 	    }
 	}

@@ -3011,15 +3120,20 @@ Utf16ToUtfProc(
 	    *dst++ = (ch & 0xFF);
 	} else if (HIGH_SURROGATE(prev) || HIGH_SURROGATE(ch)) {
 	    dst += Tcl_UniCharToUtf(ch | TCL_COMBINE, dst);
-	} else if (LOW_SURROGATE(ch) && !PROFILE_TCL8(flags)) {
+	} else if (LOW_SURROGATE(ch)) {
 	    /* Lo surrogate not preceded by Hi surrogate and not tcl8 profile */
 	    if (PROFILE_STRICT(flags)) {
 		result = TCL_CONVERT_UNKNOWN;
 		break;
+	    } else if (PROFILE_TCL8(flags)) {
+                dst += Tcl_UniCharToUtf(ch, dst);
 	    } else {
-		/* PROFILE_REPLACE */
+                /*
+                 * PROFILE_REPLACE | PROFILE_LOSSLESS. LOSSLESS treated like
+                 * REPLACE for UTF16 - see TIP 671
+                 */
 		dst += Tcl_UniCharToUtf(UNICODE_REPLACE_CHAR, dst);
-	    }
+            }
 	} else {
 	    dst += Tcl_UniCharToUtf(ch, dst);
 	}
@@ -3031,18 +3145,19 @@ Utf16ToUtfProc(
 	    src -= 2;
 	    dst--;
 	    numChars--;
-	} else if (PROFILE_REPLACE(flags)) {
+	} else if (PROFILE_TCL8(flags)) {
+	    dst += Tcl_UniCharToUtf(-1, dst);
+	} else {
+            /* PROFILE_REPLACE | PROFILE_LOSSLESS */
 	    dst--;
 	    dst += Tcl_UniCharToUtf(UNICODE_REPLACE_CHAR, dst);
-	} else {
-	    /* Bug [10c2c17c32]. If Hi surrogate, finish 3-byte UTF-8 */
-	    dst += Tcl_UniCharToUtf(-1, dst);
 	}
    }

    /*
     * If we had a truncated code unit at the end AND this is the last
-     * fragment AND profile is not "strict", stick FFFD in its place.
+     * fragment AND profile is not "strict", use the appropriate replacement
+     * strategy.
     */
    if ((flags & TCL_ENCODING_END) && (result == TCL_CONVERT_MULTIBYTE)) {
 	if (dst > dstEnd) {
@@ -3051,7 +3166,7 @@ Utf16ToUtfProc(
 	    if (PROFILE_STRICT(flags)) {
 		result = TCL_CONVERT_SYNTAX;
 	    } else {
-		/* PROFILE_REPLACE or PROFILE_TCL8 */
+		/* PROFILE_REPLACE | PROFILE_TCL8 | PROFILE_LOSSLESS */
 		result = TCL_OK;
 		dst += Tcl_UniCharToUtf(UNICODE_REPLACE_CHAR, dst);
 		numChars++;
@@ -3140,8 +3255,8 @@ UtfToUtf16Proc(
 	    if (PROFILE_STRICT(flags)) {
 		result = TCL_CONVERT_UNKNOWN;
 		break;
-	    }
-	    if (PROFILE_REPLACE(flags)) {
+	    } else if (! PROFILE_TCL8(flags)) {
+                /* PROFILE_REPLACE | PROFILE_LOSSLESS */
 		ch = UNICODE_REPLACE_CHAR;
 	    }
 	}
@@ -3251,9 +3366,14 @@ UtfToUcs2Proc(
 	    }
 	    ch = UNICODE_REPLACE_CHAR;
 	}
-	if (PROFILE_STRICT(flags) && SURROGATE(ch)) {
-	    result = TCL_CONVERT_SYNTAX;
-	    break;
+	if (SURROGATE(ch)) {
+	    if (PROFILE_STRICT(flags)) {
+		result = TCL_CONVERT_SYNTAX;
+		break;
+	    } else if (!PROFILE_TCL8(flags)) {
+                /* PROFILE_REPLACE | PROFILE_LOSSLESS */
+                ch = UNICODE_REPLACE_CHAR;
+            }
 	}

 	src += len;
@@ -3349,6 +3469,7 @@ TableToUtfProc(
 	if (prefixBytes[byte]) {
 	    if (src >= srcEnd-1) {
 		/* Prefix byte but nothing after it */
+		/* Truncated sequence ... */
 		if (!(flags & TCL_ENCODING_END)) {
 		    /* More data to come */
 		    result = TCL_CONVERT_MULTIBYTE;
@@ -3358,6 +3479,12 @@ TableToUtfProc(
 		    break;
 		} else if (PROFILE_REPLACE(flags)) {
 		    ch = UNICODE_REPLACE_CHAR;
+		} else if (PROFILE_LOSSLESS(flags)) {
+		    if (dataPtr->flags & ENCODING_ASCII_COMPATIBLE) {
+			ch = ToLossless(byte);
+		    } else {
+			ch = UNICODE_REPLACE_CHAR;
+		    }
 		} else {
 		    ch = (Tcl_UniChar)byte;
 		}
@@ -3378,6 +3505,8 @@ TableToUtfProc(
 	    }
 	    if (PROFILE_REPLACE(flags)) {
 		ch = UNICODE_REPLACE_CHAR;
+	    } else if (PROFILE_LOSSLESS(flags)) {
+		ch = ToLossless(byte);
 	    } else {
 		ch = (Tcl_UniChar)byte;
 	    }
@@ -3448,6 +3577,7 @@ TableFromUtfProc(
    Tcl_UniChar ch = 0;
    int result, len, word, numChars;
    TableEncodingData *dataPtr = (TableEncodingData *)clientData;
+    int asciiCompatible = dataPtr->flags & ENCODING_ASCII_COMPATIBLE;
    const unsigned short *const *fromUnicode;

    result = TCL_OK;
@@ -3484,14 +3614,22 @@ TableFromUtfProc(
 	word = fromUnicode[(ch >> 8)][ch & 0xFF];
 	}

+	int isWrappedLossless = 0;
 	if ((word == 0) && (ch != 0)) {
 	    if (PROFILE_STRICT(flags)) {
 		result = TCL_CONVERT_UNKNOWN;
 		break;
 	    }
-	    word = dataPtr->fallback; /* Both profiles REPLACE and TCL8 */
+	    if (PROFILE_LOSSLESS(flags) && IsLosslessWrapper(ch) &&
+		asciiCompatible) {
+		word = FromLossless(ch);
+		isWrappedLossless = 1;
+	    }
+	    else {
+		word = dataPtr->fallback; /* Both profiles REPLACE and TCL8 */
+	    }
 	}
-	if (prefixBytes[(word >> 8)] != 0) {
+	if (prefixBytes[(word >> 8)] != 0 && !isWrappedLossless) {
 	    if (dst + 1 > dstEnd) {
 		result = TCL_CONVERT_NOSPACE;
 		break;
@@ -3568,6 +3706,12 @@ Iso88591ToUtfProc(
    dstStart = dst;
    dstEnd = dst + dstLen - TCL_UTF_MAX;

+    /*
+     * Note with respect to profiles: all byte values are mapped
+     * to Unicode characters on input so there is no question of invalid
+     * 8859-1 characters.
+     */
+
    result = TCL_OK;
    for (numChars = 0; src < srcEnd && numChars <= charLimit; numChars++) {
 	Tcl_UniChar ch = 0;
@@ -3673,11 +3817,11 @@ Iso88591FromUtfProc(
 		result = TCL_CONVERT_UNKNOWN;
 		break;
 	    }
-	    /*
-	     * Plunge on, using '?' as a fallback character.
-	     */
-
-	    ch = (Tcl_UniChar) '?'; /* Profiles TCL8 and REPLACE */
+	    if (PROFILE_LOSSLESS(flags) && IsLosslessWrapper(ch)) {
+		ch = FromLossless(ch);
+	    } else {
+		ch = (Tcl_UniChar)'?';
+	    }
 	}

 	if (dst > dstEnd) {
@@ -3891,14 +4035,15 @@ EscapeToUtfProc(

 	    if ((checked == dataPtr->numSubTables + 2)
 		    || (flags & TCL_ENCODING_END)) {
-		if (!PROFILE_STRICT(flags)) {
+		if (PROFILE_STRICT(flags)) {
+		    result = TCL_CONVERT_SYNTAX;
+		} else {
 		    unsigned skip = longest > left ? left : longest;
 		    /* Unknown escape sequence */
 		    dst += Tcl_UniCharToUtf(UNICODE_REPLACE_CHAR, dst);
 		    src += skip;
 		    continue;
 		}
-		result = TCL_CONVERT_SYNTAX;
 	    } else {
 		result = TCL_CONVERT_MULTIBYTE;
 	    }
@@ -4070,7 +4215,8 @@ EscapeFromUtfProc(
 		    break;
 		}
 		encodingPtr = GetTableEncoding(dataPtr, state);
-		tableDataPtr = (const TableEncodingData *)encodingPtr->clientData;
+		tableDataPtr =
+		    (const TableEncodingData *)encodingPtr->clientData;
 		word = tableDataPtr->fallback;
 	    }

@@ -4476,6 +4622,90 @@ TclGetEncodingProfiles(Tcl_Interp *interp)
    Tcl_SetObjResult(interp, objPtr);
 }

+/*
+ *------------------------------------------------------------------------
+ *
+ * TclSystemToInternalEncoding --
+ *
+ *    Converts a string encoded in the system encoding to Tcl's internal UTF8
+ *    using the lossless profile.
+ *
+ * Results:
+ *    Tcl_OK / TCL_ERROR
+ *
+ * Side effects:
+ *    On success *dsPtr holds the converted string. On error, *dsPtr is
+ *    cleared, an error message is stored in interp (if not NULL), and a
+ *    POSIX error code stored in errno.
+ *
+ *------------------------------------------------------------------------
+ */
+int
+TclSystemToInternalEncoding(
+    Tcl_Interp *interp, /* For error messages, may be NULL */
+    const char *src,    /* String in system encoding */
+    Tcl_Size srcLen,    /* Number of bytes passed in */
+    Tcl_DString *dsPtr) /* Pointer to uninitialized or cleared Tcl_DString. */
+{
+    Tcl_Size errorLoc;
+    int ret;
+    ret = Tcl_ExternalToUtfDStringEx(interp,
+				     NULL,
+				     src,
+				     srcLen,
+				     TCL_ENCODING_PROFILE_LOSSLESS,
+				     dsPtr,
+				     &errorLoc);
+    /* On TCL_OK, caller owns *dsPtr. On failure we have to free it. */
+    if (ret != TCL_OK) {
+	Tcl_DStringFree(dsPtr);
+	ret = TCL_ERROR; /* Map TCL_CONVERT_* to TCL_ERROR */
+    }
+    return ret;
+}
+
+/*
+ *------------------------------------------------------------------------
+ *
+ * TclInternalToSystemEncoding --
+ *
+ *    Converts a string to the system encoding using the lossless profile
+ *
+ * Results:
+ *    Tcl_OK / TCL_ERROR
+ *
+ * Side effects:
+ *    On success *dsPtr holds the converted string. On error, *dsPtr is
+ *    cleared, an error message is stored in interp (if not NULL), and a
+ *    POSIX error code stored in errno.
+ *
+ *------------------------------------------------------------------------
+ */
+int
+TclInternalToSystemEncoding(
+    Tcl_Interp *interp, /* For error messages, may be NULL */
+    const char *src,    /* String in system encoding */
+    Tcl_Size srcLen,    /* Number of bytes passed in */
+    Tcl_DString *dsPtr) /* Pointer to uninitialized or cleared Tcl_DString. */
+{
+    Tcl_Size errorLoc;
+    int ret;
+    ret = Tcl_UtfToExternalDStringEx(interp,
+				     NULL,
+				     src,
+				     srcLen,
+				     TCL_ENCODING_PROFILE_LOSSLESS,
+				     dsPtr,
+				     &errorLoc);
+    /* On TCL_OK, caller owns *dsPtr. On failure we have to free it. */
+    if (ret != TCL_OK) {
+	Tcl_DStringFree(dsPtr);
+	ret = TCL_ERROR; /* Map TCL_CONVERT_* to TCL_ERROR */
+    }
+    return ret;
+}
+
+
 /*
 * Local Variables:
 * mode: c
@@ -19,20 +19,33 @@ TCL_DECLARE_MUTEX(envMutex)	/* To serialize access to environ. */

 #if defined(_WIN32)
 #  define tenviron _wenviron
-#  define tenviron2utfdstr(str, dsPtr) (Tcl_DStringInit(dsPtr), \
-		(char *)Tcl_Char16ToUtfDString((const unsigned short *)(str), -1, (dsPtr)))
-#  define utf2tenvirondstr(str, dsPtr) (Tcl_DStringInit(dsPtr), \
-		(const WCHAR *)Tcl_UtfToChar16DString((str), -1, (dsPtr)))
+static inline char *tenviron2utfdstr(const WCHAR *str, Tcl_DString *dsPtr) {
+    Tcl_DStringInit(dsPtr);
+    return Tcl_Char16ToUtfDString(str, -1, dsPtr);
+}
+static inline WCHAR *utf2tenvirondstr(const char *str, Tcl_DString *dsPtr) {
+    Tcl_DStringInit(dsPtr);
+    return Tcl_UtfToChar16DString(str, -1, dsPtr);
+}
 #  define techar WCHAR
 #  ifdef USE_PUTENV
 #    define putenv(env) _wputenv((const wchar_t *)env)
 #  endif
 #else
 #  define tenviron environ
-#  define tenviron2utfdstr(str, dsPtr) \
-		Tcl_ExternalToUtfDString(NULL, str, -1, dsPtr)
-#  define utf2tenvirondstr(str, dsPtr) \
-		Tcl_UtfToExternalDString(NULL, str, -1, dsPtr)
+static inline char *tenviron2utfdstr(const char *str, Tcl_DString *dsPtr) {
+    if (TclSystemToInternalEncoding(NULL,str,-1,dsPtr) == TCL_OK) {
+        return Tcl_DStringValue(dsPtr);
+    }
+    return NULL;
+}
+static inline char *utf2tenvirondstr(const char *str, Tcl_DString *dsPtr) {
+    Tcl_DStringInit(dsPtr);
+    if (TclInternalToSystemEncoding(NULL,str,-1,dsPtr) == TCL_OK) {
+        return Tcl_DStringValue(dsPtr);
+    }
+    return NULL;
+}
 #  define techar char
 #endif

@@ -3044,6 +3044,10 @@ TclEncodingProfileNameToId(Tcl_Interp *interp,
 MODULE_SCOPE const char *TclEncodingProfileIdToName(Tcl_Interp *interp,
 						    int profileId);
 MODULE_SCOPE void TclGetEncodingProfiles(Tcl_Interp *interp);
+MODULE_SCOPE int TclInternalToSystemEncoding(Tcl_Interp *interp,
+			const char *src, Tcl_Size srcLen, Tcl_DString *dsPtr);
+MODULE_SCOPE int TclSystemToInternalEncoding(Tcl_Interp *interp,
+			const char *src, Tcl_Size srcLen, Tcl_DString *dsPtr);

 /*
 * TIP #233 (Virtualized Time)
@@ -53,7 +53,7 @@ NewNativeObj(
    Tcl_DStringInit(&ds);
    Tcl_WCharToUtfDString(string, -1, &ds);
 #else
-    (void)Tcl_ExternalToUtfDString(NULL, (char *)string, -1, &ds);
+    (void) TclSystemToInternalEncoding(NULL, string, -1, &ds);
 #endif
    return Tcl_DStringToObj(&ds);
 }
@@ -2128,6 +2128,7 @@ static int UtfExtWrapper(
 	{"profiletcl8", TCL_ENCODING_PROFILE_TCL8},
 	{"profilestrict", TCL_ENCODING_PROFILE_STRICT},
 	{"profilereplace", TCL_ENCODING_PROFILE_REPLACE},
+	{"profilelossless", TCL_ENCODING_PROFILE_LOSSLESS},
 	{NULL, 0}
    };
    Tcl_Size i;
@@ -4242,9 +4242,9 @@ TclGetProcessGlobalValue(
 	    Tcl_MutexLock(&pgvPtr->mutex);
 	    epoch = ++pgvPtr->epoch;
 	    Tcl_UtfToExternalDStringEx(NULL, pgvPtr->encoding, pgvPtr->value,
-		pgvPtr->numBytes, TCL_ENCODING_PROFILE_TCL8, &native, NULL);
+		pgvPtr->numBytes, TCL_ENCODING_PROFILE_LOSSLESS, &native, NULL);
 	    Tcl_ExternalToUtfDStringEx(NULL, current, Tcl_DStringValue(&native),
-		Tcl_DStringLength(&native), TCL_ENCODING_PROFILE_TCL8,
+		Tcl_DStringLength(&native), TCL_ENCODING_PROFILE_LOSSLESS,
 		&newValue, NULL);
 	    Tcl_DStringFree(&native);
 	    Tcl_Free(pgvPtr->value);
@@ -643,7 +643,7 @@ SetOSTypeFromAny(
    Tcl_Size length;

    string = Tcl_GetStringFromObj(objPtr, &length);
-    Tcl_UtfToExternalDStringEx(NULL, encoding, string, length, TCL_ENCODING_PROFILE_TCL8, &ds, NULL);
+    Tcl_UtfToExternalDStringEx(NULL, encoding, string, length, TCL_ENCODING_PROFILE_LOSSLESS, &ds, NULL);

    if (Tcl_DStringLength(&ds) > 4) {
 	if (interp) {
@@ -647,8 +647,109 @@ foreach {enc str profile hex failidx ctrl comment} $encUnencodableStrings {
    }
 }

+proc ascii_compatible {enc} {
+    # All bytes under 128 should map to their ascii values
+    for {set i 0} {$i < 128} {incr i} {
+        set bin [binary format c $i]
+        if {[catch {set ch [encoding convertfrom -profile strict $enc $bin]}]} {
+            return 0
+        }
+        if {$ch ne [encoding convertfrom -profile strict ascii $bin]} {
+            return 0
+        }
+    }
+    return 1
+}
+
+#
+# Roundtrip tests for lossless profile
+foreach {enc hex profile str failidx ctrl comment} $encInvalidBytes {
+    if {"knownBug" in $ctrl} continue
+    if {$profile ne "lossless"} continue
+    # There are multiple test cases based on location of invalid bytes
+    set bytes [binary decode hex $hex]
+    set prefix A
+    set suffix B
+    set prefix_bytes [encoding convertto $enc $prefix]
+    set suffix_bytes [encoding convertto $enc $suffix]
+    set prefixLen [string length $prefix_bytes]
+    if {![ascii_compatible $enc]} {
+        # These do not implement lossless behaviors
+        test cmdAH-4.4.15.$hex.solo.$enc "Invalid byte under 128 for lossless profile" -body {
+            encoding convertfrom -profile lossless $enc $bytes
+        } -result \uFFFD* -match glob
+        continue
+    }
+    if {$ctrl eq {} || "solo" in $ctrl} {
+        test cmdAH-4.4.15.$hex.solo.$enc "Lossless roundtrip $hex for $enc" -body {
+            set decoded [encoding convertfrom -profile lossless $enc $bytes]
+            string equal $bytes [encoding convertto -profile lossless $enc $decoded]
+        } -result 1
+    }
+    if {$ctrl eq {} || "lead" in $ctrl} {
+        test cmdAH-4.4.15.$hex.lead.$enc "Lossless roundtrip $hex for $enc" -body {
+            set decoded [encoding convertfrom -profile lossless $enc $bytes$suffix_bytes]
+            string equal $bytes$suffix_bytes [encoding convertto -profile lossless $enc $decoded]
+        } -result 1
+    }
+    if {$ctrl eq {} || "tail" in $ctrl} {
+        test cmdAH-4.4.15.$hex.tail.$enc "Lossless roundtrip $hex for $enc" -body {
+            set decoded [encoding convertfrom -profile lossless $enc $prefix_bytes$bytes]
+            string equal $prefix_bytes$bytes [encoding convertto -profile lossless $enc $decoded]
+        } -result 1
+    }
+    if {$ctrl eq {} || "middle" in $ctrl} {
+        test cmdAH-4.4.15.$hex.middle.$enc "Lossless roundtrip $hex for $enc" -body {
+            set decoded [encoding convertfrom -profile lossless $enc $prefix_bytes$bytes$suffix_bytes]
+            string equal $prefix_bytes$bytes$suffix_bytes [encoding convertto -profile lossless $enc $decoded]
+        } -result 1
+    }
+}
+
+#
+# Non-ascii encoding should not output lossless wrappers
+foreach enc [encoding names] {
+    if {$enc eq "cesu-8"} {
+        test cmdAH-4.4.16.$enc "Lossless output for CESU-8" -body {
+            encoding convertto -profile lossless $enc \uDC41
+        } -result \xED\xB1\x81
+    } elseif {[ascii_compatible $enc]} {
+        test cmdAH-4.4.16.$enc "Lossless output for ascii-compatible encodings" -body {
+            encoding convertto -profile lossless $enc \uDC41
+        } -result A
+    } else {
+        test cmdAH-4.4.16.$enc "Lossless output for ascii-incompatible encodings" -body {
+            encoding convertto -profile lossless $enc \uDC41
+        } -result [encoding convertto -profile tcl8 $enc \uFFFD]
+    }
+}
+
+#
+# Invalid bytes < 128 should map to FFFD for lossless profile
+
+# Find an invalid byte within a range for the given encoding
+proc find_invalid_byte {enc {lo 0} {hi 127}} {
+    for {set i $lo} {$i <= $hi} {incr i} {
+        set bin [binary format c $i]
+        if {[catch {set ch [encoding convertfrom -profile strict $enc $bin]}]} {
+            return $bin
+        }
+    }
+    # All bytes under 128 are valid
+    return ""
+}
+foreach enc [encoding names] {
+    set byte [find_invalid_byte $enc]
+    if {$byte ne ""} {
+        test cmdAH-4.4.17.$enc "Invalid byte under 128 for lossless profile" -body {
+            encoding convertfrom -profile lossless $enc $byte
+        } -result \uFFFD
+    }
+}
+
+
 test cmdAH-4.4.xx {convertto -profile strict} -constraints {testbytestring knownBug} -body {
-    # TODO - what does testbytestring even test? Invalid UTF8 in the Tcl_Obj bytes field
+    # TODO - what does testbytestring even test? Invalid UTF8 in the Tcl_Obj bytes field?
    encoding convertto -profile strict utf-8 A[testbytestring \x80]B
 } -returnCodes error -result {unexpected byte sequence starting at index 1: '\x80'}

@@ -665,7 +766,7 @@ test cmdAH-4.5.2 {encoding names should include at least utf-8 and iso8859-1 and
 badnumargs cmdAH-4.6.1 {encoding profiles} {foo}
 test cmdAH-4.6.2 {encoding profiles} -body {
    lsort [encoding profiles]
-} -result {replace strict tcl8}
+} -result {lossless replace strict tcl8}

 #
 # file command
@@ -813,7 +813,7 @@ test encoding-24.16 {Parse valid or invalid utf-8} -constraints testbytestring -
    encoding convertto utf-8 [testbytestring "Z\u4343\x80"]
 } -returnCodes 1 -result {expected byte sequence but character 1 was '䍃' (U+004343)}
 test encoding-24.17 {Parse valid or invalid utf-8} -constraints testbytestring -body {
-    encoding convertto utf-8 [testbytestring "Z\xE0\x80"]
+    encoding convertto -profile tcl8 utf-8 [testbytestring "Z\xE0\x80"]
 } -result "Z\xC3\xA0\xE2\x82\xAC"
 test encoding-24.18 {Parse valid or invalid utf-8} -constraints testbytestring -body {
    encoding convertto utf-8 [testbytestring "Z\xE0\x80xxxxxx"]
@@ -1165,6 +1165,14 @@ test encoding-bug-66ffafd309-1-strict {Bug [66ffafd309] - truncated DBCS} -body
 test encoding-bug-66ffafd309-1-replace {Bug [66ffafd309] - truncated DBCS} -body {
    encoding convertfrom -profile replace gb12345 x
 } -result \uFFFD
+test encoding-bug-66ffafd309-1-lossless-a {Bug [66ffafd309] - truncated DBCS} -body {
+    # lossless - byte < 128
+    encoding convertfrom -profile lossless gb12345 x
+} -result \uFFFD
+test encoding-bug-66ffafd309-1-lossless-b {Bug [66ffafd309] - truncated DBCS} -body {
+    # lossless - byte > 128
+    encoding convertfrom -profile lossless gb12345 \x82
+} -result \uFFFD
 test encoding-bug-66ffafd309-2-tcl8 {Bug [66ffafd309] - invalid DBCS} -body {
    # Not truncated but invalid
    encoding convertfrom -profile tcl8 jis0208 \x78\x79
@@ -1177,6 +1185,14 @@ test encoding-bug-66ffafd309-2-replace {Bug [66ffafd309] - invalid DBCS} -body {
    # Not truncated but invalid
    encoding convertfrom -profile replace jis0208 \x78\x79
 } -result \uFFFD\uFFFD
+test encoding-bug-66ffafd309-2-lossless-a {Bug [66ffafd309] - invalid DBCS} -body {
+    # Not truncated but invalid. \x78 is invalid prefix
+    encoding convertfrom -profile lossless jis0208 \x78\x79
+} -result \uFFFD\uFFFD
+test encoding-bug-66ffafd309-2-lossless-b {Bug [66ffafd309] - invalid DBCS} -body {
+    # Not truncated but invalid. \x21 is valid prefix but FF is not valid suffix
+    encoding convertfrom -profile lossless jis0208 \x21\xFF
+} -result \uFFFD\uFFFD

 # cleanup
 namespace delete ::tcl::test::encoding
@@ -9,7 +9,7 @@
 #

 # List of defined encoding profiles
-set encProfiles {tcl8 strict replace}
+set encProfiles {tcl8 strict replace lossless}
 set encDefaultProfile strict; # Should reflect the default from implementation

 # encValidStrings - Table of valid strings.
@@ -111,6 +111,7 @@ set encInvalidBytes {}; # Reset the table
 lappend encInvalidBytes {*}{
    ascii 80 tcl8    \u20AC -1 {knownBug} {map to cp1252}
    ascii 80 replace \uFFFD -1 {} {Smallest invalid byte}
+    ascii 80 lossless \uDC80 -1 {} {Smallest invalid byte}
    ascii 80 strict  {}      0 {} {Smallest invalid byte}
    ascii 81 tcl8    \u0081 -1 {knownBug} {map to cp1252}
    ascii 82 tcl8    \u201A -1 {knownBug} {map to cp1252}
@@ -146,6 +147,7 @@ lappend encInvalidBytes {*}{

    ascii FF tcl8    \u00FF -1 {} {Largest invalid byte}
    ascii FF replace \uFFFD -1 {} {Largest invalid byte}
+    ascii FF lossless \uDCFF -1 {} {Largest invalid byte}
    ascii FF strict  {}      0 {} {Largest invalid byte}
 }

@@ -170,6 +172,7 @@ lappend encInvalidBytes {*}{
 lappend encInvalidBytes {*}{
    utf-8 80 tcl8    \u20AC -1 {} {map to cp1252}
    utf-8 80 replace \uFFFD -1 {} {Smallest invalid byte}
+    utf-8 80 lossless \uDC80 -1 {} {Smallest invalid byte}
    utf-8 80 strict  {}      0 {} {Smallest invalid byte}
    utf-8 81 tcl8    \u0081 -1 {} {map to cp1252}
    utf-8 82 tcl8    \u201A -1 {} {map to cp1252}
@@ -206,342 +209,453 @@ lappend encInvalidBytes {*}{
    utf-8 C0 tcl8    \u00C0 -1 {} {C0 is invalid anywhere}
    utf-8 C0 strict  {}      0 {} {C0 is invalid anywhere}
    utf-8 C0 replace \uFFFD -1 {} {C0 is invalid anywhere}
+    utf-8 C0 lossless \uDCC0 -1 {} {C0 is invalid anywhere}
    utf-8 C080 tcl8    \u0000 -1 {} {C080 -> U+0 in Tcl's internal modified UTF8}
    utf-8 C080 strict  {}      0 {} {C080 -> invalid}
    utf-8 C080 replace \uFFFD -1 {} {C080 -> single replacement char}
+    utf-8 C080 lossless \uDCC0\uDC80 -1 {} {C080 -> two lossless wrappers}
    utf-8 C0A2 tcl8    \u00C0\u00A2 -1 {} {websec.github.io - A}
    utf-8 C0A2 replace \uFFFD\uFFFD -1 {} {websec.github.io - A}
+    utf-8 C0A2 lossless \uDCC0\uDCA2 -1 {} {websec.github.io - A}
    utf-8 C0A2 strict  {}            0 {} {websec.github.io - A}
    utf-8 C0A7 tcl8    \u00C0\u00A7 -1 {} {websec.github.io - double quote}
    utf-8 C0A7 replace \uFFFD\uFFFD -1 {} {websec.github.io - double quote}
+    utf-8 C0A7 lossless \uDCC0\uDCA7 -1 {} {websec.github.io - A}
    utf-8 C0A7 strict  {}            0 {} {websec.github.io - double quote}
    utf-8 C0AE tcl8    \u00C0\u00AE -1 {} {websec.github.io - full stop}
    utf-8 C0AE replace \uFFFD\uFFFD -1 {} {websec.github.io - full stop}
+    utf-8 C0AE lossless \uDCC0\uDCAE -1 {} {websec.github.io - A}
    utf-8 C0AE strict  {}            0 {} {websec.github.io - full stop}
    utf-8 C0AF tcl8    \u00C0\u00AF -1 {} {websec.github.io - solidus}
    utf-8 C0AF replace \uFFFD\uFFFD -1 {} {websec.github.io - solidus}
+    utf-8 C0AF lossless \uDCC0\uDCAF -1 {} {websec.github.io - A}
    utf-8 C0AF strict  {}            0 {} {websec.github.io - solidus}

    utf-8 C1 tcl8    \u00C1 -1 {} {C1 is invalid everywhere}
    utf-8 C1 replace \uFFFD -1 {} {C1 is invalid everywhere}
+    utf-8 C1 lossless \uDCC1 -1 {} {C1 is invalid anywhere}
    utf-8 C1 strict  {}      0 {} {C1 is invalid everywhere}
    utf-8 C181 tcl8    \u00C1\u0081 -1 {} {websec.github.io - base test (A)}
    utf-8 C181 replace \uFFFD\uFFFD -1 {} {websec.github.io - base test (A)}
+    utf-8 C181 lossless \uDCC1\uDC81 -1 {} {websec.github.io - base test (A)}
    utf-8 C181 strict  {}            0 {} {websec.github.io - base test (A)}
    utf-8 C19C tcl8    \u00C1\u0153 -1 {} {websec.github.io - reverse solidus}
    utf-8 C19C replace \uFFFD\uFFFD -1 {} {websec.github.io - reverse solidus}
+    utf-8 C19C lossless \uDCC1\uDC9C -1 {} {websec.github.io - reverse solidus}
    utf-8 C19C strict  {}            0 {} {websec.github.io - reverse solidus}

    utf-8 C2 tcl8      \u00C2     -1 {} {Missing trail byte}
    utf-8 C2 replace   \uFFFD     -1 {} {Missing trail byte}
+    utf-8 C2 lossless   \uDCC2     -1 {} {Missing trail byte}
    utf-8 C2 strict    {}          0 {} {Missing trail byte}
    utf-8 C27F tcl8    \u00C2\x7F -1 {} {Trail byte must be 80:BF}
    utf-8 C27F replace \uFFFD\x7F -1 {} {Trail byte must be 80:BF}
+    utf-8 C27F lossless \uDCC2\x7F -1 {} {Trail byte must be 80:BF}
    utf-8 C27F strict  {}          0 {} {Trail byte must be 80:BF}
    utf-8 DF tcl8      \u00DF     -1 {} {Missing trail byte}
    utf-8 DF replace   \uFFFD     -1 {} {Missing trail byte}
+    utf-8 DF lossless   \uDCDF     -1 {} {Missing trail byte}
    utf-8 DF strict    {}          0 {} {Missing trail byte}
    utf-8 DF7F tcl8    \u00DF\x7F -1 {} {Trail byte must be 80:BF}
    utf-8 DF7F replace \uFFFD\x7F -1 {} {Trail byte must be 80:BF}
+    utf-8 DF7F lossless \uDCDF\x7F -1 {} {Trail byte must be 80:BF}
    utf-8 DF7F strict  {}          0 {} {Trail byte must be 80:BF}
    utf-8 DFE0A080 tcl8    \u00DF\u0800 -1 {} {Invalid trail byte is start of valid sequence}
    utf-8 DFE0A080 replace \uFFFD\u0800 -1 {} {Invalid trail byte is start of valid sequence}
+    utf-8 DFE0A080 lossless \uDCDF\u0800 -1 {} {Invalid trail byte is start of valid sequence}
    utf-8 DFE0A080 strict  {}            0 {} {Invalid trail byte is start of valid sequence}

    utf-8 E0 tcl8      \u00E0     -1 {} {Missing trail byte}
    utf-8 E0 replace   \uFFFD     -1 {} {Missing trail byte}
+    utf-8 E0 lossless  \uDCE0    -1 {} {Missing trail byte}
    utf-8 E0 strict    {}          0 {} {Missing trail byte}
    utf-8 E080 tcl8      \u00E0\u20AC   -1 {} {First trail byte must be A0:BF}
    utf-8 E080 replace   \uFFFD\uFFFD   -1 {} {First trail byte must be A0:BF}
+    utf-8 E080 lossless   \uDCE0\uDC80  -1 {} {First trail byte must be A0:BF}
    utf-8 E080 strict    {}              0 {} {First trail byte must be A0:BF}
    utf-8 E0819C tcl8    \u00E0\u0081\u0153 -1 {} {websec.github.io - reverse solidus}
    utf-8 E0819C replace \uFFFD\uFFFD\uFFFD -1 {} {websec.github.io - reverse solidus}
+    utf-8 E0819C lossless \uDCE0\uDC81\uDC9C -1 {} {websec.github.io - reverse solidus}
    utf-8 E0819C strict  {}                  0 {} {websec.github.io - reverse solidus}
    utf-8 E09F tcl8      \u00E0\u0178   -1 {} {First trail byte must be A0:BF}
    utf-8 E09F replace   \uFFFD\uFFFD   -1 {} {First trail byte must be A0:BF}
+    utf-8 E09F lossless  \uDCE0\uDC9F   -1 {} {First trail byte must be A0:BF}
    utf-8 E09F strict    {}              0 {} {First trail byte must be A0:BF}
    utf-8 E0A0 tcl8      \u00E0\u00A0   -1 {} {Missing second trail byte}
    utf-8 E0A0 replace   \uFFFD         -1 {knownW3C} {Missing second trail byte}
+    utf-8 E0A0 lossless  \uDCE0\uDCA0   -1 {} {Missing second trail byte}
    utf-8 E0A0 strict    {}              0 {} {Missing second trail byte}
    utf-8 E0BF tcl8      \u00E0\u00BF   -1 {} {Missing second trail byte}
    utf-8 E0BF replace   \uFFFD         -1 {knownW3C} {Missing second trail byte}
+    utf-8 E0BF lossless  \uDCE0\uDCBF   -1 {} {Missing second trail byte}
    utf-8 E0BF strict    {}              0 {} {Missing second trail byte}
    utf-8 E0A07F tcl8    \u00E0\u00A0\x7F   -1 {}     {Second trail byte must be 80:BF}
-    utf-8 E0A07F replace \uFFFD\u7F         -1 {knownW3C} {Second trail byte must be 80:BF}
+    utf-8 E0A07F replace \uFFFD\x7F         -1 {knownW3C} {Second trail byte must be 80:BF}
+    utf-8 E0A07F lossless \uDCE0\uDCA0\x7F  -1 {} {Second trail byte must be 80:BF}
    utf-8 E0A07F strict  {}                  0 {}         {Second trail byte must be 80:BF}
    utf-8 E0BF7F tcl8    \u00E0\u00BF\x7F   -1 {}         {Second trail byte must be 80:BF}
-    utf-8 E0BF7F replace \uFFFD\u7F         -1 {knownW3C} {Second trail byte must be 80:BF}
+    utf-8 E0BF7F replace \uFFFD\x7F         -1 {knownW3C} {Second trail byte must be 80:BF}
+    utf-8 E0BF7F lossless \uDCE0\uDCBF\x7F  -1 {} {Second trail byte must be 80:BF}
    utf-8 E0BF7F strict  {}                  0 {}         {Second trail byte must be 80:BF}

    utf-8 E1 tcl8      \u00E1     -1 {} {Missing trail byte}
    utf-8 E1 replace   \uFFFD     -1 {} {Missing trail byte}
+    utf-8 E1 lossless  \uDCE1     -1 {} {Missing trail byte}
    utf-8 E1 strict    {}          0 {} {Missing trail byte}
    utf-8 E17F tcl8    \u00E1\x7F -1 {} {Trail byte must be 80:BF}
    utf-8 E17F replace \uFFFD\x7F -1 {} {Trail byte must be 80:BF}
+    utf-8 E17F lossless \uDCE1\x7F -1 {} {Trail byte must be 80:BF}
    utf-8 E17F strict  {}          0 {} {Trail byte must be 80:BF}
    utf-8 E181 tcl8      \u00E1\u0081   -1 {} {Missing second trail byte}
    utf-8 E181 replace   \uFFFD         -1 {knownW3C} {Missing second trail byte}
+    utf-8 E181 lossless  \uDCE1\uDC81   -1 {} {Missing second trail byte}
    utf-8 E181 strict    {}              0 {} {Missing second trail byte}
    utf-8 E1BF tcl8      \u00E1\u00BF   -1 {} {Missing second trail byte}
    utf-8 E1BF replace   \uFFFD         -1 {knownW3C} {Missing second trail byte}
+    utf-8 E1BF lossless   \uDCE1\uDCBF  -1 {} {Missing second trail byte}
    utf-8 E1BF strict    {}              0 {} {Missing second trail byte}
    utf-8 E1807F tcl8    \u00E1\u20AC\x7F   -1 {} {Second trail byte must be 80:BF}
-    utf-8 E1807F replace \uFFFD\u7F         -1 {knownW3C} {Second trail byte must be 80:BF}
+    utf-8 E1807F replace \uFFFD\x7F         -1 {knownW3C} {Second trail byte must be 80:BF}
+    utf-8 E1807F lossless \uDCE1\uDC80\x7F  -1 {} {Second trail byte must be 80:BF}
    utf-8 E1807F strict  {}                  0 {}         {Second trail byte must be 80:BF}
    utf-8 E1BF7F tcl8    \u00E1\u00BF\x7F   -1 {}         {Second trail byte must be 80:BF}
-    utf-8 E1BF7F replace \uFFFD\u7F         -1 {knownW3C} {Second trail byte must be 80:BF}
+    utf-8 E1BF7F replace \uFFFD\x7F         -1 {knownW3C} {Second trail byte must be 80:BF}
+    utf-8 E1BF7F lossless \uDCE1\uDCBF\x7F  -1 {} {Second trail byte must be 80:BF}
    utf-8 E1BF7F strict  {}                  0 {}         {Second trail byte must be 80:BF}
    utf-8 EC tcl8      \u00EC     -1 {} {Missing trail byte}
    utf-8 EC replace   \uFFFD     -1 {} {Missing trail byte}
+    utf-8 EC lossless   \uDCEC    -1 {} {Missing trail byte}
    utf-8 EC strict    {}          0 {} {Missing trail byte}
    utf-8 EC7F tcl8    \u00EC\x7F -1 {} {Trail byte must be 80:BF}
    utf-8 EC7F replace \uFFFD\x7F -1 {} {Trail byte must be 80:BF}
+    utf-8 EC7F lossless \uDCEC\x7F -1 {} {Trail byte must be 80:BF}
    utf-8 EC7F strict  {}          0 {} {Trail byte must be 80:BF}
    utf-8 EC81 tcl8      \u00EC\u0081   -1 {} {Missing second trail byte}
    utf-8 EC81 replace   \uFFFD         -1 {knownW3C} {Missing second trail byte}
+    utf-8 EC81 lossless   \uDCEC\uDC81  -1 {} {Missing second trail byte}
    utf-8 EC81 strict    {}              0 {} {Missing second trail byte}
    utf-8 ECBF tcl8      \u00EC\u00BF   -1 {} {Missing second trail byte}
    utf-8 ECBF replace   \uFFFD         -1 {knownW3C} {Missing second trail byte}
+    utf-8 ECBF lossless   \uDCEC\uDCBF  -1 {} {Missing second trail byte}
    utf-8 ECBF strict    {}              0 {} {Missing second trail byte}
    utf-8 EC807F tcl8    \u00EC\u20AC\x7F   -1 {} {Second trail byte must be 80:BF}
-    utf-8 EC807F replace \uFFFD\u7F         -1 {knownW3C} {Second trail byte must be 80:BF}
+    utf-8 EC807F replace \uFFFD\x7F         -1 {knownW3C} {Second trail byte must be 80:BF}
+    utf-8 EC807F lossless \uDCEC\uDC80\x7F  -1 {} {Second trail byte must be 80:BF}
    utf-8 EC807F strict  {}                  0 {}         {Second trail byte must be 80:BF}
    utf-8 ECBF7F tcl8    \u00EC\u00BF\x7F   -1 {}         {Second trail byte must be 80:BF}
-    utf-8 ECBF7F replace \uFFFD\u7F         -1 {knownW3C} {Second trail byte must be 80:BF}
+    utf-8 ECBF7F replace \uFFFD\x7F         -1 {knownW3C} {Second trail byte must be 80:BF}
+    utf-8 ECBF7F lossless \uDCEC\uDCBF\x7F  -1 {} {Second trail byte must be 80:BF}
    utf-8 ECBF7F strict  {}                  0 {}         {Second trail byte must be 80:BF}

    utf-8 ED tcl8       \u00ED        -1 {} {Missing trail byte}
    utf-8 ED replace    \uFFFD        -1 {} {Missing trail byte}
    utf-8 ED strict     {}             0 {} {Missing trail byte}
-    utf-8 ED7F tcl8     \u00ED\u7F    -1 {} {First trail byte must be 80:9F}
-    utf-8 ED7F replace  \uFFFD\u7F    -1 {} {First trail byte must be 80:9F}
+    utf-8 ED7F tcl8     \u00ED\x7F    -1 {} {First trail byte must be 80:9F}
+    utf-8 ED7F replace  \uFFFD\x7F    -1 {} {First trail byte must be 80:9F}
+    utf-8 ED7F lossless  \uDCED\x7F    -1 {} {First trail byte must be 80:9F}
    utf-8 ED7F strict   {}             0 {} {First trail byte must be 80:9F}
    utf-8 EDA0 tcl8     \u00ED\u00A0  -1 {} {First trail byte must be 80:9F}
    utf-8 EDA0 replace  \uFFFD\uFFFD  -1 {} {First trail byte must be 80:9F}
+    utf-8 EDA0 lossless  \uDCED\uDCA0  -1 {} {First trail byte must be 80:9F}
    utf-8 EDA0 strict   {}             0 {} {First trail byte must be 80:9F}
    utf-8 ED81 tcl8      \u00ED\u0081   -1 {} {Missing second trail byte}
    utf-8 ED81 replace   \uFFFD         -1 {knownW3C} {Missing second trail byte}
+    utf-8 ED81 lossless   \uDCED\uDC81  -1 {} {Missing second trail byte}
    utf-8 ED81 strict    {}              0 {} {Missing second trail byte}
    utf-8 EDBF tcl8      \u00ED\u00BF   -1 {} {Missing second trail byte}
    utf-8 EDBF replace   \uFFFD         -1 {knownW3C} {Missing second trail byte}
+    utf-8 EDBF lossless   \uDCED\uDCBF  -1 {} {Missing second trail byte}
    utf-8 EDBF strict    {}              0 {} {Missing second trail byte}
    utf-8 ED807F tcl8      \u00ED\u20AC\x7F -1 {} {Second trail byte must be 80:BF}
-    utf-8 ED807F replace   \uFFFD\u7F       -1 {knownW3C} {Second trail byte must be 80:BF}
+    utf-8 ED807F replace   \uFFFD\x7F       -1 {knownW3C} {Second trail byte must be 80:BF}
+    utf-8 ED807F lossless   \uDCED\uDC80\x7F -1 {} {Second trail byte must be 80:BF}
    utf-8 ED807F strict    {}                0 {}  {Second trail byte must be 80:BF}
    utf-8 ED9F7F tcl8      \u00ED\u0178\x7F -1 {} {Second trail byte must be 80:BF}
-    utf-8 ED9F7F replace   \uFFFD\u7F       -1 {knownW3C} {Second trail byte must be 80:BF}
+    utf-8 ED9F7F replace   \uFFFD\x7F       -1 {knownW3C} {Second trail byte must be 80:BF}
+    utf-8 ED9F7F lossless   \uDCED\uDC9F\x7F -1 {} {Second trail byte must be 80:BF}
    utf-8 ED9F7F strict    {}                0 {}  {Second trail byte must be 80:BF}
    utf-8 EDA080 tcl8       \uD800          -1 {}  {High surrogate}
    utf-8 EDA080 replace    \uFFFD          -1 {knownBug}  {High surrogate}
+    utf-8 EDA080 lossless    \uDCED\uDCA0\uED80 -1 {knownBug}  {High surrogate}
    utf-8 EDA080 strict     {}               0 {}  {High surrogate}
    utf-8 EDAFBF tcl8       \uDBFF          -1 {}  {High surrogate}
    utf-8 EDAFBF replace    \uFFFD          -1 {knownBug}  {High surrogate}
+    utf-8 EDAFBF lossless    \uDCED\uDCAF\uDCBF -1 {knownBug}  {High surrogate}
    utf-8 EDAFBF strict     {}               0 {}  {High surrogate}
    utf-8 EDB080 tcl8       \uDC00          -1 {}  {Low surrogate}
    utf-8 EDB080 replace    \uFFFD          -1 {knownBug}  {Low surrogate}
+    utf-8 EDB080 lossless    \uDCED\uDCB0\uDC80 -1 {knownBug}  {Low surrogate}
    utf-8 EDB080 strict     {}               0 {}  {Low surrogate}
    utf-8 EDBFBF tcl8       \uDFFF          -1 {knownBug}  {Low surrogate}
    utf-8 EDBFBF replace    \uFFFD          -1 {knownBug}  {Low surrogate}
+    utf-8 EDBFBF lossless    \uDCED\uDCBF\uDCBF -1 {knownBug}  {Low surrogate}
    utf-8 EDBFBF strict     {}               0 {}  {Low surrogate}
    utf-8 EDA080EDB080 tcl8 \U00010000      -1 {knownBug}  {High low surrogate pair}
    utf-8 EDA080EDB080 replace \uFFFD\uFFFD -1 {knownBug}  {High low surrogate pair}
+    utf-8 EDA080EDB080 lossless \uDCED\uDCA0\uDC80\uDCED\uDCB0\uDC80  -1 {knownBug}  {High low surrogate pair}
    utf-8 EDA080EDB080 strict {}             0 {}  {High low surrogate pair}
    utf-8 EDAFBFEDBFBF tcl8 \U0010FFFF      -1 {knownBug}  {High low surrogate pair}
    utf-8 EDAFBFEDBFBF replace \uFFFD\uFFFD -1 {knownBug}  {High low surrogate pair}
+    utf-8 EDAFBFEDBFBF lossless \uDCED\uDCAF\uDCBF\uDCED\uDCBF\uDCBF  -1 {knownBug}  {High low surrogate pair}
    utf-8 EDAFBFEDBFBF strict {}             0 {}  {High low surrogate pair}

    utf-8 EE tcl8       \u00EE        -1 {} {Missing trail byte}
    utf-8 EE replace    \uFFFD        -1 {} {Missing trail byte}
+    utf-8 EE lossless    \uDCEE       -1 {} {Missing trail byte}
    utf-8 EE strict     {}             0 {} {Missing trail byte}
-    utf-8 EE7F tcl8     \u00EE\u7F    -1 {} {First trail byte must be 80:BF}
-    utf-8 EE7F replace  \uFFFD\u7F    -1 {} {First trail byte must be 80:BF}
+    utf-8 EE7F tcl8     \u00EE\x7F    -1 {} {First trail byte must be 80:BF}
+    utf-8 EE7F replace  \uFFFD\x7F    -1 {} {First trail byte must be 80:BF}
+    utf-8 EE7F lossless  \uDCEE\x7F     -1 {} {First trail byte must be 80:BF}
    utf-8 EE7F strict   {}             0 {} {First trail byte must be 80:BF}
    utf-8 EED0 tcl8     \u00EE\u00D0  -1 {} {First trail byte must be 80:BF}
    utf-8 EED0 replace  \uFFFD\uFFFD  -1 {} {First trail byte must be 80:BF}
+    utf-8 EED0 lossless  \uDCEE\uDCD0   -1 {} {First trail byte must be 80:BF}
    utf-8 EED0 strict   {}             0 {} {First trail byte must be 80:BF}
    utf-8 EE81 tcl8      \u00EE\u0081   -1 {} {Missing second trail byte}
    utf-8 EE81 replace   \uFFFD         -1 {knownW3C} {Missing second trail byte}
+    utf-8 EE81 lossless   \uDCEE\uDC81  -1 {} {Missing second trail byte}
    utf-8 EE81 strict    {}              0 {} {Missing second trail byte}
    utf-8 EEBF tcl8      \u00EE\u00BF   -1 {} {Missing second trail byte}
    utf-8 EEBF replace   \uFFFD         -1 {knownW3C} {Missing second trail byte}
+    utf-8 EEBF lossless   \uDCEE\uDCBF  -1 {} {Missing second trail byte}
    utf-8 EEBF strict    {}              0 {} {Missing second trail byte}
    utf-8 EE807F tcl8      \u00EE\u20AC\x7F -1 {} {Second trail byte must be 80:BF}
-    utf-8 EE807F replace   \uFFFD\u7F       -1 {knownW3C} {Second trail byte must be 80:BF}
+    utf-8 EE807F replace   \uFFFD\x7F       -1 {knownW3C} {Second trail byte must be 80:BF}
+    utf-8 EE807F lossless  \uDCEE\uDC80\x7F -1 {} {Second trail byte must be 80:BF}
    utf-8 EE807F strict    {}                0 {}  {Second trail byte must be 80:BF}
    utf-8 EEBF7F tcl8      \u00EE\u00BF\x7F -1 {} {Second trail byte must be 80:BF}
-    utf-8 EEBF7F replace   \uFFFD\u7F       -1 {knownW3C} {Second trail byte must be 80:BF}
+    utf-8 EEBF7F replace   \uFFFD\x7F       -1 {knownW3C} {Second trail byte must be 80:BF}
+    utf-8 EEBF7F lossless  \uDCEE\uDCBF\x7F -1 {} {Second trail byte must be 80:BF}
    utf-8 EEBF7F strict    {}                0 {}  {Second trail byte must be 80:BF}
    utf-8 EF tcl8       \u00EF        -1 {} {Missing trail byte}
    utf-8 EF replace    \uFFFD        -1 {} {Missing trail byte}
+    utf-8 EF lossless    \uDCEF       -1 {} {Missing trail byte}
    utf-8 EF strict     {}             0 {} {Missing trail byte}
    utf-8 EF7F tcl8     \u00EF\u7F    -1 {} {First trail byte must be 80:BF}
    utf-8 EF7F replace  \uFFFD\u7F    -1 {} {First trail byte must be 80:BF}
+    utf-8 EF7F lossless  \uDCEF\x7F -1 {} {First trail byte must be 80:BF}
    utf-8 EF7F strict   {}             0 {} {First trail byte must be 80:BF}
    utf-8 EFD0 tcl8     \u00EF\u00D0  -1 {} {First trail byte must be 80:BF}
    utf-8 EFD0 replace  \uFFFD\uFFFD  -1 {} {First trail byte must be 80:BF}
+    utf-8 EFD0 lossless  \uDCEF\uDCD0 -1 {} {First trail byte must be 80:BF}
    utf-8 EFD0 strict   {}             0 {} {First trail byte must be 80:BF}
    utf-8 EF81 tcl8      \u00EF\u0081   -1 {} {Missing second trail byte}
    utf-8 EF81 replace   \uFFFD         -1 {knownW3C} {Missing second trail byte}
+    utf-8 EF81 lossless  \uDCEF\uDC81   -1 {} {Missing second trail byte}
    utf-8 EF81 strict    {}              0 {} {Missing second trail byte}
    utf-8 EFBF tcl8      \u00EF\u00BF   -1 {} {Missing second trail byte}
    utf-8 EFBF replace   \uFFFD         -1 {knownW3C} {Missing second trail byte}
+    utf-8 EFBF lossless  \uDCEF\uDCBF   -1 {} {Missing second trail byte}
    utf-8 EFBF strict    {}              0 {} {Missing second trail byte}
    utf-8 EF807F tcl8      \u00EF\u20AC\x7F -1 {} {Second trail byte must be 80:BF}
-    utf-8 EF807F replace   \uFFFD\u7F       -1 {knownW3C} {Second trail byte must be 80:BF}
+    utf-8 EF807F replace   \uFFFD\x7F       -1 {knownW3C} {Second trail byte must be 80:BF}
+    utf-8 EF807F lossless  \uDCEF\uDC80\x7F -1 {} {Second trail byte must be 80:BF}
    utf-8 EF807F strict    {}                0 {}  {Second trail byte must be 80:BF}
    utf-8 EFBF7F tcl8      \u00EF\u00BF\x7F -1 {} {Second trail byte must be 80:BF}
-    utf-8 EFBF7F replace   \uFFFD\u7F       -1 {knownW3C} {Second trail byte must be 80:BF}
+    utf-8 EFBF7F replace   \uFFFD\x7F       -1 {knownW3C} {Second trail byte must be 80:BF}
+    utf-8 EFBF7F lossless  \uDCEF\uDCBF\x7F -1 {} {Second trail byte must be 80:BF}
    utf-8 EFBF7F strict    {}                0 {}  {Second trail byte must be 80:BF}

    utf-8 F0 tcl8       \u00F0        -1 {} {Missing trail byte}
    utf-8 F0 replace    \uFFFD        -1 {} {Missing trail byte}
+    utf-8 F0 lossless   \uDCF0       -1 {} {Missing trail byte}
    utf-8 F0 strict     {}             0 {} {Missing trail byte}
    utf-8 F080 tcl8     \u00F0\u20AC  -1 {} {First trail byte must be 90:BF}
    utf-8 F080 replace  \uFFFD        -1 {knownW3C} {First trail byte must be 90:BF}
+    utf-8 F080 lossless \uDCF0\uDC80 -1 {} {First trail byte must be 90:BF}
    utf-8 F080 strict   {}             0 {} {First trail byte must be 90:BF}
    utf-8 F08F tcl8     \u00F0\u8F    -1 {} {First trail byte must be 90:BF}
    utf-8 F08F replace  \uFFFD        -1 {knownW3C} {First trail byte must be 90:BF}
+    utf-8 F08F lossless \uDCF0\uDC8F -1 {} {First trail byte must be 90:BF}
    utf-8 F08F strict   {}             0 {} {First trail byte must be 90:BF}
    utf-8 F0D0 tcl8     \u00F0\u00D0  -1 {} {First trail byte must be 90:BF}
    utf-8 F0D0 replace  \uFFFD\uFFFD  -1 {} {First trail byte must be 90:BF}
+    utf-8 F0D0 lossless \uDCF0\uDCD0 -1 {} {First trail byte must be 90:BF}
    utf-8 F0D0 strict   {}             0 {} {First trail byte must be 90:BF}
    utf-8 F090 tcl8      \u00F0\u0090   -1 {} {Missing second trail byte}
    utf-8 F090 replace   \uFFFD         -1 {knownW3C} {Missing second trail byte}
+    utf-8 F090 lossless  \uDCF0\uDC90   -1 {} {Missing second trail byte}
    utf-8 F090 strict    {}              0 {} {Missing second trail byte}
    utf-8 F0BF tcl8      \u00F0\u00BF   -1 {} {Missing second trail byte}
    utf-8 F0BF replace   \uFFFD         -1 {knownW3C} {Missing second trail byte}
+    utf-8 F0BF lossless   \uDCF0\uDCBF  -1 {} {Missing second trail byte}
    utf-8 F0BF strict    {}              0 {} {Missing second trail byte}
    utf-8 F0907F tcl8      \u00F0\u0090\x7F -1 {} {Second trail byte must be 80:BF}
    utf-8 F0907F replace   \uFFFD\u7F       -1 {knownW3C} {Second trail byte must be 80:BF}
+    utf-8 F0907F lossless   \uDCF0\uDC90\x7F -1 {} {Second trail byte must be 80:BF}
    utf-8 F0907F strict    {}                0 {}  {Second trail byte must be 80:BF}
    utf-8 F0BF7F tcl8      \u00F0\u00BF\x7F -1 {} {Second trail byte must be 80:BF}
    utf-8 F0BF7F replace   \uFFFD\u7F       -1 {knownW3C} {Second trail byte must be 80:BF}
+    utf-8 F0BF7F lossless  \uDCF0\uDCBF\x7F -1 {} {Second trail byte must be 80:BF}
    utf-8 F0BF7F strict    {}                0 {}  {Second trail byte must be 80:BF}
    utf-8 F090BF tcl8      \u00F0\u0090\u00BF   -1 {} {Missing third trail byte}
    utf-8 F090BF replace   \uFFFD         -1 {knownW3C} {Missing third trail byte}
+    utf-8 F090BF lossless  \uDCF0\uDC90\uDCBF  -1 {} {Missing third trail byte}
    utf-8 F090BF strict    {}              0 {} {Missing third trail byte}
    utf-8 F0BF81 tcl8      \u00F0\u00BF\u0081   -1 {} {Missing third trail byte}
    utf-8 F0BF81 replace   \uFFFD         -1 {knownW3C} {Missing third trail byte}
+    utf-8 F0BF81 lossless  \uDCF0\uDCBF\uDC81  -1 {} {Missing third trail byte}
    utf-8 F0BF81 strict    {}              0 {} {Missing third trail byte}
    utf-8 F0BF807F tcl8      \u00F0\u00BF\u20AC\x7F   -1 {} {Third trail byte must be 80:BF}
    utf-8 F0BF817F replace   \uFFFD\x7F           -1 {knownW3C} {Third trail byte must be 80:BF}
+    utf-8 F0BF817F lossless   \uDCF0\uDCBF\uDC81\x7F -1 {} {Third trail byte must be 80:BF}
    utf-8 F0BF817F strict    {}              0 {} {Third trail byte must be 80:BF}
    utf-8 F090BFD0 tcl8      \u00F0\u0090\u00BF\u00D0   -1 {} {Third trail byte must be 80:BF}
    utf-8 F090BFD0 replace   \uFFFD         -1 {knownW3C} {Third trail byte must be 80:BF}
+    utf-8 F090BFD0 lossless   \uDCF0\uDC90\uDCBF\uDCD0 -1 {} {Third trail byte must be 80:BF}
    utf-8 F090BFD0 strict    {}              0 {} {Third trail byte must be 80:BF}

    utf-8 F1 tcl8       \u00F1        -1 {} {Missing trail byte}
    utf-8 F1 replace    \uFFFD        -1 {} {Missing trail byte}
+    utf-8 F1 lossless    \uDCF1       -1 {} {Missing trail byte}
    utf-8 F1 strict     {}             0 {} {Missing trail byte}
    utf-8 F17F tcl8     \u00F1\u7F    -1 {} {First trail byte must be 80:BF}
    utf-8 F17F replace  \uFFFD        -1 {knownW3C} {First trail byte must be 80:BF}
+    utf-8 F17F lossless  \uDCF1\x7F -1 {} {First trail byte must be 80:BF}
    utf-8 F17F strict   {}             0 {} {First trail byte must be 80:BF}
    utf-8 F1D0 tcl8     \u00F1\u00D0  -1 {} {First trail byte must be 80:BF}
    utf-8 F1D0 replace  \uFFFD\uFFFD  -1 {} {First trail byte must be 80:BF}
+    utf-8 F1D0 lossless  \uDCF1\uDCD0 -1 {} {First trail byte must be 80:BF}
    utf-8 F1D0 strict   {}             0 {} {First trail byte must be 80:BF}
    utf-8 F180 tcl8      \u00F1\u20AC   -1 {} {Missing second trail byte}
    utf-8 F180 replace   \uFFFD         -1 {knownW3C} {Missing second trail byte}
+    utf-8 F180 lossless   \uDCF1\uDC80  -1 {} {Missing second trail byte}
    utf-8 F180 strict    {}              0 {} {Missing second trail byte}
    utf-8 F1BF tcl8      \u00F1\u00BF   -1 {} {Missing second trail byte}
    utf-8 F1BF replace   \uFFFD         -1 {knownW3C} {Missing second trail byte}
+    utf-8 F1BF lossless   \uDCF1\uDCBF  -1 {} {Missing second trail byte}
    utf-8 F1BF strict    {}              0 {} {Missing second trail byte}
    utf-8 F1807F tcl8      \u00F1\u20AC\x7F -1 {} {Second trail byte must be 80:BF}
-    utf-8 F1807F replace   \uFFFD\u7F       -1 {knownW3C} {Second trail byte must be 80:BF}
+    utf-8 F1807F replace   \uFFFD\x7F       -1 {knownW3C} {Second trail byte must be 80:BF}
+    utf-8 F1807F lossless   \uDCF1\uDC80\x7F -1 {} {Second trail byte must be 80:BF}
    utf-8 F1807F strict    {}                0 {}  {Second trail byte must be 80:BF}
    utf-8 F1BF7F tcl8      \u00F1\u00BF\x7F -1 {} {Second trail byte must be 80:BF}
-    utf-8 F1BF7F replace   \uFFFD\u7F       -1 {knownW3C} {Second trail byte must be 80:BF}
+    utf-8 F1BF7F replace   \uFFFD\x7F       -1 {knownW3C} {Second trail byte must be 80:BF}
+    utf-8 F1BF7F lossless   \uDCF1\uDCBF\x7F -1 {} {Second trail byte must be 80:BF}
    utf-8 F1BF7F strict    {}                0 {}  {Second trail byte must be 80:BF}
    utf-8 F180BF tcl8      \u00F1\u20AC\u00BF   -1 {} {Missing third trail byte}
    utf-8 F180BF replace   \uFFFD         -1 {knownW3C} {Missing third trail byte}
+    utf-8 F180BF lossless   \uDCF1\uDC80\uDCBF -1 {} {Missing third trail byte}
    utf-8 F180BF strict    {}              0 {} {Missing third trail byte}
    utf-8 F1BF81 tcl8      \u00F1\u00BF\u0081   -1 {} {Missing third trail byte}
    utf-8 F1BF81 replace   \uFFFD         -1 {knownW3C} {Missing third trail byte}
+    utf-8 F1BF81 lossless   \uDCF1\uDCBF\uDC81  -1 {} {Missing third trail byte}
    utf-8 F1BF81 strict    {}              0 {} {Missing third trail byte}
-    utf-8 F1BF807F tcl8      \u00F1\u00BF\u20AC\x7F   -1 {} {Third trail byte must be 80:BF}
+    utf-8 F1BF807F tcl8      \u00F1\u00BF\u20AC\x7F  -1 {} {Third trail byte must be 80:BF}
    utf-8 F1BF817F replace   \uFFFD\x7F           -1 {knownW3C} {Third trail byte must be 80:BF}
+    utf-8 F1BF817F lossless   \uDCF1\uDCBF\uDC81\x7F -1 {} {Third trail byte must be 80:BF}
    utf-8 F1BF817F strict    {}              0 {} {Third trail byte must be 80:BF}
    utf-8 F180BFD0 tcl8      \u00F1\u20AC\u00BF\u00D0   -1 {} {Third trail byte must be 80:BF}
    utf-8 F180BFD0 replace   \uFFFD         -1 {knownW3C} {Third trail byte must be 80:BF}
+    utf-8 F180BFD0 lossless   \uDCF1\uDC80\uDCBF\uDCD0 -1 {} {Third trail byte must be 80:BF}
    utf-8 F180BFD0 strict    {}              0 {} {Third trail byte must be 80:BF}
    utf-8 F3 tcl8       \u00F3        -1 {} {Missing trail byte}
    utf-8 F3 replace    \uFFFD        -1 {} {Missing trail byte}
+    utf-8 F3 lossless    \uDCF3       -1 {} {Missing trail byte}
    utf-8 F3 strict     {}             0 {} {Missing trail byte}
    utf-8 F37F tcl8     \u00F3\x7F    -1 {} {First trail byte must be 80:BF}
    utf-8 F37F replace  \uFFFD        -1 {knownW3C} {First trail byte must be 80:BF}
+    utf-8 F37F lossless  \uDCF3\x7F   -1 {} {First trail byte must be 80:BF}
    utf-8 F37F strict   {}             0 {} {First trail byte must be 80:BF}
    utf-8 F3D0 tcl8     \u00F3\u00D0  -1 {} {First trail byte must be 80:BF}
    utf-8 F3D0 replace  \uFFFD\uFFFD  -1 {} {First trail byte must be 80:BF}
+    utf-8 F3D0 lossless  \uDCF3\uDCD0 -1 {} {First trail byte must be 80:BF}
    utf-8 F3D0 strict   {}             0 {} {First trail byte must be 80:BF}
    utf-8 F380 tcl8      \u00F3\u20AC   -1 {} {Missing second trail byte}
    utf-8 F380 replace   \uFFFD         -1 {knownW3C} {Missing second trail byte}
+    utf-8 F380 lossless   \uDCF3\uDC80  -1 {} {Missing second trail byte}
    utf-8 F380 strict    {}              0 {} {Missing second trail byte}
    utf-8 F3BF tcl8      \u00F3\u00BF   -1 {} {Missing second trail byte}
    utf-8 F3BF replace   \uFFFD         -1 {knownW3C} {Missing second trail byte}
+    utf-8 F3BF lossless   \uDCF3\uDCBF  -1 {} {Missing second trail byte}
    utf-8 F3BF strict    {}              0 {} {Missing second trail byte}
-    utf-8 F3807F tcl8      \u00F3\u20AC\x7F -1 {} {Second trail byte must be 80:BF}
-    utf-8 F3807F replace   \uFFFD\u7F       -1 {knownW3C} {Second trail byte must be 80:BF}
+    utf-8 F3807F tcl8     \u00F3\u20AC\x7F -1 {} {Second trail byte must be 80:BF}
+    utf-8 F3807F replace  \uFFFD\u7F       -1 {knownW3C} {Second trail byte must be 80:BF}
+    utf-8 F3807F lossless  \uDCF3\uDC80\x7F -1 {} {Second trail byte must be 80:BF}
    utf-8 F3807F strict    {}                0 {}  {Second trail byte must be 80:BF}
    utf-8 F3BF7F tcl8      \u00F3\u00BF\x7F -1 {} {Second trail byte must be 80:BF}
    utf-8 F3BF7F replace   \uFFFD\u7F       -1 {knownW3C} {Second trail byte must be 80:BF}
+    utf-8 F3BF7F lossless   \uDCF3\uDCBF\x7F -1 {} {Second trail byte must be 80:BF}
    utf-8 F3BF7F strict    {}                0 {}  {Second trail byte must be 80:BF}
    utf-8 F380BF tcl8      \u00F3\u20AC\u00BF   -1 {} {Missing third trail byte}
    utf-8 F380BF replace   \uFFFD         -1 {knownW3C} {Missing third trail byte}
+    utf-8 F380BF lossless   \uDCF3\uDC80\uDCBF -1 {} {Missing third trail byte}
    utf-8 F380BF strict    {}              0 {} {Missing third trail byte}
    utf-8 F3BF81 tcl8      \u00F3\u00BF\u0081   -1 {} {Missing third trail byte}
    utf-8 F3BF81 replace   \uFFFD         -1 {knownW3C} {Missing third trail byte}
+    utf-8 F3BF81 lossless   \uDCF3\uDCBF\uDC81 -1 {} {Missing third trail byte}
    utf-8 F3BF81 strict    {}              0 {} {Missing third trail byte}
    utf-8 F3BF807F tcl8      \u00F3\u00BF\u20AC\x7F   -1 {} {Third trail byte must be 80:BF}
    utf-8 F3BF817F replace   \uFFFD\x7F           -1 {knownW3C} {Third trail byte must be 80:BF}
+    utf-8 F3BF817F lossless   \uDCF3\uDCBF\uDC81\x7F -1 {} {Third trail byte must be 80:BF}
    utf-8 F3BF817F strict    {}              0 {} {Third trail byte must be 80:BF}
    utf-8 F380BFD0 tcl8      \u00F3\u20AC\u00BF\u00D0   -1 {} {Third trail byte must be 80:BF}
    utf-8 F380BFD0 replace   \uFFFD         -1 {knownW3C} {Third trail byte must be 80:BF}
+    utf-8 F380BFD0 lossless   \uDCF3\uDC80\uDCBF\uDCD0 -1 {} {Third trail byte must be 80:BF}
    utf-8 F380BFD0 strict    {}              0 {} {Third trail byte must be 80:BF}

    utf-8 F4 tcl8       \u00F4        -1 {} {Missing trail byte}
    utf-8 F4 replace    \uFFFD        -1 {} {Missing trail byte}
+    utf-8 F4 lossless    \uDCF4       -1 {} {Missing trail byte}
    utf-8 F4 strict     {}             0 {} {Missing trail byte}
    utf-8 F47F tcl8     \u00F4\u7F    -1 {} {First trail byte must be 80:8F}
    utf-8 F47F replace  \uFFFD\u7F    -1 {knownW3C} {First trail byte must be 80:8F}
+    utf-8 F47F lossless  \uDCF4\x7F -1 {} {First trail byte must be 80:8F}
    utf-8 F47F strict   {}             0 {} {First trail byte must be 80:8F}
    utf-8 F490 tcl8     \u00F4\u0090  -1 {} {First trail byte must be 80:8F}
    utf-8 F490 replace  \uFFFD\uFFFD  -1 {} {First trail byte must be 80:8F}
+    utf-8 F490 lossless  \uDCF4\uDC90 -1 {} {First trail byte must be 80:8F}
    utf-8 F490 strict   {}             0 {} {First trail byte must be 80:8F}
    utf-8 F480 tcl8      \u00F4\u20AC   -1 {} {Missing second trail byte}
    utf-8 F480 replace   \uFFFD         -1 {knownW3C} {Missing second trail byte}
+    utf-8 F480 lossless   \uDCF4\uDC80  -1 {} {Missing second trail byte}
    utf-8 F480 strict    {}              0 {} {Missing second trail byte}
    utf-8 F48F tcl8      \u00F4\u008F   -1 {} {Missing second trail byte}
    utf-8 F48F replace   \uFFFD         -1 {knownW3C} {Missing second trail byte}
+    utf-8 F48F lossless   \uDCF4\uDC8F  -1 {} {Missing second trail byte}
    utf-8 F48F strict    {}              0 {} {Missing second trail byte}
    utf-8 F4807F tcl8      \u00F4\u20AC\x7F -1 {} {Second trail byte must be 80:BF}
    utf-8 F4807F replace   \uFFFD\u7F       -1 {knownW3C} {Second trail byte must be 80:BF}
+    utf-8 F4807F lossless   \uDCF4\uDC80\x7F -1 {} {Second trail byte must be 80:BF}
    utf-8 F4807F strict    {}                0 {}  {Second trail byte must be 80:BF}
    utf-8 F48F7F tcl8      \u00F4\u008F\x7F -1 {} {Second trail byte must be 80:BF}
    utf-8 F48F7F replace   \uFFFD\u7F       -1 {knownW3C} {Second trail byte must be 80:BF}
+    utf-8 F48F7F lossless   \uDCF4\uDC8F\x7F -1 {} {Second trail byte must be 80:BF}
    utf-8 F48F7F strict    {}                0 {}  {Second trail byte must be 80:BF}
    utf-8 F48081 tcl8      \u00F4\u20AC\u0081   -1 {} {Missing third trail byte}
    utf-8 F48081 replace   \uFFFD         -1 {knownW3C} {Missing third trail byte}
+    utf-8 F48081 lossless   \uDCF4\uDC80\uDC81  -1 {} {Missing third trail byte}
    utf-8 F48081 strict    {}              0 {} {Missing third trail byte}
    utf-8 F48F81 tcl8      \u00F4\u008F\u0081   -1 {} {Missing third trail byte}
    utf-8 F48F81 replace   \uFFFD         -1 {knownW3C} {Missing third trail byte}
+    utf-8 F48F81 lossless   \uDCF4\uDC8F\uDC81  -1 {} {Missing third trail byte}
    utf-8 F48F81 strict    {}              0 {} {Missing third trail byte}
    utf-8 F481817F tcl8      \u00F4\u0081\u0081\x7F   -1 {} {Third trail byte must be 80:BF}
    utf-8 F480817F replace   \uFFFD\x7F           -1 {knownW3C} {Third trail byte must be 80:BF}
+    utf-8 F480817F lossless   \uDCF4\uDC80\uDC81\x7F -1 {} {Third trail byte must be 80:BF}
    utf-8 F480817F strict    {}              0 {} {Third trail byte must be 80:BF}
    utf-8 F48FBFD0 tcl8      \u00F4\u008F\u00BF\u00D0   -1 {} {Third trail byte must be 80:BF}
    utf-8 F48FBFD0 replace   \uFFFD         -1 {knownW3C} {Third trail byte must be 80:BF}
+    utf-8 F48FBFD0 lossless   \uDCF4\uDC8F\uDCBF\uDCD0 -1 {} {Third trail byte must be 80:BF}
    utf-8 F48FBFD0 strict    {}              0 {} {Third trail byte must be 80:BF}

    utf-8 F5 tcl8    \u00F5 -1 {} {F5:FF are invalid everywhere}
    utf-8 F5 replace \uFFFD -1 {} {F5:FF are invalid everywhere}
+    utf-8 F5 lossless \uDCF5 -1 {} {F5:FF are invalid everywhere}
    utf-8 F5 strict  {}      0 {} {F5:FF are invalid everywhere}
    utf-8 FF tcl8    \u00FF -1 {} {F5:FF are invalid everywhere}
    utf-8 FF replace \uFFFD -1 {} {F5:FF are invalid everywhere}
+    utf-8 FF lossless \uDCFF -1 {} {F5:FF are invalid everywhere}
    utf-8 FF strict  {}      0 {} {F5:FF are invalid everywhere}

    utf-8 C0AFE080BFF0818130 replace \uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\x30 -1 {} {Unicode Table 3-8}
+    utf-8 C0AFE080BFF0818130 lossless \uDCC0\uDCAF\uDCE0\uDC80\uDCBF\uDCF0\uDC81\uDC81\x30 -1 {} {Unicode Table 3-8}
    utf-8 EDA080EDBFBFEDAF30 replace \uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\x30 -1 {knownW3C} {Unicode Table 3-9}
+    utf-8 EDA080EDBFBFEDAF30 lossless \uD800\uDFFF\uDCED\uDCAF0 -1 {} {Unicode Table 3-9 - TODO assumes surrogates permitted in utf-8 lossless}
    utf-8 F4919293FF4180BF30 replace \uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\u0041\uFFFD\uFFFD\x30 -1 {} {Unicode Table 3-10}
+    utf-8 F4919293FF4180BF30 lossless \uDCF4\uDC91\uDC92\uDC93\uDCFF\u0041\uDC80\uDCBF\x30 -1 {} {Unicode Table 3-10}
    utf-8 E180E2F09192F1BF30 replace \uFFFD\uFFFD\uFFFD\uFFFD\x30                         -1 {knownW3C} {Unicode Table 3.11}
+    utf-8 E180E2F09192F1BF30 lossless \uDCE1\uDC80\uDCE2\uDCF0\uDC91\uDC92\uDCF1\uDCBF\x30 -1 {} {Unicode Table 3.11}
 }

 # utf16-le and utf16-be test cases. Note utf16 cases are automatically generated
@@ -552,21 +666,29 @@ lappend encInvalidBytes {*}{
    utf-16le 41      tcl8      \uFFFD -1 {solo tail} {Truncated}
    utf-16le 41      replace   \uFFFD -1 {solo tail} {Truncated}
    utf-16le 41      strict    {}      0 {solo tail} {Truncated}
+    utf-16le 41      lossless  \uFFFD -1 {solo tail} {Truncated - byte < 0x80}
+    utf-16le 80      lossless  \uFFFD -1 {solo tail} {Truncated - byte >= 0x80}
    utf-16le 00D8    tcl8      \uD800 -1 {} {Missing low surrogate}
    utf-16le 00D8    replace   \uFFFD -1 {} {Missing low surrogate}
+    utf-16le 00D8    lossless  \uFFFD -1 {} {Missing low surrogate}
    utf-16le 00D8    strict    {}      0 {knownBug} {Missing low surrogate}
    utf-16le 00DC    tcl8      \uDC00 -1 {} {Missing high surrogate}
    utf-16le 00DC    replace   \uFFFD -1 {} {Missing high surrogate}
+    utf-16le 00DC    lossless  \uFFFD -1 {} {Missing high surrogate}
    utf-16le 00DC    strict    {}      0 {knownBug} {Missing high surrogate}

    utf-16be 41      tcl8      \uFFFD -1 {solo tail} {Truncated}
    utf-16be 41      replace   \uFFFD -1 {solo tail} {Truncated}
    utf-16be 41      strict    {}      0 {solo tail} {Truncated}
+    utf-16be 41      lossless  \uFFFD -1 {solo tail} {Truncated - byte < 0x80}
+    utf-16be 80      lossless  \uFFFD -1 {solo tail} {Truncated - byte >= 0x80}
    utf-16be D800    tcl8      \uD800 -1 {} {Missing low surrogate}
    utf-16be D800    replace   \uFFFD -1 {knownBug} {Missing low surrogate}
+    utf-16be D800    lossless  \uFFFD -1 {knownBug} {Missing low surrogate}
    utf-16be D800    strict    {}      0 {knownBug} {Missing low surrogate}
    utf-16be DC00    tcl8      \uDC00 -1 {} {Missing high surrogate}
    utf-16be DC00    replace   \uFFFD -1 {knownBug} {Missing high surrogate}
+    utf-16be DC00    lossless  \uFFFD -1 {knownBug} {Missing high surrogate}
    utf-16be DC00    strict    {}      0 {knownBug} {Missing high surrogate}
 }

@@ -577,62 +699,81 @@ lappend encInvalidBytes {*}{
 lappend encInvalidBytes {*}{
    utf-32le 41      tcl8      \uFFFD  -1 {solo tail} {Truncated}
    utf-32le 41      replace   \uFFFD  -1 {solo} {Truncated}
+    utf-32le 41      lossless  \uFFFD  -1 {solo} {Truncated}
    utf-32le 41      strict    {}   0 {solo tail} {Truncated}
    utf-32le 4100    tcl8      \uFFFD  -1 {solo tail} {Truncated}
    utf-32le 4100    replace   \uFFFD  -1 {solo} {Truncated}
+    utf-32le 4100    lossless  \uFFFD  -1 {solo} {Truncated}
    utf-32le 4100    strict    {}   0 {solo tail} {Truncated}
    utf-32le 410000  tcl8      \uFFFD  -1 {solo tail} {Truncated}
    utf-32le 410000  replace   \uFFFD  -1 {solo} {Truncated}
+    utf-32le 410000  lossless  \uFFFD  -1 {solo} {Truncated}
    utf-32le 410000  strict    {}       0 {solo tail} {Truncated}
    utf-32le 00D80000 tcl8     \uD800   -1 {} {High-surrogate}
    utf-32le 00D80000 replace  \uFFFD   -1 {} {High-surrogate}
+    utf-32le 00D80000 lossless \uFFFD   -1 {} {High-surrogate}
    utf-32le 00D80000 strict   {}        0 {} {High-surrogate}
    utf-32le 00DC0000 tcl8     \uDC00   -1 {} {Low-surrogate}
    utf-32le 00DC0000 replace  \uFFFD   -1 {} {Low-surrogate}
+    utf-32le 00DC0000 lossless \uFFFD   -1 {} {Low-surrogate}
    utf-32le 00DC0000 strict   {}        0 {} {Low-surrogate}
    utf-32le 00D8000000DC0000 tcl8 \uD800\uDC00    -1 {} {High-low-surrogate-pair}
-    utf-32le 00D8000000DC0000 replace \uFFFD\uFFFD -1 {} {High-low-surrogate-pair}
+    utf-32le 00D8000000DC0000 replace  \uFFFD\uFFFD -1 {} {High-low-surrogate-pair}
+    utf-32le 00D8000000DC0000 lossless \uFFFD\uFFFD -1 {} {High-low-surrogate-pair}
    utf-32le 00D8000000DC0000 strict  {}            0 {} {High-low-surrogate-pair}
    utf-32le 00001100 tcl8 \uFFFD    -1 {} {Out of range}
-    utf-32le 00001100 replace \uFFFD -1 {} {Out of range}
+    utf-32le 00001100 replace  \uFFFD -1 {} {Out of range}
+    utf-32le 00001100 lossless \uFFFD -1 {} {Out of range}
    utf-32le 00001100 strict {}       0 {} {Out of range}
    utf-32le FFFFFFFF tcl8 \uFFFD    -1 {} {Out of range}
-    utf-32le FFFFFFFF replace \uFFFD -1 {} {Out of range}
+    utf-32le FFFFFFFF replace  \uFFFD -1 {} {Out of range}
+    utf-32le FFFFFFFF lossless \uFFFD -1 {} {Out of range}
    utf-32le FFFFFFFF strict {}       0 {} {Out of range}

    utf-32be 41      tcl8      \uFFFD  -1 {solo tail} {Truncated}
    utf-32be 41      replace   \uFFFD  -1 {solo tail} {Truncated}
+    utf-32be 41      lossless  \uFFFD  -1 {solo tail} {Truncated}
    utf-32be 41      strict    {}       0 {solo tail} {Truncated}
    utf-32be 0041    tcl8      \uFFFD  -1 {solo tail} {Truncated}
    utf-32be 0041    replace   \uFFFD  -1 {solo} {Truncated}
+    utf-32be 0041    lossless  \uFFFD  -1 {solo} {Truncated}
    utf-32be 0041    strict    {}   0 {solo tail} {Truncated}
    utf-32be 000041  tcl8      \uFFFD  -1 {solo tail} {Truncated}
    utf-32be 000041  replace   \uFFFD  -1 {solo} {Truncated}
+    utf-32be 000041  lossless  \uFFFD  -1 {solo} {Truncated}
    utf-32be 000041  strict    {}       0 {solo tail} {Truncated}
    utf-32be 0000D800 tcl8     \uD800   -1 {} {High-surrogate}
    utf-32be 0000D800 replace  \uFFFD   -1 {} {High-surrogate}
+    utf-32be 0000D800 lossless \uFFFD   -1 {} {High-surrogate}
    utf-32be 0000D800 strict   {}        0 {} {High-surrogate}
    utf-32be 0000DC00 tcl8     \uDC00   -1 {} {Low-surrogate}
    utf-32be 0000DC00 replace  \uFFFD   -1 {} {Low-surrogate}
+    utf-32be 0000DC00 lossless \uFFFD   -1 {} {Low-surrogate}
    utf-32be 0000DC00 strict   {}        0 {} {Low-surrogate}
    utf-32be 0000D8000000DC00 tcl8 \uD800\uDC00    -1 {} {High-low-surrogate-pair}
-    utf-32be 0000D8000000DC00 replace \uFFFD\uFFFD -1 {} {High-low-surrogate-pair}
+    utf-32be 0000D8000000DC00 replace  \uFFFD\uFFFD -1 {} {High-low-surrogate-pair}
+    utf-32be 0000D8000000DC00 lossless \uFFFD\uFFFD -1 {} {High-low-surrogate-pair}
    utf-32be 0000D8000000DC00 strict  {}            0 {} {High-low-surrogate-pair}
    utf-32be 00110000 tcl8 \uFFFD    -1 {} {Out of range}
-    utf-32be 00110000 replace \uFFFD -1 {} {Out of range}
+    utf-32be 00110000 replace  \uFFFD -1 {} {Out of range}
+    utf-32be 00110000 lossless \uFFFD -1 {} {Out of range}
    utf-32be 00110000 strict {}       0 {} {Out of range}
    utf-32be FFFFFFFF tcl8 \uFFFD    -1 {} {Out of range}
-    utf-32be FFFFFFFF replace \uFFFD -1 {} {Out of range}
+    utf-32be FFFFFFFF replace  \uFFFD -1 {} {Out of range}
+    utf-32be FFFFFFFF lossless \uFFFD -1 {} {Out of range}
    utf-32be FFFFFFFF strict {}       0 {} {Out of range}
 }

+# escape tables - TODO
+# This tests the EscapeToUtf code path.
+lappend encInvalidBytes {*}{
+}
+
 # Strings that cannot be encoded for specific encoding / profiles
 # <ENCODING STRING PROFILE EXPECTEDRESULT EXPECTEDFAILINDEX CTRL COMMENT>
 # <ENCODING,STRING,PROFILE> should be unique for test ids to be unique.
 # See earlier comments about CTRL field.
 #
-# Note utf-16, utf-32 missing because they are automatically
-# generated based on le/be versions.
 # TODO - out of range code point (note cannot be generated by \U notation)
 lappend encUnencodableStrings {*}{
    ascii \u00e0 tcl8    3f -1 {} {unencodable}
@@ -651,5 +792,6 @@ lappend encUnencodableStrings {*}{
 # The icuUcmTests.tcl is generated by the tools/ucm2tests.tcl script
 # and generates test vectors for the above tables for various encodings
 # based on ICU UCM files.
-# TODO - commented out for now as generating a lot of mismatches.
+# TODO - commented out for now as generating a lot of mismatches, mainly
+# due to Tcl using ? for replacement char and ICU often using ^Z.
 # source [file join [file dirname [info script]] icuUcmTests.tcl]
@@ -18,7 +18,6 @@ if {"::tcltest" ni [namespace children]} {

 source [file join [file dirname [info script]] tcltests.tcl]

-testConstraint utf8system [string equal [encoding system] utf-8]
 if {[llength [auto_execok bash]]} {
    testConstraint haveBash 1
 }
@@ -513,9 +512,9 @@ test env-9.1 {
    }
 } -result {}

-test env-10.0 {
-    Unequal environment strings test should test unequal
-} -constraints {unix haveBash utf8system knownBug} -setup {
+test env-10.1 {
+    Unequal environment strings test should test unequal (failed pre-TIP 671)
+} -constraints {unix haveBash} -setup {
    set tclScript [makeFile {
        puts [string equal $env(XX) $env(YY)]
    } tclScript]
@@ -525,10 +524,56 @@ test env-10.0 {
    }
    append shellCode "[info nameofexecutable] $tclScript\n"
    set shScript [makeFile $shellCode shScript]
+    set oldEnc [encoding system]
+    encoding system utf-8
+} -cleanup {
+    encoding system $oldEnc
 } -body {
    exec {*}[auto_execok bash] $shScript
 } -result 0

+test env-10.2 {
+    Read invalidly encoded bytes in environment value - TIP 671
+} -constraints {unix haveBash} -setup {
+    set tclScript [makeFile {
+	puts [format {%04x} [scan $env(XX) %c]]
+    } tclScript]
+    # Note following requires bash, not sh! Dunno the equivalent in sh
+    set shellCode {
+        export XX=$'\xe9'
+    }
+    append shellCode "[info nameofexecutable] $tclScript\n"
+    set shScript [makeFile $shellCode shScript]
+    set oldEnc [encoding system]
+    encoding system utf-8
+} -cleanup {
+    encoding system $oldEnc
+} -body {
+    exec {*}[auto_execok bash] $shScript
+} -result dce9
+
+test env-10.3 {
+    Write invalidly encoded bytes to environment value - TIP 671
+} -constraints {unix haveBash} -setup {
+    set tclScript [makeFile {
+        set env(YY) A$env(XX)B
+        set line [lindex [split [exec {*}[auto_execok bash] -c {echo $YY | od -t x1}] \n] 0]
+        puts [lrange $line 1 3]
+    } tclScript]
+    # Note following requires bash, not sh! Dunno the equivalent in sh
+    set shellCode {
+        export XX=$'\xe9'
+    }
+    append shellCode "[info nameofexecutable] $tclScript\n"
+    set shScript [makeFile $shellCode shScript]
+    set oldEnc [encoding system]
+    encoding system utf-8
+} -cleanup {
+    encoding system $oldEnc
+} -body {
+    exec {*}[auto_execok bash] $shScript
+} -result {41 e9 42}
+


 # cleanup
@@ -371,7 +371,7 @@ test iocmd-8.20 {fconfigure command / win console channel} -constraints {nonPort

 test iocmd-8.23 {fconfigure -profile badprofile} -body {
    fconfigure stdin -profile froboz
-} -returnCodes error -result {bad profile name "froboz": must be replace, strict, or tcl8}
+} -returnCodes error -result {bad profile name "froboz": must be lossless, replace, strict, or tcl8}

 test iocmd-9.1 {eof command} {
    list [catch {eof} msg] $msg $::errorCode
@@ -11,6 +11,10 @@ namespace eval ::tcl::test::main {
    # Is [exec] defined?
    testConstraint exec [llength [info commands exec]]

+    if {[llength [auto_execok bash]]} {
+        testConstraint haveBash 1
+    }
+
    # Is the tcl::test package loaded?
    testConstraint tcl::test [expr {
 	[llength [package provide tcl::test]]
@@ -1284,6 +1288,25 @@ namespace eval ::tcl::test::main {
 	file delete result
    } -result "1\nfoo\n"

+    test Tcl_Main-10.1 {
+        Invalidly encoded bytes in arguments - TIP 671
+    } -constraints {unix haveBash} -setup {
+        set tclScript [makeFile {
+            lassign $::argv a b
+            # a should be \udce9 (lossless map ofe9) b should be \ue9
+            puts [list [format %x [scan $a %c]] [format %x [scan $b %c]]]
+        } tclScript]
+        # Note following requires bash, not sh! Dunno the equivalent in sh
+        set shellCode [string cat "[info nameofexecutable] $tclScript" { $'\351'} { $'\303\251'} \n]
+        set shScript [makeFile $shellCode shScript]
+        set oldEnc [encoding system]
+        encoding system utf-8
+    } -cleanup {
+        encoding system $oldEnc
+    } -body {
+        exec {*}[auto_execok bash] $shScript
+    } -result {dce9 e9}
+
    cd [workingDirectory]

    cleanupTests
@@ -1201,11 +1201,11 @@ TtyGetOptionProc(
 	tcgetattr(fsPtr->fileState.fd, &iostate);
 	Tcl_DStringInit(&ds);

-	Tcl_ExternalToUtfDStringEx(NULL, NULL, (char *) &iostate.c_cc[VSTART], 1, TCL_ENCODING_PROFILE_TCL8, &ds, NULL);
+	TclSystemToInternalEncoding(NULL, (char *) &iostate.c_cc[VSTART], 1, &ds);
 	Tcl_DStringAppendElement(dsPtr, Tcl_DStringValue(&ds));
 	TclDStringClear(&ds);

-	Tcl_ExternalToUtfDStringEx(NULL, NULL, (char *) &iostate.c_cc[VSTOP], 1, TCL_ENCODING_PROFILE_TCL8, &ds, NULL);
+	TclSystemToInternalEncoding(NULL, (char *) &iostate.c_cc[VSTOP], 1, &ds);
 	Tcl_DStringAppendElement(dsPtr, Tcl_DStringValue(&ds));
 	Tcl_DStringFree(&ds);
    }
@@ -155,7 +155,7 @@ TclpFindExecutable(
 #endif
    {
 	encoding = Tcl_GetEncoding(NULL, NULL);
-	Tcl_ExternalToUtfDStringEx(NULL, encoding, name, TCL_INDEX_NONE, TCL_ENCODING_PROFILE_TCL8, &utfName, NULL);
+	Tcl_ExternalToUtfDStringEx(NULL, encoding, name, TCL_INDEX_NONE, TCL_ENCODING_PROFILE_LOSSLESS, &utfName, NULL);
 	TclSetObjNameOfExecutable(
 		Tcl_NewStringObj(Tcl_DStringValue(&utfName), TCL_INDEX_NONE), encoding);
 	Tcl_DStringFree(&utfName);
@@ -183,7 +183,7 @@ TclpFindExecutable(

    Tcl_DStringFree(&buffer);
    Tcl_UtfToExternalDStringEx(NULL, NULL, Tcl_DStringValue(&cwd),
-	    Tcl_DStringLength(&cwd), TCL_ENCODING_PROFILE_TCL8, &buffer, NULL);
+	    Tcl_DStringLength(&cwd), TCL_ENCODING_PROFILE_LOSSLESS, &buffer, NULL);
    if (Tcl_DStringValue(&cwd)[Tcl_DStringLength(&cwd) -1] != '/') {
 	TclDStringAppendLiteral(&buffer, "/");
    }
@@ -193,7 +193,7 @@ TclpFindExecutable(

    encoding = Tcl_GetEncoding(NULL, NULL);
    Tcl_ExternalToUtfDStringEx(NULL, encoding, Tcl_DStringValue(&buffer), TCL_INDEX_NONE,
-	    TCL_ENCODING_PROFILE_TCL8, &utfName, NULL);
+	    TCL_ENCODING_PROFILE_LOSSLESS, &utfName, NULL);
    TclSetObjNameOfExecutable(
 	    Tcl_NewStringObj(Tcl_DStringValue(&utfName), TCL_INDEX_NONE), encoding);
    Tcl_DStringFree(&utfName);
@@ -308,7 +308,7 @@ TclpMatchInDirectory(
 	 * Now open the directory for reading and iterate over the contents.
 	 */

-	if (Tcl_UtfToExternalDStringEx(interp, NULL, dirName, TCL_INDEX_NONE, 0, &ds, NULL) != TCL_OK) {
+	if (Tcl_UtfToExternalDStringEx(interp, NULL, dirName, TCL_INDEX_NONE, TCL_ENCODING_PROFILE_LOSSLESS, &ds, NULL) != TCL_OK) {
 	    Tcl_DStringFree(&dsOrig);
 	    Tcl_DStringFree(&ds);
 	    Tcl_DecrRefCount(fileNamePtr);
@@ -379,7 +379,7 @@ TclpMatchInDirectory(
 	     */

 	    if (Tcl_ExternalToUtfDStringEx(interp, NULL, entryPtr->d_name, TCL_INDEX_NONE,
-		    0, &utfDs, NULL) != TCL_OK) {
+		    TCL_ENCODING_PROFILE_LOSSLESS, &utfDs, NULL) != TCL_OK) {
 		matchResult = -1;
 		break;
 	    }
@@ -611,7 +611,7 @@ TclpGetUserHome(
    Tcl_DString ds;
    const char *native;

-    if (Tcl_UtfToExternalDStringEx(NULL, NULL, name, TCL_INDEX_NONE, 0, &ds, NULL) != TCL_OK) {
+    if (Tcl_UtfToExternalDStringEx(NULL, NULL, name, TCL_INDEX_NONE, TCL_ENCODING_PROFILE_LOSSLESS, &ds, NULL) != TCL_OK) {
 	Tcl_DStringFree(&ds);
 	return NULL;
    }
@@ -623,7 +623,7 @@ TclpGetUserHome(
    if (pwPtr == NULL) {
 	return NULL;
    }
-    if (Tcl_ExternalToUtfDStringEx(NULL, NULL, pwPtr->pw_dir, TCL_INDEX_NONE, 0, bufferPtr, NULL) != TCL_OK) {
+    if (Tcl_ExternalToUtfDStringEx(NULL, NULL, pwPtr->pw_dir, TCL_INDEX_NONE, TCL_ENCODING_PROFILE_LOSSLESS, bufferPtr, NULL) != TCL_OK) {
 	return NULL;
    } else {
 	return Tcl_DStringValue(bufferPtr);
@@ -805,7 +805,7 @@ TclpGetCwd(
 	}
 	return NULL;
    }
-    if (Tcl_ExternalToUtfDStringEx(interp, NULL, buffer, TCL_INDEX_NONE, 0, bufferPtr, NULL) != TCL_OK) {
+    if (Tcl_ExternalToUtfDStringEx(interp, NULL, buffer, TCL_INDEX_NONE, TCL_ENCODING_PROFILE_LOSSLESS, bufferPtr, NULL) != TCL_OK) {
 	return NULL;
    }
    return Tcl_DStringValue(bufferPtr);
@@ -843,7 +843,7 @@ TclpReadlink(
    const char *native;
    Tcl_DString ds;

-    if (Tcl_UtfToExternalDStringEx(NULL, NULL, path, TCL_INDEX_NONE, 0, &ds, NULL) != TCL_OK) {
+    if (Tcl_UtfToExternalDStringEx(NULL, NULL, path, TCL_INDEX_NONE, TCL_ENCODING_PROFILE_LOSSLESS, &ds, NULL) != TCL_OK) {
 	Tcl_DStringFree(&ds);
 	return NULL;
    }
@@ -855,7 +855,7 @@ TclpReadlink(
 	return NULL;
    }

-    if (Tcl_ExternalToUtfDStringEx(NULL, NULL, link, length, 0, linkPtr, NULL) == TCL_OK) {
+    if (Tcl_ExternalToUtfDStringEx(NULL, NULL, link, length, TCL_ENCODING_PROFILE_LOSSLESS, linkPtr, NULL) == TCL_OK) {
 	return Tcl_DStringValue(linkPtr);
    }
 #endif /* !DJGPP */
@@ -990,7 +990,7 @@ TclpObjLink(
 		return NULL;
 	    }
 	    target = Tcl_GetStringFromObj(transPtr, &length);
-	    if (Tcl_UtfToExternalDStringEx(NULL, NULL, target, length, 0, &ds, NULL) != TCL_OK) {
+	    if (Tcl_UtfToExternalDStringEx(NULL, NULL, target, length, TCL_ENCODING_PROFILE_LOSSLESS, &ds, NULL) != TCL_OK) {
 		Tcl_DStringFree(&ds);
 		return NULL;
 	    }
@@ -1029,7 +1029,7 @@ TclpObjLink(
 	    return NULL;
 	}

-	if (Tcl_ExternalToUtfDStringEx(NULL, NULL, link, (size_t)length, 0, &ds, NULL) != TCL_OK) {
+	if (Tcl_ExternalToUtfDStringEx(NULL, NULL, link, (size_t)length, TCL_ENCODING_PROFILE_LOSSLESS, &ds, NULL) != TCL_OK) {
 	    return NULL;
 	}
 	linkPtr = Tcl_DStringToObj(&ds);
@@ -1096,7 +1096,7 @@ TclpNativeToNormalized(
 {
    Tcl_DString ds;

-    Tcl_ExternalToUtfDStringEx(NULL, NULL, (const char *) clientData, TCL_INDEX_NONE, TCL_ENCODING_PROFILE_TCL8, &ds, NULL);
+    Tcl_ExternalToUtfDStringEx(NULL, NULL, (const char *) clientData, TCL_INDEX_NONE, TCL_ENCODING_PROFILE_LOSSLESS, &ds, NULL);
    return Tcl_DStringToObj(&ds);
 }

@@ -1150,7 +1150,7 @@ TclNativeCreateNativeRep(
    }

    str = Tcl_GetStringFromObj(validPathPtr, &len);
-    if (Tcl_UtfToExternalDStringEx(NULL, NULL, str, len, 0, &ds, NULL) != TCL_OK) {
+    if (Tcl_UtfToExternalDStringEx(NULL, NULL, str, len, TCL_ENCODING_PROFILE_LOSSLESS, &ds, NULL) != TCL_OK) {
 	Tcl_DecrRefCount(validPathPtr);
 	Tcl_DStringFree(&ds);
 	return NULL;
@@ -473,8 +473,12 @@ TclpInitLibraryPath(
     */

    str = getenv("TCL_LIBRARY");			/* INTL: Native. */
-    Tcl_ExternalToUtfDStringEx(NULL, NULL, str, TCL_INDEX_NONE, TCL_ENCODING_PROFILE_TCL8, &buffer, NULL);
-    str = Tcl_DStringValue(&buffer);
+    if (TclSystemToInternalEncoding(NULL, str, -1, &buffer) == TCL_OK) {
+        str = Tcl_DStringValue(&buffer);
+    } else {
+        /* Note buffer is initialized even on error so can be cleared later */
+        str = NULL;
+    }

    if ((str != NULL) && (str[0] != '\0')) {
 	Tcl_DString ds;
@@ -374,7 +374,7 @@ InitializeHostName(
 	if (gethostname(Tcl_DStringValue(&inDs),
 		Tcl_DStringLength(&inDs)) == 0) {
 	    Tcl_ExternalToUtfDStringEx(NULL, NULL, Tcl_DStringValue(&inDs),
-		    TCL_INDEX_NONE, TCL_ENCODING_PROFILE_TCL8, &ds, NULL);
+                -1, TCL_ENCODING_PROFILE_LOSSLESS, &ds, NULL);
 	}
 	Tcl_DStringFree(&inDs);
    }
Author	SHA1	Message	Date
jan.nijtmans	caf644cbe0	Rebase to tip-657	2023-09-12 14:23:43 +00:00
jan.nijtmans	d49a01c740	Rebase to tip-657	2023-08-30 13:58:17 +00:00
jan.nijtmans	658d7fba77	Merge tip-657	2023-07-19 14:59:27 +00:00
jan.nijtmans	552accf224	Merge tip-657	2023-07-19 09:03:02 +00:00
jan.nijtmans	03d9871320	Remove non-supported testcases	2023-07-16 21:29:41 +00:00
jan.nijtmans	b7c494d009	First attempt to mix TIP's #657 and #671	2023-07-14 14:51:27 +00:00
apnadkarni	633ebe176f	Merge trunk	2023-07-13 05:43:12 +00:00
apnadkarni	e17479ba10	Merge main	2023-07-05 06:13:53 +00:00
apnadkarni	958ab2623a	Distinguish test for cesu-8	2023-07-04 04:46:56 +00:00
apnadkarni	7bde1cd9a6	Disable lossless profile for ascii-incompatible encodings and merge trunk	2023-07-02 12:40:51 +00:00
apnadkarni	59668d70ae	More lossless profile tests	2023-06-29 15:26:34 +00:00
apnadkarni	c812067099	Merge trunk	2023-06-29 15:20:59 +00:00
apnadkarni	9a8004e86b	Merge trunk	2023-06-27 16:19:33 +00:00
apnadkarni	a9f158e41f	Lossless load / unload tests	2023-06-17 04:59:42 +00:00
apnadkarni	eeecd14ced	Fix lossless profile for exec	2023-06-16 16:54:55 +00:00
apnadkarni	3eaf6ded4d	Few more TIP 671 tests	2023-06-15 16:46:21 +00:00
apnadkarni	1cf2b94750	Use lossless profile for command line args. Couple of platform-specific cases.	2023-06-14 15:17:35 +00:00
apnadkarni	286c5ef81e	Merged trunk	2023-06-14 04:15:30 +00:00
apnadkarni	72e7fed2f8	Remove debug puts	2023-06-14 02:04:50 +00:00
apnadkarni	7d4143a4d6	Tests for lossless profiles. Change lossless behavior for UTF-16,32	2023-06-12 05:48:46 +00:00
apnadkarni	3f2a3c8190	Fix greedy consumption bytes in lossless profile for utf-8.	2023-06-11 17:31:29 +00:00
apnadkarni	4846aeeae8	Use lossless profile for links and globalprocessdata (exe name etc.)	2023-06-11 11:57:53 +00:00
apnadkarni	f7071952d7	Tests to read and write invalid encoded env values	2023-06-11 03:12:55 +00:00
apnadkarni	326a1b7f65	Merge trunk	2023-06-11 02:46:24 +00:00
apnadkarni	b98f4cbbeb	Lossless env - wip	2023-06-10 17:49:09 +00:00
apnadkarni	24107e4416	Refactor into common function. Add error checks that were never present.	2023-06-10 13:43:25 +00:00
apnadkarni	dfaaefd887	Work in progress	2023-06-06 22:09:02 +00:00
apnadkarni	48fe26cc2d	Implement basic lossless profile	2023-06-05 21:50:47 +00:00
apnadkarni	049c693e11	Create new branch named "tip-671"	2023-06-05 20:55:50 +00:00