Added source test with utf-8 (without -encoding param), remove too many same single byte tests

generic/tclIO.c

@@ -7203,6 +7203,16 @@ Tcl_GetChannelOption(
 	    return TCL_OK;
 	}
     }
+    if (len == 0 || HaveOpt(2, "-checkbom")) {
+	if (len == 0) {
+	    Tcl_DStringAppendElement(dsPtr, "-checkbom");
+	}
+	Tcl_DStringAppendElement(dsPtr,
+		(flags & CHANNEL_CHECKBOM) ? "1" : "0");
+	if (len > 0) {
+	    return TCL_OK;
+	}
+    }
     if (len == 0 || HaveOpt(2, "-encoding")) {
 	if (len == 0) {
 	    Tcl_DStringAppendElement(dsPtr, "-encoding");
@@ -7429,7 +7439,19 @@ Tcl_SetChannelOption(
 	    return TCL_ERROR;
 	}
 	Tcl_SetChannelBufferSize(chan, newBufferSize);
-    } else if (HaveOpt(2, "-encoding")) {
+    } else if (HaveOpt(2, "-checkbom")) {
+	int newCheckBomFlag;
+
+	if (Tcl_GetBoolean(interp, newValue, &newCheckBomFlag) == TCL_ERROR) {
+	    return TCL_ERROR;
+	}
+	if (newCheckBomFlag) {
+	    SetFlag(statePtr, CHANNEL_CHECKBOM);
+	} else {
+	    ResetFlag(statePtr, CHANNEL_CHECKBOM);
+	}
+
+   } else if (HaveOpt(2, "-encoding")) {
 	Tcl_Encoding encoding;
 
 	if ((newValue[0] == '\0') || (strcmp(newValue, "binary") == 0)) {
@@ -10859,7 +10881,7 @@ DumpFlags(
     char *str,
     int flags)
 {
-    char buf[20];
+    char buf[24];
     int i = 0;
 
 #define ChanFlag(chr,bit) (buf[i++] = ((flags & (bit)) ? (chr) : '_'))
@@ -10884,6 +10906,7 @@ DumpFlags(
     ChanFlag('H', CHANNEL_HAS_MORE_DATA);
 #endif /* TCL_IO_TRACK_OS_FOR_DRIVER_WITH_BAD_BLOCKING */
     ChanFlag('x', CHANNEL_INCLOSE);
+    ChanFlag('b', CHANNEL_CHECKBOM);
 
     buf[i] ='\0';
 

generic/tclIO.h

@@ -337,6 +337,8 @@ typedef struct ChannelState {
 					 * Used by Channel Tcl_Obj type to
 					 * determine if we have to revalidate
 					 * the channel. */
+#define CHANNEL_CHECKBOM		(1<<21)	/* Check for BOM before continuing.
+					 * If BOM is found, switch channel encoding to utf-8 */
 
 /*
  * For each channel handler registered in a call to Tcl_CreateChannelHandler,

generic/tclIOUtil.c

@@ -1784,6 +1784,12 @@ Tcl_FSEvalFileEx(
 	    Tcl_Close(interp,chan);
 	    return result;
 	}
+    } else {
+    	if (Tcl_SetChannelOption(interp, chan, "-checkbom", "1")
+    		!= TCL_OK) {
+    	    Tcl_Close(interp,chan);
+    	    return result;
+    	}
     }
 
     objPtr = Tcl_NewObj();

tests/ioCmd.test

@@ -217,7 +217,7 @@ test iocmd-8.7 {fconfigure command} {
     set x [fconfigure $f1]
     close $f1
     set x
-} {-blocking 1 -buffering full -buffersize 4096 -encoding unicode -eofchar {} -translation lf}
+} {-blocking 1 -buffering full -buffersize 4096 -checkbom 0 -encoding unicode -eofchar {} -translation lf}
 test iocmd-8.8 {fconfigure command} {
     file delete $path(test1)
     set f1 [open $path(test1) w]
@@ -228,7 +228,7 @@ test iocmd-8.8 {fconfigure command} {
     lappend x [fconfigure $f1]
     close $f1
     set x
-} {line {-blocking 1 -buffering line -buffersize 3030 -encoding unicode -eofchar {} -translation lf}}
+} {line {-blocking 1 -buffering line -buffersize 3030 -checkbom 0 -encoding unicode -eofchar {} -translation lf}}
 test iocmd-8.9 {fconfigure command} {
     file delete $path(test1)
     set f1 [open $path(test1) w]
@@ -237,7 +237,7 @@ test iocmd-8.9 {fconfigure command} {
     set x [fconfigure $f1]
     close $f1
     set x
-} {-blocking 1 -buffering none -buffersize 4040 -encoding binary -eofchar {} -translation lf}
+} {-blocking 1 -buffering none -buffersize 4040 -checkbom 0 -encoding binary -eofchar {} -translation lf}
 test iocmd-8.10 {fconfigure command} {
     list [catch {fconfigure a b} msg] $msg
 } {1 {can not find channel named "a"}}
@@ -1199,7 +1199,7 @@ test iocmd-25.1 {chan configure, cgetall, standard options} -match glob -body {
     close $c
     rename foo {}
     set res
-} -result {{-blocking 1 -buffering full -buffersize 4096 -encoding * -eofchar {{} {}} -translation {auto *}}}
+} -result {{-blocking 1 -buffering full -buffersize 4096 -checkbom 0 -encoding * -eofchar {{} {}} -translation {auto *}}}
 test iocmd-25.2 {chan configure, cgetall, no options} -match glob -body {
     set res {}
     proc foo {args} {oninit cget cgetall; onfinal; track; return ""}
@@ -1208,7 +1208,7 @@ test iocmd-25.2 {chan configure, cgetall, no options} -match glob -body {
     close $c
     rename foo {}
     set res
-} -result {{cgetall rc*} {-blocking 1 -buffering full -buffersize 4096 -encoding * -eofchar {{} {}} -translation {auto *}}}
+} -result {{cgetall rc*} {-blocking 1 -buffering full -buffersize 4096 -checkbom 0 -encoding * -eofchar {{} {}} -translation {auto *}}}
 test iocmd-25.3 {chan configure, cgetall, regular result} -match glob -body {
     set res {}
     proc foo {args} {
@@ -1220,7 +1220,7 @@ test iocmd-25.3 {chan configure, cgetall, regular result} -match glob -body {
     close $c
     rename foo {}
     set res
-} -result {{cgetall rc*} {-blocking 1 -buffering full -buffersize 4096 -encoding * -eofchar {{} {}} -translation {auto *} -bar foo -snarf x}}
+} -result {{cgetall rc*} {-blocking 1 -buffering full -buffersize 4096 -checkbom 0 -encoding * -eofchar {{} {}} -translation {auto *} -bar foo -snarf x}}
 test iocmd-25.4 {chan configure, cgetall, bad result, list of uneven length} -match glob -body {
     set res {}
     proc foo {args} {

tests/socket.test

@@ -844,7 +844,7 @@ test socket-7.3 {testing socket specific options} {socket} {
     close $s
     update
     llength $l
-} 14
+} 16
 test socket-7.4 {testing socket specific options} {socket} {
     set s [socket -server accept -myaddr 127.0.0.1 0]
     proc accept {s a p} {

tests/source.test

@@ -120,6 +120,24 @@ test source-2.7 {utf-8 with BOM} -setup {
 } -cleanup {
     removeFile $sourcefile
 } -result {new-y}
+foreach {n e} {8 utf-8 9 identity 10 cp1252} {
+test source-2.$n [list $e with BOM] -setup {
+    set sourcefile [makeFile {} source.file]
+    set saveencoding [encoding system]
+} -body {
+    encoding system $e
+    set out [open $sourcefile w]
+    fconfigure $out -encoding utf-8
+    puts $out "\ufeffset y new-y\xe9"
+    close $out
+    set y old-y
+    source $sourcefile
+    return $y
+} -cleanup {
+    encoding system $saveencoding
+    removeFile $sourcefile
+} -result "new-y\xe9"
+}
 
 test source-3.1 {return in middle of source file} -setup {
     set sourcefile [makeFile {