Keep temporary files binary on Windows

Set binary mode on the temporary file descriptor before wrapping it
in a stream. This prevents newline translation from enlarging DSP
sample files, which otherwise makes DSP edits and raster operation
tests fail on Windows.

Add a byte-for-byte temporary-file regression test covering newline
and DOS EOF bytes.

src/libbu/temp.c

@@ -210,7 +210,7 @@ mkstemp(char *file_template)
 	for (i=(int)start; i>=(int)end; i--) {
 	    file_template[i] = replace[(int)(replacelen * ((double)rand() / (double)RAND_MAX))];
 	}
-	fd = open(file_template, O_CREAT | O_EXCL | O_TRUNC | O_RDWR, S_IRUSR | S_IWUSR);
+	fd = open(file_template, O_CREAT | O_EXCL | O_TRUNC | O_RDWR | O_BINARY, S_IRUSR | S_IWUSR);
     } while ((fd == -1) && (counter++ < 1000));
 
     return fd;
@@ -285,6 +285,15 @@ bu_temp_file(char *filepath, size_t len)
 	return NULL;
     }
 
+    /* Ensure the descriptor is binary before associating it with a stream. */
+    if (UNLIKELY(setmode(fd, O_BINARY) == -1)) {
+	perror("setmode");
+	close(fd);
+	bu_file_delete(tempfile);
+	bu_log(_TF_FAIL);
+	return NULL;
+    }
+
     if (filepath) {
 	if (UNLIKELY(len < strlen(tempfile))) {
 	    bu_log("INTERNAL ERROR: bu_temp_file filepath buffer size is insufficient (%zu < %zu)\n", len, strlen(tempfile));

src/libbu/tests/CMakeLists.txt

@@ -200,6 +200,7 @@ brlcad_add_test(NAME bu_file_mime_png   COMMAND bu_test test_file_mime png 3 BU_
 #      bu_temp_filename testing    #
 ####################################
 brlcad_add_test(NAME bu_temp_filename  COMMAND bu_test test_temp_filename "single")
+brlcad_add_test(NAME bu_temp_binary    COMMAND bu_test test_temp_filename "binary")
 #BRLCAD_ADD_TEST(NAME bu_temp_filename  COMMAND bu_test test_temp_filename "parallel")
 
 #

src/libbu/tests/test_temp_filename.c

@@ -35,6 +35,57 @@
 
 #include "common.h"
 
+#include <string.h>
+
+#include "bio.h"
+#include "bu/app.h"
+#include "bu/file.h"
+
+
+static int
+temp_file_binary(void)
+{
+    /* LF and DOS EOF expose text-mode translation on Windows. */
+    const unsigned char expected[] = {0x00, 0x0a, 0x1a, 0x0d, 0xff};
+    unsigned char actual[sizeof(expected) + 1] = {0};
+    char path[MAXPATHLEN] = {0};
+    size_t count;
+    FILE *fp = bu_temp_file(path, sizeof(path));
+
+    if (!fp) {
+	bu_log("temp_filename failure: cannot create temporary file\n");
+	return 1;
+    }
+
+    count = fwrite(expected, 1, sizeof(expected), fp);
+    if (fclose(fp) != 0 || count != sizeof(expected)) {
+	bu_log("temp_filename failure: cannot write binary data\n");
+	bu_file_delete(path);
+	return 1;
+    }
+
+    fp = fopen(path, "rb");
+    if (!fp) {
+	bu_log("temp_filename failure: cannot reopen temporary file\n");
+	bu_file_delete(path);
+	return 1;
+    }
+
+    count = fread(actual, 1, sizeof(actual), fp);
+    if (fclose(fp) != 0 || count != sizeof(expected) || memcmp(actual, expected, sizeof(expected))) {
+	bu_log("temp_filename failure: binary data changed (%zu bytes read, expected %zu)\n", count, sizeof(expected));
+	bu_file_delete(path);
+	return 1;
+    }
+
+    if (!bu_file_delete(path)) {
+	bu_log("temp_filename failure: cannot delete temporary file\n");
+	return 1;
+    }
+
+    return 0;
+}
+
 
 static void
 temp_filename_thread(int cpu, void* ptr)
@@ -96,6 +147,9 @@ main(int argc, char *argv[])
 	    if (strlen(under_size) != 4)    /* buffer size -1 */
 		bu_exit(EXIT_FAILURE, "temp_filename failure: expected size [%d], got [%ld]]\n", 4, strlen(under_size));
 
+	} else if (!bu_strcmp(curr, "binary")) {
+	    if (temp_file_binary())
+		return EXIT_FAILURE;
 	} else if (!bu_strcmp(curr, "parallel")) {
 	    // test parallel / threaded -
 	    uint8_t threads = bu_avail_cpus();