fix(sgi): prevent oversized-allocation crash from bogus RLE length/offset table (#5303)

Truncating a corrupt uint32 length_tab entry into `int` could yield a
huge negative value, causing a runaway allocation and crash in
uncompress_rle_channel. Validate offset/length against the file size
instead.

Assisted-by: Claude Code / Sonnet 5

Signed-off-by: Larry Gritz <lg@larrygritz.com>

src/sgi.imageio/sgiinput.cpp

@@ -50,7 +50,7 @@ private:
 
     // read channel scanline data from file, uncompress it and save the data
     // to 'outbuf' buffer. Return true if ok, false if there was a read error.
-    bool uncompress_rle_channel(int scanline_off, int scanline_len,
+    bool uncompress_rle_channel(int64_t scanline_off, int64_t scanline_len,
                                 span<unsigned char> outbf);
 };
 
@@ -199,9 +199,9 @@ SgiInput::read_native_scanline(int subimage, int miplevel, int y, int /*z*/,
         // reading and uncompressing first channel (red in RGBA images)
         for (int c = 0; c < m_spec.nchannels; ++c) {
             // offset for this scanline/channel
-            ptrdiff_t off             = y + c * m_spec.height;
-            ptrdiff_t scanline_offset = start_tab[off];
-            ptrdiff_t scanline_length = length_tab[off];
+            ptrdiff_t off           = y + c * m_spec.height;
+            int64_t scanline_offset = start_tab[off];
+            int64_t scanline_length = length_tab[off];
             channeldata[c].resize(m_spec.width * bpc);
             if (!uncompress_rle_channel(scanline_offset, scanline_length,
                                         make_span(channeldata[c])))
@@ -245,10 +245,16 @@ SgiInput::read_native_scanline(int subimage, int miplevel, int y, int /*z*/,
 
 
 bool
-SgiInput::uncompress_rle_channel(int scanline_off, int scanline_len,
+SgiInput::uncompress_rle_channel(int64_t scanline_off, int64_t scanline_len,
                                  span<unsigned char> outbuf)
 {
-    int bpc = m_sgi_header.bpc;
+    int bpc          = m_sgi_header.bpc;
+    int64_t filesize = ioproxy() ? ioproxy()->size() : 0;
+    if (scanline_off < 0 || scanline_len < 0 || scanline_off > filesize
+        || scanline_len > filesize - scanline_off) {
+        errorfmt("Corrupt RLE offset/length table");
+        return false;
+    }
     std::unique_ptr<unsigned char[]> rle_scanline_alloc(
         new unsigned char[scanline_len]);
     auto rle_scanline = make_span(rle_scanline_alloc.get(), scanline_len);

testsuite/sgi/ref/out.txt

@@ -34,3 +34,13 @@ src/broken01.sgi :    8 x    2, 1 channel, uint8 sgi
     compression: "rle"
 iinfo ERROR: "src/broken01.sgi" : Corrupt RLE data
 iinfo ERROR: "src/broken02.sgi" : Read error: hit end of file in sgi reader
+src/broken03.sgi :   16 x   48, 1 channel, uint8 sgi
+    SHA-1: Corrupt RLE offset/length table
+    channel list: Y
+    compression: "rle"
+iinfo ERROR: "src/broken03.sgi" : Corrupt RLE offset/length table
+src/broken04.sgi :   64 x   49, 3 channel, uint16 sgi
+    SHA-1: Corrupt RLE offset/length table
+    channel list: R, G, B
+    compression: "rle"
+iinfo ERROR: "src/broken04.sgi" : Corrupt RLE offset/length table

testsuite/sgi/run.py

@@ -18,3 +18,10 @@ command += info_command ("--stats src/broken01.sgi",
 # inconsistently with the scanline indexing, causing a null-pointer crash.
 command += info_command ("--stats src/broken02.sgi",
                          info_program="iinfo", failureok=True)
+# broken03.sgi and broken04.sgi have RLE scanline length/offset table entries
+# that are bogus huge values. Truncating them to `int` produced negative
+# lengths that turned into huge allocation requests, aborting the process.
+command += info_command ("--stats src/broken03.sgi",
+                         info_program="iinfo", failureok=True)
+command += info_command ("--stats src/broken04.sgi",
+                         info_program="iinfo", failureok=True)