Merge remote-tracking branch 'upstream/master' into feature/io

Add PointIndexCodec, PointDataCodec and other Point changes

.github/workflows/weekly.yml

@@ -146,13 +146,14 @@ jobs:
       matrix:
         config:
           - { name: 'all',    image: '2026', build: 'Release', components: 'core,python,bin,view,render,test', cmake: '-DUSE_BLOSC=ON  -DUSE_ZLIB=ON  -DUSE_EXR=ON  -DUSE_PNG=ON'  }
-          - { name: 'lite',   image: '2026', build: 'Release', components: 'core,python,bin,view,render,test', cmake: '-DUSE_BLOSC=OFF -DUSE_ZLIB=OFF -DUSE_EXR=OFF -DUSE_PNG=OFF -DOPENVDB_USE_DELAYED_LOADING=OFF' }
+          - { name: 'lite',   image: '2026', build: 'Release', components: 'core,python,bin,view,render,test', cmake: '-DUSE_BLOSC=OFF -DUSE_ZLIB=OFF -DUSE_EXR=OFF -DUSE_PNG=OFF' }
           - { name: 'half',   image: '2026', build: 'Release', components: 'core,python,bin,view,render,test', cmake: '-DUSE_BLOSC=OFF -DUSE_IMATH_HALF=ON' }
           - { name: 'simd',   image: '2026', build: 'Release', components: 'core,python,bin,view,render,test', cmake: '-DOPENVDB_X86_INSTRSET=7 -DUSE_VCL=ON' } # 7=AVX
           - { name: 'pygrid', image: '2026', build: 'Release', components: 'core,python,bin,view,render,test', cmake: '-DOPENVDB_PYTHON_WRAP_ALL_GRID_TYPES=ON' }
           - { name: 'asan',   image: '2026', build: 'asan',    components: 'core,test',                        cmake: '-DDISABLE_DEPENDENCY_VERSION_CHECKS=ON -DNANOVDB_USE_OPENVDB=ON -DOPENVDB_AX_STATIC=OFF -DOPENVDB_CORE_STATIC=OFF -DUSE_BLOSC=OFF' } # We never called blosc_destroy(), so disable blosc to silence these errors
           - { name: 'ubsan',  image: '2026', build: 'ubsan',   components: 'core,test',                        cmake: '-DDISABLE_DEPENDENCY_VERSION_CHECKS=ON -DCMAKE_CXX_FLAGS="-Wno-deprecated-declarations" ' }
           - { name: 'c++20',  image: '2026', build: 'Release', components: 'core,test',                        cmake: '-DCMAKE_CXX_STANDARD=20' }
+          - { name: 'abi14',  image: '2026',           build: 'Release', components: 'core,test',              cmake: '-DOPENVDB_USE_FUTURE_ABI_14=ON -DOPENVDB_ENABLE_ASSERTS=ON' }
           - { name: 'conf',   image: '2026', build: 'Release', components: 'core,python,bin,view,render,test', cmake: '-DCMAKE_FIND_PACKAGE_PREFER_CONFIG=ON' }
       fail-fast: false
     steps:
@@ -215,7 +216,6 @@ jobs:
           --components=\"core,axcore,python,bin,render,test,axbin\"
           --cargs=\'
           -DCMAKE_CXX_STANDARD=20
-          -DOPENVDB_USE_DELAYED_LOADING=OFF
           -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/install
           ${{ matrix.config.cmake }}
           \'
@@ -420,7 +420,6 @@ jobs:
         --components="${{ matrix.config.components }}"
         --cargs=\'
         -A x64 -G \"Visual Studio 17 2022\"
-        -DOPENVDB_USE_DELAYED_LOADING=OFF
         -DUSE_BLOSC=OFF \
         -DUSE_ZLIB=OFF \
         -DVCPKG_TARGET_TRIPLET=${VCPKG_DEFAULT_TRIPLET}

CMakeLists.txt

@@ -84,7 +84,6 @@ option(OPENVDB_ENABLE_VEC_RELATIONAL_OPERATIONS [=[
 Enabled deprecated relational < and > operators on Vector types. This option
 will be switched to OFF and the feature will be subsequently removed in future
 releases.]=] ON)
-option(OPENVDB_USE_DELAYED_LOADING "Build the core OpenVDB library with delayed-loading." ON)
 option(OPENVDB_CXX_STRICT "Enable or disable pre-defined compiler warnings" OFF)
 cmake_dependent_option(OPENVDB_INSTALL_CMAKE_MODULES
   "Install the provided OpenVDB CMake modules when building the core library"
@@ -264,7 +263,6 @@ mark_as_advanced(
   OPENVDB_BUILD_HOUDINI_ABITESTS
   OPENVDB_CXX_STRICT
   OPENVDB_ENABLE_RPATH
-  OPENVDB_USE_DELAYED_LOADING
   OPENVDB_FUTURE_DEPRECATION
   OPENVDB_SIMD
   SYSTEM_LIBRARY_PATHS

cmake/FindOpenVDB.cmake

@@ -64,8 +64,6 @@ This will define the following variables:
   True if the OpenVDB Library has been built with log4cplus support
 ``OpenVDB_USES_IMATH_HALF``
   True if the OpenVDB Library has been built with Imath half support
-``OpenVDB_USES_DELAYED_LOADING``
-  True if the OpenVDB Library has been built with delayed-loading
 ``OpenVDB_ABI``
   Set if this module was able to determine the ABI number the located
   OpenVDB Library was built against. Unset otherwise.
@@ -522,7 +520,6 @@ set(OpenVDB_USES_BLOSC ${USE_BLOSC})
 set(OpenVDB_USES_ZLIB ${USE_ZLIB})
 set(OpenVDB_USES_LOG4CPLUS ${USE_LOG4CPLUS})
 set(OpenVDB_USES_IMATH_HALF ${USE_IMATH_HALF})
-set(OpenVDB_USES_DELAYED_LOADING ${OPENVDB_USE_DELAYED_LOADING})
 set(OpenVDB_DEFINITIONS)
 
 if(WIN32)
@@ -555,7 +552,6 @@ if(_OPENVDB_HAS_NEW_VERSION_HEADER)
   OPENVDB_GET_VERSION_DEFINE(${_OPENVDB_VERSION_HEADER} "OPENVDB_USE_IMATH_HALF" OpenVDB_USES_IMATH_HALF)
   OPENVDB_GET_VERSION_DEFINE(${_OPENVDB_VERSION_HEADER} "OPENVDB_USE_BLOSC" OpenVDB_USES_BLOSC)
   OPENVDB_GET_VERSION_DEFINE(${_OPENVDB_VERSION_HEADER} "OPENVDB_USE_ZLIB" OpenVDB_USES_ZLIB)
-  OPENVDB_GET_VERSION_DEFINE(${_OPENVDB_VERSION_HEADER} "OPENVDB_USE_DELAYED_LOADING" OpenVDB_USES_DELAYED_LOADING)
 elseif(NOT OPENVDB_USE_STATIC_LIBS)
   # Use GetPrerequisites to see which libraries this OpenVDB lib has linked to
   # which we can query for optional deps. This basically runs ldd/otoll/objdump
@@ -602,11 +598,6 @@ elseif(NOT OPENVDB_USE_STATIC_LIBS)
     if(NOT ${_HAS_DEP} EQUAL -1)
       set(OpenVDB_USES_IMATH_HALF ON)
     endif()
-
-    string(FIND ${PREREQUISITE} "boost_iostreams" _HAS_DEP)
-    if(NOT ${_HAS_DEP} EQUAL -1)
-      set(OpenVDB_USES_DELAYED_LOADING ON)
-    endif()
   endforeach()
 
   unset(_OPENVDB_PREREQUISITE_LIST)
@@ -628,10 +619,6 @@ if(OpenVDB_USES_IMATH_HALF)
   find_package(Imath REQUIRED CONFIG)
 endif()
 
-if(OpenVDB_USES_DELAYED_LOADING)
-  find_package(Boost REQUIRED COMPONENTS iostreams)
-endif()
-
 if(UNIX)
   find_package(Threads REQUIRED)
 endif()
@@ -643,11 +630,6 @@ endif()
 
 set(_OPENVDB_VISIBLE_DEPENDENCIES "")
 
-if(OpenVDB_USES_DELAYED_LOADING)
-  list(APPEND _OPENVDB_VISIBLE_DEPENDENCIES Boost::iostreams)
-  list(APPEND OpenVDB_DEFINITIONS OPENVDB_USE_DELAYED_LOADING)
-endif()
-
 if(OpenVDB_USES_IMATH_HALF)
   list(APPEND _OPENVDB_VISIBLE_DEPENDENCIES Imath::Imath)
 endif()

nanovdb/nanovdb/cmd/convert/nanovdb_convert.cc

@@ -257,7 +257,7 @@ int main(int argc, char* argv[])
                 }
                 if (verbose) std::cout << "Opening OpenVDB file named \"" << inputFile << "\"" << std::endl;
                 openvdb::io::File file(inputFile);
-                file.open(false); //disable delayed loading
+                file.open();
                 if (gridName.empty()) {// convert all grid in the file
                     auto grids = file.getGrids();
                     std::vector<nanovdb::GridHandle<nanovdb::HostBuffer> > handles;

nanovdb/nanovdb/examples/ex_coarsen_nanovdb_cuda/coarsen_nanovdb_cuda.cpp

@@ -50,7 +50,7 @@ int main(int argc, char *argv[])
         cpuTimer.start("Read input VDB file");
         openvdb::initialize();
         openvdb::io::File inFile(argv[1]);
-        inFile.open(false); // disable delayed loading
+        inFile.open();
         auto baseGrids = inFile.getGrids();
         inFile.close();
         auto grid = openvdb::gridPtrCast<GridT>(baseGrids->at(0));

nanovdb/nanovdb/examples/ex_dilate_nanovdb_cuda/dilate_nanovdb_cuda.cpp

@@ -44,7 +44,7 @@ int main(int argc, char *argv[])
         cpuTimer.start("Read input VDB file");
         openvdb::initialize();
         openvdb::io::File inFile(argv[1]);
-        inFile.open(false); // disable delayed loading
+        inFile.open();
         auto baseGrids = inFile.getGrids();
         inFile.close();
         auto grid = openvdb::gridPtrCast<GridT>(baseGrids->at(0));

nanovdb/nanovdb/examples/ex_merge_nanovdb_cuda/merge_nanovdb_cuda.cpp

@@ -41,7 +41,7 @@ int main(int argc, char *argv[])
         cpuTimer.start("Read first input VDB file");
         openvdb::initialize();
         openvdb::io::File inFile1(argv[1]);
-        inFile1.open(false); // disable delayed loading
+        inFile1.open();
         auto baseGrids1 = inFile1.getGrids();
         inFile1.close();
         auto grid1 = openvdb::gridPtrCast<GridT>(baseGrids1->at(0));
@@ -51,7 +51,7 @@ int main(int argc, char *argv[])
         cpuTimer.start("Read second input VDB file");
         openvdb::initialize();
         openvdb::io::File inFile2(argv[2]);
-        inFile2.open(false); // disable delayed loading
+        inFile2.open();
         auto baseGrids2 = inFile2.getGrids();
         inFile2.close();
         auto grid2 = openvdb::gridPtrCast<GridT>(baseGrids2->at(0));

nanovdb/nanovdb/examples/ex_raytrace_level_set/openvdb.cc

@@ -37,7 +37,7 @@ void runOpenVDB(nanovdb::GridHandle<nanovdb::cuda::DeviceBuffer>& handle, int nu
     openvdb::initialize();
     std::string       filename = "C:/Users/william/Downloads/dragon.vdb";
     openvdb::io::File file(filename);
-    file.open(false); //disable delayed loading
+    file.open();
     auto srcGrid = file.readGrid<BufferT>("ls_dragon");
     std::cout << "Loading OpenVDB grid[" << srcGrid->getName() << "]...\n";
 #endif

nanovdb/nanovdb/examples/ex_refine_nanovdb_cuda/refine_nanovdb_cuda.cpp

@@ -81,7 +81,7 @@ int main(int argc, char *argv[])
         cpuTimer.start("Read input VDB file");
         openvdb::initialize();
         openvdb::io::File inFile(argv[1]);
-        inFile.open(false); // disable delayed loading
+        inFile.open();
         auto baseGrids = inFile.getGrids();
         inFile.close();
         auto grid = openvdb::gridPtrCast<GridT>(baseGrids->at(0));

nanovdb/nanovdb/unittest/TestOpenVDB.cc

@@ -140,7 +140,7 @@ protected:
                 mTimer.start("Reading grid from the file \"" + fileName + "\"");
             try {
                 openvdb::io::File file(fileName);
-                file.open(false); //disable delayed loading
+                file.open();
                 grid = openvdb::gridPtrCast<openvdb::FloatGrid>(file.readGrid(file.beginName().gridName()));
             } catch(const std::exception& e) {
                 std::cerr << "An exception occurred: \"" << e.what() << "\"" << std::endl;
@@ -2860,7 +2860,7 @@ TEST_F(TestOpenVDB, LevelSetFiles)
         //mTimer.start("\nReading grid from the file \"" + fileName + "\"");
         try {
             openvdb::io::File file(fileName);
-            file.open(false); //disable delayed loading
+            file.open();
             auto srcGrid = openvdb::gridPtrCast<openvdb::FloatGrid>(file.readGrid(file.beginName().gridName()));
 
             const size_t pos = fileName.find_last_of("/\\") + 1;
@@ -2938,7 +2938,7 @@ TEST_F(TestOpenVDB, FogFiles)
         //mTimer.start("Reading grid from the file \"" + fileName + "\"");
         try {
             openvdb::io::File file(fileName);
-            file.open(false); //disable delayed loading
+            file.open();
             auto srcGrid = openvdb::gridPtrCast<openvdb::FloatGrid>(file.readGrid(file.beginName().gridName()));
 
             const size_t pos = fileName.find_last_of("/\\") + 1;
@@ -3008,7 +3008,7 @@ TEST_F(TestOpenVDB, PointFiles)
         //mTimer.start("Reading grid from the file \"" + fileName + "\"");
         try {
             openvdb::io::File file(fileName);
-            file.open(false); //disable delayed loading
+            file.open();
 
             auto srcGrid = openvdb::gridPtrCast<openvdb::points::PointDataGrid>(file.readGrid(file.beginName().gridName()));
             //std::cerr << "Read PointDataGrid named \"" << srcGrid->getName() << "\"" << std::endl;

openvdb/openvdb/CMakeLists.txt

@@ -117,19 +117,6 @@ endif()
 
 # Collect and configure lib dependencies
 
-if(OPENVDB_USE_DELAYED_LOADING)
-  find_package(Boost ${MINIMUM_BOOST_VERSION} REQUIRED COMPONENTS iostreams)
-
-  if(OPENVDB_FUTURE_DEPRECATION AND FUTURE_MINIMUM_BOOST_VERSION)
-    # The X.Y.Z boost version value isn't available until CMake 3.14
-    set(FULL_BOOST_VERSION "${Boost_MAJOR_VERSION}.${Boost_MINOR_VERSION}.${Boost_SUBMINOR_VERSION}")
-    if(${FULL_BOOST_VERSION} VERSION_LESS FUTURE_MINIMUM_BOOST_VERSION)
-      message(DEPRECATION "Support for Boost versions < ${FUTURE_MINIMUM_BOOST_VERSION} "
-        "is deprecated and will be removed.")
-    endif()
-  endif()
-endif()
-
 find_package(TBB ${MINIMUM_TBB_VERSION} REQUIRED COMPONENTS tbb)
 if(OPENVDB_FUTURE_DEPRECATION AND FUTURE_MINIMUM_TBB_VERSION)
   if(${TBB_VERSION} VERSION_LESS FUTURE_MINIMUM_TBB_VERSION)
@@ -252,27 +239,6 @@ if(UNIX)
   list(APPEND OPENVDB_CORE_DEPENDENT_LIBS Threads::Threads)
 endif()
 
-# Pull in Boost last as houdini's boost (hboost) is fully namespaced and libs
-# are renamed too. Boost can be pulled in at any time, but do it last so that,
-# if it's in a shared place (like /usr/local) it doesn't accidently pull in
-# other headers.
-
-if(OPENVDB_USE_DELAYED_LOADING)
-  list(APPEND OPENVDB_CORE_DEPENDENT_LIBS Boost::iostreams)
-  if(WIN32)
-    # Boost headers contain #pragma commands on Windows which causes Boost
-    # libraries to be linked in automatically. Custom boost installations
-    # may find that these naming conventions don't always match and can
-    # cause linker errors. This option disables this feature of Boost. Note
-    # -DBOOST_ALL_NO_LIB can also be provided manually.
-    if(OPENVDB_DISABLE_BOOST_IMPLICIT_LINKING)
-        list(APPEND OPENVDB_CORE_DEPENDENT_LIBS
-          Boost::disable_autolinking  # add -DBOOST_ALL_NO_LIB
-        )
-    endif()
-  endif()
-endif()
-
 ##########################################################################
 
 ##### Core library configuration
@@ -374,13 +340,12 @@ configure_file(version.h.in openvdb/version.h)
 set(OPENVDB_LIBRARY_SOURCE_FILES
   Grid.cc
   io/Archive.cc
+  io/Codec.cc
   io/Compression.cc
-  io/DelayedLoadMetadata.cc
   io/File.cc
   io/GridDescriptor.cc
   io/Queue.cc
   io/Stream.cc
-  io/TempFile.cc
   math/Maps.cc
   math/Proximity.cc
   math/QuantizedUnitVec.cc
@@ -411,16 +376,28 @@ set(OPENVDB_LIBRARY_INCLUDE_FILES
   TypeList.h
 )
 
+set(OPENVDB_LIBRARY_CODECS_INCLUDE_FILES
+  codecs/BoolCodec.h
+  codecs/PointDataCodec.h
+  codecs/PointIndexCodec.h
+  codecs/ScalarCodec.h
+  codecs/TopologyCodec.h
+  codecs/ValueMaskCodec.h
+)
+
+set(OPENVDB_LIBRARY_CODECS_IMPL_INCLUDE_FILES
+  codecs/impl/ScalarLeafCodec.h
+)
+
 set(OPENVDB_LIBRARY_IO_INCLUDE_FILES
   io/Archive.h
+  io/Codec.h
   io/Compression.h
-  io/DelayedLoadMetadata.h
   io/File.h
   io/GridDescriptor.h
   io/io.h
   io/Queue.h
   io/Stream.h
-  io/TempFile.h
 )
 
 set(OPENVDB_LIBRARY_MATH_INCLUDE_FILES
@@ -461,6 +438,7 @@ set(OPENVDB_LIBRARY_POINTS_INCLUDE_FILES
   points/PointConversion.h
   points/PointCount.h
   points/PointDataGrid.h
+  points/PointDataIO.h
   points/PointDelete.h
   points/PointGroup.h
   points/PointMask.h
@@ -612,6 +590,8 @@ if(USE_EXPLICIT_INSTANTIATION)
   # inexpensive to generate in this case.
   set(OPENVDB_LIBRARY_INSTANTIATION_HEADERS
     ${OPENVDB_LIBRARY_TOOLS_INCLUDE_FILES}
+    ${OPENVDB_LIBRARY_CODECS_INCLUDE_FILES}
+    ${OPENVDB_LIBRARY_CODECS_IMPL_INCLUDE_FILES}
   )
 
   # A new source file is generated by CMake for every header
@@ -686,9 +666,6 @@ endif()
 if(USE_LOG4CPLUS)
   list(APPEND OPENVDB_CORE_PUBLIC_DEFINES -DOPENVDB_USE_LOG4CPLUS)
 endif()
-if(OPENVDB_USE_DELAYED_LOADING)
-  list(APPEND OPENVDB_CORE_PUBLIC_DEFINES -DOPENVDB_USE_DELAYED_LOADING)
-endif()
 
 ##########################################################################
 
@@ -799,6 +776,8 @@ endif()
 install(FILES ${OPENVDB_LIBRARY_INCLUDE_FILES} DESTINATION ${OPENVDB_INSTALL_INCLUDEDIR}/openvdb)
 install(FILES ${CMAKE_CURRENT_BINARY_DIR}/openvdb/version.h DESTINATION ${OPENVDB_INSTALL_INCLUDEDIR}/openvdb)
 install(FILES ${OPENVDB_LIBRARY_IO_INCLUDE_FILES} DESTINATION ${OPENVDB_INSTALL_INCLUDEDIR}/openvdb/io)
+install(FILES ${OPENVDB_LIBRARY_CODECS_INCLUDE_FILES} DESTINATION ${OPENVDB_INSTALL_INCLUDEDIR}/openvdb/codecs)
+install(FILES ${OPENVDB_LIBRARY_CODECS_IMPL_INCLUDE_FILES} DESTINATION ${OPENVDB_INSTALL_INCLUDEDIR}/openvdb/codecs/impl)
 install(FILES ${OPENVDB_LIBRARY_MATH_INCLUDE_FILES} DESTINATION ${OPENVDB_INSTALL_INCLUDEDIR}/openvdb/math)
 install(FILES ${OPENVDB_LIBRARY_POINTS_INCLUDE_FILES} DESTINATION ${OPENVDB_INSTALL_INCLUDEDIR}/openvdb/points)
 install(FILES ${OPENVDB_LIBRARY_POINTS_IMPL_INCLUDE_FILES} DESTINATION ${OPENVDB_INSTALL_INCLUDEDIR}/openvdb/points/impl)

openvdb/openvdb/Grid.h

@@ -457,12 +457,12 @@ public:
     virtual void readBuffers(std::istream&) = 0;
     /// Read all of this grid's data buffers that intersect the given index-space bounding box.
     virtual void readBuffers(std::istream&, const CoordBBox&) = 0;
-    /// @brief Read all of this grid's data buffers that are not yet resident in memory
-    /// (because delayed loading is in effect).
-    /// @details If this grid was read from a memory-mapped file, this operation
-    /// disconnects the grid from the file.
-    /// @sa io::File::open, io::MappedFile
+
+#if OPENVDB_ABI_VERSION_NUMBER < 14
+    OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.")
     virtual void readNonresidentBuffers() const = 0;
+#endif
+
     /// Write out all data buffers for this grid.
     virtual void writeBuffers(std::ostream&) const = 0;
 
@@ -941,12 +941,12 @@ public:
     void readBuffers(std::istream&) override;
     /// Read all of this grid's data buffers that intersect the given index-space bounding box.
     void readBuffers(std::istream&, const CoordBBox&) override;
-    /// @brief Read all of this grid's data buffers that are not yet resident in memory
-    /// (because delayed loading is in effect).
-    /// @details If this grid was read from a memory-mapped file, this operation
-    /// disconnects the grid from the file.
-    /// @sa io::File::open, io::MappedFile
-    void readNonresidentBuffers() const override;
+
+#if OPENVDB_ABI_VERSION_NUMBER < 14
+    OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.")
+    void readNonresidentBuffers() const override { }
+#endif
+
     /// Write out all data buffers for this grid.
     void writeBuffers(std::ostream&) const override;
 
@@ -1183,14 +1183,34 @@ struct TreeAdapter<tree::ValueAccessor<_TreeType> >
 ////////////////////////////////////////
 
 
+namespace points {
+
+template<typename T, Index Log2Dim> class PointDataLeafNode;
+
+/// @brief Type trait that evaluates to true only for @c PointDataLeafNode instantiations.
+template<typename T>
+struct IsPointDataLeafNode : std::false_type {};
+
+template<typename T, Index Log2Dim>
+struct IsPointDataLeafNode<PointDataLeafNode<T, Log2Dim>> : std::true_type {};
+
+} // namespace points
+
+
 /// @brief Metafunction that specifies whether a given leaf node, tree, or grid type
-/// requires multiple passes to read and write voxel data
-/// @details Multi-pass I/O allows one to optimize the data layout of leaf nodes
-/// for certain access patterns during delayed loading.
-/// @sa io::MultiPass
+/// requires multiple passes to read and write voxel data.
+/// @details Multi-pass I/O allows leaf nodes to optimize their serialization layout
+/// for delayed-load access patterns. Only @c PointDataLeafNode supports multi-pass I/O.
+/// Defining a custom leaf node that inherits @c io::PointDataGridMultiPass is no longer permitted.
+/// @sa points::IsPointDataLeafNode
 template<typename LeafNodeType>
 struct HasMultiPassIO {
-    static const bool value = std::is_base_of<io::MultiPass, LeafNodeType>::value;
+    static_assert(
+        !std::is_base_of<io::PointDataGridMultiPass, LeafNodeType>::value
+        || points::IsPointDataLeafNode<LeafNodeType>::value,
+        "Only PointDataLeafNode may inherit from io::PointDataGridMultiPass; "
+        "use points::IsPointDataLeafNode to test for multi-pass I/O support.");
+    static const bool value = points::IsPointDataLeafNode<LeafNodeType>::value;
 };
 
 // Partial specialization for Tree types
@@ -1672,14 +1692,6 @@ Grid<TreeT>::readBuffers(std::istream& is, const CoordBBox& bbox)
 }
 
 
-template<typename TreeT>
-inline void
-Grid<TreeT>::readNonresidentBuffers() const
-{
-    tree().readNonresidentBuffers();
-}
-
-
 template<typename TreeT>
 inline void
 Grid<TreeT>::writeBuffers(std::ostream& os) const

openvdb/openvdb/codecs/BoolCodec.h

@@ -0,0 +1,154 @@
+// Copyright Contributors to the OpenVDB Project
+// SPDX-License-Identifier: Apache-2.0
+
+#ifndef OPENVDB_IO_CODECS_BOOLCODEC_HAS_BEEN_INCLUDED
+#define OPENVDB_IO_CODECS_BOOLCODEC_HAS_BEEN_INCLUDED
+
+#include <openvdb/io/Codec.h>
+
+#include <openvdb/openvdb.h>
+
+#include "TopologyCodec.h"
+
+namespace openvdb {
+OPENVDB_USE_VERSION_NAMESPACE
+namespace OPENVDB_VERSION_NAME {
+namespace codecs {
+namespace internal {
+
+template <typename GridT>
+struct ReadBoolBuffersOp
+{
+    using TreeT = typename GridT::TreeType;
+    using RootT = typename TreeT::RootNodeType;
+    using LeafT = typename TreeT::LeafNodeType;
+    using ValueT = typename TreeT::ValueType;
+
+    ReadBoolBuffersOp(std::istream& _is, bool _saveFloatAsHalf,
+        const ValueT& _background, const CoordBBox* _clipBBox)
+        : is(_is)
+        , saveFloatAsHalf(_saveFloatAsHalf)
+        , background(_background)
+        , clipBBox(_clipBBox) { }
+
+    void operator()(RootT& root, size_t)
+    {
+        // Clip root-level tiles and prune children that were clipped.
+        if (clipBBox) {
+            root.clip(*clipBBox);
+        }
+    }
+
+    template <typename NodeT>
+    void operator()(NodeT& node, size_t)
+    {
+        // Clip internal node tiles and prune children that were clipped.
+        if (clipBBox) {
+            node.clip(*clipBBox, background);
+        }
+    }
+
+    void operator()(LeafT& leaf, size_t)
+    {
+        // Read in the value mask.
+        leaf.getValueMask().load(is);
+        // Read in the origin.
+        Coord origin;
+        is.read(reinterpret_cast<char*>(&origin), sizeof(Coord::ValueType) * 3);
+        leaf.setOrigin(origin);
+
+        // Read in the mask for the voxel values.
+        typename LeafT::Buffer::NodeMaskType nodeMask;
+        nodeMask.load(is);
+        typename LeafT::Buffer temp(nodeMask);
+        leaf.swap(temp);
+    }
+
+    std::istream& is;
+    const bool saveFloatAsHalf;
+    const ValueT& background;
+    const CoordBBox* clipBBox = nullptr;
+}; // struct ReadBoolBuffersOp
+
+template <typename GridT>
+struct WriteBoolBuffersOp
+{
+    using TreeT = typename GridT::TreeType;
+    using LeafT = typename TreeT::LeafNodeType;
+
+    WriteBoolBuffersOp(std::ostream& _os, bool _saveFloatAsHalf)
+        : os(_os)
+        , saveFloatAsHalf(_saveFloatAsHalf) { }
+
+    template <typename NodeT>
+    void operator()(const NodeT&, size_t) { }
+
+    void operator()(const LeafT& leaf, size_t)
+    {
+        // Write out the value mask.
+        leaf.getValueMask().save(os);
+
+        // Write out the origin.
+        os.write(reinterpret_cast<const char*>(&leaf.origin()), sizeof(Coord::ValueType) * 3);
+
+        // Write out the voxel values.
+        leaf.buffer().storage().save(os);
+    }
+
+    std::ostream& os;
+    const bool saveFloatAsHalf;
+}; // struct WriteBoolBuffersOp
+
+} // namespace internal
+
+template <typename GridT>
+struct BoolCodec final: public TopologyCodec<GridT>
+{
+    using Ptr = std::unique_ptr<BoolCodec<GridT>>;
+
+    ~BoolCodec() noexcept = default;
+
+    static inline std::string name() { return GridT::gridType(); }
+
+    void readBuffers(std::istream& is, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) final
+    {
+        GridT& grid = static_cast<GridT&>(*data.grid);
+
+        if (grid.hasMultiPassIO()) {
+            OPENVDB_THROW(IoError, "Multi-pass IO is not supported in BoolCodec");
+        }
+
+        io::checkFormatVersion(is);
+
+        bool saveFloatAsHalf = grid.saveFloatAsHalf();
+
+        auto& tree = grid.tree();
+        tree.clearAllAccessors();
+
+        std::unique_ptr<CoordBBox> clipIndexBBox;
+        if (options.clipBBox.isSorted()) {
+            clipIndexBBox = std::make_unique<CoordBBox>(grid.constTransform().worldToIndexNodeCentered(options.clipBBox));
+        }
+
+        internal::ReadBoolBuffersOp<GridT> readBuffersOp(is, saveFloatAsHalf, tree.background(), clipIndexBBox.get());
+        tools::visitNodesDepthFirst(grid.tree(), readBuffersOp, /*idx=*/0, /*topDown=*/false);
+    }
+
+    void writeBuffers(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) final
+    {
+        const GridT& grid = static_cast<const GridT&>(gridBase);
+
+        if (grid.hasMultiPassIO()) {
+            OPENVDB_THROW(IoError, "Multi-pass IO is not supported in BoolCodec");
+        }
+
+        internal::WriteBoolBuffersOp<GridT> writeBuffersOp(os, grid.saveFloatAsHalf());
+        tools::visitNodesDepthFirst(grid.tree(), writeBuffersOp);
+    }
+}; // struct BoolCodec
+
+} // namespace codecs
+} // namespace OPENVDB_VERSION_NAME
+} // namespace openvdb
+
+#endif // OPENVDB_IO_CODECS_BOOLCODEC_HAS_BEEN_INCLUDED

openvdb/openvdb/codecs/PointDataCodec.h

@@ -0,0 +1,451 @@
+// Copyright Contributors to the OpenVDB Project
+// SPDX-License-Identifier: Apache-2.0
+
+#ifndef OPENVDB_IO_CODECS_POINTDATACODEC_HAS_BEEN_INCLUDED
+#define OPENVDB_IO_CODECS_POINTDATACODEC_HAS_BEEN_INCLUDED
+
+#include <openvdb/io/Codec.h>
+
+#include <openvdb/tools/Clip.h>
+
+#include <openvdb/points/PointDataGrid.h>
+#include <openvdb/points/AttributeSet.h>
+#include <openvdb/points/StreamCompression.h>
+
+#include "impl/ScalarLeafCodec.h"
+#include "TopologyCodec.h"
+
+#include <set>
+#include <vector>
+
+namespace openvdb {
+OPENVDB_USE_VERSION_NAMESPACE
+namespace OPENVDB_VERSION_NAME {
+namespace codecs {
+namespace internal {
+
+/// Look up an existing paged stream or create a new one for the given attribute index
+template <typename PagedStreamPtrT>
+inline typename PagedStreamPtrT::element_type* getOrCreatePagedStream(
+    std::map<Index, PagedStreamPtrT>& pagedStreams, Index attributeIndex)
+{
+    auto it = pagedStreams.find(attributeIndex);
+    if (it != pagedStreams.end()) return it->second.get();
+    using PagedStreamT = typename PagedStreamPtrT::element_type;
+    auto ptr = std::make_shared<PagedStreamT>();
+    auto& stored = (pagedStreams[attributeIndex] = std::move(ptr));
+    return stored.get();
+}
+
+////////////////////////////////////////
+// Read-side functions
+
+template <typename LeafT>
+inline void readPointDataVoxelSizes(const std::vector<LeafT*>& leaves,
+    std::istream& is, std::unordered_map<Coord, uint16_t>& voxelBufferSizes)
+{
+    for (auto* leaf : leaves) {
+        uint16_t voxelBufferSize;
+        is.read(reinterpret_cast<char*>(&voxelBufferSize), sizeof(uint16_t));
+        voxelBufferSizes[leaf->origin()] = voxelBufferSize;
+    }
+}
+
+template <typename LeafT>
+inline void readPointDataDescriptors(const std::vector<LeafT*>& leaves,
+    std::istream& is)
+{
+    points::AttributeSet::Descriptor::Ptr sharedDescriptor;
+    for (auto* leaf : leaves) {
+        points::AttributeSet::UniquePtr attrSet = leaf->stealAttributeSet();
+        if (sharedDescriptor) {
+            // Reuse shared descriptor from first leaf
+            attrSet->resetDescriptor(sharedDescriptor, /*allowMismatchingDescriptors=*/true);
+        }
+        else {
+            uint8_t header;
+            is.read(reinterpret_cast<char*>(&header), sizeof(uint8_t));
+            attrSet->readDescriptor(is);
+            if (header & uint8_t(1)) {
+                // Store descriptor for subsequent leaves
+                sharedDescriptor = attrSet->descriptorPtr();
+            }
+            // a forwards-compatibility mechanism for future use,
+            // if a 0x2 bit is set, read and skip over a specific number of bytes
+            if (header & uint8_t(2)) {
+                uint64_t bytesToSkip;
+                is.read(reinterpret_cast<char*>(&bytesToSkip), sizeof(uint64_t));
+                if (bytesToSkip > uint64_t(0)) {
+                    std::vector<uint8_t> tempData(bytesToSkip);
+                    is.read(reinterpret_cast<char*>(&tempData[0]), bytesToSkip);
+                }
+            }
+            // this reader is only able to read headers with 0x1 and 0x2 bits set
+            if (header > uint8_t(3)) {
+                OPENVDB_THROW(IoError, "Unrecognised header flags in PointDataLeafNode");
+            }
+        }
+        attrSet->readMetadata(is);
+        leaf->replaceAttributeSet(attrSet.release(), /*allowMismatchingDescriptors=*/true);
+    }
+}
+
+template <typename LeafT>
+inline void readPointDataAttributeSizes(const std::vector<LeafT*>& leaves,
+    std::istream& is, Index attributeIndex,
+    std::map<Index, compression::PagedInputStream::Ptr>& pagedStreams)
+{
+    for (auto* leaf : leaves) {
+        points::AttributeArray* array = attributeIndex < leaf->attributeSet().size() ?
+            &leaf->attributeArray(attributeIndex) : nullptr;
+        if (array) {
+            auto* pagedStream = getOrCreatePagedStream(pagedStreams, attributeIndex);
+            pagedStream->setInputStream(is);
+            pagedStream->setSizeOnly(true);
+            array->readPagedBuffers(*pagedStream);
+        }
+    }
+}
+
+template <typename LeafT>
+inline void readPointDataVoxelData(const std::vector<LeafT*>& leaves,
+    std::istream& is, bool saveFloatAsHalf,
+    const typename LeafT::ValueType& background,
+    [[maybe_unused]] const std::unordered_map<Coord, uint16_t>& voxelBufferSizes)
+{
+    using BaseLeaf = typename LeafT::BaseLeaf;
+    for (auto* leaf : leaves) {
+        OPENVDB_ASSERT(voxelBufferSizes.find(leaf->origin()) != voxelBufferSizes.end());
+        BaseLeaf& baseLeaf = static_cast<BaseLeaf&>(*leaf);
+        readScalarLeafBuffers(baseLeaf, is, saveFloatAsHalf, background);
+    }
+}
+
+template <typename LeafT>
+inline void readPointDataAttributeData(const std::vector<LeafT*>& leaves,
+    std::istream& is, Index attributeIndex,
+    std::map<Index, compression::PagedInputStream::Ptr>& pagedStreams)
+{
+    for (auto* leaf : leaves) {
+        points::AttributeArray* array = attributeIndex < leaf->attributeSet().size() ?
+            &leaf->attributeArray(attributeIndex) : nullptr;
+        if (array) {
+            auto* pagedStream = getOrCreatePagedStream(pagedStreams, attributeIndex);
+            pagedStream->setInputStream(is);
+            pagedStream->setSizeOnly(false);
+            array->readPagedBuffers(*pagedStream);
+        }
+    }
+}
+
+template <typename LeafT>
+inline void skipPointDataAttributeSizes(const std::vector<LeafT*>& leaves,
+    std::istream& is, Index attributeIndex,
+    std::map<Index, compression::PagedInputStream::Ptr>& pagedStreams)
+{
+    for (auto* leaf : leaves) {
+        points::AttributeArray* array = attributeIndex < leaf->attributeSet().size() ?
+            &leaf->attributeArray(attributeIndex) : nullptr;
+        if (array) {
+            auto* pagedStream = getOrCreatePagedStream(pagedStreams, attributeIndex);
+            pagedStream->setInputStream(is);
+            pagedStream->setSizeOnly(true);
+            array->skipPagedBuffers(*pagedStream);
+        }
+    }
+}
+
+template <typename LeafT>
+inline void skipPointDataAttributeData(const std::vector<LeafT*>& leaves,
+    std::istream& is, Index attributeIndex,
+    std::map<Index, compression::PagedInputStream::Ptr>& pagedStreams)
+{
+    for (auto* leaf : leaves) {
+        points::AttributeArray* array = attributeIndex < leaf->attributeSet().size() ?
+            &leaf->attributeArray(attributeIndex) : nullptr;
+        if (array) {
+            auto* pagedStream = getOrCreatePagedStream(pagedStreams, attributeIndex);
+            pagedStream->setInputStream(is);
+            pagedStream->setSizeOnly(false);
+            array->skipPagedBuffers(*pagedStream);
+        }
+    }
+}
+
+////////////////////////////////////////
+// Write-side functions
+
+template <typename LeafT>
+inline Index countPointDataPasses(const std::vector<const LeafT*>& leaves)
+{
+    Index maxRequiredPasses = 0;
+    for (const auto* leaf : leaves) {
+        const Index requiredPasses = leaf->buffers();
+        if (requiredPasses > maxRequiredPasses) {
+            maxRequiredPasses = requiredPasses;
+        }
+    }
+    return maxRequiredPasses;
+}
+
+template <typename LeafT>
+inline void writePointDataVoxelSizes(const std::vector<const LeafT*>& leaves,
+    std::ostream& os, bool& matching,
+    points::AttributeSet::Descriptor::Ptr& sharedDescriptor)
+{
+    bool descriptorChecked = false;
+    matching = true;
+    for (const auto* leaf : leaves) {
+        io::writeCompressedValuesSize(os, leaf->buffer().data(), LeafT::SIZE);
+
+        // Track descriptor matching
+        const auto& descriptor = leaf->attributeSet().descriptorPtr();
+        if (!descriptorChecked) {
+            // First leaf - store descriptor
+            descriptorChecked = true;
+            sharedDescriptor = descriptor;
+        }
+        else if (matching && *sharedDescriptor != *descriptor) {
+            matching = false;
+        }
+    }
+}
+
+template <typename LeafT>
+inline void writePointDataDescriptors(const std::vector<const LeafT*>& leaves,
+    std::ostream& os, bool matching,
+    const points::AttributeSet::Descriptor::Ptr&)
+{
+    bool firstWrite = true;
+    for (const auto* leaf : leaves) {
+        const points::AttributeSet& attributeSet = leaf->attributeSet();
+        if (matching) {
+            // Shared descriptor - only write on first leaf
+            if (firstWrite) {
+                firstWrite = false;
+                uint8_t header(1);
+                os.write(reinterpret_cast<const char*>(&header), sizeof(uint8_t));
+                attributeSet.writeDescriptor(os, /*transient=*/false);
+            }
+        }
+        else {
+            // Non-shared descriptor - write on every leaf
+            uint8_t header(0);
+            os.write(reinterpret_cast<const char*>(&header), sizeof(uint8_t));
+            attributeSet.writeDescriptor(os, /*transient=*/false);
+        }
+        attributeSet.writeMetadata(os, /*transient=*/false, /*paged=*/true);
+    }
+}
+
+template <typename LeafT>
+inline void writePointDataAttributeSizes(const std::vector<const LeafT*>& leaves,
+    std::ostream& os, Index attributeIndex)
+{
+    std::map<Index, compression::PagedOutputStream::Ptr> pagedStreams;
+    for (const auto* leaf : leaves) {
+        const points::AttributeArray* array = attributeIndex < leaf->attributeSet().size() ?
+            leaf->attributeSet().getConst(attributeIndex) : nullptr;
+        if (array) {
+            auto* pagedStream = getOrCreatePagedStream(pagedStreams, attributeIndex);
+            pagedStream->setOutputStream(os);
+            pagedStream->setSizeOnly(true);
+            array->writePagedBuffers(*pagedStream, /*outputTransient*/false);
+        }
+    }
+    // Flush paged streams to write any remaining buffered page headers
+    for (auto& pair : pagedStreams) {
+        pair.second->flush();
+    }
+}
+
+template <typename LeafT>
+inline void writePointDataVoxelData(const std::vector<const LeafT*>& leaves,
+    std::ostream& os, bool saveFloatAsHalf)
+{
+    using BaseLeaf = typename LeafT::BaseLeaf;
+    for (const auto* leaf : leaves) {
+        const BaseLeaf& baseLeaf = static_cast<const BaseLeaf&>(*leaf);
+        writeScalarLeafBuffers(baseLeaf, os, saveFloatAsHalf);
+    }
+}
+
+template <typename LeafT>
+inline void writePointDataAttributeData(const std::vector<const LeafT*>& leaves,
+    std::ostream& os, Index attributeIndex)
+{
+    std::map<Index, compression::PagedOutputStream::Ptr> pagedStreams;
+    for (const auto* leaf : leaves) {
+        const points::AttributeArray* array = attributeIndex < leaf->attributeSet().size() ?
+            leaf->attributeSet().getConst(attributeIndex) : nullptr;
+        if (array) {
+            auto* pagedStream = getOrCreatePagedStream(pagedStreams, attributeIndex);
+            pagedStream->setOutputStream(os);
+            pagedStream->setSizeOnly(false);
+            array->writePagedBuffers(*pagedStream, /*outputTransient*/false);
+        }
+    }
+    // Flush paged streams to write any remaining buffered page data
+    for (auto& pair : pagedStreams) {
+        pair.second->flush();
+    }
+}
+
+} // namespace internal
+
+/// Per-grid-type codec-specific options for PointDataCodec
+/// Contains point attribute filtering options
+struct OPENVDB_API PointDataCodecTypeData : public io::ReadTypedOptions
+{
+    // Point Attribute Options - which attributes to read
+    std::vector<std::string> pointAttributeNames;
+}; // struct PointDataCodecTypeData
+
+template <typename GridT>
+struct PointDataCodec final: public TopologyCodec<GridT>
+{
+    using Ptr = std::unique_ptr<PointDataCodec<GridT>>;
+
+    ~PointDataCodec() noexcept = default;
+
+    static inline std::string name() { return GridT::gridType(); }
+
+    void readBuffers(std::istream& is, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) final
+    {
+        OPENVDB_ASSERT(dynamic_cast<GridT*>(data.grid.get()));
+
+        GridT& grid = static_cast<GridT&>(*data.grid);
+
+        std::vector<std::string> pointAttributeNames;
+
+        // Look up point-specific options if provided
+        auto it = options.typeData.find(name());
+        if (it != options.typeData.end()) {
+            auto& pointTypeData = io::ReadTypedOptions::cast<PointDataCodecTypeData>(it->second);
+            pointAttributeNames = pointTypeData.pointAttributeNames;
+        }
+
+        io::checkFormatVersion(is);
+
+        bool saveFloatAsHalf = grid.saveFloatAsHalf();
+
+        auto& tree = grid.tree();
+        tree.clearAllAccessors();
+
+        uint16_t numPasses = 1;
+        is.read(reinterpret_cast<char*>(&numPasses), sizeof(uint16_t));
+        const Index attributes = (numPasses - 4) / 2;
+
+        using LeafT = typename GridT::TreeType::LeafNodeType;
+        std::vector<LeafT*> leaves;
+        tree.getNodes(leaves);
+
+        // Pass 0: read voxel data sizes
+        std::unordered_map<Coord, uint16_t> voxelBufferSizes;
+        internal::readPointDataVoxelSizes(leaves, is, voxelBufferSizes);
+
+        // Pass 1: read descriptor and attribute metadata
+        internal::readPointDataDescriptors(leaves, is);
+
+        // Build set of attribute indices to skip based on pointAttributeNames.
+        // An empty pointAttributeNames means no filtering (read all attributes).
+        std::set<Index> skipIndices;
+        if (!pointAttributeNames.empty() && !leaves.empty()) {
+            const auto& nameMap = leaves[0]->attributeSet().descriptor().map();
+            const std::set<std::string> wantedNames(
+                pointAttributeNames.begin(),
+                pointAttributeNames.end());
+            for (const auto& namePos : nameMap) {
+                if (wantedNames.find(namePos.first) == wantedNames.end()) {
+                    skipIndices.insert(static_cast<Index>(namePos.second));
+                }
+            }
+        }
+
+        // Passes 2..N+1: read attribute buffer sizes
+        std::map<Index, compression::PagedInputStream::Ptr> pagedStreams;
+        for (Index i = 0; i < attributes; ++i) {
+            if (skipIndices.count(i)) {
+                internal::skipPointDataAttributeSizes(leaves, is, i, pagedStreams);
+            } else {
+                internal::readPointDataAttributeSizes(leaves, is, i, pagedStreams);
+            }
+        }
+
+        // Pass N+2: read voxel data
+        internal::readPointDataVoxelData(leaves, is, saveFloatAsHalf,
+            tree.background(), voxelBufferSizes);
+
+        // Passes N+3..2N+2: read attribute data buffers
+        for (Index i = 0; i < attributes; ++i) {
+            if (skipIndices.count(i)) {
+                internal::skipPointDataAttributeData(leaves, is, i, pagedStreams);
+            } else {
+                internal::readPointDataAttributeData(leaves, is, i, pagedStreams);
+            }
+        }
+
+        // Drop skipped attributes from each leaf's AttributeSet
+        if (!skipIndices.empty() && !leaves.empty()) {
+            const std::vector<size_t> dropPositions(skipIndices.begin(), skipIndices.end());
+            auto filteredDescriptor =
+                leaves[0]->attributeSet().descriptorPtr()->duplicateDrop(dropPositions);
+            for (auto* leaf : leaves) {
+                leaf->dropAttributes(dropPositions,
+                    leaf->attributeSet().descriptor(), filteredDescriptor);
+            }
+        }
+
+        // PointDataGrid uses multiple passes, so clip after reading
+        // the buffers if bbox is not infinite.
+        if (options.clipBBox.isSorted()) {
+            CoordBBox indexBBox =
+                grid.constTransform().worldToIndexNodeCentered(options.clipBBox);
+            grid.tree().root().clip(indexBBox);
+        }
+    }
+
+    void writeBuffers(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) final
+    {
+        const GridT& grid = static_cast<const GridT&>(gridBase);
+        bool saveFloatAsHalf = grid.saveFloatAsHalf();
+
+        using LeafT = typename GridT::TreeType::LeafNodeType;
+        std::vector<const LeafT*> leaves;
+        grid.tree().getNodes(leaves);
+
+        // Determine how many leaf buffer passes are required for this grid
+        uint16_t numPasses =
+            static_cast<uint16_t>(internal::countPointDataPasses(leaves));
+        os.write(reinterpret_cast<const char*>(&numPasses), sizeof(uint16_t));
+
+        const Index attributes = (numPasses - 4) / 2;
+
+        // Pass 0: write voxel data sizes + descriptor tracking
+        bool matching = true;
+        points::AttributeSet::Descriptor::Ptr sharedDescriptor;
+        internal::writePointDataVoxelSizes(leaves, os, matching, sharedDescriptor);
+
+        // Pass 1: write descriptor and attribute metadata
+        internal::writePointDataDescriptors(leaves, os, matching, sharedDescriptor);
+
+        // Passes 2..N+1: write attribute buffer sizes (page headers)
+        for (Index i = 0; i < attributes; ++i) {
+            internal::writePointDataAttributeSizes(leaves, os, i);
+        }
+
+        // Pass N+2: write voxel data
+        internal::writePointDataVoxelData(leaves, os, saveFloatAsHalf);
+
+        // Passes N+3..2N+2: write attribute data buffers (page data)
+        for (Index i = 0; i < attributes; ++i) {
+            internal::writePointDataAttributeData(leaves, os, i);
+        }
+    }
+}; // struct PointDataCodec
+
+} // namespace codecs
+} // namespace OPENVDB_VERSION_NAME
+} // namespace openvdb
+
+#endif // OPENVDB_IO_CODECS_POINTDATACODEC_HAS_BEEN_INCLUDED

openvdb/openvdb/codecs/PointIndexCodec.h

@@ -0,0 +1,159 @@
+// Copyright Contributors to the OpenVDB Project
+// SPDX-License-Identifier: Apache-2.0
+
+#ifndef OPENVDB_IO_CODECS_POINTINDEXCODEC_HAS_BEEN_INCLUDED
+#define OPENVDB_IO_CODECS_POINTINDEXCODEC_HAS_BEEN_INCLUDED
+
+#include <openvdb/io/Codec.h>
+
+#include <openvdb/tools/PointIndexGrid.h>
+
+#include "impl/ScalarLeafCodec.h"
+#include "TopologyCodec.h"
+
+namespace openvdb {
+OPENVDB_USE_VERSION_NAMESPACE
+namespace OPENVDB_VERSION_NAME {
+namespace codecs {
+namespace internal {
+
+template <typename GridT>
+struct ReadPointIndexBuffersOp
+{
+    using TreeT = typename GridT::TreeType;
+    using RootT = typename TreeT::RootNodeType;
+    using LeafT = typename TreeT::LeafNodeType;
+    using ValueT = typename TreeT::ValueType;
+
+    ReadPointIndexBuffersOp(std::istream& _is, bool _saveFloatAsHalf,
+        const ValueT& _background)
+        : is(_is)
+        , saveFloatAsHalf(_saveFloatAsHalf)
+        , background(_background) { }
+
+    template <typename NodeT>
+    void operator()(NodeT&, size_t) { }
+
+    void operator()(LeafT& leaf, size_t)
+    {
+        using BaseLeaf = typename LeafT::BaseLeaf;
+
+        // Read the value mask and voxel data via base class
+        BaseLeaf& baseLeaf = static_cast<BaseLeaf&>(leaf);
+        readScalarLeafBuffers(baseLeaf, is, saveFloatAsHalf, background, /*skip=*/false, /*clipBBox=*/nullptr);
+
+        // Read the number of indices.
+        Index64 numIndices = Index64(0);
+        is.read(reinterpret_cast<char*>(&numIndices), sizeof(Index64));
+
+        // Read the indices data.
+        leaf.indices().resize(size_t(numIndices));
+        is.read(reinterpret_cast<char*>(leaf.indices().data()), numIndices * sizeof(ValueT));
+
+        // Reserved for future use.
+        Index64 auxDataBytes = Index64(0);
+        is.read(reinterpret_cast<char*>(&auxDataBytes), sizeof(Index64));
+        if (auxDataBytes > 0) {
+            // For now, read and discard any auxiliary data.
+            std::unique_ptr<char[]> auxData{new char[auxDataBytes]};
+            is.read(auxData.get(), auxDataBytes);
+        }
+    }
+
+    std::istream& is;
+    const bool saveFloatAsHalf;
+    const ValueT& background;
+}; // struct ReadPointIndexBuffersOp
+
+template <typename GridT>
+struct WritePointIndexBuffersOp
+{
+    using TreeT = typename GridT::TreeType;
+    using LeafT = typename TreeT::LeafNodeType;
+    using ValueT = typename TreeT::ValueType;
+
+    WritePointIndexBuffersOp(std::ostream& _os, bool _saveFloatAsHalf)
+        : os(_os)
+        , saveFloatAsHalf(_saveFloatAsHalf) { }
+
+    template <typename NodeT>
+    void operator()(const NodeT&, size_t) { }
+
+    void operator()(const LeafT& leaf, size_t)
+    {
+        using BaseLeaf = typename LeafT::BaseLeaf;
+
+        // Write out the value mask and voxel values via base class
+        const BaseLeaf& baseLeaf = static_cast<const BaseLeaf&>(leaf);
+        writeScalarLeafBuffers(baseLeaf, os, saveFloatAsHalf);
+
+        // Write the number of indices.
+        Index64 numIndices = Index64(leaf.indices().size());
+        os.write(reinterpret_cast<const char*>(&numIndices), sizeof(Index64));
+
+        // Write the indices data.
+        os.write(reinterpret_cast<const char*>(leaf.indices().data()), numIndices * sizeof(ValueT));
+
+        // Reserved for future use.
+        const Index64 auxDataBytes = Index64(0);
+        os.write(reinterpret_cast<const char*>(&auxDataBytes), sizeof(Index64));
+    }
+
+    std::ostream& os;
+    const bool saveFloatAsHalf;
+}; // struct WritePointIndexBuffersOp
+
+} // namespace internal
+
+template <typename GridT>
+struct PointIndexCodec final: public TopologyCodec<GridT>
+{
+    using Ptr = std::unique_ptr<PointIndexCodec<GridT>>;
+
+    ~PointIndexCodec() noexcept = default;
+
+    static inline std::string name() { return GridT::gridType(); }
+
+    void readBuffers(std::istream& is, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics& diagnostics) final
+    {
+        OPENVDB_ASSERT(dynamic_cast<GridT*>(data.grid.get()));
+
+        GridT& grid = static_cast<GridT&>(*data.grid);
+
+        if (grid.hasMultiPassIO()) {
+            OPENVDB_THROW(IoError, "Multi-pass IO is not supported in PointIndexCodec");
+        }
+
+        io::checkFormatVersion(is);
+
+        bool saveFloatAsHalf = grid.saveFloatAsHalf();
+
+        auto& tree = grid.tree();
+        tree.clearAllAccessors();
+
+        if (options.clipBBox.isSorted()) {
+            diagnostics.addWarning(grid.getName(), "bounding box clipping is not supported for PointIndexGrids");
+        }
+
+        internal::ReadPointIndexBuffersOp<GridT> readBuffersOp(is, saveFloatAsHalf, tree.background());
+        tools::visitNodesDepthFirst(grid.tree(), readBuffersOp, /*idx=*/0, /*topDown=*/false);
+    }
+
+    void writeBuffers(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) final
+    {
+        const GridT& grid = static_cast<const GridT&>(gridBase);
+
+        if (grid.hasMultiPassIO()) {
+            OPENVDB_THROW(IoError, "Multi-pass IO is not supported in PointIndexCodec");
+        }
+
+        internal::WritePointIndexBuffersOp<GridT> writeBuffersOp(os, grid.saveFloatAsHalf());
+        tools::visitNodesDepthFirst(grid.tree(), writeBuffersOp);
+    }
+}; // struct PointIndexCodec
+
+} // namespace codecs
+} // namespace OPENVDB_VERSION_NAME
+} // namespace openvdb
+
+#endif // OPENVDB_IO_CODECS_POINTINDEXCODEC_HAS_BEEN_INCLUDED

openvdb/openvdb/codecs/ScalarCodec.h

@@ -0,0 +1,173 @@
+// Copyright Contributors to the OpenVDB Project
+// SPDX-License-Identifier: Apache-2.0
+
+#ifndef OPENVDB_IO_CODECS_SCALARCODEC_HAS_BEEN_INCLUDED
+#define OPENVDB_IO_CODECS_SCALARCODEC_HAS_BEEN_INCLUDED
+
+#include <openvdb/openvdb.h>
+#include <openvdb/tree/Tree.h>
+#include <openvdb/tools/Clip.h>
+#include <openvdb/tools/NodeVisitor.h>
+#include <openvdb/io/Codec.h>
+
+#include "impl/ScalarLeafCodec.h"
+#include "TopologyCodec.h"
+
+namespace openvdb {
+OPENVDB_USE_VERSION_NAMESPACE
+namespace OPENVDB_VERSION_NAME {
+namespace codecs {
+namespace internal {
+
+template <typename TreeT>
+struct WriteBuffersOp
+{
+    using LeafT = typename TreeT::LeafNodeType;
+
+    WriteBuffersOp(std::ostream& _os, bool _saveFloatAsHalf)
+        : os(_os)
+        , saveFloatAsHalf(_saveFloatAsHalf) { }
+
+    template <typename NodeT>
+    void operator()(const NodeT&, size_t) { }
+
+    void operator()(const LeafT& leaf, size_t)
+    {
+        writeScalarLeafBuffers(leaf, os, saveFloatAsHalf);
+    }
+
+    std::ostream& os;
+    const bool saveFloatAsHalf;
+}; // struct WriteBuffersOp
+
+
+template <typename TreeT, typename StorageTreeT>
+struct ReadBuffersOp
+{
+    using RootT = typename TreeT::RootNodeType;
+    using LeafT = typename TreeT::LeafNodeType;
+    using ValueT = typename TreeT::ValueType;
+    using StorageLeafT = typename StorageTreeT::LeafNodeType;
+
+    ReadBuffersOp(std::istream& _is, bool _saveFloatAsHalf, const ValueT& _background,
+        const CoordBBox* _clipBBox)
+        : is(_is)
+        , saveFloatAsHalf(_saveFloatAsHalf)
+        , background(_background)
+        , clipBBox(_clipBBox) { }
+
+    void operator()(RootT& root, size_t)
+    {
+        if (clipBBox) {
+            root.clip(*clipBBox);
+        }
+    }
+
+    template <typename NodeT>
+    void operator()(NodeT& node, size_t)
+    {
+        if (clipBBox) {
+            node.clip(*clipBBox, background);
+        }
+    }
+
+    void operator()(LeafT& leaf, size_t)
+    {
+        readScalarLeafBuffers<LeafT, StorageLeafT>(leaf, is, saveFloatAsHalf, background, /*skip=*/false, clipBBox);
+    }
+
+    std::istream& is;
+    const bool saveFloatAsHalf;
+    const ValueT& background;
+    const CoordBBox* clipBBox = nullptr;
+}; // struct ReadBuffersOp
+
+
+// Free-standing function for both standard and conversion codec cases
+// Uses StorageGridT = GridT by default, but allows different storage type for conversions
+template<typename GridT, typename StorageGridT = GridT>
+void scalarCodecReadBuffers(GridT& grid, std::istream& is, const io::ReadOptions& options)
+{
+    if (grid.hasMultiPassIO()) {
+        OPENVDB_THROW(IoError, "Multi-pass IO is not supported in ScalarCodec");
+    }
+
+    using TreeT = typename GridT::TreeType;
+    using StorageTreeT = typename StorageGridT::TreeType;
+
+    io::checkFormatVersion(is);
+
+    bool saveFloatAsHalf = grid.saveFloatAsHalf();
+
+    auto& tree = grid.tree();
+    tree.clearAllAccessors();
+
+    std::unique_ptr<CoordBBox> clipIndexBBox;
+    if (options.clipBBox.isSorted()) {
+        clipIndexBBox = std::make_unique<CoordBBox>(grid.constTransform().worldToIndexNodeCentered(options.clipBBox));
+    }
+
+    // Works for both standard (TreeT == StorageTreeT) and conversion cases
+    ReadBuffersOp<TreeT, StorageTreeT> readBuffersOp(is, saveFloatAsHalf, tree.background(),
+        clipIndexBBox.get());
+    tools::visitNodesDepthFirst(grid.tree(), readBuffersOp, /*idx=*/0, /*topDown=*/false);
+}
+
+// Free-standing function for write case (no StorageGridT needed)
+template<typename GridT>
+void scalarCodecWriteBuffers(const GridT& grid, std::ostream& os)
+{
+    using TreeType = typename GridT::TreeType;
+
+    if (grid.hasMultiPassIO()) {
+        OPENVDB_THROW(IoError, "Multi-pass IO is not supported in ScalarCodec");
+    }
+
+    WriteBuffersOp<TreeType> writeBuffersOp(os, grid.saveFloatAsHalf());
+    tools::visitNodesDepthFirst(grid.tree(), writeBuffersOp);
+}
+
+} // namespace internal
+
+template <typename GridT, typename StorageGridT = GridT, io::CodecMode Mode = io::CodecMode::ReadWrite>
+struct ScalarCodec final: public TopologyCodec<GridT, StorageGridT, Mode>
+{
+    static_assert(GridT::TreeType::RootNodeType::template SameConfiguration<
+        typename StorageGridT::TreeType::RootNodeType>::value,
+        "GridT and StorageGridT must have the same configuration");
+
+    using Ptr = std::unique_ptr<ScalarCodec<GridT, StorageGridT, Mode>>;
+
+    ~ScalarCodec() noexcept = default;
+
+    static inline std::string name()
+    {
+        if constexpr (std::is_same_v<GridT, StorageGridT>) {
+            return GridT::gridType();
+        } else {
+            std::string buildType = typeNameAsString<typename GridT::BuildType>();
+            return StorageGridT::gridType() + "_to_" + buildType;
+        }
+    }
+
+    void readBuffers(std::istream& is, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) final
+    {
+        GridT& grid = static_cast<GridT&>(*data.grid);
+        internal::scalarCodecReadBuffers<GridT, StorageGridT>(grid, is, options);
+    }
+
+    void writeBuffers(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) final
+    {
+        if constexpr (Mode == io::CodecMode::ReadOnly) return;
+
+        const GridT& grid = static_cast<const GridT&>(gridBase);
+        internal::scalarCodecWriteBuffers(grid, os);
+    }
+}; // struct ScalarCodec
+
+
+} // namespace codecs
+} // namespace OPENVDB_VERSION_NAME
+} // namespace openvdb
+
+#endif // OPENVDB_IO_CODECS_SCALARCODEC_HAS_BEEN_INCLUDED

openvdb/openvdb/codecs/TopologyCodec.h

@@ -0,0 +1,341 @@
+// Copyright Contributors to the OpenVDB Project
+// SPDX-License-Identifier: Apache-2.0
+
+#ifndef OPENVDB_IO_CODECS_TOPOLOGYCODEC_HAS_BEEN_INCLUDED
+#define OPENVDB_IO_CODECS_TOPOLOGYCODEC_HAS_BEEN_INCLUDED
+
+#include <openvdb/openvdb.h>
+#include <openvdb/tree/Tree.h>
+#include <openvdb/tree/NodeManager.h>
+#include <openvdb/tools/NodeVisitor.h>
+#include <openvdb/io/Codec.h>
+#include <openvdb/Grid.h>
+
+namespace openvdb {
+OPENVDB_USE_VERSION_NAMESPACE
+namespace OPENVDB_VERSION_NAME {
+namespace codecs {
+namespace internal {
+
+template <typename TreeT>
+struct WriteTopologyOp
+{
+    using ValueT = typename TreeT::ValueType;
+    using RootT = typename TreeT::RootNodeType;
+    using LeafT = typename TreeT::LeafNodeType;
+
+    WriteTopologyOp(std::ostream& _os, bool _saveFloatAsHalf)
+        : os(_os)
+        , saveFloatAsHalf(_saveFloatAsHalf) { }
+
+    void operator()(const RootT& root, size_t)
+    {
+        int32_t bufferCount = 1;
+        os.write(reinterpret_cast<char*>(&bufferCount), sizeof(int32_t));
+
+        background = &root.background();
+        if (!saveFloatAsHalf) {
+            os.write(reinterpret_cast<const char*>(background), sizeof(ValueT));
+        } else {
+            ValueT truncatedVal = io::truncateRealToHalf(*background);
+            os.write(reinterpret_cast<const char*>(&truncatedVal), sizeof(ValueT));
+        }
+        io::setGridBackgroundValuePtr(os, background);
+
+        const Index numTiles = root.tileCount(), numChildren = root.childCount();
+        os.write(reinterpret_cast<const char*>(&numTiles), sizeof(Index));
+        os.write(reinterpret_cast<const char*>(&numChildren), sizeof(Index));
+
+        // Write tiles.
+        for (auto iter = root.cbeginValueAll(); iter; ++iter) {
+            const auto& ijk = iter.getCoord();
+            os.write(reinterpret_cast<const char*>(ijk.asPointer()), 3 * sizeof(Int32));
+            ValueT value;
+            bool active = root.getTileValueUnsafe(ijk, value);
+            os.write(reinterpret_cast<const char*>(&value), sizeof(ValueT));
+            os.write(reinterpret_cast<const char*>(&active), sizeof(bool));
+        }
+
+        rootChildLevel = RootT::LEVEL-1;
+    }
+
+    template <typename NodeT>
+    void operator()(const NodeT& node, size_t)
+    {
+        // Write origin for RootNode children only
+        if (rootChildLevel == NodeT::LEVEL) {
+            const auto& ijk = node.origin();
+            os.write(reinterpret_cast<const char*>(ijk.asPointer()), 3 * sizeof(Int32));
+        }
+
+        const auto& childMask = node.getChildMask();
+        const auto& valueMask = node.getValueMask();
+
+        childMask.save(os);
+        valueMask.save(os);
+
+        {
+            // Copy all of this node's values into an array.
+            std::unique_ptr<ValueT[]> valuePtr(new ValueT[NodeT::NUM_VALUES]);
+            ValueT* values = valuePtr.get();
+            const ValueT zero = zeroVal<ValueT>();
+            for (Index i = 0; i < NodeT::NUM_VALUES; ++i) {
+                values[i] = (node.isChildMaskOff(i) ? node.getValueUnsafe(i) : zero);
+            }
+            // Compress (optionally) and write out the contents of the array.
+            io::writeCompressedValues(os, values, NodeT::NUM_VALUES, valueMask, childMask, saveFloatAsHalf);
+        }
+    }
+
+    void operator()(const LeafT& leaf, size_t)
+    {
+        leaf.getValueMask().save(os);
+    }
+
+    std::ostream& os;
+    const bool saveFloatAsHalf;
+    const ValueT* background = nullptr;
+    Index rootChildLevel = std::numeric_limits<Index>::max();
+}; // struct WriteTopologyOp
+
+
+template <typename TreeT, typename StorageTreeT>
+struct ReadTopologyOp
+{
+    using ValueT = typename TreeT::ValueType;
+    using RootT = typename TreeT::RootNodeType;
+    using LeafT = typename TreeT::LeafNodeType;
+    using StorageValueT = typename StorageTreeT::ValueType;
+
+    ReadTopologyOp(std::istream& _is, bool _saveFloatAsHalf, io::ReadDiagnostics& _diagnostics,
+            const std::string& _gridName)
+        : is(_is)
+        , saveFloatAsHalf(_saveFloatAsHalf)
+        , diagnostics(_diagnostics)
+        , gridName(_gridName) { }
+
+    void operator()(RootT& root)
+    {
+        using ChildT = typename RootT::ChildNodeType;
+
+        int32_t bufferCount;
+        is.read(reinterpret_cast<char*>(&bufferCount), sizeof(int32_t));
+        if (bufferCount != 1) {
+            diagnostics.addWarning(gridName, "multi-buffer trees are no longer supported");
+        }
+
+        // Delete the existing tree.
+        root.clear();
+
+        // Read a RootNode that was stored in the current format.
+
+        if constexpr (std::is_same_v<ValueT, StorageValueT>) {
+            is.read(reinterpret_cast<char*>(&background), sizeof(ValueT));
+        } else {
+            StorageValueT _background;
+            is.read(reinterpret_cast<char*>(&_background), sizeof(StorageValueT));
+            background = static_cast<ValueT>(_background);
+        }
+
+        root.setBackground(background, false);
+        io::setGridBackgroundValuePtr(is, &root.background());
+
+        Index numTiles = 0, numChildren = 0;
+        is.read(reinterpret_cast<char*>(&numTiles), sizeof(Index));
+        is.read(reinterpret_cast<char*>(&numChildren), sizeof(Index));
+
+        Int32 vec[3];
+        StorageValueT value;
+        bool active;
+
+        // Read tiles.
+        for (Index n = 0; n < numTiles; ++n) {
+            is.read(reinterpret_cast<char*>(vec), 3 * sizeof(Int32));
+            is.read(reinterpret_cast<char*>(&value), sizeof(StorageValueT));
+            is.read(reinterpret_cast<char*>(&active), sizeof(bool));
+            Coord origin(vec);
+            if constexpr (std::is_same_v<ValueT, StorageValueT>) {
+                root.addTile(origin, value, active);
+            } else {
+                root.addTile(origin, static_cast<ValueT>(value), active);
+            }
+        }
+
+        // Read child nodes.
+        for (Index n = 0; n < numChildren; ++n) {
+            is.read(reinterpret_cast<char*>(vec), 3 * sizeof(Int32));
+            Coord origin(vec);
+            ChildT* child = new ChildT(PartialCreate(), origin, background);
+            (*this)(*child);
+            root.addChild(child);
+        }
+    }
+
+    template <typename NodeT>
+    void operator()(NodeT& node)
+    {
+        using ChildT = typename NodeT::ChildNodeType;
+        using NodeMaskT = typename NodeT::NodeMaskType;
+
+        NodeMaskT childMask, valueMask;
+        childMask.load(is);
+        valueMask.load(is);
+        node.setValueMaskUnsafe(valueMask);
+
+        const bool oldVersion =
+            (io::getFormatVersion(is) < OPENVDB_FILE_VERSION_NODE_MASK_COMPRESSION);
+        const Index numValues = (oldVersion ? childMask.countOff() : NodeT::NUM_VALUES);
+        {
+            // Read in (and uncompress, if necessary) all of this node's values
+            // into a contiguous array.
+            std::unique_ptr<StorageValueT[]> valuePtr(new StorageValueT[numValues]);
+            StorageValueT* values = valuePtr.get();
+            io::readCompressedValues(is, values, numValues, valueMask, saveFloatAsHalf);
+
+            // Copy values from the array into this node's table.
+            if (oldVersion) {
+                Index n = 0;
+                for (auto iter = node.beginValueAll(); iter; ++iter) {
+                    node.setValueOnlyUnsafe(iter.pos(), static_cast<ValueT>(values[n++]));
+                }
+                OPENVDB_ASSERT(n == numValues);
+            } else {
+                for (auto iter = node.beginValueAll(); iter; ++iter) {
+                    node.setValueOnlyUnsafe(iter.pos(), static_cast<ValueT>(values[iter.pos()]));
+                }
+            }
+        }
+
+        // Read in all child nodes and insert them into the table at their proper locations.
+        for (auto iter = childMask.beginOn(); iter; ++iter) {
+            Coord origin = node.offsetToGlobalCoord(iter.pos());
+            auto* child = new ChildT(PartialCreate(), origin, background);
+            (*this)(*child);
+            node.setChildUnsafe(iter.pos(), child);
+        }
+    }
+
+    void operator()(LeafT& leaf)
+    {
+        typename LeafT::NodeMaskType valueMask;
+        valueMask.load(is);
+        leaf.setValueMask(valueMask);
+    }
+
+    std::istream& is;
+    bool saveFloatAsHalf;
+    ValueT background;
+    io::ReadDiagnostics& diagnostics;
+    std::string gridName;
+}; // struct ReadTopologyOp
+
+template <typename TreeT>
+struct SetTilesToBackgroundOp
+{
+    using RootT = typename TreeT::RootNodeType;
+    using LeafT = typename TreeT::LeafNodeType;
+    using ValueT = typename TreeT::ValueType;
+
+    explicit SetTilesToBackgroundOp(const ValueT& background)
+        : mBackground(background) { }
+
+    bool operator()(RootT& root, size_t) const
+    {
+        for (auto it = root.beginValueAll(); it; ++it) {
+            it.setValue(mBackground);
+        }
+        return true;
+    }
+
+    template<typename NodeT>
+    bool operator()(NodeT& node, size_t) const
+    {
+        for (Index i = 0; i < NodeT::NUM_VALUES; ++i) {
+            if (node.isChildMaskOff(i)) {
+                node.setValueOnlyUnsafe(i, mBackground);
+            }
+        }
+        return true;
+    }
+
+    bool operator()(LeafT&, size_t) const { return true; }
+
+private:
+    const ValueT mBackground;
+}; // struct SetTilesToBackgroundOp
+
+template<typename TreeT>
+void setTilesToBackground(TreeT& tree)
+{
+    const typename TreeT::ValueType& background = tree.root().background();
+    SetTilesToBackgroundOp<TreeT> op(background);
+    tree::DynamicNodeManager<TreeT> nodeManager(tree);
+    nodeManager.foreachTopDown(op);
+}
+
+// Free-standing function for read case (supports type conversion via StorageGridT)
+template<typename GridT, typename StorageGridT = GridT>
+void topologyCodecReadTopology(GridBase& gridBase, std::istream& is, const io::ReadOptions& options, io::ReadDiagnostics& diagnostics)
+{
+    io::checkFormatVersion(is);
+
+    GridT& grid = static_cast<GridT&>(gridBase);
+    grid.tree().clearAllAccessors();
+
+    internal::ReadTopologyOp<typename GridT::TreeType, typename StorageGridT::TreeType> readTopologyOp(is, grid.saveFloatAsHalf(), diagnostics, grid.getName());
+    readTopologyOp(grid.tree().root());
+
+    if (options.readMode == io::ReadMode::TopologyOnly) {
+        internal::setTilesToBackground(grid.tree());
+        return;
+    }
+}
+
+// Free-standing function for write case (no StorageGridT needed)
+template<typename GridT>
+void topologyCodecWriteTopology(const GridBase& gridBase, std::ostream& os)
+{
+    const GridT& grid = static_cast<const GridT&>(gridBase);
+    using TreeType = typename GridT::TreeType;
+
+    internal::WriteTopologyOp<TreeType> writeTopologyOp(os, grid.saveFloatAsHalf());
+    tools::visitNodesDepthFirst(grid.tree(), writeTopologyOp);
+}
+
+} // namespace internal
+
+template <typename GridT, typename StorageGridT = GridT, io::CodecMode Mode = io::CodecMode::ReadWrite>
+struct TopologyCodec : public io::Codec
+{
+    using Ptr = std::unique_ptr<TopologyCodec<GridT, StorageGridT, Mode>>;
+
+    ~TopologyCodec() noexcept = default;
+
+    io::CodecData::Ptr createData() override
+    {
+        auto data = std::make_unique<io::CodecData>();
+        data->grid = GridT::create();
+        return data;
+    }
+
+    void readTopology(std::istream& is, io::CodecData& data, const io::ReadOptions& options,
+        io::ReadDiagnostics& diagnostics) final
+    {
+        internal::topologyCodecReadTopology<GridT, StorageGridT>(*data.grid, is, options, diagnostics);
+    }
+
+    void writeTopology(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) final
+    {
+        // disable implementation when read only
+        if constexpr (Mode == io::CodecMode::ReadOnly) return;
+
+        internal::topologyCodecWriteTopology<GridT>(gridBase, os);
+    }
+}; // struct TopologyCodec
+
+
+} // namespace codecs
+} // namespace OPENVDB_VERSION_NAME
+} // namespace openvdb
+
+#endif // OPENVDB_IO_CODECS_TOPOLOGYCODEC_HAS_BEEN_INCLUDED

openvdb/openvdb/codecs/ValueMaskCodec.h

@@ -0,0 +1,146 @@
+// Copyright Contributors to the OpenVDB Project
+// SPDX-License-Identifier: Apache-2.0
+
+#ifndef OPENVDB_IO_CODECS_VALUEMASKCODEC_HAS_BEEN_INCLUDED
+#define OPENVDB_IO_CODECS_VALUEMASKCODEC_HAS_BEEN_INCLUDED
+
+#include <openvdb/io/Codec.h>
+
+#include <openvdb/openvdb.h>
+#include <openvdb/tools/Clip.h>
+
+#include "TopologyCodec.h"
+
+namespace openvdb {
+OPENVDB_USE_VERSION_NAMESPACE
+namespace OPENVDB_VERSION_NAME {
+namespace codecs {
+namespace internal {
+
+template <typename GridT>
+struct ReadValueMaskBuffersOp
+{
+    using TreeT = typename GridT::TreeType;
+    using RootT = typename TreeT::RootNodeType;
+    using LeafT = typename TreeT::LeafNodeType;
+    using ValueT = typename TreeT::ValueType;
+
+    ReadValueMaskBuffersOp(std::istream& _is, bool _saveFloatAsHalf,
+        const ValueT& _background, const CoordBBox* _clipBBox)
+        : is(_is)
+        , saveFloatAsHalf(_saveFloatAsHalf)
+        , background(_background)
+        , clipBBox(_clipBBox) { }
+
+    void operator()(RootT& root, size_t)
+    {
+        // Clip root-level tiles and prune children that were clipped.
+        if (clipBBox) {
+            root.clip(*clipBBox);
+        }
+    }
+
+    template <typename NodeT>
+    void operator()(NodeT& node, size_t)
+    {
+        // Clip internal node tiles and prune children that were clipped.
+        if (clipBBox) {
+            node.clip(*clipBBox, background);
+        }
+    }
+
+    void operator()(LeafT& leaf, size_t)
+    {
+        // Read in the value mask.
+        leaf.getValueMask().load(is);
+        // Read in the origin.
+        Coord origin;
+        origin.read(is);
+        leaf.setOrigin(origin);
+    }
+
+    std::istream& is;
+    const bool saveFloatAsHalf;
+    const ValueT& background;
+    const CoordBBox* clipBBox = nullptr;
+}; // struct ReadValueMaskBuffersOp
+
+template <typename GridT>
+struct WriteValueMaskBuffersOp
+{
+    using TreeT = typename GridT::TreeType;
+    using LeafT = typename TreeT::LeafNodeType;
+
+    WriteValueMaskBuffersOp(std::ostream& _os, bool _saveFloatAsHalf)
+        : os(_os)
+        , saveFloatAsHalf(_saveFloatAsHalf) { }
+
+    template <typename NodeT>
+    void operator()(const NodeT&, size_t) { }
+
+    void operator()(const LeafT& leaf, size_t)
+    {
+        // Write out the value mask.
+        leaf.getValueMask().save(os);
+
+        // Write out the origin.
+        leaf.origin().write(os);
+    }
+
+    std::ostream& os;
+    const bool saveFloatAsHalf;
+}; // struct WriteValueMaskBuffersOp
+
+} // namespace internal
+
+template <typename GridT>
+struct ValueMaskCodec final: public TopologyCodec<GridT>
+{
+    using Ptr = std::unique_ptr<ValueMaskCodec<GridT>>;
+
+    ~ValueMaskCodec() noexcept = default;
+
+    static inline std::string name() { return GridT::gridType(); }
+
+    void readBuffers(std::istream& is, io::CodecData& data, const io::ReadOptions& options, io::ReadDiagnostics&) final
+    {
+        GridT& grid = static_cast<GridT&>(*data.grid);
+
+        if (grid.hasMultiPassIO()) {
+            OPENVDB_THROW(IoError, "Multi-pass IO is not supported in ValueMaskCodec");
+        }
+
+        io::checkFormatVersion(is);
+
+        bool saveFloatAsHalf = grid.saveFloatAsHalf();
+
+        auto& tree = grid.tree();
+        tree.clearAllAccessors();
+
+        std::unique_ptr<CoordBBox> clipIndexBBox;
+        if (options.clipBBox.isSorted()) {
+            clipIndexBBox = std::make_unique<CoordBBox>(grid.constTransform().worldToIndexNodeCentered(options.clipBBox));
+        }
+
+        internal::ReadValueMaskBuffersOp<GridT> readBuffersOp(is, saveFloatAsHalf, tree.background(), clipIndexBBox.get());
+        tools::visitNodesDepthFirst(grid.tree(), readBuffersOp, /*idx=*/0, /*topDown=*/false);
+    }
+
+    void writeBuffers(std::ostream& os, const GridBase& gridBase, const io::WriteOptions&) final
+    {
+        const GridT& grid = static_cast<const GridT&>(gridBase);
+
+        if (grid.hasMultiPassIO()) {
+            OPENVDB_THROW(IoError, "Multi-pass IO is not supported in ValueMaskCodec");
+        }
+
+        internal::WriteValueMaskBuffersOp<GridT> writeBuffersOp(os, grid.saveFloatAsHalf());
+        tools::visitNodesDepthFirst(grid.tree(), writeBuffersOp);
+    }
+}; // struct ValueMaskCodec
+
+} // namespace codecs
+} // namespace OPENVDB_VERSION_NAME
+} // namespace openvdb
+
+#endif // OPENVDB_IO_CODECS_VALUEMASKCODEC_HAS_BEEN_INCLUDED

openvdb/openvdb/codecs/impl/ScalarLeafCodec.h

@@ -0,0 +1,123 @@
+// Copyright Contributors to the OpenVDB Project
+// SPDX-License-Identifier: Apache-2.0
+
+#ifndef OPENVDB_IO_CODECS_IMPL_SCALARLEAFCODEC_HAS_BEEN_INCLUDED
+#define OPENVDB_IO_CODECS_IMPL_SCALARLEAFCODEC_HAS_BEEN_INCLUDED
+
+#include <openvdb/io/Compression.h>
+
+namespace openvdb {
+OPENVDB_USE_VERSION_NAMESPACE
+namespace OPENVDB_VERSION_NAME {
+namespace codecs {
+namespace internal {
+
+template <typename LeafT>
+void writeScalarLeafBuffers(const LeafT& leaf, std::ostream& os, bool saveFloatAsHalf)
+{
+    using NodeMaskT = typename LeafT::NodeMaskType;
+
+    // Write out the value mask.
+    leaf.getValueMask().save(os);
+
+    leaf.buffer().data(); // load values
+
+    io::writeCompressedValues(os, leaf.buffer().data(), LeafT::SIZE,
+        leaf.getValueMask(), /*childMask=*/NodeMaskT(), saveFloatAsHalf);
+}
+
+
+template <typename LeafT, typename StorageLeafT = LeafT>
+void readScalarLeafBuffers(LeafT& leaf, std::istream& is, bool saveFloatAsHalf,
+    const typename LeafT::ValueType& background, bool skip = false,
+    const math::CoordBBox* clipBBox = nullptr)
+{
+    using ValueT = typename LeafT::ValueType;
+    using NodeMaskT = typename LeafT::NodeMaskType;
+    using StorageBufferT = typename StorageLeafT::Buffer;
+    using StorageValueT = typename StorageLeafT::ValueType;
+
+    constexpr Index SIZE = LeafT::SIZE;
+
+    SharedPtr<io::StreamMetadata> meta = io::getStreamMetadataPtr(is);
+    const bool seekable = meta && meta->seekable();
+
+    auto& valueMask = leaf.getValueMask();
+
+    // Load or seek the value mask
+    if (seekable)   valueMask.seek(is);
+    else            valueMask.load(is);
+
+    // Pre-node-mask-compression format stored the origin and buffer count
+    // inline after the value mask.
+    int8_t numBuffers = 1;
+    if (io::getFormatVersion(is) < OPENVDB_FILE_VERSION_NODE_MASK_COMPRESSION) {
+        Coord origin;
+        is.read(reinterpret_cast<char*>(&origin), sizeof(Coord::ValueType) * 3);
+        leaf.setOrigin(origin);
+
+        is.read(reinterpret_cast<char*>(&numBuffers), sizeof(int8_t));
+
+        if (numBuffers > 1) {
+            OPENVDB_THROW(IoError, "Old file format 221 (FLOAT_FRUSTUM_BBOX) with multiple buffers is not supported");
+        }
+    }
+
+    if (skip) {
+        if (seekable) {
+            io::readCompressedValues<StorageValueT, NodeMaskT>(is, nullptr, SIZE, valueMask, saveFloatAsHalf);
+        } else {
+            StorageBufferT storageTemp;
+            io::readCompressedValues(is, storageTemp.data(), SIZE, valueMask, saveFloatAsHalf);
+        }
+        // Clear the value mask so that the skipped leaf has no active
+        // voxels. Without this, the leaf retains its on-disk active
+        // topology even though no data was read into its buffer.
+        valueMask.setOff();
+        return;
+    }
+
+    if constexpr (std::is_same_v<typename LeafT::BuildType, ValueMask>) {
+        // ValueMask leaf: value == active state, already captured in the value mask above.
+        // Seek/consume past the storage buffer without populating any separate leaf buffer.
+        if (seekable) {
+            io::readCompressedValues<StorageValueT, NodeMaskT>(is, nullptr, SIZE, valueMask, saveFloatAsHalf);
+        } else {
+            StorageBufferT storageTemp;
+            io::readCompressedValues(is, storageTemp.data(), SIZE, valueMask, saveFloatAsHalf);
+        }
+    } else if constexpr (std::is_same_v<ValueT, bool>) {
+        // Bool leaf: must read storage values regardless of seekability, then convert.
+        if constexpr (std::is_same_v<StorageValueT, bool>) {
+            io::readCompressedValues(is, leaf.buffer().data(), SIZE, valueMask, saveFloatAsHalf);
+        } else {
+            StorageBufferT storageTemp;
+            io::readCompressedValues(is, storageTemp.data(), SIZE, valueMask, saveFloatAsHalf);
+            for (Index i = 0; i < SIZE; ++i) {
+                leaf.buffer().setValue(i, static_cast<bool>(storageTemp.getValue(i)));
+            }
+        }
+    } else {
+        leaf.buffer().allocate();
+        if constexpr (std::is_same_v<StorageValueT, ValueT>) {
+            io::readCompressedValues(is, leaf.buffer().data(), SIZE, valueMask, saveFloatAsHalf);
+        } else {
+            StorageBufferT storageTemp;
+            io::readCompressedValues(is, storageTemp.data(), SIZE, valueMask, saveFloatAsHalf);
+            for (Index i = 0; i < SIZE; ++i) {
+                leaf.buffer().setValue(i, static_cast<ValueT>(storageTemp.getValue(i)));
+            }
+        }
+    }
+
+    if (clipBBox) {
+        leaf.clip(*clipBBox, background);
+    }
+}
+
+} // namespace internal
+} // namespace codecs
+} // namespace OPENVDB_VERSION_NAME
+} // namespace openvdb
+
+#endif // OPENVDB_IO_CODECS_IMPL_SCALARLEAFCODEC_HAS_BEEN_INCLUDED

openvdb/openvdb/io/Archive.cc

@@ -4,7 +4,6 @@
 #include "Archive.h"
 
 #include "GridDescriptor.h"
-#include "DelayedLoadMetadata.h"
 #include "io.h"
 
 #include <openvdb/Exceptions.h>
@@ -13,35 +12,6 @@
 #include <openvdb/util/logging.h>
 #include <openvdb/openvdb.h>
 
-#ifdef OPENVDB_USE_DELAYED_LOADING
-// Boost.Interprocess uses a header-only portion of Boost.DateTime
-#ifdef __clang__
-#pragma clang diagnostic push
-#pragma clang diagnostic ignored "-Wunused-macros"
-#endif
-#define BOOST_DATE_TIME_NO_LIB
-#ifdef __clang__
-#pragma clang diagnostic pop
-#endif
-#include <boost/interprocess/file_mapping.hpp>
-#include <boost/interprocess/mapped_region.hpp>
-#include <boost/iostreams/device/array.hpp>
-#include <boost/iostreams/stream.hpp>
-
-#ifdef _WIN32
-#include <boost/interprocess/detail/os_file_functions.hpp> // open_existing_file(), close_file()
-extern "C" __declspec(dllimport) bool __stdcall GetFileTime(
-    void* fh, void* ctime, void* atime, void* mtime);
-// boost::interprocess::detail was renamed to boost::interprocess::ipcdetail in Boost 1.48.
-// Ensure that both namespaces exist.
-namespace boost { namespace interprocess { namespace detail {} namespace ipcdetail {} } }
-#else
-#include <sys/types.h> // for struct stat
-#include <sys/stat.h> // for stat()
-#include <unistd.h> // for unlink()
-#endif
-#endif // OPENVDB_USE_DELAYED_LOADING
-
 #include <atomic>
 
 #include <algorithm> // for std::find_if()
@@ -209,7 +179,7 @@ struct StreamMetadata::Impl
     uint32_t mPass = 0;
     MetaMap mGridMetadata;
     AuxDataMap mAuxData;
-    bool mDelayedLoadMeta = DelayedLoadMetadata::isRegisteredType();
+    bool mDelayedLoadMeta = false;
     uint64_t mLeaf = 0;
     uint32_t mTest = 0; // for testing only
 }; // struct StreamMetadata
@@ -274,7 +244,6 @@ bool            StreamMetadata::seekable() const        { return mImpl->mSeekabl
 bool            StreamMetadata::delayedLoadMeta() const { return mImpl->mDelayedLoadMeta; }
 bool            StreamMetadata::countingPasses() const  { return mImpl->mCountingPasses; }
 uint32_t        StreamMetadata::pass() const            { return mImpl->mPass; }
-uint64_t        StreamMetadata::leaf() const            { return mImpl->mLeaf; }
 MetaMap&        StreamMetadata::gridMetadata()          { return mImpl->mGridMetadata; }
 const MetaMap&  StreamMetadata::gridMetadata() const    { return mImpl->mGridMetadata; }
 uint32_t        StreamMetadata::__test() const          { return mImpl->mTest; }
@@ -292,7 +261,6 @@ void StreamMetadata::setWriteGridStats(bool b)          { mImpl->mWriteGridStats
 void StreamMetadata::setSeekable(bool b)                { mImpl->mSeekable = b; }
 void StreamMetadata::setCountingPasses(bool b)          { mImpl->mCountingPasses = b; }
 void StreamMetadata::setPass(uint32_t i)                { mImpl->mPass = i; }
-void StreamMetadata::setLeaf(uint64_t i)                { mImpl->mLeaf = i; }
 void StreamMetadata::__setTest(uint32_t t)              { mImpl->mTest = t; }
 
 std::string
@@ -306,7 +274,6 @@ StreamMetadata::str() const
     ostr << "compression: " << compressionToString(compression()) << "\n";
     ostr << "half_float: " << halfFloat() << "\n";
     ostr << "seekable: " << seekable() << "\n";
-    ostr << "delayed_load_meta: " << delayedLoadMeta() << "\n";
     ostr << "pass: " << pass() << "\n";
     ostr << "counting_passes: " << countingPasses() << "\n";
     ostr << "write_grid_stats_metadata: " << writeGridStats() << "\n";
@@ -339,73 +306,6 @@ writeAsType(std::ostream& os, const std::any& val)
     return false;
 }
 
-struct PopulateDelayedLoadMetadataOp
-{
-    DelayedLoadMetadata& metadata;
-    uint32_t compression;
-
-    PopulateDelayedLoadMetadataOp(DelayedLoadMetadata& _metadata, uint32_t _compression)
-        : metadata(_metadata)
-        , compression(_compression) { }
-
-    template<typename GridT>
-    void operator()(const GridT& grid) const
-    {
-        using TreeT = typename GridT::TreeType;
-        using ValueT = typename TreeT::ValueType;
-        using LeafT = typename TreeT::LeafNodeType;
-        using MaskT = typename LeafT::NodeMaskType;
-
-        const TreeT& tree = grid.constTree();
-        const Index64 leafCount = tree.leafCount();
-
-        // early exit if not leaf nodes
-        if (leafCount == Index64(0))    return;
-
-        metadata.resizeMask(leafCount);
-
-        if (compression & (COMPRESS_BLOSC | COMPRESS_ZIP)) {
-            metadata.resizeCompressedSize(leafCount);
-        }
-
-        const auto background = tree.background();
-        const bool saveFloatAsHalf = grid.saveFloatAsHalf();
-
-        tree::LeafManager<const TreeT> leafManager(tree);
-
-        leafManager.foreach(
-            [&](const LeafT& leaf, size_t idx) {
-                // set mask value
-                MaskCompress<ValueT, MaskT> maskCompressData(
-                    leaf.valueMask(), /*childMask=*/MaskT(), leaf.buffer().data(), background);
-                metadata.setMask(idx, maskCompressData.metadata);
-
-                if (compression & (COMPRESS_BLOSC | COMPRESS_ZIP)) {
-                    // set compressed size value
-                    size_t sizeBytes(8);
-                    size_t compressedSize = io::writeCompressedValuesSize(
-                        leaf.buffer().data(), LeafT::SIZE,
-                        leaf.valueMask(), maskCompressData.metadata, saveFloatAsHalf, compression);
-                    metadata.setCompressedSize(idx, compressedSize+sizeBytes);
-                }
-            }
-        );
-    }
-};
-
-bool populateDelayedLoadMetadata(DelayedLoadMetadata& metadata,
-    const GridBase& gridBase, uint32_t compression)
-{
-    PopulateDelayedLoadMetadataOp op(metadata, compression);
-
-    using AllowedTypes = TypeList<
-        Int32Grid, Int64Grid,
-        FloatGrid, DoubleGrid,
-        Vec3IGrid, Vec3SGrid, Vec3DGrid>;
-
-    return gridBase.apply<AllowedTypes>(op);
-}
-
 } // unnamed namespace
 
 std::ostream&
@@ -443,117 +343,6 @@ operator<<(std::ostream& os, const StreamMetadata::AuxDataMap& auxData)
 ////////////////////////////////////////
 
 
-#ifdef OPENVDB_USE_DELAYED_LOADING
-
-
-// Memory-mapping a VDB file permits threaded input (and output, potentially,
-// though that might not be practical for compressed files or files containing
-// multiple grids).  In particular, a memory-mapped file can be loaded lazily,
-// meaning that the voxel buffers of the leaf nodes of a grid's tree are not allocated
-// until they are actually accessed.  When access to its buffer is requested,
-// a leaf node allocates memory for the buffer and then streams in (and decompresses)
-// its contents from the memory map, starting from a stream offset that was recorded
-// at the time the node was constructed.  The memory map must persist as long as
-// there are unloaded leaf nodes; this is ensured by storing a shared pointer
-// to the map in each unloaded node.
-
-class MappedFile::Impl
-{
-public:
-    Impl(const std::string& filename, bool autoDelete)
-        : mMap(filename.c_str(), boost::interprocess::read_only)
-        , mRegion(mMap, boost::interprocess::read_only)
-        , mAutoDelete(autoDelete)
-    {
-        if (mAutoDelete) {
-#ifndef _WIN32
-            // On Unix systems, unlink the file so that it gets deleted once it is closed.
-            ::unlink(mMap.get_name());
-#endif
-        }
-    }
-
-    ~Impl()
-    {
-        std::string filename;
-        if (const char* s = mMap.get_name()) filename = s;
-        OPENVDB_LOG_DEBUG_RUNTIME("closing memory-mapped file " << filename);
-        if (mNotifier) mNotifier(filename);
-        if (mAutoDelete) {
-            if (!boost::interprocess::file_mapping::remove(filename.c_str())) {
-                if (errno != ENOENT) {
-                    // Warn if the file exists but couldn't be removed.
-                    std::string mesg = getErrorString();
-                    if (!mesg.empty()) mesg = " (" + mesg + ")";
-                    OPENVDB_LOG_WARN("failed to remove temporary file " << filename << mesg);
-                }
-            }
-        }
-    }
-
-    boost::interprocess::file_mapping mMap;
-    boost::interprocess::mapped_region mRegion;
-    bool mAutoDelete;
-    Notifier mNotifier;
-#if OPENVDB_ABI_VERSION_NUMBER <= 12
-    mutable std::atomic<Index64> mLastWriteTime;
-#endif
-
-private:
-    Impl(const Impl&); // not copyable
-    Impl& operator=(const Impl&); // not copyable
-};
-
-
-MappedFile::MappedFile(const std::string& filename, bool autoDelete):
-    mImpl(new Impl(filename, autoDelete))
-{
-}
-
-
-MappedFile::~MappedFile()
-{
-}
-
-
-std::string
-MappedFile::filename() const
-{
-    std::string result;
-    if (const char* s = mImpl->mMap.get_name()) result = s;
-    return result;
-}
-
-
-SharedPtr<std::streambuf>
-MappedFile::createBuffer() const
-{
-    return SharedPtr<std::streambuf>{
-        new boost::iostreams::stream_buffer<boost::iostreams::array_source>{
-            static_cast<const char*>(mImpl->mRegion.get_address()), mImpl->mRegion.get_size()}};
-}
-
-
-void
-MappedFile::setNotifier(const Notifier& notifier)
-{
-    mImpl->mNotifier = notifier;
-}
-
-
-void
-MappedFile::clearNotifier()
-{
-    mImpl->mNotifier = nullptr;
-}
-
-
-#endif // OPENVDB_USE_DELAYED_LOADING
-
-
-////////////////////////////////////////
-
-
 std::string
 getErrorString(int errorNum)
 {
@@ -595,6 +384,35 @@ Archive::copy() const
 }
 
 
+void
+Archive::enableReadDiagnostics()
+{
+    mReadDiagnostics.enable();
+}
+
+
+void
+Archive::disableReadDiagnostics()
+{
+    mReadDiagnostics.disable();
+    mReadDiagnostics.clear();
+}
+
+
+const ReadDiagnostics&
+Archive::readDiagnostics() const
+{
+    return mReadDiagnostics;
+}
+
+
+void
+Archive::clearReadDiagnostics()
+{
+    mReadDiagnostics.clear();
+}
+
+
 ////////////////////////////////////////
 
 
@@ -888,25 +706,6 @@ setGridBackgroundValuePtr(std::ios_base& strm, const void* background)
 }
 
 
-#ifdef OPENVDB_USE_DELAYED_LOADING
-MappedFile::Ptr
-getMappedFilePtr(std::ios_base& strm)
-{
-    if (const void* ptr = strm.pword(GetSteamState().mappedFile)) {
-        return *static_cast<const MappedFile::Ptr*>(ptr);
-    }
-    return MappedFile::Ptr();
-}
-
-
-void
-setMappedFilePtr(std::ios_base& strm, io::MappedFile::Ptr& mappedFile)
-{
-    strm.pword(GetSteamState().mappedFile) = &mappedFile;
-}
-#endif // OPENVDB_USE_DELAYED_LOADING
-
-
 StreamMetadata::Ptr
 getStreamMetadataPtr(std::ios_base& strm)
 {
@@ -1095,6 +894,27 @@ Archive::readGridCount(std::istream& is)
 ////////////////////////////////////////
 
 
+io::Codec*
+Archive::findCodec(const std::string& gridType, const io::ReadOptions& options)
+{
+    // if readMode is Half, then search for a codec that converts
+    // from the storage grid type to the grid type
+    if (options.readMode == ReadMode::Half) {
+        return io::CodecRegistry::get(gridType + "_to_half");
+    } else if (options.readMode == ReadMode::Bool) {
+        return io::CodecRegistry::get(gridType + "_to_bool");
+    } else if (options.readMode == ReadMode::Mask) {
+        return io::CodecRegistry::get(gridType + "_to_mask");
+    }
+
+    // Determine the I/O codec to use to read this grid
+    return io::CodecRegistry::get(gridType);
+}
+
+
+////////////////////////////////////////
+
+
 void
 Archive::connectInstance(const GridDescriptor& gd, const NamedGridMap& grids) const
 {
@@ -1126,35 +946,12 @@ Archive::connectInstance(const GridDescriptor& gd, const NamedGridMap& grids) co
 ////////////////////////////////////////
 
 
-//static
-bool
-Archive::isDelayedLoadingEnabled()
+GridBase::Ptr
+Archive::readGrid(const GridDescriptor& gd, std::istream& is, const io::ReadOptions& readOptions, ReadDiagnostics& diagnostics)
 {
-#ifdef OPENVDB_USE_DELAYED_LOADING
-    return (nullptr == std::getenv("OPENVDB_DISABLE_DELAYED_LOAD"));
-#else
-    return false;
-#endif
-}
-
-
-namespace {
-
-struct NoBBox {};
-
-template<typename BoxType>
-void
-doReadGrid(GridBase::Ptr grid, const GridDescriptor& gd, std::istream& is, const BoxType& bbox)
-{
-    struct Local {
-        static void readBuffers(GridBase& g, std::istream& istrm, NoBBox) { g.readBuffers(istrm); }
-        static void readBuffers(GridBase& g, std::istream& istrm, const CoordBBox& indexBBox) {
-            g.readBuffers(istrm, indexBBox);
-        }
-        static void readBuffers(GridBase& g, std::istream& istrm, const BBoxd& worldBBox) {
-            g.readBuffers(istrm, g.constTransform().worldToIndexNodeCentered(worldBBox));
-        }
-    };
+    // Read the compression settings for this grid and tag the stream with them
+    // so that downstream functions can reference them.
+    readGridCompression(is);
 
     // Restore the file-level stream metadata on exit.
     struct OnExit {
@@ -1165,6 +962,27 @@ doReadGrid(GridBase::Ptr grid, const GridDescriptor& gd, std::istream& is, const
     };
     OnExit restore(is);
 
+    // Find the codec for the grid type and options.
+    io::Codec* codec = findCodec(gd.gridType(), readOptions);
+
+    GridBase::Ptr grid;
+    io::CodecData::Ptr codecData;
+
+    // Create the grid.
+    if (codec) {
+        codecData = codec->createData();
+        if (codecData->grid)    grid = codecData->grid;
+    } else {
+        if (!GridBase::isRegistered(gd.gridType())) {
+            OPENVDB_THROW(KeyError, "Cannot read grid "
+                << GridDescriptor::nameAsString(gd.uniqueName())
+                << ": grid type " << gd.gridType() << " is not registered");
+        }
+
+        grid = GridBase::createGrid(gd.gridType());
+    }
+    grid->setSaveFloatAsHalf(gd.saveFloatAsHalf());
+
     // Stream metadata varies per grid, and it needs to persist
     // in case delayed load is in effect.
     io::StreamMetadata::Ptr streamMetadata;
@@ -1182,75 +1000,47 @@ doReadGrid(GridBase::Ptr grid, const GridDescriptor& gd, std::istream& is, const
 
     grid->readMeta(is);
 
-    // Add a description of the compression settings to the grid as metadata.
-    /// @todo Would this be useful?
-    //const uint32_t c = getDataCompression(is);
-    //grid->insertMeta(GridBase::META_FILE_COMPRESSION,
-    //    StringMetadata(compressionToString(c)));
-
-    const VersionId version = getLibraryVersion(is);
-    if (version.first < 6 || (version.first == 6 && version.second <= 1)) {
-        // If delay load metadata exists, but the file format version does not support
-        // delay load metadata, this likely means the original grid was read and then
-        // written using a prior version of OpenVDB and ABI>=5 where unknown metadata
-        // can be blindly copied. This means that it is possible for the metadata to
-        // no longer be in sync with the grid, so we remove it to ensure correctness.
-
-        if ((*grid)[GridBase::META_FILE_DELAYED_LOAD]) {
-            grid->removeMeta(GridBase::META_FILE_DELAYED_LOAD);
-        }
+    // Delayed loading is no longer supported - always remove metadata related to delayed loading if it exists
+    if ((*grid)[GridBase::META_FILE_DELAYED_LOAD]) {
+        grid->removeMeta(GridBase::META_FILE_DELAYED_LOAD);
     }
 
     streamMetadata->gridMetadata() = static_cast<MetaMap&>(*grid);
     const GridClass gridClass = grid->getGridClass();
     io::setGridClass(is, gridClass);
 
-    // reset leaf value to zero
-    streamMetadata->setLeaf(0);
-
-    // drop DelayedLoadMetadata from the grid as it is only useful for IO
-    // a stream metadata non-zero value disables this behaviour for testing
-
-    if (streamMetadata->__test() == uint32_t(0)) {
-        if ((*grid)[GridBase::META_FILE_DELAYED_LOAD]) {
-            grid->removeMeta(GridBase::META_FILE_DELAYED_LOAD);
+    grid->readTransform(is);
+    if (readOptions.readMode != io::ReadMode::TopologyOnly && !gd.isInstance()) {
+        // read topology
+        if (codec) {
+            codec->readTopology(is, *codecData, readOptions, diagnostics);
+        } else {
+            grid->readTopology(is);
+        }
+        // read buffers
+        if (codec) {
+            codec->readBuffers(is, *codecData, readOptions, diagnostics);
+        } else {
+            const auto& worldBBox = readOptions.clipBBox;
+            const bool clip = worldBBox.isSorted();
+            if (clip) {
+                const auto indexBBox = grid->constTransform().worldToIndexNodeCentered(worldBBox);
+                grid->readBuffers(is, indexBBox);
+            } else {
+                grid->readBuffers(is);
+            }
         }
     }
 
-    grid->readTransform(is);
-    if (!gd.isInstance()) {
-        grid->readTopology(is);
-        Local::readBuffers(*grid, is, bbox);
-    }
+    return grid;
 }
 
-} // unnamed namespace
 
-
-void
-Archive::readGrid(GridBase::Ptr grid, const GridDescriptor& gd, std::istream& is)
+GridBase::Ptr
+Archive::readGrid(const GridDescriptor& gd, std::istream& is, const io::ReadOptions& readOptions)
 {
-    // Read the compression settings for this grid and tag the stream with them
-    // so that downstream functions can reference them.
-    readGridCompression(is);
-
-    doReadGrid(grid, gd, is, NoBBox());
-}
-
-void
-Archive::readGrid(GridBase::Ptr grid, const GridDescriptor& gd,
-    std::istream& is, const BBoxd& worldBBox)
-{
-    readGridCompression(is);
-    doReadGrid(grid, gd, is, worldBBox);
-}
-
-void
-Archive::readGrid(GridBase::Ptr grid, const GridDescriptor& gd,
-    std::istream& is, const CoordBBox& indexBBox)
-{
-    readGridCompression(is);
-    doReadGrid(grid, gd, is, indexBBox);
+    ReadDiagnostics nullDiagnostics;
+    return readGrid(gd, is, readOptions, nullDiagnostics);
 }
 
 
@@ -1259,15 +1049,15 @@ Archive::readGrid(GridBase::Ptr grid, const GridDescriptor& gd,
 
 void
 Archive::write(std::ostream& os, const GridPtrVec& grids, bool seekable,
-    const MetaMap& metadata) const
+    const MetaMap& metadata, const io::WriteOptions& writeOptions) const
 {
-    this->write(os, GridCPtrVec(grids.begin(), grids.end()), seekable, metadata);
+    this->write(os, GridCPtrVec(grids.begin(), grids.end()), seekable, metadata, writeOptions);
 }
 
 
 void
 Archive::write(std::ostream& os, const GridCPtrVec& grids, bool seekable,
-    const MetaMap& metadata) const
+    const MetaMap& metadata, const io::WriteOptions& writeOptions) const
 {
     // Set stream flags so that downstream functions can reference them.
     io::StreamMetadata::Ptr streamMetadata = io::getStreamMetadataPtr(os);
@@ -1339,7 +1129,7 @@ Archive::write(std::ostream& os, const GridCPtrVec& grids, bool seekable,
                 // Get the name of the other grid.
                 gd.setInstanceParentName(mapIter->second.uniqueName());
                 // Write out this grid's descriptor and metadata, but not its tree.
-                writeGridInstance(gd, grid, os, seekable);
+                writeGridInstance(gd, grid, os, seekable, writeOptions);
 
                 OPENVDB_LOG_DEBUG_RUNTIME("io::Archive::write(): "
                     << GridDescriptor::nameAsString(gd.uniqueName())
@@ -1348,7 +1138,7 @@ Archive::write(std::ostream& os, const GridCPtrVec& grids, bool seekable,
                     << GridDescriptor::nameAsString(gd.instanceParentName()));
             } else {
                 // Write out the grid descriptor and its associated grid.
-                writeGrid(gd, grid, os, seekable);
+                writeGrid(gd, grid, os, seekable, writeOptions);
                 // Record the grid's tree pointer so that the tree doesn't get written
                 // more than once.
                 treeMap[treePtr] = gd;
@@ -1364,7 +1154,7 @@ Archive::write(std::ostream& os, const GridCPtrVec& grids, bool seekable,
 
 void
 Archive::writeGrid(GridDescriptor& gd, GridBase::ConstPtr grid,
-    std::ostream& os, bool seekable) const
+    std::ostream& os, bool seekable, const io::WriteOptions& writeOptions) const
 {
     // Restore file-level stream metadata on exit.
     struct OnExit {
@@ -1375,6 +1165,9 @@ Archive::writeGrid(GridDescriptor& gd, GridBase::ConstPtr grid,
     };
     OnExit restore(os);
 
+    // Find the codec for the grid type and options.
+    io::Codec* codec = findCodec(gd.gridType());
+
     // Stream metadata varies per grid, so make a copy of the file-level stream metadata.
     io::StreamMetadata::Ptr streamMetadata;
     if (io::StreamMetadata::Ptr meta = io::getStreamMetadataPtr(os)) {
@@ -1403,35 +1196,36 @@ Archive::writeGrid(GridDescriptor& gd, GridBase::ConstPtr grid,
     // Save the compression settings for this grid.
     setGridCompression(os, *grid);
 
-    // copy grid and add delay load metadata
-    const auto copyOfGrid = grid->copyGrid(); // shallow copy
-    const auto nonConstCopyOfGrid = ConstPtrCast<GridBase>(copyOfGrid);
-    nonConstCopyOfGrid->insertMeta(GridBase::META_FILE_DELAYED_LOAD,
-        DelayedLoadMetadata());
-    DelayedLoadMetadata::Ptr delayLoadMeta =
-        nonConstCopyOfGrid->getMetadata<DelayedLoadMetadata>(GridBase::META_FILE_DELAYED_LOAD);
-    if (!populateDelayedLoadMetadata(*delayLoadMeta, *grid, compression())) {
-        nonConstCopyOfGrid->removeMeta(GridBase::META_FILE_DELAYED_LOAD);
-    }
-
     // Save the grid's metadata and transform.
     if (getWriteGridStatsMetadata(os)) {
         // Compute and add grid statistics metadata.
+        const auto copyOfGrid = grid->copyGrid(); // shallow copy
+        const auto nonConstCopyOfGrid = ConstPtrCast<GridBase>(copyOfGrid);
         nonConstCopyOfGrid->addStatsMetadata();
         nonConstCopyOfGrid->insertMeta(GridBase::META_FILE_COMPRESSION,
             StringMetadata(compressionToString(getDataCompression(os))));
+        copyOfGrid->writeMeta(os);
+    } else {
+        grid->writeMeta(os);
     }
-    copyOfGrid->writeMeta(os);
     grid->writeTransform(os);
 
     // Save the grid's structure.
-    grid->writeTopology(os);
+    if (codec) {
+        codec->writeTopology(os, *grid, writeOptions);
+    } else {
+        grid->writeTopology(os);
+    }
 
     // Now we know the grid block storage position.
     if (seekable) gd.setBlockPos(os.tellp());
 
     // Save out the data blocks of the grid.
-    grid->writeBuffers(os);
+    if (codec) {
+        codec->writeBuffers(os, *grid, writeOptions);
+    } else {
+        grid->writeBuffers(os);
+    }
 
     // Now we know the end position of this grid.
     if (seekable) gd.setEndPos(os.tellp());
@@ -1450,7 +1244,7 @@ Archive::writeGrid(GridDescriptor& gd, GridBase::ConstPtr grid,
 
 void
 Archive::writeGridInstance(GridDescriptor& gd, GridBase::ConstPtr grid,
-    std::ostream& os, bool seekable) const
+    std::ostream& os, bool seekable, const io::WriteOptions&) const
 {
     // Write out the Descriptor's header information (grid name, type
     // and instance parent name).

openvdb/openvdb/io/Archive.h

@@ -5,11 +5,14 @@
 #define OPENVDB_IO_ARCHIVE_HAS_BEEN_INCLUDED
 
 #include <openvdb/version.h>
-#include "Compression.h" // for COMPRESS_ZIP, etc.
 #include <openvdb/Grid.h>
 #include <openvdb/MetaMap.h>
 #include <openvdb/Platform.h>
 #include <openvdb/version.h> // for VersionId
+
+#include "Codec.h"
+#include "Compression.h" // for COMPRESS_ZIP, etc.
+
 #include <cstdint>
 #include <iosfwd>
 #include <map>
@@ -89,14 +92,23 @@ public:
     void setGridStatsMetadataEnabled(bool b) { mEnableGridStats = b; }
 
     /// @brief Write the grids in the given container to this archive's output stream.
-    virtual void write(const GridCPtrVec&, const MetaMap& = MetaMap()) const {}
+    virtual void write(const GridCPtrVec&, const MetaMap& = MetaMap(),
+        const io::WriteOptions& = io::WriteOptions{}) const {}
 
-    /// @brief Return @c true if delayed loading is enabled.
-    /// @details If enabled, delayed loading can be disabled for individual files,
-    /// but not vice-versa.
-    /// @note Define the environment variable @c OPENVDB_DISABLE_DELAYED_LOAD
-    /// to disable delayed loading unconditionally.
-    static bool isDelayedLoadingEnabled();
+    /// @brief Return @c false (delayed loading has been removed).
+    static bool isDelayedLoadingEnabled() { return false; }
+
+    /// @brief Enable collection of read diagnostics (warnings, etc.) during I/O operations.
+    void enableReadDiagnostics();
+
+    /// @brief Disable collection of read diagnostics.
+    void disableReadDiagnostics();
+
+    /// @brief Return a const reference to the diagnostics collector.
+    const ReadDiagnostics& readDiagnostics() const;
+
+    /// @brief Clear the diagnostics list while keeping collection active.
+    void clearReadDiagnostics();
 
 protected:
     /// @brief Return @c true if the input stream contains grid offsets
@@ -131,14 +143,15 @@ protected:
     /// Read in and return the number of grids on the input stream.
     static int32_t readGridCount(std::istream&);
 
-    /// Populate the given grid from the input stream.
-    static void readGrid(GridBase::Ptr, const GridDescriptor&, std::istream&);
-    /// @brief Populate the given grid from the input stream, but only where it
-    /// intersects the given world-space bounding box.
-    static void readGrid(GridBase::Ptr, const GridDescriptor&, std::istream&, const BBoxd&);
-    /// @brief Populate the given grid from the input stream, but only where it
-    /// intersects the given index-space bounding box.
-    static void readGrid(GridBase::Ptr, const GridDescriptor&, std::istream&, const CoordBBox&);
+    /// @brief Find the codec for the given grid type and options.
+    static io::Codec* findCodec(const std::string& gridType, const io::ReadOptions& options = io::ReadOptions{});
+
+    /// @brief Read in and create the grid represented by the given grid descriptor using the
+    /// given input stream, using the provided options if given.
+    static GridBase::Ptr readGrid(const GridDescriptor&, std::istream&,
+        const io::ReadOptions& readOptions, ReadDiagnostics& diagnostics);
+    static GridBase::Ptr readGrid(const GridDescriptor&, std::istream&,
+        const io::ReadOptions& readOptions = io::ReadOptions{});
 
     using NamedGridMap = std::map<Name /*uniqueName*/, GridBase::Ptr>;
 
@@ -149,13 +162,14 @@ protected:
     /// Write the given grid descriptor and grid to an output stream
     /// and update the GridDescriptor offsets.
     /// @param seekable  if true, the output stream supports seek operations
-    void writeGrid(GridDescriptor&, GridBase::ConstPtr, std::ostream&, bool seekable) const;
+    void writeGrid(GridDescriptor&, GridBase::ConstPtr, std::ostream&, bool seekable,
+        const io::WriteOptions& writeOptions = io::WriteOptions{}) const;
     /// Write the given grid descriptor and grid metadata to an output stream
     /// and update the GridDescriptor offsets, but don't write the grid's tree,
     /// since it is shared with another grid.
     /// @param seekable  if true, the output stream supports seek operations
     void writeGridInstance(GridDescriptor&, GridBase::ConstPtr,
-        std::ostream&, bool seekable) const;
+        std::ostream&, bool seekable, const io::WriteOptions& writeOptions = io::WriteOptions{}) const;
 
     /// @brief Read the magic number, version numbers, UUID, etc. from the given input stream.
     /// @return @c true if the input UUID differs from the previously-read UUID.
@@ -167,10 +181,15 @@ protected:
 
     //@{
     /// Write the given grids to an output stream.
-    void write(std::ostream&, const GridPtrVec&, bool seekable, const MetaMap& = MetaMap()) const;
-    void write(std::ostream&, const GridCPtrVec&, bool seekable, const MetaMap& = MetaMap()) const;
+    void write(std::ostream&, const GridPtrVec&, bool seekable, const MetaMap&,
+        const io::WriteOptions& writeOptions = io::WriteOptions{}) const;
+    void write(std::ostream&, const GridCPtrVec&, bool seekable, const MetaMap&,
+        const io::WriteOptions& writeOptions = io::WriteOptions{}) const;
     //@}
 
+    /// Diagnostics collector for read operations (always valid; enabled/disabled via flag)
+    mutable ReadDiagnostics mReadDiagnostics;
+
 private:
     friend class ::TestFile;
 

openvdb/openvdb/io/Codec.cc

@@ -0,0 +1,134 @@
+// Copyright Contributors to the OpenVDB Project
+// SPDX-License-Identifier: Apache-2.0
+
+#include "Codec.h"
+
+#include <openvdb/codecs/ScalarCodec.h>
+#include <openvdb/codecs/BoolCodec.h>
+#include <openvdb/codecs/PointDataCodec.h>
+#include <openvdb/codecs/PointIndexCodec.h>
+#include <openvdb/codecs/ValueMaskCodec.h>
+
+#include <openvdb/openvdb.h>
+
+namespace openvdb {
+OPENVDB_USE_VERSION_NAMESPACE
+namespace OPENVDB_VERSION_NAME {
+namespace io {
+
+namespace {
+
+using CodecFactoryMap = std::map<std::string, Codec::Ptr>;
+using CodecFactoryMapCIter = CodecFactoryMap::const_iterator;
+
+struct LockedCodecRegistry {
+    LockedCodecRegistry() {}
+    ~LockedCodecRegistry() {}
+    std::mutex mMutex;
+    CodecFactoryMap mMap;
+};
+
+// Global function for accessing the registry
+static LockedCodecRegistry*
+getCodecRegistry()
+{
+    static LockedCodecRegistry registry;
+    return &registry;
+}
+
+} // unnamed namespace
+
+
+namespace internal {
+
+template <typename GridT>
+struct RegisterCodec { inline void operator()() { CodecRegistry::registerCodec<codecs::ScalarCodec<GridT>>(); } };
+
+template <typename GridT>
+struct RegisterConvertCodec {
+    inline void operator()()
+    {
+        CodecRegistry::registerCodec<codecs::ScalarCodec<MaskGrid, GridT, CodecMode::ReadOnly>>();
+        CodecRegistry::registerCodec<codecs::ScalarCodec<BoolGrid, GridT, CodecMode::ReadOnly>>();
+    }
+}; // struct RegisterConvertCodec
+
+void initialize()
+{
+    NumericGridTypes::foreach<RegisterCodec>();
+    Vec3GridTypes::foreach<RegisterCodec>();
+
+    CodecRegistry::registerCodec<codecs::BoolCodec<BoolGrid>>();
+    CodecRegistry::registerCodec<codecs::ValueMaskCodec<MaskGrid>>();
+    CodecRegistry::registerCodec<codecs::PointIndexCodec<tools::PointIndexGrid>>();
+    CodecRegistry::registerCodec<codecs::PointDataCodec<points::PointDataGrid>>();
+
+    // register the plugin that converts from scalar to mask/bool
+    NumericGridTypes::foreach<RegisterConvertCodec>();
+
+    // register the plugin that converts from float to half
+    CodecRegistry::registerCodec<codecs::ScalarCodec<HalfGrid, FloatGrid, CodecMode::ReadOnly>>();
+}
+
+void uninitialize()
+{
+    CodecRegistry::clear();
+}
+
+} // namespace internal
+
+
+////////////////////////////////////////
+
+
+bool
+CodecRegistry::isRegistered(const std::string& name)
+{
+    LockedCodecRegistry* registry = getCodecRegistry();
+    std::lock_guard<std::mutex> lock(registry->mMutex);
+
+    return (registry->mMap.find(name) != registry->mMap.end());
+}
+
+
+void
+CodecRegistry::registerCodecByName(const std::string& name, Codec::Ptr&& codec)
+{
+    LockedCodecRegistry* registry = getCodecRegistry();
+    std::lock_guard<std::mutex> lock(registry->mMutex);
+
+    if (registry->mMap.find(name) != registry->mMap.end()) {
+        OPENVDB_THROW(KeyError,
+            "Cannot register codec " << name << ". Codec is already registered");
+    }
+
+    registry->mMap[name] = std::move(codec);
+}
+
+
+Codec*
+CodecRegistry::get(const std::string& name)
+{
+    LockedCodecRegistry* registry = getCodecRegistry();
+    std::lock_guard<std::mutex> lock(registry->mMutex);
+
+    CodecFactoryMapCIter iter = registry->mMap.find(name);
+
+    return (iter != registry->mMap.end()) ? iter->second.get() : nullptr;
+}
+
+
+void
+CodecRegistry::clear()
+{
+    LockedCodecRegistry* registry = getCodecRegistry();
+    std::lock_guard<std::mutex> lock(registry->mMutex);
+
+    registry->mMap.clear();
+}
+
+
+} // namespace io
+} // namespace OPENVDB_VERSION_NAME
+} // namespace openvdb
+

openvdb/openvdb/io/Codec.h

@@ -0,0 +1,473 @@
+// Copyright Contributors to the OpenVDB Project
+// SPDX-License-Identifier: Apache-2.0
+
+#ifndef OPENVDB_IO_CODEC_HAS_BEEN_INCLUDED
+#define OPENVDB_IO_CODEC_HAS_BEEN_INCLUDED
+
+#include <string>
+#include <unordered_map>
+#include <memory>
+#include <iostream>
+#include <vector>
+#include <mutex>
+
+#include <openvdb/Types.h>
+
+namespace openvdb {
+OPENVDB_USE_VERSION_NAMESPACE
+namespace OPENVDB_VERSION_NAME {
+
+// Forward declaration
+class GridBase;
+
+namespace io {
+
+namespace internal {
+
+// Global registration of codecs. These initialization functions are NOT intended
+// to be called directly by the user.
+// While they are thread-safe, they lack the early-exit logic of openvdb::initialize() -
+// this can result in exceptions being thrown when attempting to register a codec that is
+// already registered.
+// They are also used extensively in unit tests.
+
+OPENVDB_API void initialize();
+OPENVDB_API void uninitialize();
+
+} // namespace internal
+
+////////////////////////////////////////
+
+/// @brief Controls which operations a codec exposes.
+///
+/// Passed to codec registration to restrict how the codec may be used at
+/// runtime. A @c ReadOnly codec can still be used to read existing files but
+/// will not be selected as the write codec for new grids.
+enum class CodecMode {
+    /// Both @c readTopology()/@c readBuffers() and
+    /// @c writeTopology()/@c writeBuffers() are enabled. This is the default.
+    ReadWrite,
+    /// Only @c readTopology() and @c readBuffers() are enabled; the codec
+    /// will not be offered as a write target.
+    ReadOnly
+};
+
+/// @brief Controls how grid data is read.
+///
+/// Passed via @c ReadOptions::readMode to influence which portions of a grid
+/// are deserialized and whether any on-the-fly conversion is performed during read.
+/// Codecs are expected to honour this setting in their @c readTopology() and
+/// @c readBuffers() implementations.
+enum class ReadMode {
+    /// Deserialize both topology and all value buffers. This is the default
+    /// behaviour and produces a fully populated grid identical to the one
+    /// that was written.
+    Original,
+    /// Deserialize topology and value buffers, converting each voxel value
+    /// to half-precision floating point on the fly. Codecs that do not
+    /// support native half conversion should fall back to @c Original and
+    /// record a @c ReadDiagnostic warning.
+    Half,
+    /// Deserialize topology and value buffers, converting each voxel value
+    /// to @c bool (non-zero to @c true, zero to @c false). Produces a
+    /// @c BoolGrid whose active set matches the source grid's active set.
+    Bool,
+    /// Deserialize topology only and promote the result to a mask grid,
+    /// discarding all value data. Equivalent to reading @c TopologyOnly and
+    /// then constructing a @c MaskGrid from the active-voxel set, but may
+    /// be performed more efficiently inside the codec.
+    Mask,
+    /// Deserialize topology only; value buffers are skipped. The resulting
+    /// grid has a valid tree structure (active/inactive state, node
+    /// hierarchy) but leaf buffer data is left at its default (background)
+    /// value. Useful when only the active-voxel mask is needed and
+    /// avoiding the cost of reading large value buffers is desirable.
+    TopologyOnly
+};
+
+/// @brief Base class for per-grid-type, codec-specific read options.
+///
+/// Codecs that require type-specific read configuration should derive from
+/// this class and store an instance in @c ReadOptions::typeData, keyed by
+/// the grid type string (e.g. @c "Vec3SGrid").
+///
+/// @par Example
+/// @code
+/// struct MyCodecReadOptions : public ReadTypedOptions {
+///     bool someFlag = false;
+/// };
+///
+/// ReadOptions options;
+/// auto data = std::make_shared<MyCodecReadOptions>();
+/// data->someFlag = true;
+/// options.typeData["Vec3SGrid"] = data;
+/// @endcode
+struct OPENVDB_API ReadTypedOptions
+{
+    using Ptr = std::shared_ptr<ReadTypedOptions>;
+
+    virtual ~ReadTypedOptions() = default;
+
+    /// @brief Downcast a @c ReadTypedOptions::Ptr to a concrete derived type.
+    ///
+    /// Performs a @c static_cast to @c T after verifying via @c dynamic_cast
+    /// (when @c OPENVDB_ASSERT is enabled) that @p data actually holds an
+    /// instance of @c T. Calling this with a mismatched type is undefined
+    /// behaviour in release builds.
+    ///
+    /// @tparam T  The concrete @c ReadTypedOptions subclass to cast to.
+    /// @param  data  A shared pointer to the base @c ReadTypedOptions object.
+    /// @return A reference to the underlying @c T instance.
+    template<typename T>
+    static T& cast(const Ptr& data) {
+        OPENVDB_ASSERT(dynamic_cast<T*>(data.get()) != nullptr);
+        return *static_cast<T*>(data.get());
+    }
+}; // struct ReadTypedOptions
+
+/// @brief Global read configuration passed to every codec during deserialization.
+///
+/// An instance of this struct is threaded through all codec @c readTopology() and
+/// @c readBuffers() calls so that site-wide policy (clipping, read mode) and any
+/// type-specific overrides are available in a single place.
+///
+/// @par Clipping
+/// When @c clipBBox is non-empty (i.e. @c !clipBBox.empty()), codecs should
+/// restrict the voxel data they load to the region that intersects that world-space
+/// bounding box. An empty bbox (the default) means no clipping is applied.
+///
+/// @par Read mode
+/// @c readMode controls the granularity and on-the-fly conversion applied
+/// during deserialization. The default, @c ReadMode::Original, loads both
+/// topology and value buffers without modification.
+/// @c ReadMode::Half, @c ReadMode::Bool, and @c ReadMode::Mask request
+/// in-place type conversion as data is read.
+/// @c ReadMode::TopologyOnly skips value buffers entirely, which can be
+/// significantly faster when only the active-voxel mask is needed.
+///
+/// @par Per-type options
+/// @c typeData allows callers to attach codec-specific configuration for
+/// individual grid types. Entries are keyed by the grid type string
+/// (e.g. @c "Vec3SGrid") and hold a @c ReadTypedOptions-derived object.
+/// Codecs retrieve their entry via @c ReadTypedOptions::cast<T>().
+struct OPENVDB_API ReadOptions
+{
+    /// World-space bounding box used to spatially clip the read.
+    /// An empty bbox (the default) disables clipping.
+    BBoxd clipBBox = BBoxd();
+
+    /// Controls which portions of each grid are deserialized.
+    /// Defaults to @c ReadMode::Original (full topology + value buffers).
+    ReadMode readMode = ReadMode::Original;
+
+    /// Optional per-grid-type codec configuration, keyed by grid type string.
+    /// Values are @c ReadTypedOptions subclass instances; use
+    /// @c ReadTypedOptions::cast<T>() to retrieve the concrete type.
+    std::unordered_map<std::string, ReadTypedOptions::Ptr> typeData;
+}; // struct ReadOptions
+
+/// @brief Global write configuration passed to every codec during serialization.
+///
+/// Currently carries no fields, but is provided for forward compatibility:
+/// future write-time options (e.g. compression hints, metadata policies) can
+/// be added here without changing the codec interface.
+struct OPENVDB_API WriteOptions
+{
+}; // struct WriteOptions
+
+/// @brief Severity level for a read diagnostic.
+/// @note Currently only @c Warning is defined.
+enum class DiagnosticSeverity { Warning };
+
+/// @brief A single read diagnostic message.
+/// @note @c context is typically the grid name or another identifier that
+///   locates the source of the message.
+struct ReadDiagnostic {
+    DiagnosticSeverity severity;
+    std::string context;
+    std::string message;
+};
+
+/// @brief Thread-safe collection of read diagnostics accumulated during a codec read.
+///
+/// @details Diagnostics report situations where a requested read option could
+/// not be honoured by the codec — for example, clipping that was not applied
+/// natively and had to fall back to a post-process, or an option that the codec
+/// ignores entirely. The object must be explicitly enabled before any messages
+/// are recorded; it is disabled by default.
+struct OPENVDB_API ReadDiagnostics {
+    ReadDiagnostics() = default;
+    ReadDiagnostics(const ReadDiagnostics& other) {
+        std::lock_guard<std::mutex> lock(other.mMutex);
+        mEnabled = other.mEnabled;
+        mDiagnostics = other.mDiagnostics;
+    }
+    ReadDiagnostics& operator=(const ReadDiagnostics& other) {
+        if (this != &other) {
+            std::scoped_lock lock(mMutex, other.mMutex);
+            mEnabled = other.mEnabled;
+            mDiagnostics = other.mDiagnostics;
+        }
+        return *this;
+    }
+    /// @brief Append a warning diagnostic. No-op if the object is disabled.
+    /// @note This method is thread-safe.
+    void addWarning(const std::string& context, const std::string& message) {
+        if (!mEnabled) return;
+        std::lock_guard<std::mutex> lock(mMutex);
+        mDiagnostics.push_back({DiagnosticSeverity::Warning, context, message});
+    }
+    /// @brief Remove all recorded diagnostics.
+    /// @note This method is thread-safe.
+    void clear() {
+        std::lock_guard<std::mutex> lock(mMutex);
+        mDiagnostics.clear();
+    }
+    /// @brief Enable diagnostic recording.
+    void enable() { mEnabled = true; }
+    /// @brief Disable diagnostic recording.
+    void disable() { mEnabled = false; }
+    /// @brief Return @c true if diagnostic recording is enabled.
+    bool enabled() const { return mEnabled; }
+    /// @brief Return the list of recorded diagnostics.
+    const std::vector<ReadDiagnostic>& diagnostics() const { return mDiagnostics; }
+    /// @brief Return @c true if no diagnostics have been recorded.
+    bool empty() const { return mDiagnostics.empty(); }
+private:
+    bool mEnabled = false;
+    std::vector<ReadDiagnostic> mDiagnostics;
+    mutable std::mutex mMutex;
+}; // struct ReadDiagnostics
+
+/// @brief Mutable per-operation state created by @c Codec::createData() and
+///   passed into every read and write call on a given codec.
+///
+/// @details The main purpose of this class is to provide a mechanism for passing
+/// additional state between the different methods in the codec. @c ReadOptions is
+/// immutable and cannot carry per-operation state; the codec itself is stateless
+/// by design (to allow sharing across threads) and therefore also cannot be used
+/// for this purpose.
+///
+/// The base class provides the single field that every codec requires:
+/// a shared pointer to the @c GridBase being populated (on read) or inspected
+/// (on write). Derived codecs that need additional transient state should
+/// subclass @c CodecData and add those fields.
+///
+/// A @c CodecData instance is created once per grid per read or write operation
+/// by @c Codec::createData(), which also allocates the concrete @c GridBase
+/// subclass and stores it in @c grid.
+///
+/// @par Deriving from CodecData
+/// @code
+/// struct MyCodecData : public io::CodecData {
+///     // Additional per-operation state goes here
+///     SomeCache intermediateBuffer;
+/// };
+/// @endcode
+///
+/// Inside @c Codec::createData(), the derived type is allocated and returned:
+/// @code
+/// io::CodecData::Ptr createData() override {
+///     auto data = std::make_unique<MyCodecData>();
+///     data->grid = MyGridType::create();
+///     return data;
+/// }
+/// @endcode
+///
+/// In the read and write methods, downcast the reference back to the concrete
+/// type to access the extra fields:
+/// @code
+/// void readBuffers(std::istream& is, io::CodecData& data, ...) override {
+///     auto& myData = static_cast<MyCodecData&>(data);
+///     MyGridType& grid = static_cast<MyGridType&>(*myData.grid);
+///     // Use myData.intermediateBuffer, grid, etc.
+/// }
+/// @endcode
+struct OPENVDB_API CodecData
+{
+    using Ptr = std::unique_ptr<CodecData>;
+
+    virtual ~CodecData() = default;
+
+    /// The grid being populated on read, or being serialized on write.
+    SharedPtr<GridBase> grid;
+}; // struct CodecData
+
+/// @brief Abstract base class for grid I/O codecs.
+///
+/// @details A codec encapsulates the serialization and deserialization logic
+/// for a specific grid type (or family of grid types). The I/O subsystem
+/// selects a codec by name at runtime, calls @c createData() to allocate
+/// per-operation state, and then dispatches to the appropriate read or write
+/// methods.
+///
+/// Codecs must be stateless: all mutable per-operation state is stored in a
+/// @c CodecData object (see above) that is created fresh for each grid read
+/// or write. This ensures that a single registered codec instance can safely
+/// handle concurrent I/O operations.
+///
+/// @par Implementing a new codec
+///
+/// 1. **Declare the codec struct**, deriving publicly from @c Codec (or from a
+///    convenience intermediate that already handles topology, if one exists for
+///    the grid family in question):
+///    @code
+///    struct MyCodec : public io::Codec { ... };
+///    @endcode
+///
+/// 2. **Provide a static @c name() method** that returns the unique string
+///    identifier under which the codec will be registered. Any globally
+///    unique ASCII string is acceptable:
+///    @code
+///    static std::string name() { return "mycodec"; }
+///    @endcode
+///
+/// 3. **Override @c createData()** (the only pure-virtual method). Allocate a
+///    @c CodecData (or a derived subclass if extra per-operation state is
+///    needed), create the appropriate @c GridBase subclass, assign it to
+///    @c CodecData::grid, and return the object:
+///    @code
+///    io::CodecData::Ptr createData() override {
+///        auto data = std::make_unique<io::CodecData>();
+///        data->grid = MyGridType::create();
+///        return data;
+///    }
+///    @endcode
+///
+/// 4. **Override the read and/or write methods** as required. All four
+///    methods have default no-op implementations, so override only those that
+///    the codec actually uses. The I/O layer separates topology (the tree
+///    structure and active/inactive voxel mask) from value buffers (the actual
+///    voxel data), allowing callers to request topology-only reads via
+///    @c ReadOptions::readMode. Implementations should honour that setting:
+///    @code
+///    void readTopology(std::istream& is, io::CodecData& data,
+///                      const io::ReadOptions& options,
+///                      io::ReadDiagnostics& diagnostics) override
+///    {
+///        MyGridType& grid = static_cast<MyGridType&>(*data.grid);
+///        // Deserialize the tree structure into grid...
+///    }
+///
+///    void readBuffers(std::istream& is, io::CodecData& data,
+///                     const io::ReadOptions& options,
+///                     io::ReadDiagnostics& diagnostics) override
+///    {
+///        if (options.readMode == io::ReadMode::TopologyOnly) return;
+///        MyGridType& grid = static_cast<MyGridType&>(*data.grid);
+///        // Deserialize voxel values into grid...
+///    }
+///    @endcode
+///    If a requested option (e.g. spatial clipping) cannot be honoured natively,
+///    record a warning via @c diagnostics.addWarning() rather than silently ignoring it.
+///
+/// 5. **Register the codec** once at start-up, typically from the library's
+///    @c initialize() function, using @c CodecRegistry::registerCodec<MyCodec>():
+///    @code
+///    io::CodecRegistry::registerCodec<MyCodec>();
+///    @endcode
+///
+/// @note The codec struct itself is never copied; the registry takes ownership
+///   of a single heap-allocated instance via @c Codec::Ptr
+///   (@c std::unique_ptr<Codec>).
+struct OPENVDB_API Codec
+{
+    using Ptr = std::unique_ptr<Codec>;
+
+    virtual ~Codec() = default;
+
+    /// @brief Allocate per-operation codec state, including the target grid.
+    ///
+    /// This is the only pure-virtual method. Implementations must create a
+    /// concrete @c GridBase subclass appropriate for the codec, store it in
+    /// @c CodecData::grid, and return the @c CodecData (or a derived subclass
+    /// carrying additional state). The returned object is passed by reference
+    /// to every subsequent read or write call for the same grid.
+    ///
+    /// @return A fully initialized @c CodecData whose @c grid field is non-null.
+    virtual CodecData::Ptr createData() = 0;
+
+    /// @brief Deserialize the grid topology (tree structure and active-voxel
+    ///   mask) from @a is into the grid held by @a data.
+    virtual void readTopology(std::istream& /*is*/, CodecData& /*data*/,
+        const ReadOptions& /*options*/, ReadDiagnostics& /*diagnostics*/) { }
+
+    /// @brief Deserialize all voxel-value buffers from @a is into the grid
+    ///   held by @a data.
+    ///
+    /// The default implementation is a no-op. Override to populate leaf-node
+    /// value buffers after topology has been established. When
+    /// @c options.readMode is @c ReadMode::TopologyOnly this method will not
+    /// be called by the I/O layer, so implementations may also guard against
+    /// that mode internally for safety. If @c options.clipBBox is non-empty,
+    /// restrict the loaded data to the region that intersects it; if the codec
+    /// cannot honour clipping natively, fall back to a post-process and record
+    /// a warning via @a diagnostics.
+    virtual void readBuffers(std::istream& /*is*/, CodecData& /*data*/,
+        const ReadOptions& /*options*/, ReadDiagnostics& /*diagnostics*/) { }
+
+    /// @brief Serialize the grid topology (tree structure and active-voxel
+    ///   mask) from @a grid to @a os.
+    ///
+    /// The default implementation is a no-op. Override when the codec stores
+    /// topology as a distinct section that precedes the value buffers in the
+    /// stream.
+    virtual void writeTopology(std::ostream& /*os*/, const GridBase& /*grid*/,
+        const WriteOptions& /*options*/) { }
+
+    /// @brief Serialize all voxel-value buffers from @a grid to @a os.
+    ///
+    /// The default implementation is a no-op. Override to write the leaf-node
+    /// value buffers that follow the topology section.
+    virtual void writeBuffers(std::ostream& /*os*/, const GridBase& /*grid*/,
+        const WriteOptions& /*options*/) { }
+}; // struct Codec
+
+/// @brief A thread-safe, process-global registry that maps codec names to
+///   @c Codec instances.
+///
+/// Codecs are identified by a unique string name (typically provided by @c Codec::name())
+/// and stored as @c Codec::Ptr (i.e. @c std::unique_ptr<Codec>).
+///
+/// @par Typical usage
+/// @code
+/// // Register a custom codec once at start-up (e.g. from initialize()):
+/// CodecRegistry::registerCodec<MyCodec>();
+/// @endcode
+///
+/// @note Attempting to register a codec whose name is already present throws
+///   @c openvdb::KeyError. Call @c isRegistered() first when the caller
+///   cannot guarantee uniqueness.
+struct OPENVDB_API CodecRegistry
+{
+    /// Return @c true if a codec with the given @a name has been registered.
+    static bool isRegistered(const std::string& name);
+
+    /// Register a codec under the explicit string @a name, transferring
+    /// ownership of @a codec to the registry.
+    /// @throw KeyError if @a name is already registered.
+    static void registerCodecByName(const std::string& name, Codec::Ptr&& codec);
+
+    /// Convenience wrapper that registers @c CodecT using the name returned
+    /// by @c CodecT::name(), constructing the instance internally.
+    /// @throw KeyError if the codec is already registered.
+    template <typename CodecT>
+    static void registerCodec()
+    {
+        registerCodecByName(CodecT::name(), std::make_unique<CodecT>());
+    }
+
+    /// Return a raw (non-owning) pointer to the codec registered under
+    /// @a name, or @c nullptr if no such codec exists.
+    /// The returned pointer remains valid for the lifetime of the registry.
+    static Codec* get(const std::string& name);
+
+    /// Deregister all codecs and reset the registry to an empty state.
+    static void clear();
+}; // struct CodecRegistry
+
+} // namespace io
+} // namespace OPENVDB_VERSION_NAME
+} // namespace openvdb
+
+#endif // OPENVDB_IO_CODEC_HAS_BEEN_INCLUDED

openvdb/openvdb/io/Compression.h

@@ -9,7 +9,6 @@
 #include <openvdb/math/Math.h> // for negative()
 #include <openvdb/util/Assert.h>
 #include "io.h" // for getDataCompression(), etc.
-#include "DelayedLoadMetadata.h"
 #include <algorithm>
 #include <iostream>
 #include <memory>
@@ -236,28 +235,20 @@ OPENVDB_API void bloscFromStream(std::istream&, char* data, size_t numBytes);
 /// @param count        the number of elements to read in
 /// @param compression  whether and how the data is compressed (either COMPRESS_NONE,
 ///                     COMPRESS_ZIP, COMPRESS_ACTIVE_MASK or COMPRESS_BLOSC)
-/// @param metadata     optional pointer to a DelayedLoadMetadata object that stores
-///                     the size of the compressed buffer
-/// @param metadataOffset offset into DelayedLoadMetadata, ignored if pointer is null
 /// @throw IoError if @a compression is COMPRESS_BLOSC but OpenVDB was compiled
 /// without Blosc support.
 /// @details This default implementation is instantiated only for types
 /// whose size can be determined by the sizeof() operator.
 template<typename T>
 inline void
-readData(std::istream& is, T* data, Index count, uint32_t compression,
-    DelayedLoadMetadata* metadata = nullptr, size_t metadataOffset = size_t(0))
+readData(std::istream& is, T* data, Index count, uint32_t compression)
 {
     const bool seek = data == nullptr;
     if (seek) {
         OPENVDB_ASSERT(!getStreamMetadataPtr(is) || getStreamMetadataPtr(is)->seekable());
     }
-    const bool hasCompression = compression & (COMPRESS_BLOSC | COMPRESS_ZIP);
 
-    if (metadata && seek && hasCompression) {
-        size_t compressedSize = metadata->getCompressedSize(metadataOffset);
-        is.seekg(compressedSize, std::ios_base::cur);
-    } else if (compression & COMPRESS_BLOSC) {
+    if (compression & COMPRESS_BLOSC) {
         bloscFromStream(is, reinterpret_cast<char*>(data), sizeof(T) * count);
     } else if (compression & COMPRESS_ZIP) {
         unzipFromStream(is, reinterpret_cast<char*>(data), sizeof(T) * count);
@@ -271,8 +262,7 @@ readData(std::istream& is, T* data, Index count, uint32_t compression,
 /// Specialization for std::string input
 template<>
 inline void
-readData<std::string>(std::istream& is, std::string* data, Index count, uint32_t /*compression*/,
-    DelayedLoadMetadata* /*metadata*/, size_t /*metadataOffset*/)
+readData<std::string>(std::istream& is, std::string* data, Index count, uint32_t /*compression*/)
 {
     for (Index i = 0; i < count; ++i) {
         size_t len = 0;
@@ -294,25 +284,22 @@ template<bool IsReal, typename T> struct HalfReader;
 /// Partial specialization for non-floating-point types (no half to float promotion)
 template<typename T>
 struct HalfReader</*IsReal=*/false, T> {
-    static inline void read(std::istream& is, T* data, Index count, uint32_t compression,
-        DelayedLoadMetadata* metadata = nullptr, size_t metadataOffset = size_t(0)) {
-        readData(is, data, count, compression, metadata, metadataOffset);
+    static inline void read(std::istream& is, T* data, Index count, uint32_t compression) {
+        readData(is, data, count, compression);
     }
 };
 /// Partial specialization for floating-point types
 template<typename T>
 struct HalfReader</*IsReal=*/true, T> {
     using HalfT = typename RealToHalf<T>::HalfT;
-    static inline void read(std::istream& is, T* data, Index count, uint32_t compression,
-        DelayedLoadMetadata* metadata = nullptr, size_t metadataOffset = size_t(0)) {
+    static inline void read(std::istream& is, T* data, Index count, uint32_t compression) {
         if (count < 1) return;
         if (data == nullptr) {
             // seek mode - pass through null pointer
-            readData<HalfT>(is, nullptr, count, compression, metadata, metadataOffset);
+            readData<HalfT>(is, nullptr, count, compression);
         } else {
             std::vector<HalfT> halfData(count); // temp buffer into which to read half float values
-            readData<HalfT>(is, reinterpret_cast<HalfT*>(&halfData[0]), count, compression,
-                metadata, metadataOffset);
+            readData<HalfT>(is, reinterpret_cast<HalfT*>(&halfData[0]), count, compression);
             // Copy half float values from the temporary buffer to the full float output array.
             std::copy(halfData.begin(), halfData.end(), data);
         }
@@ -476,16 +463,6 @@ readCompressedValues(std::istream& is, ValueT* destBuf, Index destCount,
     const bool seek = (destBuf == nullptr);
     OPENVDB_ASSERT(!seek || (!meta || meta->seekable()));
 
-    // Get delayed load metadata if it exists
-
-    DelayedLoadMetadata::Ptr delayLoadMeta;
-    uint64_t leafIndex(0);
-    if (seek && meta && meta->delayedLoadMeta()) {
-        delayLoadMeta =
-            meta->gridMetadata().getMetadata<DelayedLoadMetadata>("file_delayed_load");
-        leafIndex = meta->leaf();
-    }
-
     int8_t metadata = NO_MASK_AND_ALL_VALS;
 
     if (getFormatVersion(is) >= OPENVDB_FILE_VERSION_NODE_MASK_COMPRESSION) {
@@ -493,9 +470,6 @@ readCompressedValues(std::istream& is, ValueT* destBuf, Index destCount,
         // (selection mask and/or inactive value(s)) is saved.
         if (seek && !maskCompressed) {
             is.seekg(/*bytes=*/1, std::ios_base::cur);
-        } else if (seek && delayLoadMeta) {
-            metadata = delayLoadMeta->getMask(leafIndex);
-            is.seekg(/*bytes=*/1, std::ios_base::cur);
         } else {
             is.read(reinterpret_cast<char*>(&metadata), /*bytes=*/1);
         }
@@ -562,10 +536,10 @@ readCompressedValues(std::istream& is, ValueT* destBuf, Index destCount,
     // Read in the buffer.
     if (fromHalf) {
         HalfReader<RealToHalf<ValueT>::isReal, ValueT>::read(
-            is, (seek ? nullptr : tempBuf), tempCount, compression, delayLoadMeta.get(), leafIndex);
+            is, (seek ? nullptr : tempBuf), tempCount, compression);
     } else {
         readData<ValueT>(
-            is, (seek ? nullptr : tempBuf), tempCount, compression, delayLoadMeta.get(), leafIndex);
+            is, (seek ? nullptr : tempBuf), tempCount, compression);
     }
 
     // If mask compression is enabled and the number of active values read into

openvdb/openvdb/io/DelayedLoadMetadata.cc

@@ -1,300 +0,0 @@
-// Copyright Contributors to the OpenVDB Project
-// SPDX-License-Identifier: Apache-2.0
-
-#include "DelayedLoadMetadata.h"
-
-#include <openvdb/points/StreamCompression.h>
-#include <openvdb/util/Assert.h>
-
-#ifdef OPENVDB_USE_BLOSC
-#include <blosc.h>
-
-namespace {
-
-inline size_t padMask(size_t bytes)
-{
-    return size_t(std::ceil(static_cast<float>(bytes+1) /
-        sizeof(openvdb::io::DelayedLoadMetadata::MaskType)));
-}
-
-inline size_t padCompressedSize(size_t bytes)
-{
-    return size_t(std::ceil(static_cast<float>(bytes+1) /
-        sizeof(openvdb::io::DelayedLoadMetadata::CompressedSizeType)));
-}
-
-} // namespace
-
-#endif
-
-namespace openvdb {
-OPENVDB_USE_VERSION_NAMESPACE
-namespace OPENVDB_VERSION_NAME {
-namespace io {
-
-DelayedLoadMetadata::DelayedLoadMetadata(const DelayedLoadMetadata& other)
-    : Metadata()
-    , mMask(other.mMask)
-    , mCompressedSize(other.mCompressedSize)
-{
-}
-
-Name DelayedLoadMetadata::typeName() const
-{
-    return DelayedLoadMetadata::staticTypeName();
-}
-
-Metadata::Ptr DelayedLoadMetadata::copy() const
-{
-    Metadata::Ptr metadata(new DelayedLoadMetadata());
-    metadata->copy(*this);
-    return metadata;
-}
-
-void DelayedLoadMetadata::copy(const Metadata& other)
-{
-    const DelayedLoadMetadata* t = dynamic_cast<const DelayedLoadMetadata*>(&other);
-    if (t == nullptr) OPENVDB_THROW(TypeError, "Incompatible type during copy");
-    mMask = t->mMask;
-    mCompressedSize = t->mCompressedSize;
-}
-
-std::string DelayedLoadMetadata::str() const
-{
-    return "";
-}
-
-bool DelayedLoadMetadata::asBool() const
-{
-    return false;
-}
-
-Index32 DelayedLoadMetadata::size() const
-{
-    if (mMask.empty() && mCompressedSize.empty())   return Index32(0);
-
-    // count
-    size_t size = sizeof(Index32);
-
-    { // mask
-        size += sizeof(Index32);
-        size_t compressedSize = compression::bloscCompressedSize(
-            reinterpret_cast<const char*>(mMask.data()), mMask.size()*sizeof(MaskType));
-
-        if (compressedSize > 0)     size += compressedSize;
-        else                        size += mMask.size()*sizeof(MaskType);
-    }
-    { // compressed size
-        size += sizeof(Index32);
-        if (!mCompressedSize.empty()) {
-            size_t compressedSize = compression::bloscCompressedSize(
-                reinterpret_cast<const char*>(mCompressedSize.data()), mCompressedSize.size()*sizeof(CompressedSizeType));
-
-            if (compressedSize > 0)     size += compressedSize;
-            else                        size += mCompressedSize.size()*sizeof(CompressedSizeType);
-        }
-    }
-
-    return static_cast<Index32>(size);
-}
-
-void DelayedLoadMetadata::clear()
-{
-    mMask.clear();
-    mCompressedSize.clear();
-}
-
-bool DelayedLoadMetadata::empty() const
-{
-    return mMask.empty() && mCompressedSize.empty();
-}
-
-void DelayedLoadMetadata::resizeMask(size_t size)
-{
-    mMask.resize(size);
-}
-
-void DelayedLoadMetadata::resizeCompressedSize(size_t size)
-{
-    mCompressedSize.resize(size);
-}
-
-DelayedLoadMetadata::MaskType DelayedLoadMetadata::getMask(size_t index) const
-{
-    OPENVDB_ASSERT(DelayedLoadMetadata::isRegisteredType());
-    OPENVDB_ASSERT(index < mMask.size());
-    return mMask[index];
-}
-
-void DelayedLoadMetadata::setMask(size_t index, const MaskType& value)
-{
-    OPENVDB_ASSERT(index < mMask.size());
-    mMask[index] = value;
-}
-
-DelayedLoadMetadata::CompressedSizeType DelayedLoadMetadata::getCompressedSize(size_t index) const
-{
-    OPENVDB_ASSERT(DelayedLoadMetadata::isRegisteredType());
-    OPENVDB_ASSERT(index < mCompressedSize.size());
-    return mCompressedSize[index];
-}
-
-void DelayedLoadMetadata::setCompressedSize(size_t index, const CompressedSizeType& value)
-{
-    OPENVDB_ASSERT(index < mCompressedSize.size());
-    mCompressedSize[index] = value;
-}
-
-void DelayedLoadMetadata::readValue(std::istream& is, Index32 numBytes)
-{
-    if (numBytes == 0)  return;
-
-    // initial header size
-    size_t total = sizeof(Index32);
-
-    Index32 count = 0;
-    is.read(reinterpret_cast<char*>(&count), sizeof(Index32));
-    total += sizeof(Index32);
-
-    Index32 bytes = 0;
-    is.read(reinterpret_cast<char*>(&bytes), sizeof(Index32));
-    total += sizeof(Index32);
-
-    if (bytes > Index32(0)) {
-        std::unique_ptr<char[]> compressedBuffer(new char[bytes]);
-        is.read(reinterpret_cast<char*>(compressedBuffer.get()), bytes);
-
-        total += bytes;
-
-#ifdef OPENVDB_USE_BLOSC
-        // pad to include BLOSC_MAX_OVERHEAD
-        size_t uncompressedBytes = openvdb::compression::bloscUncompressedSize(compressedBuffer.get());
-        const size_t paddedCount = padMask(uncompressedBytes + BLOSC_MAX_OVERHEAD);
-
-        mMask.reserve(paddedCount);
-        mMask.resize(count);
-
-        // resize should never modify capacity for smaller vector sizes
-        OPENVDB_ASSERT(mMask.capacity() >= paddedCount);
-
-        compression::bloscDecompress(reinterpret_cast<char*>(mMask.data()), count*sizeof(MaskType), mMask.capacity()*sizeof(MaskType), compressedBuffer.get());
-#endif
-    } else {
-        mMask.resize(count);
-        is.read(reinterpret_cast<char*>(mMask.data()), count*sizeof(MaskType));
-        total += count*sizeof(MaskType);
-    }
-
-    is.read(reinterpret_cast<char*>(&bytes), sizeof(Index32));
-
-    if (bytes != std::numeric_limits<Index32>::max()) {
-        if (bytes > Index32(0)) {
-            std::unique_ptr<char[]> compressedBuffer(new char[bytes]);
-            is.read(reinterpret_cast<char*>(compressedBuffer.get()), bytes);
-
-            total += size_t(bytes);
-
-#ifdef OPENVDB_USE_BLOSC
-            // pad to include BLOSC_MAX_OVERHEAD
-            size_t uncompressedBytes = openvdb::compression::bloscUncompressedSize(compressedBuffer.get());
-            const size_t paddedCount = padCompressedSize(uncompressedBytes + BLOSC_MAX_OVERHEAD);
-
-            mCompressedSize.reserve(paddedCount);
-            mCompressedSize.resize(count);
-
-            // resize should never modify capacity for smaller vector sizes
-            OPENVDB_ASSERT(mCompressedSize.capacity() >= paddedCount);
-
-            compression::bloscDecompress(reinterpret_cast<char*>(mCompressedSize.data()), count*sizeof(CompressedSizeType), mCompressedSize.capacity()*sizeof(CompressedSizeType), compressedBuffer.get());
-#endif
-        } else {
-            mCompressedSize.resize(count);
-            is.read(reinterpret_cast<char*>(mCompressedSize.data()), count*sizeof(CompressedSizeType));
-            total += count*sizeof(CompressedSizeType);
-        }
-    }
-
-    Index32 totalBytes = static_cast<Index32>(total);
-
-    if (totalBytes < numBytes) {
-        // Read and discard any unknown bytes at the end of the metadata for forwards-compatibility
-        // (without seeking, because the stream might not be seekable).
-        const size_t BUFFER_SIZE = 1024;
-        std::vector<char> buffer(BUFFER_SIZE);
-        for (Index32 bytesRemaining = numBytes - totalBytes; bytesRemaining > 0; ) {
-            const Index32 bytesToSkip = std::min<Index32>(bytesRemaining, BUFFER_SIZE);
-            is.read(&buffer[0], bytesToSkip);
-            bytesRemaining -= bytesToSkip;
-        }
-    }
-}
-
-void DelayedLoadMetadata::writeValue(std::ostream& os) const
-{
-    // metadata has a limit of 2^32 bytes
-    OPENVDB_ASSERT(mMask.size() < std::numeric_limits<Index32>::max());
-    OPENVDB_ASSERT(mCompressedSize.size() < std::numeric_limits<Index32>::max());
-
-    if (mMask.empty() && mCompressedSize.empty())     return;
-
-    OPENVDB_ASSERT(mCompressedSize.empty() || (mMask.size() == mCompressedSize.size()));
-
-    Index32 count = static_cast<Index32>(mMask.size());
-    os.write(reinterpret_cast<const char*>(&count), sizeof(Index32));
-
-    const Index32 zeroSize(0);
-    const Index32 maxSize(std::numeric_limits<Index32>::max());
-
-    { // mask buffer
-        size_t compressedBytes(0);
-        std::unique_ptr<char[]> compressedBuffer;
-        if (compression::bloscCanCompress()) {
-            compressedBuffer = compression::bloscCompress(
-                reinterpret_cast<const char*>(mMask.data()),
-                mMask.size()*sizeof(MaskType), compressedBytes, /*resize=*/false);
-        }
-
-        if (compressedBuffer) {
-            OPENVDB_ASSERT(compressedBytes < std::numeric_limits<Index32>::max());
-            Index32 bytes(static_cast<Index32>(compressedBytes));
-            os.write(reinterpret_cast<const char*>(&bytes), sizeof(Index32));
-            os.write(reinterpret_cast<const char*>(compressedBuffer.get()), compressedBytes);
-        }
-        else {
-            os.write(reinterpret_cast<const char*>(&zeroSize), sizeof(Index32));
-            os.write(reinterpret_cast<const char*>(mMask.data()),
-                mMask.size()*sizeof(MaskType));
-        }
-    }
-
-    // compressed size buffer
-
-    if (mCompressedSize.empty()) {
-        // write out maximum Index32 value to denote no compressed sizes stored
-        os.write(reinterpret_cast<const char*>(&maxSize), sizeof(Index32));
-    } else {
-        size_t compressedBytes(0);
-        std::unique_ptr<char[]> compressedBuffer;
-        if (compression::bloscCanCompress()) {
-            compressedBuffer = compression::bloscCompress(
-                reinterpret_cast<const char*>(mCompressedSize.data()),
-                mCompressedSize.size()*sizeof(CompressedSizeType), compressedBytes, /*resize=*/false);
-        }
-
-        if (compressedBuffer) {
-            OPENVDB_ASSERT(compressedBytes < std::numeric_limits<Index32>::max());
-            Index32 bytes(static_cast<Index32>(compressedBytes));
-            os.write(reinterpret_cast<const char*>(&bytes), sizeof(Index32));
-            os.write(reinterpret_cast<const char*>(compressedBuffer.get()), compressedBytes);
-        }
-        else {
-            os.write(reinterpret_cast<const char*>(&zeroSize), sizeof(Index32));
-            os.write(reinterpret_cast<const char*>(mCompressedSize.data()),
-                mCompressedSize.size()*sizeof(CompressedSizeType));
-        }
-    }
-}
-
-} // namespace io
-} // namespace OPENVDB_VERSION_NAME
-} // namespace openvdb

openvdb/openvdb/io/DelayedLoadMetadata.h

@@ -1,102 +0,0 @@
-// Copyright Contributors to the OpenVDB Project
-// SPDX-License-Identifier: Apache-2.0
-
-#ifndef OPENVDB_DELAYED_LOAD_METADATA_HAS_BEEN_INCLUDED
-#define OPENVDB_DELAYED_LOAD_METADATA_HAS_BEEN_INCLUDED
-
-#include <openvdb/Metadata.h>
-#include <cstdint>
-#include <iostream>
-#include <string>
-#include <vector>
-
-
-namespace openvdb {
-OPENVDB_USE_VERSION_NAMESPACE
-namespace OPENVDB_VERSION_NAME {
-namespace io {
-
-/// @brief Store a buffer of data that can be optionally used
-/// during reading for faster delayed-load I/O performance
-class OPENVDB_API DelayedLoadMetadata: public Metadata
-{
-public:
-    using Ptr = SharedPtr<DelayedLoadMetadata>;
-    using ConstPtr = SharedPtr<const DelayedLoadMetadata>;
-    using MaskType = int8_t;
-    using CompressedSizeType = int64_t;
-
-    DelayedLoadMetadata() = default;
-    DelayedLoadMetadata(const DelayedLoadMetadata& other);
-    ~DelayedLoadMetadata() override = default;
-
-    Name typeName() const override;
-    Metadata::Ptr copy() const override;
-    void copy(const Metadata&) override;
-    std::string str() const override;
-    bool asBool() const override;
-    Index32 size() const override;
-
-    static Name staticTypeName() { return "__delayedload"; }
-
-    static Metadata::Ptr createMetadata()
-    {
-        Metadata::Ptr ret(new DelayedLoadMetadata);
-        return ret;
-    }
-
-    static void registerType()
-    {
-        Metadata::registerType(DelayedLoadMetadata::staticTypeName(),
-                               DelayedLoadMetadata::createMetadata);
-    }
-
-    static void unregisterType()
-    {
-        Metadata::unregisterType(DelayedLoadMetadata::staticTypeName());
-    }
-
-    static bool isRegisteredType()
-    {
-        return Metadata::isRegisteredType(DelayedLoadMetadata::staticTypeName());
-    }
-
-    /// @brief Delete the contents of the mask and compressed size arrays
-    void clear();
-    /// @brief Return @c true if both arrays are empty
-    bool empty() const;
-
-    /// @brief Resize the mask array
-    void resizeMask(size_t size);
-    /// @brief Resize the compressed size array
-    void resizeCompressedSize(size_t size);
-
-    /// @brief Return the mask value for a specific index
-    /// @note throws if index is out-of-range or DelayedLoadMask not registered
-    MaskType getMask(size_t index) const;
-    /// @brief Set the mask value for a specific index
-    /// @note throws if index is out-of-range
-    void setMask(size_t index, const MaskType& value);
-
-    /// @brief Return the compressed size value for a specific index
-    /// @note throws if index is out-of-range or DelayedLoadMask not registered
-    CompressedSizeType getCompressedSize(size_t index) const;
-    /// @brief Set the compressed size value for a specific index
-    /// @note throws if index is out-of-range
-    void setCompressedSize(size_t index, const CompressedSizeType& value);
-
-protected:
-    void readValue(std::istream&, Index32 numBytes) override;
-    void writeValue(std::ostream&) const override;
-
-private:
-    std::vector<MaskType> mMask;
-    std::vector<CompressedSizeType> mCompressedSize;
-}; // class DelayedLoadMetadata
-
-
-} // namespace io
-} // namespace OPENVDB_VERSION_NAME
-} // namespace openvdb
-
-#endif // OPENVDB_DELAYED_LOAD_METADATA_HAS_BEEN_INCLUDED

openvdb/openvdb/io/File.cc

@@ -5,20 +5,11 @@
 
 #include "File.h"
 
-#include "TempFile.h"
 #include <openvdb/Exceptions.h>
 #include <openvdb/util/logging.h>
 #include <openvdb/util/Assert.h>
 #include <cstdint>
 
-#ifdef OPENVDB_USE_DELAYED_LOADING
-#include <boost/iostreams/copy.hpp>
-#ifndef _WIN32
-#include <sys/types.h>
-#include <unistd.h>
-#endif
-#endif // OPENVDB_USE_DELAYED_LOADING
-
 #include <sys/stat.h> // stat()
 
 #include <cstdlib> // for getenv(), strtoul()
@@ -34,106 +25,24 @@ OPENVDB_USE_VERSION_NAMESPACE
 namespace OPENVDB_VERSION_NAME {
 namespace io {
 
-// Implementation details of the File class
-struct File::Impl
+
+File::File(const std::string& filename)
+    : Archive()
+    , mFilename(filename)
 {
-    enum { DEFAULT_COPY_MAX_BYTES = 500000000 }; // 500 MB
-
-    struct NoBBox {};
-
-    // Common implementation of the various File::readGrid() overloads,
-    // with and without bounding box clipping
-    template<typename BoxType>
-    static GridBase::Ptr readGrid(const File& file, const GridDescriptor& gd, const BoxType& bbox)
-    {
-        // This method should not be called for files that don't contain grid offsets.
-        OPENVDB_ASSERT(file.inputHasGridOffsets());
-
-        GridBase::Ptr grid = file.createGrid(gd);
-        gd.seekToGrid(file.inputStream());
-        unarchive(file, grid, gd, bbox);
-        return grid;
-    }
-
-    static void unarchive(const File& file, GridBase::Ptr& grid,
-        const GridDescriptor& gd, NoBBox)
-    {
-        file.Archive::readGrid(grid, gd, file.inputStream());
-    }
-
-    static void unarchive(const File& file, GridBase::Ptr& grid,
-        const GridDescriptor& gd, const CoordBBox& indexBBox)
-    {
-        file.Archive::readGrid(grid, gd, file.inputStream(), indexBBox);
-    }
-
-    static void unarchive(const File& file, GridBase::Ptr& grid,
-        const GridDescriptor& gd, const BBoxd& worldBBox)
-    {
-        file.Archive::readGrid(grid, gd, file.inputStream(), worldBBox);
-    }
-
-    static Index64 getDefaultCopyMaxBytes()
-    {
-        Index64 result = DEFAULT_COPY_MAX_BYTES;
-        if (const char* s = std::getenv("OPENVDB_DELAYED_LOAD_COPY_MAX_BYTES")) {
-            char* endptr = nullptr;
-            result = std::strtoul(s, &endptr, /*base=*/10);
-        }
-        return result;
-    }
-
-    std::string mFilename;
-    // The file-level metadata
-    MetaMap::Ptr mMeta;
-    // The file stream that is open for reading
-    std::unique_ptr<std::istream> mInStream;
-    // File-level stream metadata (file format, compression, etc.)
-    StreamMetadata::Ptr mStreamMetadata;
-    // Flag indicating if we have read in the global information (header,
-    // metadata, and grid descriptors) for this VDB file
-    bool mIsOpen;
-    // Grid descriptors for all grids stored in the file, indexed by grid name
-    NameMap mGridDescriptors;
-    // All grids, indexed by unique name (used only when mHasGridOffsets is false)
-    Archive::NamedGridMap mNamedGrids;
-    // All grids stored in the file (used only when mHasGridOffsets is false)
-    GridPtrVecPtr mGrids;
-#ifdef OPENVDB_USE_DELAYED_LOADING
-    // The memory-mapped file
-    MappedFile::Ptr mFileMapping;
-    // The buffer for the input stream, if it is a memory-mapped file
-    SharedPtr<std::streambuf> mStreamBuf;
-    // File size limit for copying during delayed loading
-    Index64 mCopyMaxBytes;
-#endif
-}; // class File::Impl
-
-
-////////////////////////////////////////
-
-
-File::File(const std::string& filename): mImpl(new Impl)
-{
-    mImpl->mFilename = filename;
-    mImpl->mIsOpen = false;
-#ifdef OPENVDB_USE_DELAYED_LOADING
-    mImpl->mCopyMaxBytes = Impl::getDefaultCopyMaxBytes();
-#endif
     setInputHasGridOffsets(true);
 }
 
 
-File::~File()
-{
-}
-
-
 File::File(const File& other)
     : Archive(other)
-    , mImpl(new Impl)
+    , mFilename(other.mFilename)
+    , mMeta(other.mMeta)
+    , mIsOpen(false)
+    , mGridDescriptors(other.mGridDescriptors)
+    , mNamedGrids(other.mNamedGrids)
+    , mGrids(other.mGrids)
 {
-    *this = other;
 }
 
 
@@ -142,16 +51,12 @@ File::operator=(const File& other)
 {
     if (&other != this) {
         Archive::operator=(other);
-        const Impl& otherImpl = *other.mImpl;
-        mImpl->mFilename = otherImpl.mFilename;
-        mImpl->mMeta = otherImpl.mMeta;
-        mImpl->mIsOpen = false; // don't want two file objects reading from the same stream
-#ifdef OPENVDB_USE_DELAYED_LOADING
-        mImpl->mCopyMaxBytes = otherImpl.mCopyMaxBytes;
-#endif
-        mImpl->mGridDescriptors = otherImpl.mGridDescriptors;
-        mImpl->mNamedGrids = otherImpl.mNamedGrids;
-        mImpl->mGrids = otherImpl.mGrids;
+        mFilename = other.mFilename;
+        mMeta = other.mMeta;
+        mIsOpen = false; // don't want two file objects reading from the same stream
+        mGridDescriptors = other.mGridDescriptors;
+        mNamedGrids = other.mNamedGrids;
+        mGrids = other.mGrids;
     }
     return *this;
 }
@@ -170,43 +75,43 @@ File::copy() const
 const std::string&
 File::filename() const
 {
-    return mImpl->mFilename;
+    return mFilename;
 }
 
 
 MetaMap::Ptr
 File::fileMetadata()
 {
-    return mImpl->mMeta;
+    return mMeta;
 }
 
 MetaMap::ConstPtr
 File::fileMetadata() const
 {
-    return mImpl->mMeta;
+    return mMeta;
 }
 
 
 const File::NameMap&
 File::gridDescriptors() const
 {
-    return mImpl->mGridDescriptors;
+    return mGridDescriptors;
 }
 
 File::NameMap&
 File::gridDescriptors()
 {
-    return mImpl->mGridDescriptors;
+    return mGridDescriptors;
 }
 
 
 std::istream&
 File::inputStream() const
 {
-    if (!mImpl->mInStream) {
-        OPENVDB_THROW(IoError, filename() << " is not open for reading");
+    if (!mInStream) {
+        OPENVDB_THROW(IoError, mFilename << " is not open for reading");
     }
-    return *mImpl->mInStream;
+    return *mInStream;
 }
 
 
@@ -222,11 +127,11 @@ File::getSize() const
 
     Index64 result = std::numeric_limits<Index64>::max();
 
-    std::string mesg = "could not get size of file " + filename();
+    std::string mesg = "could not get size of file " + mFilename;
 
 #ifdef _WIN32
     // Get the file size by seeking to the end of the file.
-    std::ifstream fstrm(filename());
+    std::ifstream fstrm(mFilename);
     if (fstrm) {
         fstrm.seekg(0, fstrm.end);
         result = static_cast<Index64>(fstrm.tellg());
@@ -236,7 +141,7 @@ File::getSize() const
 #else
     // Get the file size using the stat() system call.
     struct stat info;
-    if (0 != ::stat(filename().c_str(), &info)) {
+    if (0 != ::stat(mFilename.c_str(), &info)) {
         std::string s = getErrorString();
         if (!s.empty()) mesg += " (" + s + ")";
         OPENVDB_THROW(IoError, mesg);
@@ -252,96 +157,31 @@ File::getSize() const
 }
 
 
-#ifdef OPENVDB_USE_DELAYED_LOADING
-Index64
-File::copyMaxBytes() const
-{
-    return mImpl->mCopyMaxBytes;
-}
-
-
-void
-File::setCopyMaxBytes(Index64 bytes)
-{
-    mImpl->mCopyMaxBytes = bytes;
-}
-#endif
-
-
 ////////////////////////////////////////
 
 
 bool
 File::isOpen() const
 {
-    return mImpl->mIsOpen;
+    return mIsOpen;
 }
 
 
 bool
-#ifdef OPENVDB_USE_DELAYED_LOADING
-File::open(bool delayLoad, const MappedFile::Notifier& notifier)
-#else
-File::open(bool /*delayLoad = true*/)
-#endif // OPENVDB_USE_DELAYED_LOADING
+File::open()
 {
-    if (isOpen()) {
-        OPENVDB_THROW(IoError, filename() << " is already open");
+    if (mIsOpen) {
+        OPENVDB_THROW(IoError, mFilename << " is already open");
     }
-    mImpl->mInStream.reset();
+    mInStream.reset();
 
-    // Open the file.
+    // Open the file using standard I/O (delayed loading has been removed)
     std::unique_ptr<std::istream> newStream;
-    SharedPtr<std::streambuf> newStreamBuf;
-#ifdef OPENVDB_USE_DELAYED_LOADING
-    MappedFile::Ptr newFileMapping;
-    if (!delayLoad || !Archive::isDelayedLoadingEnabled()) {
-#endif
-        newStream.reset(new std::ifstream(
-            filename().c_str(), std::ios_base::in | std::ios_base::binary));
-#ifdef OPENVDB_USE_DELAYED_LOADING
-    } else {
-        bool isTempFile = false;
-        std::string fname = filename();
-        if (getSize() < copyMaxBytes()) {
-            // If the file is not too large, make a temporary private copy of it
-            // and open the copy instead.  The original file can then be modified
-            // or removed without affecting delayed load.
-            try {
-                TempFile tempFile;
-                std::ifstream fstrm(filename().c_str(),
-                    std::ios_base::in | std::ios_base::binary);
-                boost::iostreams::copy(fstrm, tempFile);
-                fname = tempFile.filename();
-                isTempFile = true;
-            } catch (std::exception& e) {
-                std::string mesg;
-                if (e.what()) mesg = std::string(" (") + e.what() + ")";
-                OPENVDB_LOG_WARN("failed to create a temporary copy of " << filename()
-                    << " for delayed loading" << mesg
-                    << "; will read directly from " << filename() << " instead");
-            }
-        }
-
-        // While the file is open, its mapping, stream buffer and stream
-        // must all be maintained.  Once the file is closed, the buffer and
-        // the stream can be discarded, but the mapping needs to persist
-        // if any grids were lazily loaded.
-        try {
-            newFileMapping.reset(new MappedFile(fname, /*autoDelete=*/isTempFile));
-            newStreamBuf = newFileMapping->createBuffer();
-            newStream.reset(new std::istream(newStreamBuf.get()));
-        } catch (std::exception& e) {
-            std::ostringstream ostr;
-            ostr << "could not open file " << filename();
-            if (e.what() != nullptr) ostr << " (" << e.what() << ")";
-            OPENVDB_THROW(IoError, ostr.str());
-        }
-    }
-#endif // OPENVDB_USE_DELAYED_LOADING
+    newStream.reset(new std::ifstream(
+        mFilename.c_str(), std::ios_base::in | std::ios_base::binary));
 
     if (newStream->fail()) {
-        OPENVDB_THROW(IoError, "could not open file " << filename());
+        OPENVDB_THROW(IoError, "could not open file " << mFilename);
     }
 
     // Read in the file header.
@@ -351,63 +191,67 @@ File::open(bool /*delayLoad = true*/)
     } catch (IoError& e) {
         if (e.what() && std::string("not a VDB file") == e.what()) {
             // Rethrow, adding the filename.
-            OPENVDB_THROW(IoError, filename() << " is not a VDB file");
+            OPENVDB_THROW(IoError, mFilename << " is not a VDB file");
         }
         throw;
     }
 
-#ifdef OPENVDB_USE_DELAYED_LOADING
-    mImpl->mFileMapping = newFileMapping;
-    if (mImpl->mFileMapping) mImpl->mFileMapping->setNotifier(notifier);
-    mImpl->mStreamBuf = newStreamBuf;
-#endif
-    mImpl->mInStream.swap(newStream);
+    mInStream.swap(newStream);
 
     // Tag the input stream with the file format and library version numbers
     // and other metadata.
-    mImpl->mStreamMetadata.reset(new StreamMetadata);
-    mImpl->mStreamMetadata->setSeekable(true);
-    io::setStreamMetadataPtr(inputStream(), mImpl->mStreamMetadata, /*transfer=*/false);
+    mStreamMetadata.reset(new StreamMetadata);
+    mStreamMetadata->setSeekable(true);
+    io::setStreamMetadataPtr(inputStream(), mStreamMetadata, /*transfer=*/false);
     Archive::setFormatVersion(inputStream());
     Archive::setLibraryVersion(inputStream());
     Archive::setDataCompression(inputStream());
-#ifdef OPENVDB_USE_DELAYED_LOADING
-    io::setMappedFilePtr(inputStream(), mImpl->mFileMapping);
-#endif
 
     // Read in the VDB metadata.
-    mImpl->mMeta = MetaMap::Ptr(new MetaMap);
-    mImpl->mMeta->readMeta(inputStream());
+    mMeta = MetaMap::Ptr(new MetaMap);
+    mMeta->readMeta(inputStream());
 
     if (!inputHasGridOffsets()) {
-        OPENVDB_LOG_DEBUG_RUNTIME("file " << filename() << " does not support partial reading");
+        OPENVDB_LOG_DEBUG_RUNTIME("file " << mFilename << " does not support partial reading");
 
-        mImpl->mGrids.reset(new GridPtrVec);
-        mImpl->mNamedGrids.clear();
+        mGrids.reset(new GridPtrVec);
+        mNamedGrids.clear();
 
         // Stream in the entire contents of the file and append all grids to mGrids.
         const int32_t gridCount = readGridCount(inputStream());
         for (int32_t i = 0; i < gridCount; ++i) {
             GridDescriptor gd;
-            gd.read(inputStream());
+            gd.readHeader(inputStream());
+            gd.readStreamPos(inputStream());
 
-            GridBase::Ptr grid = createGrid(gd);
-            Archive::readGrid(grid, gd, inputStream());
+            GridBase::Ptr grid = Archive::readGrid(gd, inputStream(), io::ReadOptions{});
 
-            gridDescriptors().insert(std::make_pair(gd.gridName(), gd));
-            mImpl->mGrids->push_back(grid);
-            mImpl->mNamedGrids[gd.uniqueName()] = grid;
+            mGridDescriptors.insert(std::make_pair(gd.gridName(), gd));
+            mGrids->push_back(grid);
+            mNamedGrids[gd.uniqueName()] = grid;
         }
         // Connect instances (grids that share trees with other grids).
-        for (NameMapCIter it = gridDescriptors().begin(); it != gridDescriptors().end(); ++it) {
-            Archive::connectInstance(it->second, mImpl->mNamedGrids);
+        for (NameMapCIter it = mGridDescriptors.begin(); it != mGridDescriptors.end(); ++it) {
+            Archive::connectInstance(it->second, mNamedGrids);
         }
     } else {
-        // Read in just the grid descriptors.
-        readGridDescriptors(inputStream());
+        mGridDescriptors.clear();
+
+        for (int32_t i = 0, N = readGridCount(inputStream()); i < N; ++i) {
+            // Read the grid descriptor.
+            GridDescriptor gd;
+            gd.readHeader(inputStream());
+            gd.readStreamPos(inputStream());
+
+            // Add the descriptor to the dictionary.
+            mGridDescriptors.insert(std::make_pair(gd.gridName(), gd));
+
+            // Skip forward to the next descriptor.
+            gd.seekToEnd(inputStream());
+        }
     }
 
-    mImpl->mIsOpen = true;
+    mIsOpen = true;
     return newFile; // true if file is not identical to opened file
 }
 
@@ -416,18 +260,14 @@ void
 File::close()
 {
     // Reset all data.
-    mImpl->mMeta.reset();
-    mImpl->mGridDescriptors.clear();
-    mImpl->mGrids.reset();
-    mImpl->mNamedGrids.clear();
-    mImpl->mInStream.reset();
-    mImpl->mStreamMetadata.reset();
-#ifdef OPENVDB_USE_DELAYED_LOADING
-    mImpl->mStreamBuf.reset();
-    mImpl->mFileMapping.reset();
-#endif
+    mMeta.reset();
+    mGridDescriptors.clear();
+    mGrids.reset();
+    mNamedGrids.clear();
+    mInStream.reset();
+    mStreamMetadata.reset();
 
-    mImpl->mIsOpen = false;
+    mIsOpen = false;
     setInputHasGridOffsets(true);
 }
 
@@ -438,52 +278,54 @@ File::close()
 bool
 File::hasGrid(const Name& name) const
 {
-    if (!isOpen()) {
-        OPENVDB_THROW(IoError, filename() << " is not open for reading");
+    if (!mIsOpen) {
+        OPENVDB_THROW(IoError, mFilename << " is not open for reading");
     }
-    return (findDescriptor(name) != gridDescriptors().end());
+    return (findDescriptor(name) != mGridDescriptors.end());
 }
 
 
 MetaMap::Ptr
 File::getMetadata() const
 {
-    if (!isOpen()) {
-        OPENVDB_THROW(IoError, filename() << " is not open for reading");
+    if (!mIsOpen) {
+        OPENVDB_THROW(IoError, mFilename << " is not open for reading");
     }
     // Return a deep copy of the file-level metadata, which was read
     // when the file was opened.
-    return MetaMap::Ptr(new MetaMap(*mImpl->mMeta));
+    return MetaMap::Ptr(new MetaMap(*mMeta));
 }
 
 
 GridPtrVecPtr
-File::getGrids() const
+File::getGrids(const io::ReadOptions& readOptions) const
 {
-    if (!isOpen()) {
-        OPENVDB_THROW(IoError, filename() << " is not open for reading");
+    if (!mIsOpen) {
+        OPENVDB_THROW(IoError, mFilename << " is not open for reading");
     }
 
     GridPtrVecPtr ret;
     if (!inputHasGridOffsets()) {
         // If the input file doesn't have grid offsets, then all of the grids
         // have already been streamed in and stored in mGrids.
-        ret = mImpl->mGrids;
+        ret = mGrids;
     } else {
         ret.reset(new GridPtrVec);
 
         Archive::NamedGridMap namedGrids;
 
         // Read all grids represented by the GridDescriptors.
-        for (NameMapCIter i = gridDescriptors().begin(), e = gridDescriptors().end(); i != e; ++i) {
+        for (NameMapCIter i = mGridDescriptors.begin(), e = mGridDescriptors.end(); i != e; ++i) {
             const GridDescriptor& gd = i->second;
-            GridBase::Ptr grid = readGrid(gd);
+            // Seek to the grid in the file.
+            gd.seekToGrid(inputStream());
+            GridBase::Ptr grid = Archive::readGrid(gd, inputStream(), readOptions, mReadDiagnostics);
             ret->push_back(grid);
             namedGrids[gd.uniqueName()] = grid;
         }
 
         // Connect instances (grids that share trees with other grids).
-        for (NameMapCIter i = gridDescriptors().begin(), e = gridDescriptors().end(); i != e; ++i) {
+        for (NameMapCIter i = mGridDescriptors.begin(), e = mGridDescriptors.end(); i != e; ++i) {
             Archive::connectInstance(i->second, namedGrids);
         }
     }
@@ -503,11 +345,11 @@ File::retrieveCachedGrid(const Name& name) const
 
     // Search by unique name.
     Archive::NamedGridMap::const_iterator it =
-        mImpl->mNamedGrids.find(GridDescriptor::stringAsUniqueName(name));
+        mNamedGrids.find(GridDescriptor::stringAsUniqueName(name));
     // If not found, search by grid name.
-    if (it == mImpl->mNamedGrids.end()) it = mImpl->mNamedGrids.find(name);
-    if (it == mImpl->mNamedGrids.end()) {
-        OPENVDB_THROW(KeyError, filename() << " has no grid named \"" << name << "\"");
+    if (it == mNamedGrids.end()) it = mNamedGrids.find(name);
+    if (it == mNamedGrids.end()) {
+        OPENVDB_THROW(KeyError, mFilename << " has no grid named \"" << name << "\"");
     }
     return it->second;
 }
@@ -519,8 +361,8 @@ File::retrieveCachedGrid(const Name& name) const
 GridPtrVecPtr
 File::readAllGridMetadata()
 {
-    if (!isOpen()) {
-        OPENVDB_THROW(IoError, filename() << " is not open for reading");
+    if (!mIsOpen) {
+        OPENVDB_THROW(IoError, mFilename << " is not open for reading");
     }
 
     if (fileVersion() < OPENVDB_FILE_VERSION_FLOAT_FRUSTUM_BBOX) {
@@ -533,18 +375,22 @@ File::readAllGridMetadata()
     if (!inputHasGridOffsets()) {
         // If the input file doesn't have grid offsets, then all of the grids
         // have already been streamed in and stored in mGrids.
-        for (size_t i = 0, N = mImpl->mGrids->size(); i < N; ++i) {
+        for (size_t i = 0, N = mGrids->size(); i < N; ++i) {
             // Return copies of the grids, but with empty trees.
-            ret->push_back((*mImpl->mGrids)[i]->copyGridWithNewTree());
+            ret->push_back((*mGrids)[i]->copyGridWithNewTree());
         }
     } else {
         // Read just the metadata and transforms for all grids.
-        for (NameMapCIter i = gridDescriptors().begin(), e = gridDescriptors().end(); i != e; ++i) {
+        for (NameMapCIter i = mGridDescriptors.begin(), e = mGridDescriptors.end(); i != e; ++i) {
             const GridDescriptor& gd = i->second;
-            GridBase::ConstPtr grid = readGridPartial(gd, /*readTopology=*/false);
+            // Seek to the grid in the file.
+            gd.seekToGrid(inputStream());
+            io::ReadOptions readOptions;
+            readOptions.readMode = io::ReadMode::TopologyOnly;
+            GridBase::ConstPtr grid = Archive::readGrid(gd, inputStream(), readOptions);
             // Return copies of the grids, but with empty trees.
             // (As of 0.98.0, at least, it would suffice to just const cast
-            // the grid pointers returned by readGridPartial(), but shallow
+            // the grid pointers returned by readGrid(partial=true), but shallow
             // copying the grids helps to ensure future compatibility.)
             ret->push_back(grid->copyGridWithNewTree());
         }
@@ -556,8 +402,8 @@ File::readAllGridMetadata()
 GridBase::Ptr
 File::readGridMetadata(const Name& name)
 {
-    if (!isOpen()) {
-        OPENVDB_THROW(IoError, filename() << " is not open for reading.");
+    if (!mIsOpen) {
+        OPENVDB_THROW(IoError, mFilename << " is not open for reading.");
     }
 
     if (fileVersion() < OPENVDB_FILE_VERSION_FLOAT_FRUSTUM_BBOX) {
@@ -572,13 +418,16 @@ File::readGridMetadata(const Name& name)
         ret = readGrid(name);
     } else {
         NameMapCIter it = findDescriptor(name);
-        if (it == gridDescriptors().end()) {
-            OPENVDB_THROW(KeyError, filename() << " has no grid named \"" << name << "\"");
+        if (it == mGridDescriptors.end()) {
+            OPENVDB_THROW(KeyError, mFilename << " has no grid named \"" << name << "\"");
         }
 
         // Seek to and read in the grid from the file.
         const GridDescriptor& gd = it->second;
-        ret = readGridPartial(gd, /*readTopology=*/false);
+        gd.seekToGrid(inputStream());
+        io::ReadOptions readOptions;
+        readOptions.readMode = io::ReadMode::TopologyOnly;
+        ret = Archive::readGrid(gd, inputStream(), readOptions);
     }
     return ret->copyGridWithNewTree();
 }
@@ -587,34 +436,29 @@ File::readGridMetadata(const Name& name)
 ////////////////////////////////////////
 
 
-GridBase::Ptr
-File::readGrid(const Name& name)
-{
-    return readGridByName(name, BBoxd());
-}
-
-
 GridBase::Ptr
 File::readGrid(const Name& name, const BBoxd& bbox)
 {
-    return readGridByName(name, bbox);
+    io::ReadOptions readOptions;
+    readOptions.clipBBox = bbox;
+    return readGrid(name, readOptions);
 }
 
 
 GridBase::Ptr
-File::readGridByName(const Name& name, const BBoxd& bbox)
+File::readGrid(const Name& name, const io::ReadOptions& readOptions)
 {
-    if (!isOpen()) {
-        OPENVDB_THROW(IoError, filename() << " is not open for reading.");
+    if (!mIsOpen) {
+        OPENVDB_THROW(IoError, mFilename << " is not open for reading.");
     }
 
-    const bool clip = bbox.isSorted();
-
     // If a grid with the given name was already read and cached
     // (along with the entire contents of the file, because the file
     // doesn't support random access), retrieve and return it.
     GridBase::Ptr grid = retrieveCachedGrid(name);
     if (grid) {
+        const auto& bbox = readOptions.clipBBox;
+        const bool clip = bbox.isSorted();
         if (clip) {
             grid = grid->deepCopyGrid();
             grid->clipGrid(bbox);
@@ -623,32 +467,33 @@ File::readGridByName(const Name& name, const BBoxd& bbox)
     }
 
     NameMapCIter it = findDescriptor(name);
-    if (it == gridDescriptors().end()) {
-        OPENVDB_THROW(KeyError, filename() << " has no grid named \"" << name << "\"");
+    if (it == mGridDescriptors.end()) {
+        OPENVDB_THROW(KeyError, mFilename << " has no grid named \"" << name << "\"");
     }
 
     // Seek to and read in the grid from the file.
     const GridDescriptor& gd = it->second;
-    grid = (clip ? readGrid(gd, bbox) : readGrid(gd));
+    // This method should not be called for files that don't contain grid offsets.
+    OPENVDB_ASSERT(inputHasGridOffsets());
+    // Seek to the grid in the file.
+    gd.seekToGrid(inputStream());
+    grid = Archive::readGrid(gd, inputStream(), readOptions, mReadDiagnostics);
 
     if (gd.isInstance()) {
         /// @todo Refactor to share code with Archive::connectInstance()?
         NameMapCIter parentIt =
             findDescriptor(GridDescriptor::nameAsString(gd.instanceParentName()));
-        if (parentIt == gridDescriptors().end()) {
+        if (parentIt == mGridDescriptors.end()) {
             OPENVDB_THROW(KeyError, "missing instance parent \""
                 << GridDescriptor::nameAsString(gd.instanceParentName())
                 << "\" for grid " << GridDescriptor::nameAsString(gd.uniqueName())
-                << " in file " << filename());
+                << " in file " << mFilename);
         }
 
         GridBase::Ptr parent;
-        if (clip) {
-            const CoordBBox indexBBox = grid->constTransform().worldToIndexNodeCentered(bbox);
-            parent = readGrid(parentIt->second, indexBBox);
-        } else {
-            parent = readGrid(parentIt->second);
-        }
+        OPENVDB_ASSERT(inputHasGridOffsets());
+        parentIt->second.seekToGrid(inputStream());
+        parent = Archive::readGrid(parentIt->second, inputStream(), readOptions, mReadDiagnostics);
         if (parent) grid->setTree(parent->baseTreePtr());
     }
     return grid;
@@ -659,53 +504,29 @@ File::readGridByName(const Name& name, const BBoxd& bbox)
 
 
 void
-File::writeGrids(const GridCPtrVec& grids, const MetaMap& meta) const
+File::writeGrids(const GridCPtrVec& grids, const MetaMap& meta, const io::WriteOptions& writeOptions) const
 {
-    if (isOpen()) {
+    if (mIsOpen) {
         OPENVDB_THROW(IoError,
-            filename() << " cannot be written because it is open for reading");
+            mFilename << " cannot be written because it is open for reading");
     }
 
     // Create a file stream and write it out.
     std::ofstream file;
-    file.open(filename().c_str(),
+    file.open(mFilename.c_str(),
         std::ios_base::out | std::ios_base::binary | std::ios_base::trunc);
 
     if (file.fail()) {
-        OPENVDB_THROW(IoError, "could not open " << filename() << " for writing");
+        OPENVDB_THROW(IoError, "could not open " << mFilename << " for writing");
     }
 
     // Write out the vdb.
-    Archive::write(file, grids, /*seekable=*/true, meta);
+    Archive::write(file, grids, /*seekable=*/true, meta, writeOptions);
 
     file.close();
 }
 
 
-////////////////////////////////////////
-
-
-void
-File::readGridDescriptors(std::istream& is)
-{
-    // This method should not be called for files that don't contain grid offsets.
-    OPENVDB_ASSERT(inputHasGridOffsets());
-
-    gridDescriptors().clear();
-
-    for (int32_t i = 0, N = readGridCount(is); i < N; ++i) {
-        // Read the grid descriptor.
-        GridDescriptor gd;
-        gd.read(is);
-
-        // Add the descriptor to the dictionary.
-        gridDescriptors().insert(std::make_pair(gd.gridName(), gd));
-
-        // Skip forward to the next descriptor.
-        gd.seekToEnd(is);
-    }
-}
-
 
 ////////////////////////////////////////
 
@@ -716,20 +537,20 @@ File::findDescriptor(const Name& name) const
     const Name uniqueName = GridDescriptor::stringAsUniqueName(name);
 
     // Find all descriptors with the given grid name.
-    std::pair<NameMapCIter, NameMapCIter> range = gridDescriptors().equal_range(name);
+    std::pair<NameMapCIter, NameMapCIter> range = mGridDescriptors.equal_range(name);
 
     if (range.first == range.second) {
         // If no descriptors were found with the given grid name, the name might have
         // a suffix ("name[N]").  In that case, remove the "[N]" suffix and search again.
-        range = gridDescriptors().equal_range(GridDescriptor::stripSuffix(uniqueName));
+        range = mGridDescriptors.equal_range(GridDescriptor::stripSuffix(uniqueName));
     }
 
     const size_t count = size_t(std::distance(range.first, range.second));
     if (count > 1 && name == uniqueName) {
-        OPENVDB_LOG_WARN(filename() << " has more than one grid named \"" << name << "\"");
+        OPENVDB_LOG_WARN(mFilename << " has more than one grid named \"" << name << "\"");
     }
 
-    NameMapCIter ret = gridDescriptors().end();
+    NameMapCIter ret = mGridDescriptors.end();
 
     if (count > 0) {
         if (name == uniqueName) {
@@ -755,109 +576,23 @@ File::findDescriptor(const Name& name) const
 ////////////////////////////////////////
 
 
-GridBase::Ptr
-File::createGrid(const GridDescriptor& gd) const
-{
-    // Create the grid.
-    if (!GridBase::isRegistered(gd.gridType())) {
-        OPENVDB_THROW(KeyError, "Cannot read grid "
-            << GridDescriptor::nameAsString(gd.uniqueName())
-            << " from " << filename() << ": grid type "
-            << gd.gridType() << " is not registered");
-    }
-
-    GridBase::Ptr grid = GridBase::createGrid(gd.gridType());
-    if (grid) grid->setSaveFloatAsHalf(gd.saveFloatAsHalf());
-
-    return grid;
-}
-
-
-GridBase::ConstPtr
-File::readGridPartial(const GridDescriptor& gd, bool readTopology) const
-{
-    // This method should not be called for files that don't contain grid offsets.
-    OPENVDB_ASSERT(inputHasGridOffsets());
-
-    GridBase::Ptr grid = createGrid(gd);
-
-    // Seek to grid.
-    gd.seekToGrid(inputStream());
-
-    // Read the grid partially.
-    readGridPartial(grid, inputStream(), gd.isInstance(), readTopology);
-
-    // Promote to a const grid.
-    GridBase::ConstPtr constGrid = grid;
-
-    return constGrid;
-}
-
-
-GridBase::Ptr
-File::readGrid(const GridDescriptor& gd) const
-{
-    return Impl::readGrid(*this, gd, Impl::NoBBox());
-}
-
-
-GridBase::Ptr
-File::readGrid(const GridDescriptor& gd, const BBoxd& bbox) const
-{
-    return Impl::readGrid(*this, gd, bbox);
-}
-
-
-GridBase::Ptr
-File::readGrid(const GridDescriptor& gd, const CoordBBox& bbox) const
-{
-    return Impl::readGrid(*this, gd, bbox);
-}
-
-
-void
-File::readGridPartial(GridBase::Ptr grid, std::istream& is,
-    bool isInstance, bool readTopology) const
-{
-    // This method should not be called for files that don't contain grid offsets.
-    OPENVDB_ASSERT(inputHasGridOffsets());
-
-    // This code needs to stay in sync with io::Archive::readGrid(), in terms of
-    // the order of operations.
-    readGridCompression(is);
-    grid->readMeta(is);
-
-    // drop DelayedLoadMetadata from the grid as it is only useful for IO
-    if ((*grid)[GridBase::META_FILE_DELAYED_LOAD]) {
-        grid->removeMeta(GridBase::META_FILE_DELAYED_LOAD);
-    }
-
-    grid->readTransform(is);
-    if (!isInstance && readTopology) {
-        grid->readTopology(is);
-    }
-}
-
-
-////////////////////////////////////////
-
-
 File::NameIterator
 File::beginName() const
 {
-    if (!isOpen()) {
-        OPENVDB_THROW(IoError, filename() << " is not open for reading");
+    if (!mIsOpen) {
+        OPENVDB_THROW(IoError, mFilename << " is not open for reading");
     }
-    return File::NameIterator(gridDescriptors().begin());
+    return File::NameIterator(mGridDescriptors.begin());
 }
 
 
 File::NameIterator
 File::endName() const
 {
-    return File::NameIterator(gridDescriptors().end());
+    return File::NameIterator(mGridDescriptors.end());
 }
 
+
 } // namespace io
 } // namespace OPENVDB_VERSION_NAME
 } // namespace openvdb

openvdb/openvdb/io/File.h

@@ -34,7 +34,7 @@ public:
     using NameMapCIter = NameMap::const_iterator;
 
     explicit File(const std::string& filename);
-    ~File() override;
+    ~File() override { }
 
     /// @brief Copy constructor
     /// @details The copy will be closed and will not reference the same
@@ -54,20 +54,16 @@ public:
     /// @details The file does not necessarily exist on disk yet.
     const std::string& filename() const;
 
-#ifdef OPENVDB_USE_DELAYED_LOADING
     /// @brief Open the file, read the file header and the file-level metadata,
     /// and populate the grid descriptors, but do not load any grids into memory.
-    /// @details If @a delayLoad is true, map the file into memory and enable delayed loading
-    /// of grids, and if a notifier is provided, call it when the file gets unmapped.
-    /// @note Define the environment variable @c OPENVDB_DISABLE_DELAYED_LOAD to disable
-    /// delayed loading unconditionally.
     /// @throw IoError if the file is not a valid VDB file.
     /// @return @c true if the file's UUID has changed since it was last read.
-    /// @see setCopyMaxBytes
-    bool open(bool delayLoad = true, const MappedFile::Notifier& = MappedFile::Notifier());
-#else
-    bool open(bool /*delayLoad*/ = false);
-#endif
+    bool open();
+
+    OPENVDB_DEPRECATED_MESSAGE("Use File::open() instead.This method is deprecated and will be removed. Delayed loading is no longer supported.")
+    bool open(bool /*delayLoad*/) { return open(); }
+    OPENVDB_DEPRECATED_MESSAGE("Use File::open() instead. This method is deprecated and will be removed. Delayed loading is no longer supported.")
+    bool open(bool /*delayLoad*/, const MappedFile::Notifier& /*notifier*/) { return open(); }
 
     /// Return @c true if the file has been opened for reading.
     bool isOpen() const;
@@ -79,23 +75,10 @@ public:
     /// @throw IoError if the file size cannot be determined.
     Index64 getSize() const;
 
-#ifdef OPENVDB_USE_DELAYED_LOADING
-    /// @brief Return the size in bytes above which this file will not be
-    /// automatically copied during delayed loading.
-    Index64 copyMaxBytes() const;
-    /// @brief If this file is opened with delayed loading enabled, make a private copy
-    /// of the file if its size in bytes is less than the specified value.
-    /// @details Making a private copy ensures that the file can't change on disk
-    /// before it has been fully read.
-    /// @warning If the file is larger than this size, it is the user's responsibility
-    /// to ensure that it does not change on disk before it has been fully read.
-    /// Undefined behavior and/or a crash might result otherwise.
-    /// @note Copying is enabled by default, but it can be disabled for individual files
-    /// by setting the maximum size to zero bytes.  A default size limit can be specified
-    /// by setting the environment variable @c OPENVDB_DELAYED_LOAD_COPY_MAX_BYTES
-    /// to the desired number of bytes.
-    void setCopyMaxBytes(Index64 bytes);
-#endif
+    OPENVDB_DEPRECATED_MESSAGE("Always returns 0. This method is deprecated and will be removed. Delayed loading is no longer supported.")
+    Index64 copyMaxBytes() const { return 0; }
+    OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.")
+    void setCopyMaxBytes(Index64 /*bytes*/) { }
 
     /// Return @c true if a grid of the given name exists in this file.
     bool hasGrid(const Name&) const;
@@ -104,7 +87,7 @@ public:
     MetaMap::Ptr getMetadata() const;
 
     /// Read the entire contents of the file and return a list of grid pointers.
-    GridPtrVecPtr getGrids() const;
+    GridPtrVecPtr getGrids(const io::ReadOptions& readOptions = io::ReadOptions{}) const;
 
     /// @brief Read just the grid metadata and transforms from the file and return a list
     /// of pointers to grids that are empty except for their metadata and transforms.
@@ -117,8 +100,8 @@ public:
     /// @throw KeyError if no grid with the given name exists in this file.
     GridBase::Ptr readGridMetadata(const Name&);
 
-    /// Read an entire grid, including all of its data blocks.
-    GridBase::Ptr readGrid(const Name&);
+    /// Read an entire grid, including all of its data blocks, using the provided options if given.
+    GridBase::Ptr readGrid(const Name&, const io::ReadOptions& readOptions = io::ReadOptions{});
     /// @brief Read a grid, including its data blocks, but only where it
     /// intersects the given world-space bounding box.
     GridBase::Ptr readGrid(const Name&, const BBoxd&);
@@ -127,12 +110,14 @@ public:
 
     /// @brief Write the grids in the given container to the file whose name
     /// was given in the constructor.
-    void write(const GridCPtrVec&, const MetaMap& = MetaMap()) const override;
+    void write(const GridCPtrVec&, const MetaMap& = MetaMap(),
+        const io::WriteOptions& = io::WriteOptions{}) const override;
 
     /// @brief Write the grids in the given container to the file whose name
     /// was given in the constructor.
     template<typename GridPtrContainerT>
-    void write(const GridPtrContainerT&, const MetaMap& = MetaMap()) const;
+    void write(const GridPtrContainerT&, const MetaMap& = MetaMap(),
+        const io::WriteOptions& = io::WriteOptions{}) const;
 
     /// A const iterator that iterates over all names in the file. This is only
     /// valid once the file has been opened.
@@ -163,42 +148,16 @@ public:
     NameIterator endName() const;
 
 private:
-    /// Read in all grid descriptors that are stored in the given stream.
-    void readGridDescriptors(std::istream&);
-
     /// @brief Return an iterator to the descriptor for the grid with the given name.
     /// If the name is non-unique, return an iterator to the first matching descriptor.
     NameMapCIter findDescriptor(const Name&) const;
 
-    /// Return a newly created, empty grid of the type specified by the given grid descriptor.
-    GridBase::Ptr createGrid(const GridDescriptor&) const;
-
-    /// @brief Read a grid, including its data blocks, but only where it
-    /// intersects the given world-space bounding box.
-    GridBase::Ptr readGridByName(const Name&, const BBoxd&);
-
-    /// Read in and return the partially-populated grid specified by the given grid descriptor.
-    GridBase::ConstPtr readGridPartial(const GridDescriptor&, bool readTopology) const;
-
-    /// Read in and return the grid specified by the given grid descriptor.
-    GridBase::Ptr readGrid(const GridDescriptor&) const;
-    /// Read in and return the region of the grid specified by the given grid descriptor
-    /// that intersects the given world-space bounding box.
-    GridBase::Ptr readGrid(const GridDescriptor&, const BBoxd&) const;
-    /// Read in and return the region of the grid specified by the given grid descriptor
-    /// that intersects the given index-space bounding box.
-    GridBase::Ptr readGrid(const GridDescriptor&, const CoordBBox&) const;
-
-    /// @brief Partially populate the given grid by reading its metadata and transform and,
-    /// if the grid is not an instance, its tree structure, but not the tree's leaf nodes.
-    void readGridPartial(GridBase::Ptr, std::istream&, bool isInstance, bool readTopology) const;
-
     /// @brief Retrieve a grid from @c mNamedGrids.  Return a null pointer
     /// if @c mNamedGrids was not populated (because this file is random-access).
     /// @throw KeyError if no grid with the given name exists in this file.
     GridBase::Ptr retrieveCachedGrid(const Name&) const;
 
-    void writeGrids(const GridCPtrVec&, const MetaMap&) const;
+    void writeGrids(const GridCPtrVec&, const MetaMap&, const io::WriteOptions&) const;
 
     MetaMap::Ptr fileMetadata();
     MetaMap::ConstPtr fileMetadata() const;
@@ -211,8 +170,22 @@ private:
     friend class ::TestFile;
     friend class ::TestStream;
 
-    struct Impl;
-    std::unique_ptr<Impl> mImpl;
+    std::string mFilename;
+    // The file-level metadata
+    MetaMap::Ptr mMeta;
+    // The file stream that is open for reading
+    std::unique_ptr<std::istream> mInStream;
+    // File-level stream metadata (file format, compression, etc.)
+    StreamMetadata::Ptr mStreamMetadata;
+    // Flag indicating if we have read in the global information (header,
+    // metadata, and grid descriptors) for this VDB file
+    bool mIsOpen = false;
+    // Grid descriptors for all grids stored in the file, indexed by grid name
+    NameMap mGridDescriptors;
+    // All grids, indexed by unique name (used only when mHasGridOffsets is false)
+    Archive::NamedGridMap mNamedGrids;
+    // All grids stored in the file (used only when mHasGridOffsets is false)
+    GridPtrVecPtr mGrids;
 };
 
 
@@ -220,19 +193,21 @@ private:
 
 
 inline void
-File::write(const GridCPtrVec& grids, const MetaMap& meta) const
+File::write(const GridCPtrVec& grids, const MetaMap& meta,
+    const io::WriteOptions& writeOptions) const
 {
-    this->writeGrids(grids, meta);
+    this->writeGrids(grids, meta, writeOptions);
 }
 
 
 template<typename GridPtrContainerT>
 inline void
-File::write(const GridPtrContainerT& container, const MetaMap& meta) const
+File::write(const GridPtrContainerT& container, const MetaMap& meta,
+    const io::WriteOptions& writeOptions) const
 {
     GridCPtrVec grids;
     std::copy(container.begin(), container.end(), std::back_inserter(grids));
-    this->writeGrids(grids, meta);
+    this->writeGrids(grids, meta, writeOptions);
 }
 
 } // namespace io

openvdb/openvdb/io/GridDescriptor.cc

@@ -69,8 +69,8 @@ GridDescriptor::writeStreamPos(std::ostream &os) const
     os.write(reinterpret_cast<const char*>(&mEndPos), sizeof(int64_t));
 }
 
-GridBase::Ptr
-GridDescriptor::read(std::istream &is)
+void
+GridDescriptor::readHeader(std::istream &is)
 {
     checkFormatVersion(is);
 
@@ -86,21 +86,28 @@ GridDescriptor::read(std::istream &is)
     }
 
     mInstanceParentName = readString(is);
+}
+
+void
+GridDescriptor::readStreamPos(std::istream &is)
+{
+    is.read(reinterpret_cast<char*>(&mGridPos), sizeof(int64_t));
+    is.read(reinterpret_cast<char*>(&mBlockPos), sizeof(int64_t));
+    is.read(reinterpret_cast<char*>(&mEndPos), sizeof(int64_t));
+}
+
+GridBase::Ptr
+GridDescriptor::read(std::istream &is)
+{
+    readHeader(is);
+    readStreamPos(is);
 
-    // Create the grid of the type if it has been registered.
     if (!GridBase::isRegistered(mGridType)) {
         OPENVDB_THROW(LookupError, "Cannot read grid." <<
             " Grid type " << mGridType << " is not registered.");
     }
-    // else
     GridBase::Ptr grid = GridBase::createGrid(mGridType);
     if (grid) grid->setSaveFloatAsHalf(mSaveFloatAsHalf);
-
-    // Read in the offsets.
-    is.read(reinterpret_cast<char*>(&mGridPos), sizeof(int64_t));
-    is.read(reinterpret_cast<char*>(&mBlockPos), sizeof(int64_t));
-    is.read(reinterpret_cast<char*>(&mEndPos), sizeof(int64_t));
-
     return grid;
 }
 

openvdb/openvdb/io/GridDescriptor.h

@@ -61,8 +61,18 @@ public:
     /// written out separately.
     void writeStreamPos(std::ostream&) const;
 
+    /// @brief Read this descriptor's header information (all data except for
+    /// stream offsets) from the given stream.
+    void readHeader(std::istream&);
+
+    /// @brief Read stream positions (grid, block, and end offsets) from the
+    /// given stream.
+    void readStreamPos(std::istream&);
+
     /// @brief Read a grid descriptor from the given stream.
     /// @return an empty grid of the type specified by the grid descriptor.
+    /// @deprecated Use readHeader() followed by readStreamPos() instead.
+    OPENVDB_DEPRECATED_MESSAGE("Use readHeader() followed by readStreamPos() instead.")
     GridBase::Ptr read(std::istream&);
 
     /// @brief Append the number @a n to the given name (separated by an ASCII

openvdb/openvdb/io/Stream.cc

@@ -5,14 +5,9 @@
 
 #include "File.h" ///< @todo refactor
 #include "GridDescriptor.h"
-#include "TempFile.h"
 #include <openvdb/Exceptions.h>
 #include <cstdint>
 
-#ifdef OPENVDB_USE_DELAYED_LOADING
-#include <boost/iostreams/copy.hpp>
-#endif
-
 #include <cstdio> // for remove()
 #include <functional> // for std::bind()
 #include <iostream>
@@ -24,140 +19,62 @@ OPENVDB_USE_VERSION_NAMESPACE
 namespace OPENVDB_VERSION_NAME {
 namespace io {
 
-struct Stream::Impl
-{
-    Impl(): mOutputStream{nullptr} {}
-    Impl(const Impl& other) { *this = other; }
-    Impl& operator=(const Impl& other)
-    {
-        if (&other != this) {
-            mMeta = other.mMeta; ///< @todo deep copy?
-            mGrids = other.mGrids; ///< @todo deep copy?
-            mOutputStream = other.mOutputStream;
-            mFile.reset();
-        }
-        return *this;
-    }
 
-    MetaMap::Ptr mMeta;
-    GridPtrVecPtr mGrids;
-    std::ostream* mOutputStream;
-    std::unique_ptr<File> mFile;
-};
-
-
-////////////////////////////////////////
-
-
-#ifdef OPENVDB_USE_DELAYED_LOADING
-
-namespace {
-
-/// @todo Use MappedFile auto-deletion instead.
-void
-removeTempFile(const std::string expectedFilename, const std::string& filename)
-{
-    if (filename == expectedFilename) {
-        if (0 != std::remove(filename.c_str())) {
-            std::string mesg = getErrorString();
-            if (!mesg.empty()) mesg = " (" + mesg + ")";
-            OPENVDB_LOG_WARN("failed to remove temporary file " << filename << mesg);
-        }
-    }
-}
-
-}
-
-#endif // OPENVDB_USE_DELAYED_LOADING
-
-
-Stream::Stream(std::istream& is, bool delayLoad): mImpl(new Impl)
+Stream::Stream(std::istream& is)
 {
     if (!is) return;
 
-    (void) delayLoad;
+    // Delayed loading has been removed - always read directly from the stream
+    readHeader(is);
 
-#ifdef OPENVDB_USE_DELAYED_LOADING
-    if (delayLoad && Archive::isDelayedLoadingEnabled()) {
-        // Copy the contents of the stream to a temporary private file
-        // and open the file instead.
-        std::unique_ptr<TempFile> tempFile;
-        try {
-            tempFile.reset(new TempFile);
-        } catch (std::exception& e) {
-            std::string mesg;
-            if (e.what()) mesg = std::string(" (") + e.what() + ")";
-            OPENVDB_LOG_WARN("failed to create a temporary file for delayed loading" << mesg
-                << "; will read directly from the input stream instead");
-        }
-        if (tempFile) {
-            boost::iostreams::copy(is, *tempFile);
-            const std::string& filename = tempFile->filename();
-            mImpl->mFile.reset(new File(filename));
-            mImpl->mFile->setCopyMaxBytes(0); // don't make a copy of the temporary file
-            /// @todo Need to pass auto-deletion flag to MappedFile.
-            mImpl->mFile->open(delayLoad,
-                std::bind(&removeTempFile, filename, std::placeholders::_1));
-        }
+    // Tag the input stream with the library and file format version numbers
+    // and the compression options specified in the header.
+    StreamMetadata::Ptr streamMetadata(new StreamMetadata);
+    io::setStreamMetadataPtr(is, streamMetadata, /*transfer=*/false);
+    io::setVersion(is, libraryVersion(), fileVersion());
+    io::setDataCompression(is, compression());
+
+    // Read in the VDB metadata.
+    mMeta.reset(new MetaMap);
+    mMeta->readMeta(is);
+
+    // Read in the number of grids.
+    const int32_t gridCount = readGridCount(is);
+
+    // Read in all grids and insert them into mGrids.
+    mGrids.reset(new GridPtrVec);
+    std::vector<GridDescriptor> descriptors;
+    descriptors.reserve(gridCount);
+    Archive::NamedGridMap namedGrids;
+    for (int32_t i = 0; i < gridCount; ++i) {
+        GridDescriptor gd;
+        gd.readHeader(is);
+        gd.readStreamPos(is);
+        descriptors.push_back(gd);
+        GridBase::Ptr grid = Archive::readGrid(gd, is, io::ReadOptions{});
+        mGrids->push_back(grid);
+        namedGrids[gd.uniqueName()] = grid;
     }
-#endif // OPENVDB_USE_DELAYED_LOADING
 
-    if (!mImpl->mFile) {
-        readHeader(is);
-
-        // Tag the input stream with the library and file format version numbers
-        // and the compression options specified in the header.
-        StreamMetadata::Ptr streamMetadata(new StreamMetadata);
-        io::setStreamMetadataPtr(is, streamMetadata, /*transfer=*/false);
-        io::setVersion(is, libraryVersion(), fileVersion());
-        io::setDataCompression(is, compression());
-
-        // Read in the VDB metadata.
-        mImpl->mMeta.reset(new MetaMap);
-        mImpl->mMeta->readMeta(is);
-
-        // Read in the number of grids.
-        const int32_t gridCount = readGridCount(is);
-
-        // Read in all grids and insert them into mGrids.
-        mImpl->mGrids.reset(new GridPtrVec);
-        std::vector<GridDescriptor> descriptors;
-        descriptors.reserve(gridCount);
-        Archive::NamedGridMap namedGrids;
-        for (int32_t i = 0; i < gridCount; ++i) {
-            GridDescriptor gd;
-            gd.read(is);
-            descriptors.push_back(gd);
-            GridBase::Ptr grid = readGrid(gd, is);
-            mImpl->mGrids->push_back(grid);
-            namedGrids[gd.uniqueName()] = grid;
-        }
-
-        // Connect instances (grids that share trees with other grids).
-        for (size_t i = 0, N = descriptors.size(); i < N; ++i) {
-            Archive::connectInstance(descriptors[i], namedGrids);
-        }
+    // Connect instances (grids that share trees with other grids).
+    for (size_t i = 0, N = descriptors.size(); i < N; ++i) {
+        Archive::connectInstance(descriptors[i], namedGrids);
     }
 }
 
 
-Stream::Stream(): mImpl(new Impl)
+Stream::Stream(std::ostream& os)
+    : Archive()
+    , mOutputStream(&os)
 {
 }
 
 
-Stream::Stream(std::ostream& os): mImpl(new Impl)
-{
-    mImpl->mOutputStream = &os;
-}
-
-
-Stream::~Stream()
-{
-}
-
-
-Stream::Stream(const Stream& other): Archive(other), mImpl(new Impl(*other.mImpl))
+Stream::Stream(const Stream& other)
+    : Archive(other)
+    , mMeta(other.mMeta)
+    , mGrids(other.mGrids)
+    , mOutputStream(other.mOutputStream)
 {
 }
 
@@ -166,7 +83,9 @@ Stream&
 Stream::operator=(const Stream& other)
 {
     if (&other != this) {
-        mImpl.reset(new Impl(*other.mImpl));
+        mMeta = other.mMeta;
+        mGrids = other.mGrids;
+        mOutputStream = other.mOutputStream;
     }
     return *this;
 }
@@ -182,39 +101,22 @@ Stream::copy() const
 ////////////////////////////////////////
 
 
-GridBase::Ptr
-Stream::readGrid(const GridDescriptor& gd, std::istream& is) const
-{
-    GridBase::Ptr grid;
-
-    if (!GridBase::isRegistered(gd.gridType())) {
-        OPENVDB_THROW(TypeError, "can't read grid \""
-            << GridDescriptor::nameAsString(gd.uniqueName()) <<
-            "\" from input stream because grid type " << gd.gridType() << " is unknown");
-    } else {
-        grid = GridBase::createGrid(gd.gridType());
-        if (grid) grid->setSaveFloatAsHalf(gd.saveFloatAsHalf());
-
-        Archive::readGrid(grid, gd, is);
-    }
-    return grid;
-}
-
-
 void
-Stream::write(const GridCPtrVec& grids, const MetaMap& metadata) const
+Stream::write(const GridCPtrVec& grids, const MetaMap& metadata,
+    const io::WriteOptions& writeOptions) const
 {
-    if (mImpl->mOutputStream == nullptr) {
+    if (mOutputStream == nullptr) {
         OPENVDB_THROW(ValueError, "no output stream was specified");
     }
-    this->writeGrids(*mImpl->mOutputStream, grids, metadata);
+    this->writeGrids(*mOutputStream, grids, metadata, writeOptions);
 }
 
 
 void
-Stream::writeGrids(std::ostream& os, const GridCPtrVec& grids, const MetaMap& metadata) const
+Stream::writeGrids(std::ostream& os, const GridCPtrVec& grids, const MetaMap& metadata,
+    const io::WriteOptions& writeOptions) const
 {
-    Archive::write(os, grids, /*seekable=*/false, metadata);
+    Archive::write(os, grids, /*seekable=*/false, metadata, writeOptions);
 }
 
 
@@ -225,12 +127,10 @@ MetaMap::Ptr
 Stream::getMetadata() const
 {
     MetaMap::Ptr result;
-    if (mImpl->mFile) {
-        result = mImpl->mFile->getMetadata();
-    } else if (mImpl->mMeta) {
+    if (mMeta) {
         // Return a deep copy of the file-level metadata
         // that was read when this object was constructed.
-        result.reset(new MetaMap(*mImpl->mMeta));
+        result.reset(new MetaMap(*mMeta));
     }
     return result;
 }
@@ -239,10 +139,7 @@ Stream::getMetadata() const
 GridPtrVecPtr
 Stream::getGrids()
 {
-    if (mImpl->mFile) {
-        return mImpl->mFile->getGrids();
-    }
-    return mImpl->mGrids;
+    return mGrids;
 }
 
 } // namespace io

openvdb/openvdb/io/Stream.h

@@ -22,21 +22,21 @@ class OPENVDB_API Stream: public Archive
 {
 public:
     /// @brief Read grids from an input stream.
-    /// @details If @a delayLoad is true, map the contents of the input stream
-    /// into memory and enable delayed loading of grids.
-    /// @note Define the environment variable @c OPENVDB_DISABLE_DELAYED_LOAD
-    /// to disable delayed loading unconditionally.
-    explicit Stream(std::istream&, bool delayLoad = true);
+    /// @param is The input stream to read from
+    explicit Stream(std::istream&);
+
+    OPENVDB_DEPRECATED_MESSAGE("Use Stream(std::istream&) instead. This method is deprecated and will be removed. Delayed loading is no longer supported.")
+    Stream(std::istream& is, bool /*delayLoad*/) : Stream(is) { }
 
     /// Construct an archive for stream output.
-    Stream();
+    Stream() = default;
     /// Construct an archive for output to the given stream.
     explicit Stream(std::ostream&);
 
     Stream(const Stream&);
     Stream& operator=(const Stream&);
 
-    ~Stream() override;
+    ~Stream() override { }
 
     /// @brief Return a copy of this archive.
     Archive::Ptr copy() const override;
@@ -49,24 +49,22 @@ public:
 
     /// @brief Write the grids in the given container to this archive's output stream.
     /// @throw ValueError if this archive was constructed without specifying an output stream.
-    void write(const GridCPtrVec&, const MetaMap& = MetaMap()) const override;
+    void write(const GridCPtrVec&, const MetaMap& = MetaMap(),
+        const io::WriteOptions& = io::WriteOptions{}) const override;
 
     /// @brief Write the grids in the given container to this archive's output stream.
     /// @throw ValueError if this archive was constructed without specifying an output stream.
     template<typename GridPtrContainerT>
-    void write(const GridPtrContainerT&, const MetaMap& = MetaMap()) const;
+    void write(const GridPtrContainerT&, const MetaMap& = MetaMap(),
+        const io::WriteOptions& = io::WriteOptions{}) const;
 
 private:
-    /// Create a new grid of the type specified by the given descriptor,
-    /// then populate the grid from the given input stream.
-    /// @return the newly created grid.
-    GridBase::Ptr readGrid(const GridDescriptor&, std::istream&) const;
+    void writeGrids(std::ostream&, const GridCPtrVec&, const MetaMap&,
+        const io::WriteOptions&) const;
 
-    void writeGrids(std::ostream&, const GridCPtrVec&, const MetaMap&) const;
-
-
-    struct Impl;
-    std::unique_ptr<Impl> mImpl;
+    MetaMap::Ptr mMeta;
+    GridPtrVecPtr mGrids;
+    std::ostream* mOutputStream = nullptr;
 };
 
 
@@ -75,11 +73,12 @@ private:
 
 template<typename GridPtrContainerT>
 inline void
-Stream::write(const GridPtrContainerT& container, const MetaMap& metadata) const
+Stream::write(const GridPtrContainerT& container, const MetaMap& metadata,
+    const io::WriteOptions& writeOptions) const
 {
     GridCPtrVec grids;
     std::copy(container.begin(), container.end(), std::back_inserter(grids));
-    this->write(grids, metadata);
+    this->write(grids, metadata, writeOptions);
 }
 
 } // namespace io

openvdb/openvdb/io/TempFile.cc

@@ -1,144 +0,0 @@
-// Copyright Contributors to the OpenVDB Project
-// SPDX-License-Identifier: Apache-2.0
-
-/// @file TempFile.cc
-
-#ifdef OPENVDB_USE_DELAYED_LOADING
-
-#include "TempFile.h"
-
-#include <openvdb/Exceptions.h>
-#ifndef _WIN32
-#include <boost/iostreams/stream.hpp>
-#include <boost/iostreams/device/file_descriptor.hpp>
-#include <cstdlib> // for std::getenv(), mkstemp()
-#include <sys/types.h> // for mode_t
-#include <sys/stat.h> // for mkdir(), umask()
-#include <unistd.h> // for access()
-#else
-#include <fstream> // for std::filebuf
-#endif
-#include <cstdio> // for std::tmpnam(), L_tmpnam, P_tmpdir
-#include <iostream>
-#include <sstream>
-#include <string>
-#include <vector>
-
-
-namespace openvdb {
-OPENVDB_USE_VERSION_NAMESPACE
-namespace OPENVDB_VERSION_NAME {
-namespace io {
-
-struct TempFile::TempFileImpl
-{
-    const std::string& filename() const { return mPath; }
-
-    bool is_open() const { return mBuffer.is_open(); }
-
-    /// @internal boost::filesystem::unique_path(), etc. might be useful here,
-    /// but as of 9/2014, Houdini ships without the Boost.Filesystem library,
-    /// which makes it much less convenient to use that library.
-#ifndef _WIN32
-    TempFileImpl(std::ostream& os): mFileDescr(-1) { this->init(os); }
-
-    void init(std::ostream& os)
-    {
-        std::string fn = this->getTempDir() + "/openvdb_temp_XXXXXX";
-        std::vector<char> fnbuf(fn.begin(), fn.end());
-        fnbuf.push_back(char(0));
-
-        //const mode_t savedMode = ::umask(~(S_IRUSR | S_IWUSR));
-        mFileDescr = ::mkstemp(&fnbuf[0]);
-        //::umask(savedMode);
-        if (mFileDescr < 0) {
-            OPENVDB_THROW(IoError, "failed to generate temporary file");
-        }
-
-        mPath.assign(&fnbuf[0]);
-
-        mDevice = DeviceType(mFileDescr, boost::iostreams::never_close_handle);
-        mBuffer.open(mDevice);
-        os.rdbuf(&mBuffer);
-
-        if (!os.good()) {
-            OPENVDB_THROW(IoError, "failed to open temporary file " + mPath);
-        }
-    }
-
-    void close() { mBuffer.close(); if (mFileDescr >= 0) ::close(mFileDescr); }
-
-    static std::string getTempDir()
-    {
-        if (const char* dir = std::getenv("OPENVDB_TEMP_DIR")) {
-            if (0 != ::access(dir, F_OK)) {
-#ifdef _WIN32
-                ::mkdir(dir);
-#else
-                ::mkdir(dir, S_IRUSR | S_IWUSR | S_IXUSR);
-#endif
-                if (0 != ::access(dir, F_OK)) {
-                    OPENVDB_THROW(IoError,
-                        "failed to create OPENVDB_TEMP_DIR (" + std::string(dir) + ")");
-                }
-            }
-            return dir;
-        }
-        if (const char* dir = std::getenv("TMPDIR")) return dir;
-        return P_tmpdir;
-    }
-
-    using DeviceType = boost::iostreams::file_descriptor_sink;
-    using BufferType = boost::iostreams::stream_buffer<boost::iostreams::file_descriptor_sink>;
-
-    std::string mPath;
-    DeviceType mDevice;
-    BufferType mBuffer;
-    int mFileDescr;
-#else // _WIN32
-    // Use only standard library routines; no POSIX.
-
-    TempFileImpl(std::ostream& os) { this->init(os); }
-
-    void init(std::ostream& os)
-    {
-        char fnbuf[L_tmpnam];
-        const char* filename = std::tmpnam(fnbuf);
-        if (!filename) {
-            OPENVDB_THROW(IoError, "failed to generate name for temporary file");
-        }
-        /// @todo This is not safe, since another process could open a file
-        /// with this name before we do.  Unfortunately, there is no safe,
-        /// portable way to create a temporary file.
-        mPath = filename;
-
-        const std::ios_base::openmode mode = (std::ios_base::out | std::ios_base::binary);
-        os.rdbuf(mBuffer.open(mPath.c_str(), mode));
-        if (!os.good()) {
-            OPENVDB_THROW(IoError, "failed to open temporary file " + mPath);
-        }
-    }
-
-    void close() { mBuffer.close(); }
-
-    std::string mPath;
-    std::filebuf mBuffer;
-#endif // _WIN32
-
-private:
-    TempFileImpl(const TempFileImpl&); // disable copying
-    TempFileImpl& operator=(const TempFileImpl&); // disable assignment
-};
-
-
-TempFile::TempFile(): std::ostream(nullptr), mImpl(new TempFileImpl(*this)) {}
-TempFile::~TempFile() { this->close(); }
-const std::string& TempFile::filename() const { return mImpl->filename(); }
-bool TempFile::is_open() const { return mImpl->is_open(); }
-void TempFile::close() { mImpl->close(); }
-
-} // namespace io
-} // namespace OPENVDB_VERSION_NAME
-} // namespace openvdb
-
-#endif // OPENVDB_USE_DELAYED_LOADING

openvdb/openvdb/io/TempFile.h

@@ -1,54 +0,0 @@
-// Copyright Contributors to the OpenVDB Project
-// SPDX-License-Identifier: Apache-2.0
-
-/// @file TempFile.h
-
-#ifdef OPENVDB_USE_DELAYED_LOADING
-
-#ifndef OPENVDB_IO_TEMPFILE_HAS_BEEN_INCLUDED
-#define OPENVDB_IO_TEMPFILE_HAS_BEEN_INCLUDED
-
-#include <openvdb/version.h>
-#include <memory>
-#include <ostream>
-
-
-namespace openvdb {
-OPENVDB_USE_VERSION_NAMESPACE
-namespace OPENVDB_VERSION_NAME {
-namespace io {
-
-/// Output stream to a unique temporary file
-class OPENVDB_API TempFile: public std::ostream
-{
-public:
-    /// @brief Create and open a unique file.
-    /// @details On UNIX systems, the file is created in the directory specified by
-    /// the environment variable @c OPENVDB_TEMP_DIR, if that variable is defined,
-    /// or else in the directory specified by @c TMPDIR, if that variable is defined.
-    /// Otherwise (and on non-UNIX systems), the file is created in the system default
-    /// temporary directory.
-    TempFile();
-    ~TempFile();
-
-    /// Return the path to the temporary file.
-    const std::string& filename() const;
-
-    /// Return @c true if the file is open for writing.
-    bool is_open() const;
-
-    /// Close the file.
-    void close();
-
-private:
-    struct TempFileImpl;
-    std::unique_ptr<TempFileImpl> mImpl;
-};
-
-} // namespace io
-} // namespace OPENVDB_VERSION_NAME
-} // namespace openvdb
-
-#endif // OPENVDB_IO_TEMPFILE_HAS_BEEN_INCLUDED
-
-#endif // OPENVDB_USE_DELAYED_LOADING

openvdb/openvdb/io/io.h

@@ -76,9 +76,6 @@ public:
     uint32_t pass() const;
     void setPass(uint32_t);
 
-    uint64_t leaf() const;
-    void setLeaf(uint64_t);
-
     //@{
     /// @brief Return a (reference to a) copy of the metadata of the grid
     /// currently being read or written.
@@ -119,9 +116,9 @@ std::ostream& operator<<(std::ostream&, const StreamMetadata::AuxDataMap&);
 ////////////////////////////////////////
 
 
-/// @brief Leaf nodes that require multi-pass I/O must inherit from this struct.
-/// @sa Grid::hasMultiPassIO()
-struct MultiPass {};
+/// @brief Tag for multi-pass I/O. Only points::PointDataLeafNode may inherit from this.
+/// @sa Grid::hasMultiPassIO(), points::IsPointDataLeafNode
+struct PointDataGridMultiPass {};
 
 
 ////////////////////////////////////////
@@ -129,50 +126,37 @@ struct MultiPass {};
 
 class File;
 
-#ifdef OPENVDB_USE_DELAYED_LOADING
 
-/// @brief Handle to control the lifetime of a memory-mapped .vdb file
+// This class is deprecated and will be removed. Delayed loading is no longer supported.
+// It is retained for API backwards compatibility only, but all methods are no-ops.
 class OPENVDB_API MappedFile
 {
 public:
     using Ptr = SharedPtr<MappedFile>;
+    using Notifier = std::function<void(std::string /*filename*/)>;
 
-    ~MappedFile();
-    MappedFile(const MappedFile&) = delete; // not copyable
+    OPENVDB_DEPRECATED_MESSAGE("This class is deprecated and will be removed. Delayed loading is no longer supported.")
+    explicit MappedFile(const std::string& /*filename*/, bool /*autoDelete*/ = false) {}
+
+    /// @brief Destructor
+    ~MappedFile() = default;
+
+    MappedFile(const MappedFile&) = delete;
     MappedFile& operator=(const MappedFile&) = delete;
 
-    /// Return the filename of the mapped file.
-    std::string filename() const;
+    OPENVDB_DEPRECATED_MESSAGE("Always returns an empty string. This method is deprecated and will be removed. Delayed loading is no longer supported.")
+    std::string filename() const { return std::string(); }
 
-    /// @brief Return a new stream buffer for the mapped file.
-    /// @details Typical usage is
-    /// @code
-    /// openvdb::io::MappedFile::Ptr mappedFile = ...;
-    /// auto buf = mappedFile->createBuffer();
-    /// std::istream istrm{buf.get()};
-    /// // Read from istrm...
-    /// @endcode
-    /// The buffer must persist as long as the stream is open.
-    SharedPtr<std::streambuf> createBuffer() const;
+    OPENVDB_DEPRECATED_MESSAGE("Always returns a null stream buffer pointer. This method is deprecated and will be removed. Delayed loading is no longer supported.")
+    SharedPtr<std::streambuf> createBuffer() const { return SharedPtr<std::streambuf>(); }
 
-    using Notifier = std::function<void(std::string /*filename*/)>;
-    /// @brief Register a function that will be called with this file's name
-    /// when the file is unmapped.
-    void setNotifier(const Notifier&);
-    /// Deregister the notifier.
-    void clearNotifier();
+    OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.")
+    void setNotifier(const Notifier&) {}
 
-private:
-    friend class File;
-    friend class ::TestMappedFile;
-
-    explicit MappedFile(const std::string& filename, bool autoDelete = false);
-
-    class Impl;
-    std::unique_ptr<Impl> mImpl;
+    OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.")
+    void clearNotifier() {}
 }; // class MappedFile
 
-#endif // OPENVDB_USE_DELAYED_LOADING
 
 ////////////////////////////////////////
 
@@ -249,15 +233,10 @@ OPENVDB_API bool getWriteGridStatsMetadata(std::ios_base&);
 /// and store them as grid metadata when writing to the given stream.
 OPENVDB_API void setWriteGridStatsMetadata(std::ios_base&, bool writeGridStats);
 
-#ifdef OPENVDB_USE_DELAYED_LOADING
-/// @brief Return a shared pointer to the memory-mapped file with which the given stream
-/// is associated, or a null pointer if the stream is not associated with a memory-mapped file.
-OPENVDB_API SharedPtr<MappedFile> getMappedFilePtr(std::ios_base&);
-/// @brief Associate the given stream with (a shared pointer to) a memory-mapped file.
-/// @note The shared pointer object (not just the io::MappedFile object to which it points)
-/// must remain valid until the file is closed.
-OPENVDB_API void setMappedFilePtr(std::ios_base&, SharedPtr<MappedFile>&);
-#endif // OPENVDB_USE_DELAYED_LOADING
+OPENVDB_DEPRECATED_MESSAGE("Always returns a null pointer. This function is deprecated and will be removed. Delayed loading is no longer supported.")
+inline SharedPtr<MappedFile> getMappedFilePtr(std::ios_base&) { return SharedPtr<MappedFile>(); }
+OPENVDB_DEPRECATED_MESSAGE("This function is deprecated and will be removed. Delayed loading is no longer supported.")
+inline void setMappedFilePtr(std::ios_base&, SharedPtr<MappedFile>&) { }
 
 /// @brief Return a shared pointer to an object that stores metadata (file format,
 /// compression scheme, etc.) for use when reading from or writing to the given stream.

openvdb/openvdb/openvdb.cc

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: Apache-2.0
 
 #include "openvdb.h"
-#include "io/DelayedLoadMetadata.h"
+#include "io/Codec.h"
 #include "points/PointDataGrid.h"
 #include "tools/PointIndexGrid.h"
 #include "util/logging.h"
@@ -71,6 +71,8 @@ initialize()
 
     logging::initialize();
 
+    io::internal::initialize();
+
     // Register metadata.
     Metadata::clearRegistry();
     MetaTypes::foreach<RegisterMeta>();
@@ -131,6 +133,7 @@ __pragma(warning(default:1711))
     Metadata::clearRegistry();
     GridBase::clearRegistry();
     math::MapRegistry::clear();
+    io::internal::uninitialize();
     points::internal::uninitialize();
 
 #ifdef OPENVDB_USE_BLOSC

openvdb/openvdb/openvdb.h

@@ -46,9 +46,6 @@ OPENVDB_API void initialize();
 OPENVDB_API void uninitialize();
 
 
-// foward declare some default types
-namespace io { class DelayedLoadMetadata; }
-
 /// Common tree types
 using BoolTree     = tree::Tree4<bool,        5, 4, 3>::Type;
 using DoubleTree   = tree::Tree4<double,      5, 4, 3>::Type;
@@ -222,8 +219,7 @@ using MetaTypes = TypeList<
     Vec4SMetadata,
     Vec4DMetadata,
     Mat4SMetadata,
-    Mat4DMetadata,
-    io::DelayedLoadMetadata>;
+    Mat4DMetadata>;
 
 
 } // namespace OPENVDB_VERSION_NAME

openvdb/openvdb/points/AttributeArray.cc

@@ -61,7 +61,6 @@ AttributeArray::AttributeArray(const AttributeArray& rhs, const tbb::spin_mutex:
     : mIsUniform(rhs.mIsUniform)
     , mFlags(rhs.mFlags)
     , mUsePagedRead(rhs.mUsePagedRead)
-    , mOutOfCore(rhs.mOutOfCore.load())
     , mPageHandle()
 {
     if (mFlags & PARTIALREAD)       mCompressedBytes = rhs.mCompressedBytes;
@@ -78,7 +77,6 @@ AttributeArray::operator=(const AttributeArray& rhs)
     mIsUniform = rhs.mIsUniform;
     mFlags = rhs.mFlags;
     mUsePagedRead = rhs.mUsePagedRead;
-    mOutOfCore.store(rhs.mOutOfCore);
     if (mFlags & PARTIALREAD)       mCompressedBytes = rhs.mCompressedBytes;
     else if (rhs.mPageHandle)       mPageHandle = rhs.mPageHandle->copy();
     else                            mPageHandle.reset();
@@ -190,9 +188,6 @@ AttributeArray::setConstantStride(bool state)
 bool
 AttributeArray::operator==(const AttributeArray& other) const
 {
-    this->loadData();
-    other.loadData();
-
     if (this->mUsePagedRead != other.mUsePagedRead ||
         this->mFlags != other.mFlags) return false;
     return this->isEqual(other);

openvdb/openvdb/points/AttributeArray.h

@@ -130,7 +130,7 @@ public:
 
     template <typename ValueType, typename CodecType> friend class AttributeHandle;
 
-    AttributeArray(): mPageHandle() { mOutOfCore = 0; }
+    AttributeArray(): mPageHandle() { }
     virtual ~AttributeArray()
     {
         // if this AttributeArray has been partially read, zero the compressed bytes,
@@ -189,10 +189,7 @@ public:
     /// Return the number of bytes of memory used by this attribute.
     virtual size_t memUsage() const = 0;
 
-    /// Return the number of bytes of memory used by this attribute array once it
-    /// has been deserialized (this may be different to memUsage() if delay-loading
-    /// is in use). Note that this method does NOT consider the fact that a
-    /// uniform attribute could be expanded and only deals with delay-loading.
+    OPENVDB_DEPRECATED_MESSAGE("Use memUsage() instead. This method is deprecated and will be removed. Delayed loading is no longer supported.")
     virtual size_t memUsageIfLoaded() const = 0;
 
     /// Create a new attribute array of the given (registered) type, length and stride.
@@ -311,14 +308,17 @@ public:
 
     /// Read attribute buffers from a paged stream.
     virtual void readPagedBuffers(compression::PagedInputStream&) = 0;
+    /// Skip attribute buffers in a paged stream without reading or
+    /// allocating any data.
+    void skipPagedBuffers(compression::PagedInputStream&);
     /// Write attribute buffers to a paged stream.
     /// @param outputTransient if true, write out transient attributes
     virtual void writePagedBuffers(compression::PagedOutputStream&, bool outputTransient) const = 0;
 
-    /// Ensures all data is in-core
+    OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.")
     virtual void loadData() const = 0;
 
-    /// Return @c true if all data has been loaded
+    OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.")
     virtual bool isDataLoaded() const = 0;
 
     /// Check the compressed bytes and flags. If they are equal, perform a deeper
@@ -366,10 +366,12 @@ protected:
     mutable tbb::spin_mutex mMutex;
     uint8_t mFlags = 0;
     uint8_t mUsePagedRead = 0;
-    std::atomic<Index32> mOutOfCore; // interpreted as bool
+#if OPENVDB_ABI_VERSION_NUMBER < 14
+    std::atomic<Index32> mOutOfCore{0}; // interpreted as bool
+#endif
     /// used for out-of-core, paged reading
     union {
-        compression::PageHandle::Ptr mPageHandle;
+        std::unique_ptr<compression::PageHandle> mPageHandle;
         size_t mCompressedBytes;
     };
 }; // class AttributeArray
@@ -611,11 +613,8 @@ public:
     /// Return the number of bytes of memory used by this attribute.
     size_t memUsage() const override;
 
-    /// Return the number of bytes of memory used by this attribute array once it
-    /// has been deserialized (this may be different to memUsage() if delay-loading
-    /// is in use). Note that this method does NOT consider the fact that a
-    /// uniform attribute could be expanded and only deals with delay-loading.
-    size_t memUsageIfLoaded() const override;
+    OPENVDB_DEPRECATED_MESSAGE("Use memUsage() instead. This method is deprecated and will be removed. Delayed loading is no longer supported.")
+    size_t memUsageIfLoaded() const override { return memUsage(); }
 
     /// Return the value at index @a n (assumes in-core)
     ValueType getUnsafe(Index n) const;
@@ -696,14 +695,14 @@ public:
     /// @param outputTransient if true, write out transient attributes
     void writePagedBuffers(compression::PagedOutputStream& os, bool outputTransient) const override;
 
-    /// Return @c true if this buffer's values have not yet been read from disk.
-    inline bool isOutOfCore() const;
+    OPENVDB_DEPRECATED_MESSAGE("Always returns false. This method is deprecated and will be removed. Delayed loading is no longer supported.")
+    bool isOutOfCore() const { return false; }
 
-    /// Ensures all data is in-core
-    void loadData() const override;
+    OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.")
+    void loadData() const override { }
 
-    /// Return @c true if all data has been loaded
-    bool isDataLoaded() const override;
+    OPENVDB_DEPRECATED_MESSAGE("Always returns true. This method is deprecated and will be removed. Delayed loading is no longer supported.")
+    bool isDataLoaded() const override { return true; }
 
     /// Return the raw data buffer
     inline const StorageType* constData() const { return this->data(); }
@@ -716,18 +715,13 @@ protected:
     inline const StorageType* data() const { OPENVDB_ASSERT(validData()); return mData.get(); }
 
     /// Verify that data is not out-of-core or in a partially-read state
-    inline bool validData() const { return !(isOutOfCore() || (flags() & PARTIALREAD)); }
+    inline bool validData() const { return !((flags() & PARTIALREAD)); }
 
 private:
     friend class ::TestAttributeArray;
 
     TypedAttributeArray(const TypedAttributeArray&, const tbb::spin_mutex::scoped_lock&);
 
-    /// Load data from memory-mapped file.
-    inline void doLoad() const;
-    /// Load data from memory-mapped file (unsafe as this function is not protected by a mutex).
-    inline void doLoadUnsafe() const;
-
     /// Toggle out-of-core state
     inline void setOutOfCore(const bool);
 
@@ -962,8 +956,6 @@ void AttributeArray::doCopyValues(const AttributeArray& sourceArray, const IterT
 {
     // ensure both arrays have float-float or integer-integer value types
     OPENVDB_ASSERT(sourceArray.valueTypeIsFloatingPoint() == this->valueTypeIsFloatingPoint());
-    // ensure both arrays have been loaded from disk (if delay-loaded)
-    OPENVDB_ASSERT(sourceArray.isDataLoaded() && this->isDataLoaded());
     // ensure storage size * stride matches on both arrays
     OPENVDB_ASSERT(this->storageTypeSize()*this->stride() ==
         sourceArray.storageTypeSize()*sourceArray.stride());
@@ -1024,10 +1016,6 @@ void AttributeArray::copyValues(const AttributeArray& sourceArray, const IterT&
         OPENVDB_THROW(TypeError, "Cannot copy array data due to mis-match in storage type sizes.");
     }
 
-    // ensure both arrays have been loaded from disk
-    sourceArray.loadData();
-    this->loadData();
-
     // if the target array is uniform, expand it first
     this->expand();
 
@@ -1206,8 +1194,6 @@ template<typename ValueType_, typename Codec_>
 size_t
 TypedAttributeArray<ValueType_, Codec_>::arrayMemUsage() const
 {
-    if (this->isOutOfCore())        return 0;
-
     return (mIsUniform ? 1 : this->dataSize()) * sizeof(StorageType);
 }
 
@@ -1232,11 +1218,6 @@ template<typename ValueType_, typename Codec_>
 void
 TypedAttributeArray<ValueType_, Codec_>::deallocate()
 {
-    // detach from file if delay-loaded
-    if (this->isOutOfCore()) {
-        this->setOutOfCore(false);
-        this->mPageHandle.reset();
-    }
     if (mData)      mData.reset();
 }
 
@@ -1304,14 +1285,6 @@ TypedAttributeArray<ValueType_, Codec_>::memUsage() const
 }
 
 
-template<typename ValueType_, typename Codec_>
-size_t
-TypedAttributeArray<ValueType_, Codec_>::memUsageIfLoaded() const
-{
-    return sizeof(*this) + (mIsUniform ? 1 : this->dataSize()) * sizeof(StorageType);
-}
-
-
 template<typename ValueType_, typename Codec_>
 typename TypedAttributeArray<ValueType_, Codec_>::ValueType
 TypedAttributeArray<ValueType_, Codec_>::getUnsafe(Index n) const
@@ -1329,7 +1302,6 @@ typename TypedAttributeArray<ValueType_, Codec_>::ValueType
 TypedAttributeArray<ValueType_, Codec_>::get(Index n) const
 {
     if (n >= this->dataSize())           OPENVDB_THROW(IndexError, "Out-of-range access.");
-    if (this->isOutOfCore())            this->doLoad();
 
     return this->getUnsafe(n);
 }
@@ -1366,7 +1338,6 @@ void
 TypedAttributeArray<ValueType_, Codec_>::setUnsafe(Index n, const ValueType& val)
 {
     OPENVDB_ASSERT(n < this->dataSize());
-    OPENVDB_ASSERT(!this->isOutOfCore());
     OPENVDB_ASSERT(!this->isUniform());
 
     // this unsafe method assumes the data is not uniform, however if it is, this redirects the index
@@ -1381,7 +1352,6 @@ void
 TypedAttributeArray<ValueType_, Codec_>::set(Index n, const ValueType& val)
 {
     if (n >= this->dataSize())           OPENVDB_THROW(IndexError, "Out-of-range access.");
-    if (this->isOutOfCore())            this->doLoad();
     if (this->isUniform())              this->expand();
 
     this->setUnsafe(n, val);
@@ -1486,12 +1456,6 @@ template<typename ValueType_, typename Codec_>
 void
 TypedAttributeArray<ValueType_, Codec_>::fill(const ValueType& value)
 {
-    if (this->isOutOfCore()) {
-        tbb::spin_mutex::scoped_lock lock(mMutex);
-        this->deallocate();
-        this->allocate();
-    }
-
     const Index size = mIsUniform ? 1 : this->dataSize();
     for (Index i = 0; i < size; ++i)  {
         Codec::encode(value, this->data()[i]);
@@ -1507,54 +1471,6 @@ TypedAttributeArray<ValueType_, Codec_>::fill(AttributeArray* array, const Value
 }
 
 
-template<typename ValueType_, typename Codec_>
-bool
-TypedAttributeArray<ValueType_, Codec_>::isOutOfCore() const
-{
-    return mOutOfCore;
-}
-
-
-template<typename ValueType_, typename Codec_>
-void
-TypedAttributeArray<ValueType_, Codec_>::setOutOfCore(const bool b)
-{
-    mOutOfCore = b;
-}
-
-
-template<typename ValueType_, typename Codec_>
-void
-TypedAttributeArray<ValueType_, Codec_>::doLoad() const
-{
-    if (!(this->isOutOfCore()))     return;
-
-    TypedAttributeArray<ValueType_, Codec_>* self =
-        const_cast<TypedAttributeArray<ValueType_, Codec_>*>(this);
-
-    // This lock will be contended at most once, after which this buffer
-    // will no longer be out-of-core.
-    tbb::spin_mutex::scoped_lock lock(self->mMutex);
-    this->doLoadUnsafe();
-}
-
-
-template<typename ValueType_, typename Codec_>
-void
-TypedAttributeArray<ValueType_, Codec_>::loadData() const
-{
-    this->doLoad();
-}
-
-
-template<typename ValueType_, typename Codec_>
-bool
-TypedAttributeArray<ValueType_, Codec_>::isDataLoaded() const
-{
-    return !this->isOutOfCore();
-}
-
-
 template<typename ValueType_, typename Codec_>
 void
 TypedAttributeArray<ValueType_, Codec_>::read(std::istream& is)
@@ -1624,8 +1540,6 @@ TypedAttributeArray<ValueType_, Codec_>::readBuffers(std::istream& is)
         OPENVDB_THROW(IoError, "Cannot read paged AttributeArray buffers.");
     }
 
-    tbb::spin_mutex::scoped_lock lock(mMutex);
-
     this->deallocate();
 
     uint8_t bloscCompressed(0);
@@ -1663,13 +1577,6 @@ TypedAttributeArray<ValueType_, Codec_>::readPagedBuffers(compression::PagedInpu
         return;
     }
 
-#ifdef OPENVDB_USE_DELAYED_LOADING
-    // If this array is being read from a memory-mapped file, delay loading of its data
-    // until the data is actually accessed.
-    io::MappedFile::Ptr mappedFile = io::getMappedFilePtr(is.getInputStream());
-    const bool delayLoad = (mappedFile.get() != nullptr);
-#endif
-
     if (is.sizeOnly())
     {
         size_t compressedBytes(mCompressedBytes);
@@ -1682,26 +1589,49 @@ TypedAttributeArray<ValueType_, Codec_>::readPagedBuffers(compression::PagedInpu
 
     OPENVDB_ASSERT(mPageHandle);
 
-    tbb::spin_mutex::scoped_lock lock(mMutex);
-
     this->deallocate();
 
-#ifdef OPENVDB_USE_DELAYED_LOADING
-    this->setOutOfCore(delayLoad);
-    is.read(mPageHandle, std::streamsize(mPageHandle->size()), delayLoad);
-#else
     is.read(mPageHandle, std::streamsize(mPageHandle->size()), false);
-#endif // OPENVDB_USE_DELAYED_LOADING
+    std::unique_ptr<char[]> buffer = mPageHandle->read();
+    mData.reset(reinterpret_cast<StorageType*>(buffer.release()));
+    mPageHandle.reset();
 
-#ifdef OPENVDB_USE_DELAYED_LOADING
-    if (!delayLoad) {
-#endif
-        std::unique_ptr<char[]> buffer = mPageHandle->read();
-        mData.reset(reinterpret_cast<StorageType*>(buffer.release()));
-        mPageHandle.reset();
-#ifdef OPENVDB_USE_DELAYED_LOADING
+    // clear page state
+
+    mUsePagedRead = 0;
+}
+
+
+inline void
+AttributeArray::skipPagedBuffers(compression::PagedInputStream& is)
+{
+    if (!mUsePagedRead) {
+        if (!is.sizeOnly()) {
+            // for non-paged data, seek past the raw data in the stream
+            std::istream& inputStream = is.getInputStream();
+            uint8_t bloscCompressed(0);
+            if (!mIsUniform)    inputStream.read(reinterpret_cast<char*>(&bloscCompressed), sizeof(uint8_t));
+            inputStream.seekg(mCompressedBytes, std::ios_base::cur);
+            mCompressedBytes = 0;
+            mFlags = static_cast<uint8_t>(mFlags & ~PARTIALREAD);
+        }
+        return;
     }
-#endif
+
+    if (is.sizeOnly())
+    {
+        size_t compressedBytes(mCompressedBytes);
+        mCompressedBytes = 0;
+        mFlags = static_cast<uint8_t>(mFlags & ~PARTIALREAD);
+        OPENVDB_ASSERT(!mPageHandle);
+        mPageHandle = is.createHandle(compressedBytes);
+        return;
+    }
+
+    OPENVDB_ASSERT(mPageHandle);
+
+    is.skip(mPageHandle, std::streamsize(mPageHandle->size()));
+    mPageHandle.reset();
 
     // clear page state
 
@@ -1744,9 +1674,6 @@ TypedAttributeArray<ValueType_, Codec_>::writeMetadata(std::ostream& os, bool ou
 
     bool bloscCompression = io::getDataCompression(os) & io::COMPRESS_BLOSC;
 
-    // any compressed data needs to be loaded if out-of-core
-    if (bloscCompression)    this->doLoad();
-
     size_t compressedBytes = 0;
 
     if (!strideOfOne)
@@ -1795,8 +1722,6 @@ TypedAttributeArray<ValueType_, Codec_>::writeBuffers(std::ostream& os, bool out
         OPENVDB_THROW(IoError, "Cannot write out a partially-read AttributeArray.");
     }
 
-    this->doLoad();
-
     if (this->isUniform()) {
         os.write(reinterpret_cast<const char*>(this->data()), sizeof(StorageType));
     }
@@ -1844,37 +1769,10 @@ TypedAttributeArray<ValueType_, Codec_>::writePagedBuffers(compression::PagedOut
         OPENVDB_THROW(IoError, "Cannot write out a partially-read AttributeArray.");
     }
 
-    this->doLoad();
-
     os.write(reinterpret_cast<const char*>(this->data()), this->arrayMemUsage());
 }
 
 
-template<typename ValueType_, typename Codec_>
-void
-TypedAttributeArray<ValueType_, Codec_>::doLoadUnsafe() const
-{
-    if (!(this->isOutOfCore())) return;
-
-    // this function expects the mutex to already be locked
-
-    auto* self = const_cast<TypedAttributeArray<ValueType_, Codec_>*>(this);
-
-    OPENVDB_ASSERT(self->mPageHandle);
-    OPENVDB_ASSERT(!(self->mFlags & PARTIALREAD));
-
-    std::unique_ptr<char[]> buffer = self->mPageHandle->read();
-
-    self->mData.reset(reinterpret_cast<StorageType*>(buffer.release()));
-
-    self->mPageHandle.reset();
-
-    // clear all write and out-of-core flags
-
-    self->mOutOfCore = false;
-}
-
-
 template<typename ValueType_, typename Codec_>
 AttributeArray::AccessorBasePtr
 TypedAttributeArray<ValueType_, Codec_>::getAccessor() const
@@ -1901,9 +1799,6 @@ TypedAttributeArray<ValueType_, Codec_>::isEqual(const AttributeArray& other) co
        this->mIsUniform != otherT->mIsUniform ||
        this->attributeType() != this->attributeType()) return false;
 
-    this->doLoad();
-    otherT->doLoad();
-
     const StorageType *target = this->data(), *source = otherT->data();
     if (!target && !source) return true;
     if (!target || !source) return false;
@@ -1995,10 +1890,6 @@ AttributeHandle<ValueType, CodecType>::AttributeHandle(const AttributeArray& arr
         OPENVDB_THROW(TypeError, "Cannot bind handle due to incompatible type of AttributeArray.");
     }
 
-    // load data if delay-loaded
-
-    mArray->loadData();
-
     // bind getter and setter methods
 
     AttributeArray::AccessorBasePtr accessor = mArray->getAccessor();

openvdb/openvdb/points/AttributeGroup.cc

@@ -23,10 +23,6 @@ GroupHandle::GroupHandle(const GroupAttributeArray& array, const GroupType& offs
         , mBitMask(static_cast<GroupType>(1 << offset))
 {
     OPENVDB_ASSERT(isGroup(mArray));
-
-    // load data if delay-loaded
-
-    mArray.loadData();
 }
 
 
@@ -36,10 +32,6 @@ GroupHandle::GroupHandle(const GroupAttributeArray& array, const GroupType& bitM
     , mBitMask(bitMask)
 {
     OPENVDB_ASSERT(isGroup(mArray));
-
-    // load data if delay-loaded
-
-    mArray.loadData();
 }
 
 

openvdb/openvdb/points/AttributeSet.cc

@@ -140,17 +140,6 @@ AttributeSet::memUsage() const
 }
 
 
-size_t
-AttributeSet::memUsageIfLoaded() const
-{
-    size_t bytes = sizeof(*this) + mDescr->memUsage();
-    for (const auto& attr : mAttrs) {
-        bytes += attr->memUsageIfLoaded();
-    }
-    return bytes;
-}
-
-
 size_t
 AttributeSet::find(const std::string& name) const
 {

openvdb/openvdb/points/AttributeSet.h

@@ -114,10 +114,8 @@ public:
     /// Return the number of bytes of memory used by this attribute set.
     size_t memUsage() const;
 
-    /// Return the number of bytes of memory used by this attribute set once it
-    /// has been deserialized (this may be different to memUsage() if delay-loading
-    /// is in use).
-    size_t memUsageIfLoaded() const;
+    OPENVDB_DEPRECATED_MESSAGE("Use memUsage() instead. This method is deprecated and will be removed. Delayed loading is no longer supported.")
+    size_t memUsageIfLoaded() const { return memUsage(); }
 
     /// @brief  Return the position of the attribute array whose name is @a name,
     ///         or @c INVALID_POS if no match is found.

openvdb/openvdb/points/PointConversion.h

@@ -134,7 +134,6 @@ populateAttribute(  PointDataTreeT& tree,
 /// @param pointOffsets         a vector of cumulative point offsets for each leaf
 /// @param startOffset          a value to shift all the point offsets by
 /// @param filter               an index filter
-/// @param inCoreOnly           true if out-of-core leaf nodes are to be ignored
 ///
 
 template <typename PositionAttribute, typename PointDataGridT, typename FilterT = NullFilter>
@@ -143,8 +142,17 @@ convertPointDataGridPosition(   PositionAttribute& positionAttribute,
                                 const PointDataGridT& grid,
                                 const std::vector<Index64>& pointOffsets,
                                 const Index64 startOffset,
-                                const FilterT& filter = NullFilter(),
-                                const bool inCoreOnly = false);
+                                const FilterT& filter = NullFilter());
+
+template <typename PositionAttribute, typename PointDataGridT, typename FilterT = NullFilter>
+OPENVDB_DEPRECATED_MESSAGE("Use convertPointDataGridPosition() without inCoreOnly parameter instead. This method is deprecated and will be removed. Delayed loading is no longer supported.")
+inline void
+convertPointDataGridPosition(   PositionAttribute& positionAttribute,
+                                const PointDataGridT& grid,
+                                const std::vector<Index64>& pointOffsets,
+                                const Index64 startOffset,
+                                const FilterT& filter,
+                                const bool /*inCoreOnly*/);
 
 
 /// @brief Convert the attribute from a PointDataGrid
@@ -156,7 +164,6 @@ convertPointDataGridPosition(   PositionAttribute& positionAttribute,
 /// @param arrayIndex           the index in the Descriptor of the array to be converted.
 /// @param stride               the stride of the attribute
 /// @param filter               an index filter
-/// @param inCoreOnly           true if out-of-core leaf nodes are to be ignored
 template <typename TypedAttribute, typename PointDataTreeT, typename FilterT = NullFilter>
 inline void
 convertPointDataGridAttribute(  TypedAttribute& attribute,
@@ -165,8 +172,19 @@ convertPointDataGridAttribute(  TypedAttribute& attribute,
                                 const Index64 startOffset,
                                 const unsigned arrayIndex,
                                 const Index stride = 1,
-                                const FilterT& filter = NullFilter(),
-                                const bool inCoreOnly = false);
+                                const FilterT& filter = NullFilter());
+
+template <typename TypedAttribute, typename PointDataTreeT, typename FilterT = NullFilter>
+OPENVDB_DEPRECATED_MESSAGE("Use convertPointDataGridAttribute() without inCoreOnly parameter instead. This method is deprecated and will be removed. Delayed loading is no longer supported.")
+inline void
+convertPointDataGridAttribute(  TypedAttribute& attribute,
+                                const PointDataTreeT& tree,
+                                const std::vector<Index64>& pointOffsets,
+                                const Index64 startOffset,
+                                const unsigned arrayIndex,
+                                const Index stride,
+                                const FilterT& filter,
+                                const bool /*inCoreOnly*/);
 
 
 /// @brief Convert the group from a PointDataGrid
@@ -177,9 +195,7 @@ convertPointDataGridAttribute(  TypedAttribute& attribute,
 /// @param startOffset          a value to shift all the point offsets by
 /// @param index                the group index to be converted.
 /// @param filter               an index filter
-/// @param inCoreOnly           true if out-of-core leaf nodes are to be ignored
 ///
-
 template <typename Group, typename PointDataTreeT, typename FilterT = NullFilter>
 inline void
 convertPointDataGridGroup(  Group& group,
@@ -187,8 +203,18 @@ convertPointDataGridGroup(  Group& group,
                             const std::vector<Index64>& pointOffsets,
                             const Index64 startOffset,
                             const AttributeSet::Descriptor::GroupIndex index,
-                            const FilterT& filter = NullFilter(),
-                            const bool inCoreOnly = false);
+                            const FilterT& filter = NullFilter());
+
+template <typename Group, typename PointDataTreeT, typename FilterT = NullFilter>
+OPENVDB_DEPRECATED_MESSAGE("Use convertPointDataGridGroup() without inCoreOnly parameter instead. This method is deprecated and will be removed. Delayed loading is no longer supported.")
+inline void
+convertPointDataGridGroup(  Group& group,
+                            const PointDataTreeT& tree,
+                            const std::vector<Index64>& pointOffsets,
+                            const Index64 startOffset,
+                            const AttributeSet::Descriptor::GroupIndex index,
+                            const FilterT& filter,
+                            const bool /*inCoreOnly*/);
 
 // for internal use only - this traits class extracts T::value_type if defined,
 // otherwise falls back to using Vec3R

openvdb/openvdb/points/PointCount.h

@@ -28,28 +28,39 @@ namespace points {
 /// @brief Count the total number of points in a PointDataTree
 /// @param tree         the PointDataTree in which to count the points
 /// @param filter       an optional index filter
-/// @param inCoreOnly   if true, points in out-of-core leaf nodes are not counted
 /// @param threaded     enable or disable threading  (threading is enabled by default)
 template <typename PointDataTreeT, typename FilterT = NullFilter>
 inline Index64 pointCount(  const PointDataTreeT& tree,
                             const FilterT& filter = NullFilter(),
-                            const bool inCoreOnly = false,
                             const bool threaded = true);
 
+template <typename PointDataTreeT, typename FilterT = NullFilter>
+OPENVDB_DEPRECATED_MESSAGE("Use pointCount() without inCoreOnly parameter instead. This method is deprecated and will be removed. Delayed loading is no longer supported.")
+inline Index64 pointCount(  const PointDataTreeT& tree,
+                            const FilterT& filter,
+                            const bool /*inCoreOnly*/,
+                            const bool threaded);
+
 /// @brief Populate an array of cumulative point offsets per leaf node.
 /// @param pointOffsets     array of offsets to be populated
 /// @param tree             the PointDataTree from which to populate the offsets
 /// @param filter           an optional index filter
-/// @param inCoreOnly       if true, points in out-of-core leaf nodes are ignored
 /// @param threaded         enable or disable threading  (threading is enabled by default)
 /// @return The final cumulative point offset.
 template <typename PointDataTreeT, typename FilterT = NullFilter>
 inline Index64 pointOffsets(std::vector<Index64>& pointOffsets,
                             const PointDataTreeT& tree,
                             const FilterT& filter = NullFilter(),
-                            const bool inCoreOnly = false,
                             const bool threaded = true);
 
+template <typename PointDataTreeT, typename FilterT = NullFilter>
+OPENVDB_DEPRECATED_MESSAGE("Use pointOffsets() without inCoreOnly parameter instead. This method is deprecated and will be removed. Delayed loading is no longer supported.")
+inline Index64 pointOffsets(std::vector<Index64>& pointOffsets,
+                            const PointDataTreeT& tree,
+                            const FilterT& filter,
+                            const bool /*inCoreOnly*/,
+                            const bool threaded);
+
 /// @brief Generate a new grid with voxel values to store the number of points per voxel
 /// @param grid             the PointDataGrid to use to compute the count grid
 /// @param filter           an optional index filter

openvdb/openvdb/points/PointDataGrid.h

@@ -31,140 +31,14 @@
 #include <utility> // std::pair, std::make_pair
 #include <vector>
 
+#include <openvdb/points/PointDataIO.h> // io::readCompressedValues(), io::writeCompressedValues(), io::writeCompressedValuesSize()
+
 class TestPointDataLeaf;
 
 namespace openvdb {
 OPENVDB_USE_VERSION_NAMESPACE
 namespace OPENVDB_VERSION_NAME {
 
-namespace io
-{
-
-/// @brief openvdb::io::readCompressedValues specialized on PointDataIndex32 arrays to
-/// ignore the value mask, use a larger block size and use 16-bit size instead of 64-bit
-template<>
-inline void
-readCompressedValues(   std::istream& is, PointDataIndex32* destBuf, Index destCount,
-                        const util::NodeMask<3>& /*valueMask*/, bool /*fromHalf*/)
-{
-    using compression::bloscDecompress;
-
-    const bool seek = destBuf == nullptr;
-
-    const size_t destBytes = destCount*sizeof(PointDataIndex32);
-    const size_t maximumBytes = std::numeric_limits<uint16_t>::max();
-    if (destBytes >= maximumBytes) {
-        OPENVDB_THROW(openvdb::IoError, "Cannot read more than " <<
-                                maximumBytes << " bytes in voxel values.")
-    }
-
-    uint16_t bytes16;
-
-    const io::StreamMetadata::Ptr meta = io::getStreamMetadataPtr(is);
-
-    if (seek && meta) {
-        // buffer size temporarily stored in the StreamMetadata pass
-        // to avoid having to perform an expensive disk read for 2-bytes
-        bytes16 = static_cast<uint16_t>(meta->pass());
-        // seek over size of the compressed buffer
-        is.seekg(sizeof(uint16_t), std::ios_base::cur);
-    }
-    else {
-        // otherwise read from disk
-        is.read(reinterpret_cast<char*>(&bytes16), sizeof(uint16_t));
-    }
-
-    if (bytes16 == std::numeric_limits<uint16_t>::max()) {
-        // read or seek uncompressed data
-        if (seek) {
-            is.seekg(destBytes, std::ios_base::cur);
-        }
-        else {
-            is.read(reinterpret_cast<char*>(destBuf), destBytes);
-        }
-    }
-    else {
-        // read or seek uncompressed data
-        if (seek) {
-            is.seekg(int(bytes16), std::ios_base::cur);
-        }
-        else {
-            // decompress into the destination buffer
-            std::unique_ptr<char[]> bloscBuffer(new char[int(bytes16)]);
-            is.read(bloscBuffer.get(), bytes16);
-            std::unique_ptr<char[]> buffer = bloscDecompress(   bloscBuffer.get(),
-                                                                destBytes,
-                                                                /*resize=*/false);
-            std::memcpy(destBuf, buffer.get(), destBytes);
-        }
-    }
-}
-
-/// @brief openvdb::io::writeCompressedValues specialized on PointDataIndex32 arrays to
-/// ignore the value mask, use a larger block size and use 16-bit size instead of 64-bit
-template<>
-inline void
-writeCompressedValues(  std::ostream& os, const PointDataIndex32* srcBuf, Index srcCount,
-                        const util::NodeMask<3>& /*valueMask*/,
-                        const util::NodeMask<3>& /*childMask*/, bool /*toHalf*/)
-{
-    using compression::bloscCompress;
-
-    const size_t srcBytes = srcCount*sizeof(PointDataIndex32);
-    const size_t maximumBytes = std::numeric_limits<uint16_t>::max();
-    if (srcBytes >= maximumBytes) {
-        OPENVDB_THROW(openvdb::IoError, "Cannot write more than " <<
-                                maximumBytes << " bytes in voxel values.")
-    }
-
-    const char* charBuffer = reinterpret_cast<const char*>(srcBuf);
-
-    size_t compressedBytes;
-    std::unique_ptr<char[]> buffer = bloscCompress( charBuffer, srcBytes,
-                                                    compressedBytes, /*resize=*/false);
-
-    if (compressedBytes > 0) {
-        auto bytes16 = static_cast<uint16_t>(compressedBytes); // clamp to 16-bit unsigned integer
-        os.write(reinterpret_cast<const char*>(&bytes16), sizeof(uint16_t));
-        os.write(reinterpret_cast<const char*>(buffer.get()), compressedBytes);
-    }
-    else {
-        auto bytes16 = static_cast<uint16_t>(maximumBytes); // max value indicates uncompressed
-        os.write(reinterpret_cast<const char*>(&bytes16), sizeof(uint16_t));
-        os.write(reinterpret_cast<const char*>(srcBuf), srcBytes);
-    }
-}
-
-template <typename T>
-inline void
-writeCompressedValuesSize(std::ostream& os, const T* srcBuf, Index srcCount)
-{
-    using compression::bloscCompressedSize;
-
-    const size_t srcBytes = srcCount*sizeof(T);
-    const size_t maximumBytes = std::numeric_limits<uint16_t>::max();
-    if (srcBytes >= maximumBytes) {
-        OPENVDB_THROW(openvdb::IoError, "Cannot write more than " <<
-                                maximumBytes << " bytes in voxel values.")
-    }
-
-    const char* charBuffer = reinterpret_cast<const char*>(srcBuf);
-
-    // calculate voxel buffer size after compression
-    size_t compressedBytes = bloscCompressedSize(charBuffer, srcBytes);
-
-    if (compressedBytes > 0) {
-        auto bytes16 = static_cast<uint16_t>(compressedBytes); // clamp to 16-bit unsigned integer
-        os.write(reinterpret_cast<const char*>(&bytes16), sizeof(uint16_t));
-    }
-    else {
-        auto bytes16 = static_cast<uint16_t>(maximumBytes); // max value indicates uncompressed
-        os.write(reinterpret_cast<const char*>(&bytes16), sizeof(uint16_t));
-    }
-}
-
-} // namespace io
-
 
 // forward declaration
 namespace tree {
@@ -218,26 +92,22 @@ makeDescriptorUnique(PointDataTreeT& tree);
 ///
 /// @note   Multiple threads cannot safely access the same AttributeArray when using streaming.
 template <typename PointDataTreeT>
+OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.")
 inline void
-setStreamingMode(PointDataTreeT& tree, bool on = true);
+setStreamingMode(PointDataTreeT&, bool /*on*/ = true) { }
 
 
-/// @brief  Sequentially pre-fetch all delayed-load voxel and attribute data from disk in order
-///         to accelerate subsequent random access.
-///
-/// @param  tree                the PointDataTree.
-/// @param  position            if enabled, prefetch the position attribute (default is on)
-/// @param  otherAttributes     if enabled, prefetch all other attributes (default is on)
 template <typename PointDataTreeT>
+OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.")
 inline void
-prefetch(PointDataTreeT& tree, bool position = true, bool otherAttributes = true);
+prefetch(PointDataTreeT&, bool /*position*/ = true, bool /*otherAttributes*/ = true) { }
 
 
 ////////////////////////////////////////
 
 
 template <typename T, Index Log2Dim>
-class PointDataLeafNode : public tree::LeafNode<T, Log2Dim>, io::MultiPass {
+class PointDataLeafNode : public tree::LeafNode<T, Log2Dim>, io::PointDataGridMultiPass {
 
 public:
     using LeafNodeType  = PointDataLeafNode<T, Log2Dim>;
@@ -504,7 +374,8 @@ public:
 
 
     Index64 memUsage() const;
-    Index64 memUsageIfLoaded() const;
+    OPENVDB_DEPRECATED_MESSAGE("Use memUsage() instead. This method is deprecated and will be removed. Delayed loading is no longer supported.")
+    Index64 memUsageIfLoaded() const { return memUsage(); }
 
     void evalActiveBoundingBox(CoordBBox& bbox, bool visitVoxels = true) const;
 
@@ -1522,13 +1393,6 @@ PointDataLeafNode<T, Log2Dim>::memUsage() const
     return BaseLeaf::memUsage() + mAttributeSet->memUsage();
 }
 
-template<typename T, Index Log2Dim>
-inline Index64
-PointDataLeafNode<T, Log2Dim>::memUsageIfLoaded() const
-{
-    return BaseLeaf::memUsageIfLoaded() + mAttributeSet->memUsageIfLoaded();
-}
-
 template<typename T, Index Log2Dim>
 inline void
 PointDataLeafNode<T, Log2Dim>::evalActiveBoundingBox(CoordBBox& bbox, bool visitVoxels) const
@@ -1598,63 +1462,6 @@ makeDescriptorUnique(PointDataTreeT& tree)
 }
 
 
-template <typename PointDataTreeT>
-inline void
-setStreamingMode(PointDataTreeT& tree, bool on)
-{
-    auto leafIter = tree.beginLeaf();
-    for (; leafIter; ++leafIter) {
-        for (size_t i = 0; i < leafIter->attributeSet().size(); i++) {
-            leafIter->attributeArray(i).setStreaming(on);
-        }
-    }
-}
-
-
-template <typename PointDataTreeT>
-inline void
-prefetch(PointDataTreeT& tree, bool position, bool otherAttributes)
-{
-    // NOTE: the following is intentionally not multi-threaded, as the I/O
-    // is faster if done in the order in which it is stored in the file
-
-    auto leaf = tree.cbeginLeaf();
-    if (!leaf)  return;
-
-    const auto& attributeSet = leaf->attributeSet();
-
-    // pre-fetch leaf data
-
-    for ( ; leaf; ++leaf) {
-        leaf->buffer().data();
-    }
-
-    // pre-fetch position attribute data (position will typically have index 0)
-
-    size_t positionIndex = attributeSet.find("P");
-
-    if (position && positionIndex != AttributeSet::INVALID_POS) {
-        for (leaf = tree.cbeginLeaf(); leaf; ++leaf) {
-            OPENVDB_ASSERT(leaf->hasAttribute(positionIndex));
-            leaf->constAttributeArray(positionIndex).loadData();
-        }
-    }
-
-    // pre-fetch other attribute data
-
-    if (otherAttributes) {
-        const size_t attributes = attributeSet.size();
-        for (size_t attributeIndex = 0; attributeIndex < attributes; attributeIndex++) {
-            if (attributeIndex == positionIndex)     continue;
-            for (leaf = tree.cbeginLeaf(); leaf; ++leaf) {
-                OPENVDB_ASSERT(leaf->hasAttribute(attributeIndex));
-                leaf->constAttributeArray(attributeIndex).loadData();
-            }
-        }
-    }
-}
-
-
 namespace internal {
 
 /// @brief Global registration of point data-related types

openvdb/openvdb/points/PointDataIO.h

@@ -0,0 +1,148 @@
+// Copyright Contributors to the OpenVDB Project
+// SPDX-License-Identifier: Apache-2.0
+
+#ifndef OPENVDB_POINTS_POINT_DATA_IO_HAS_BEEN_INCLUDED
+#define OPENVDB_POINTS_POINT_DATA_IO_HAS_BEEN_INCLUDED
+
+
+namespace openvdb {
+OPENVDB_USE_VERSION_NAMESPACE
+namespace OPENVDB_VERSION_NAME {
+
+
+////////////////////////////////////////
+
+
+namespace io
+{
+
+/// @brief openvdb::io::readCompressedValues specialized on PointDataIndex32 arrays to
+/// ignore the value mask, use a larger block size and use 16-bit size instead of 64-bit
+template<>
+inline void
+readCompressedValues(   std::istream& is, PointDataIndex32* destBuf, Index destCount,
+                        const util::NodeMask<3>& /*valueMask*/, bool /*fromHalf*/)
+{
+    using compression::bloscDecompress;
+
+    const bool seek = destBuf == nullptr;
+
+    const size_t destBytes = destCount*sizeof(PointDataIndex32);
+    const size_t maximumBytes = std::numeric_limits<uint16_t>::max();
+    if (destBytes >= maximumBytes) {
+        OPENVDB_THROW(openvdb::IoError, "Cannot read more than " <<
+                                maximumBytes << " bytes in voxel values.")
+    }
+
+    uint16_t bytes16;
+
+    const io::StreamMetadata::Ptr meta = io::getStreamMetadataPtr(is);
+
+    if (seek && meta) {
+        // buffer size temporarily stored in the StreamMetadata pass
+        // to avoid having to perform an expensive disk read for 2-bytes
+        bytes16 = static_cast<uint16_t>(meta->pass());
+        // seek over size of the compressed buffer
+        is.seekg(sizeof(uint16_t), std::ios_base::cur);
+    }
+    else {
+        // otherwise read from disk
+        is.read(reinterpret_cast<char*>(&bytes16), sizeof(uint16_t));
+    }
+
+    if (bytes16 == std::numeric_limits<uint16_t>::max()) {
+        // read or seek uncompressed data
+        if (seek) {
+            is.seekg(destBytes, std::ios_base::cur);
+        }
+        else {
+            is.read(reinterpret_cast<char*>(destBuf), destBytes);
+        }
+    }
+    else {
+        // read or seek uncompressed data
+        if (seek) {
+            is.seekg(int(bytes16), std::ios_base::cur);
+        }
+        else {
+            // decompress into the destination buffer
+            std::unique_ptr<char[]> bloscBuffer(new char[int(bytes16)]);
+            is.read(bloscBuffer.get(), bytes16);
+            std::unique_ptr<char[]> buffer = bloscDecompress(   bloscBuffer.get(),
+                                                                destBytes,
+                                                                /*resize=*/false);
+            std::memcpy(destBuf, buffer.get(), destBytes);
+        }
+    }
+}
+
+/// @brief openvdb::io::writeCompressedValues specialized on PointDataIndex32 arrays to
+/// ignore the value mask, use a larger block size and use 16-bit size instead of 64-bit
+template<>
+inline void
+writeCompressedValues(  std::ostream& os, const PointDataIndex32* srcBuf, Index srcCount,
+                        const util::NodeMask<3>& /*valueMask*/,
+                        const util::NodeMask<3>& /*childMask*/, bool /*toHalf*/)
+{
+    using compression::bloscCompress;
+
+    const size_t srcBytes = srcCount*sizeof(PointDataIndex32);
+    const size_t maximumBytes = std::numeric_limits<uint16_t>::max();
+    if (srcBytes >= maximumBytes) {
+        OPENVDB_THROW(openvdb::IoError, "Cannot write more than " <<
+                                maximumBytes << " bytes in voxel values.")
+    }
+
+    const char* charBuffer = reinterpret_cast<const char*>(srcBuf);
+
+    size_t compressedBytes;
+    std::unique_ptr<char[]> buffer = bloscCompress( charBuffer, srcBytes,
+                                                    compressedBytes, /*resize=*/false);
+
+    if (compressedBytes > 0) {
+        auto bytes16 = static_cast<uint16_t>(compressedBytes); // clamp to 16-bit unsigned integer
+        os.write(reinterpret_cast<const char*>(&bytes16), sizeof(uint16_t));
+        os.write(reinterpret_cast<const char*>(buffer.get()), compressedBytes);
+    }
+    else {
+        auto bytes16 = static_cast<uint16_t>(maximumBytes); // max value indicates uncompressed
+        os.write(reinterpret_cast<const char*>(&bytes16), sizeof(uint16_t));
+        os.write(reinterpret_cast<const char*>(srcBuf), srcBytes);
+    }
+}
+
+template <typename T>
+inline void
+writeCompressedValuesSize(std::ostream& os, const T* srcBuf, Index srcCount)
+{
+    using compression::bloscCompressedSize;
+
+    const size_t srcBytes = srcCount*sizeof(T);
+    const size_t maximumBytes = std::numeric_limits<uint16_t>::max();
+    if (srcBytes >= maximumBytes) {
+        OPENVDB_THROW(openvdb::IoError, "Cannot write more than " <<
+                                maximumBytes << " bytes in voxel values.")
+    }
+
+    const char* charBuffer = reinterpret_cast<const char*>(srcBuf);
+
+    // calculate voxel buffer size after compression
+    size_t compressedBytes = bloscCompressedSize(charBuffer, srcBytes);
+
+    if (compressedBytes > 0) {
+        auto bytes16 = static_cast<uint16_t>(compressedBytes); // clamp to 16-bit unsigned integer
+        os.write(reinterpret_cast<const char*>(&bytes16), sizeof(uint16_t));
+    }
+    else {
+        auto bytes16 = static_cast<uint16_t>(maximumBytes); // max value indicates uncompressed
+        os.write(reinterpret_cast<const char*>(&bytes16), sizeof(uint16_t));
+    }
+}
+
+} // namespace io
+
+
+} // namespace OPENVDB_VERSION_NAME
+} // namespace openvdb
+
+#endif // OPENVDB_POINTS_POINT_DATA_IO_HAS_BEEN_INCLUDED

openvdb/openvdb/points/StreamCompression.cc

@@ -279,13 +279,6 @@ bloscDecompress(const char*, const size_t, const bool)
 ////////////////////////////////////////
 
 
-void
-Page::load() const
-{
-    this->doLoad();
-}
-
-
 long
 Page::uncompressedBytes() const
 {
@@ -297,10 +290,6 @@ Page::uncompressedBytes() const
 const char*
 Page::buffer(const int index) const
 {
-#ifdef OPENVDB_USE_DELAYED_LOADING
-    if (this->isOutOfCore())   this->load();
-#endif
-
     return mData.get() + index;
 }
 
@@ -336,51 +325,31 @@ Page::readBuffers(std::istream&is, bool delayed)
 
     bool isCompressed = mInfo->compressedBytes > 0;
 
-#ifdef OPENVDB_USE_DELAYED_LOADING
-    io::MappedFile::Ptr mappedFile = io::getMappedFilePtr(is);
+    std::unique_ptr<char[]> buffer(new char[
+        (isCompressed ? mInfo->compressedBytes : -mInfo->compressedBytes)]);
+    is.read(buffer.get(), (isCompressed ? mInfo->compressedBytes : -mInfo->compressedBytes));
 
-    if (delayed && mappedFile) {
-        SharedPtr<io::StreamMetadata> meta = io::getStreamMetadataPtr(is);
-        OPENVDB_ASSERT(meta);
-
-        std::streamoff filepos = is.tellg();
-
-        // seek over the page
-        is.seekg((isCompressed ? mInfo->compressedBytes : -mInfo->compressedBytes),
-            std::ios_base::cur);
-
-        mInfo->mappedFile = mappedFile;
-        mInfo->meta = meta;
-        mInfo->filepos = filepos;
-
-        OPENVDB_ASSERT(mInfo->mappedFile);
+    if (mInfo->compressedBytes > 0) {
+        this->decompress(buffer);
+    } else {
+        this->copy(buffer, -static_cast<int>(mInfo->compressedBytes));
     }
-    else {
-#endif
-        std::unique_ptr<char[]> buffer(new char[
-            (isCompressed ? mInfo->compressedBytes : -mInfo->compressedBytes)]);
-        is.read(buffer.get(), (isCompressed ? mInfo->compressedBytes : -mInfo->compressedBytes));
-
-        if (mInfo->compressedBytes > 0) {
-            this->decompress(buffer);
-        } else {
-            this->copy(buffer, -static_cast<int>(mInfo->compressedBytes));
-        }
-        mInfo.reset();
-#ifdef OPENVDB_USE_DELAYED_LOADING
-    }
-#endif
+    mInfo.reset();
 }
 
 
-bool
-Page::isOutOfCore() const
+void
+Page::skipBuffers(std::istream& is)
 {
-#ifdef OPENVDB_USE_DELAYED_LOADING
-    return bool(mInfo);
-#else
-    return false;
-#endif
+    OPENVDB_ASSERT(mInfo);
+
+    bool isCompressed = mInfo->compressedBytes > 0;
+    std::streamsize bytes = isCompressed ?
+        mInfo->compressedBytes : -mInfo->compressedBytes;
+
+    is.seekg(bytes, std::ios_base::cur);
+
+    mInfo.reset();
 }
 
 
@@ -406,47 +375,6 @@ Page::decompress(const std::unique_ptr<char[]>& temp)
 }
 
 
-void
-Page::doLoad() const
-{
-#ifdef OPENVDB_USE_DELAYED_LOADING
-    if (!this->isOutOfCore())  return;
-
-    Page* self = const_cast<Page*>(this);
-
-    // This lock will be contended at most once, after which this buffer
-    // will no longer be out-of-core.
-    tbb::spin_mutex::scoped_lock lock(self->mMutex);
-    if (!this->isOutOfCore()) return;
-
-    OPENVDB_ASSERT(self->mInfo);
-
-    int compressedBytes = static_cast<int>(self->mInfo->compressedBytes);
-    bool compressed = compressedBytes > 0;
-    if (!compressed) compressedBytes = -compressedBytes;
-
-    OPENVDB_ASSERT(compressedBytes);
-
-    std::unique_ptr<char[]> temp(new char[compressedBytes]);
-
-    OPENVDB_ASSERT(self->mInfo->mappedFile);
-    SharedPtr<std::streambuf> buf = self->mInfo->mappedFile->createBuffer();
-    OPENVDB_ASSERT(buf);
-
-    std::istream is(buf.get());
-    io::setStreamMetadataPtr(is, self->mInfo->meta, /*transfer=*/true);
-    is.seekg(self->mInfo->filepos);
-
-    is.read(temp.get(), compressedBytes);
-
-    if (compressed)     self->decompress(temp);
-    else                self->copy(temp, compressedBytes);
-
-    self->mInfo.reset();
-#endif
-}
-
-
 ////////////////////////////////////////
 
 
@@ -525,6 +453,23 @@ PagedInputStream::read(PageHandle::Ptr& pageHandle, std::streamsize n, bool dela
 }
 
 
+void
+PagedInputStream::skip(PageHandle::Ptr& pageHandle, std::streamsize n)
+{
+    OPENVDB_ASSERT(mByteIndex <= mUncompressedBytes);
+
+    Page& page = pageHandle->page();
+
+    if (mByteIndex == mUncompressedBytes) {
+        mUncompressedBytes = static_cast<int>(page.uncompressedBytes());
+        page.skipBuffers(*mIs);
+        mByteIndex = 0;
+    }
+
+    mByteIndex += int(n);
+}
+
+
 ////////////////////////////////////////
 
 

openvdb/openvdb/points/StreamCompression.h

@@ -114,10 +114,6 @@ class OPENVDB_API Page
 private:
     struct Info
     {
-#ifdef OPENVDB_USE_DELAYED_LOADING
-        io::MappedFile::Ptr mappedFile;
-#endif
-        SharedPtr<io::StreamMetadata> meta;
         std::streamoff filepos;
         long compressedBytes;
         long uncompressedBytes;
@@ -128,8 +124,8 @@ public:
 
     Page() = default;
 
-    /// @brief load the Page into memory
-    void load() const;
+    OPENVDB_DEPRECATED_MESSAGE("Always returns false. This method is deprecated and will be removed. Delayed loading is no longer supported.")
+    void load() const { }
 
     /// @brief Uncompressed bytes of the Paged data, available
     /// when the header has been read.
@@ -146,8 +142,12 @@ public:
     /// pointers will be stored to load the data lazily.
     void readBuffers(std::istream&, bool delayed);
 
-    /// @brief Test if the data is out-of-core
-    bool isOutOfCore() const;
+    /// @brief Skip the Page buffers by seeking past the compressed data
+    /// without reading or decompressing it.
+    void skipBuffers(std::istream&);
+
+    OPENVDB_DEPRECATED_MESSAGE("Always returns false. This method is deprecated and will be removed. Delayed loading is no longer supported.")
+    bool isOutOfCore() const { return false; }
 
 private:
     /// @brief Convenience method to store a copy of the supplied buffer
@@ -156,9 +156,6 @@ private:
     /// @brief Decompress and store the supplied data
     void decompress(const std::unique_ptr<char[]>& temp);
 
-    /// @brief Thread-safe loading of the data
-    void doLoad() const;
-
     std::unique_ptr<Info> mInfo = std::unique_ptr<Info>(new Info);
     std::unique_ptr<char[]> mData;
     tbb::spin_mutex mMutex;
@@ -229,6 +226,10 @@ public:
     /// an immediate read of the data.
     void read(PageHandle::Ptr& pageHandle, std::streamsize n, bool delayed = true);
 
+    /// @brief Skip past the page data referenced by @a pageHandle without
+    /// reading or decompressing it.
+    void skip(PageHandle::Ptr& pageHandle, std::streamsize n);
+
 private:
     int mByteIndex = 0;
     int mUncompressedBytes = 0;

openvdb/openvdb/points/impl/PointConversionImpl.h

@@ -133,15 +133,13 @@ struct ConvertPointDataGridPositionOp {
                                     const Index64 startOffset,
                                     const math::Transform& transform,
                                     const size_t index,
-                                    const FilterT& filter,
-                                    const bool inCoreOnly)
+                                    const FilterT& filter)
         : mAttribute(attribute)
         , mPointOffsets(pointOffsets)
         , mStartOffset(startOffset)
         , mTransform(transform)
         , mIndex(index)
         , mFilter(filter)
-        , mInCoreOnly(inCoreOnly)
     {
         // only accept Vec3f as ValueType
         static_assert(VecTraits<ValueType>::Size == 3 &&
@@ -169,8 +167,6 @@ struct ConvertPointDataGridPositionOp {
 
             OPENVDB_ASSERT(leaf.pos() < mPointOffsets.size());
 
-            if (mInCoreOnly && leaf->buffer().isOutOfCore())    continue;
-
             Index64 offset = mStartOffset;
 
             if (leaf.pos() > 0) offset += mPointOffsets[leaf.pos() - 1];
@@ -196,7 +192,6 @@ struct ConvertPointDataGridPositionOp {
     const math::Transform&                  mTransform;
     const size_t                            mIndex;
     const FilterT&                          mFilter;
-    const bool                              mInCoreOnly;
 }; // ConvertPointDataGridPositionOp
 
 
@@ -215,15 +210,13 @@ struct ConvertPointDataGridAttributeOp {
                                     const Index64 startOffset,
                                     const size_t index,
                                     const Index stride,
-                                    const FilterT& filter,
-                                    const bool inCoreOnly)
+                                    const FilterT& filter)
         : mAttribute(attribute)
         , mPointOffsets(pointOffsets)
         , mStartOffset(startOffset)
         , mIndex(index)
         , mStride(stride)
-        , mFilter(filter)
-        , mInCoreOnly(inCoreOnly) { }
+        , mFilter(filter) { }
 
     template <typename IterT>
     void convert(IterT& iter, HandleT& targetHandle,
@@ -257,8 +250,6 @@ struct ConvertPointDataGridAttributeOp {
 
             OPENVDB_ASSERT(leaf.pos() < mPointOffsets.size());
 
-            if (mInCoreOnly && leaf->buffer().isOutOfCore())    continue;
-
             Index64 offset = mStartOffset;
 
             if (leaf.pos() > 0) offset += mPointOffsets[leaf.pos() - 1];
@@ -284,7 +275,6 @@ struct ConvertPointDataGridAttributeOp {
     const size_t                            mIndex;
     const Index                             mStride;
     const FilterT&                          mFilter;
-    const bool                              mInCoreOnly;
 }; // ConvertPointDataGridAttributeOp
 
 template<typename PointDataTreeType, typename Group, typename FilterT>
@@ -299,14 +289,12 @@ struct ConvertPointDataGridGroupOp {
                                 const std::vector<Index64>& pointOffsets,
                                 const Index64 startOffset,
                                 const AttributeSet::Descriptor::GroupIndex index,
-                                const FilterT& filter,
-                                const bool inCoreOnly)
+                                const FilterT& filter)
         : mGroup(group)
         , mPointOffsets(pointOffsets)
         , mStartOffset(startOffset)
         , mIndex(index)
-        , mFilter(filter)
-        , mInCoreOnly(inCoreOnly) { }
+        , mFilter(filter) { }
 
     template <typename IterT>
     void convert(IterT& iter, const GroupAttributeArray& groupArray, Index64& offset) const
@@ -337,8 +325,6 @@ struct ConvertPointDataGridGroupOp {
 
             OPENVDB_ASSERT(leaf.pos() < mPointOffsets.size());
 
-            if (mInCoreOnly && leaf->buffer().isOutOfCore())    continue;
-
             Index64 offset = mStartOffset;
 
             if (leaf.pos() > 0)     offset += mPointOffsets[leaf.pos() - 1];
@@ -365,7 +351,6 @@ struct ConvertPointDataGridGroupOp {
     const Index64                           mStartOffset;
     const GroupIndex                        mIndex;
     const FilterT&                          mFilter;
-    const bool                              mInCoreOnly;
 }; // ConvertPointDataGridGroupOp
 
 template<typename PositionArrayT, typename VecT = Vec3R>
@@ -582,8 +567,7 @@ convertPointDataGridPosition(   PositionAttribute& positionAttribute,
                                 const PointDataGridT& grid,
                                 const std::vector<Index64>& pointOffsets,
                                 const Index64 startOffset,
-                                const FilterT& filter,
-                                const bool inCoreOnly)
+                                const FilterT& filter)
 {
     using TreeType      = typename PointDataGridT::TreeType;
     using LeafManagerT  = typename tree::LeafManager<const TreeType>;
@@ -601,12 +585,26 @@ convertPointDataGridPosition(   PositionAttribute& positionAttribute,
     LeafManagerT leafManager(tree);
     ConvertPointDataGridPositionOp<TreeType, PositionAttribute, FilterT> convert(
                     positionAttribute, pointOffsets, startOffset, grid.transform(), positionIndex,
-                    filter, inCoreOnly);
+                    filter);
     tbb::parallel_for(leafManager.leafRange(), convert);
     positionAttribute.compact();
 }
 
 
+template <typename PositionAttribute, typename PointDataGridT, typename FilterT>
+OPENVDB_DEPRECATED_MESSAGE("Use convertPointDataGridPosition() without inCoreOnly parameter instead. This method is deprecated and will be removed. Delayed loading is no longer supported.")
+void
+convertPointDataGridPosition(   PositionAttribute& positionAttribute,
+                                const PointDataGridT& grid,
+                                const std::vector<Index64>& pointOffsets,
+                                const Index64 startOffset,
+                                const FilterT& filter,
+                                const bool /*inCoreOnly*/)
+{
+    convertPointDataGridPosition(positionAttribute, grid, pointOffsets, startOffset, filter);
+}
+
+
 ////////////////////////////////////////
 
 
@@ -618,8 +616,7 @@ convertPointDataGridAttribute(  TypedAttribute& attribute,
                                 const Index64 startOffset,
                                 const unsigned arrayIndex,
                                 const Index stride,
-                                const FilterT& filter,
-                                const bool inCoreOnly)
+                                const FilterT& filter)
 {
     using LeafManagerT = typename tree::LeafManager<const PointDataTreeT>;
 
@@ -633,12 +630,28 @@ convertPointDataGridAttribute(  TypedAttribute& attribute,
     LeafManagerT leafManager(tree);
     ConvertPointDataGridAttributeOp<PointDataTreeT, TypedAttribute, FilterT> convert(
                         attribute, pointOffsets, startOffset, arrayIndex, stride,
-                        filter, inCoreOnly);
+                        filter);
         tbb::parallel_for(leafManager.leafRange(), convert);
     attribute.compact();
 }
 
 
+template <typename TypedAttribute, typename PointDataTreeT, typename FilterT>
+OPENVDB_DEPRECATED_MESSAGE("Use convertPointDataGridAttribute() without inCoreOnly parameter instead. This method is deprecated and will be removed. Delayed loading is no longer supported.")
+void
+convertPointDataGridAttribute(  TypedAttribute& attribute,
+                                const PointDataTreeT& tree,
+                                const std::vector<Index64>& pointOffsets,
+                                const Index64 startOffset,
+                                const unsigned arrayIndex,
+                                const Index stride,
+                                const FilterT& filter,
+                                const bool /*inCoreOnly*/)
+{
+    convertPointDataGridAttribute(attribute, tree, pointOffsets, startOffset, arrayIndex, stride, filter);
+}
+
+
 ////////////////////////////////////////
 
 
@@ -649,8 +662,7 @@ convertPointDataGridGroup(  Group& group,
                             const std::vector<Index64>& pointOffsets,
                             const Index64 startOffset,
                             const AttributeSet::Descriptor::GroupIndex index,
-                            const FilterT& filter,
-                            const bool inCoreOnly)
+                            const FilterT& filter)
 {
     using LeafManagerT= typename tree::LeafManager<const PointDataTreeT>;
 
@@ -662,7 +674,7 @@ convertPointDataGridGroup(  Group& group,
     LeafManagerT leafManager(tree);
     ConvertPointDataGridGroupOp<PointDataTreeT, Group, FilterT> convert(
                     group, pointOffsets, startOffset, index,
-                    filter, inCoreOnly);
+                    filter);
     tbb::parallel_for(leafManager.leafRange(), convert);
 
     // must call this after modifying point groups in parallel
@@ -670,6 +682,22 @@ convertPointDataGridGroup(  Group& group,
     group.finalize();
 }
 
+
+template <typename Group, typename PointDataTreeT, typename FilterT>
+OPENVDB_DEPRECATED_MESSAGE("Use convertPointDataGridGroup() without inCoreOnly parameter instead. This method is deprecated and will be removed. Delayed loading is no longer supported.")
+void
+convertPointDataGridGroup(  Group& group,
+                            const PointDataTreeT& tree,
+                            const std::vector<Index64>& pointOffsets,
+                            const Index64 startOffset,
+                            const AttributeSet::Descriptor::GroupIndex index,
+                            const FilterT& filter,
+                            const bool /*inCoreOnly*/)
+{
+    convertPointDataGridGroup(group, tree, pointOffsets, startOffset, index, filter);
+}
+
+
 template<typename PositionWrapper, typename InterrupterT, typename VecT>
 inline float
 computeVoxelSize(  const PositionWrapper& positions,

openvdb/openvdb/points/impl/PointCountImpl.h

@@ -17,16 +17,14 @@ namespace points {
 template <typename PointDataTreeT, typename FilterT>
 Index64 pointCount(const PointDataTreeT& tree,
                    const FilterT& filter,
-                   const bool inCoreOnly,
                    const bool threaded)
 {
     using LeafManagerT = tree::LeafManager<const PointDataTreeT>;
     using LeafRangeT = typename LeafManagerT::LeafRange;
 
     auto countLambda =
-        [&filter, &inCoreOnly] (const LeafRangeT& range, Index64 sum) -> Index64 {
+        [&filter] (const LeafRangeT& range, Index64 sum) -> Index64 {
             for (const auto& leaf : range) {
-                if (inCoreOnly && leaf.buffer().isOutOfCore())  continue;
                 auto state = filter.state(leaf);
                 if (state == index::ALL) {
                     sum += leaf.pointCount();
@@ -48,11 +46,21 @@ Index64 pointCount(const PointDataTreeT& tree,
 }
 
 
+template <typename PointDataTreeT, typename FilterT>
+OPENVDB_DEPRECATED_MESSAGE("Use pointCount() without inCoreOnly parameter instead. This method is deprecated and will be removed. Delayed loading is no longer supported.")
+Index64 pointCount(const PointDataTreeT& tree,
+                   const FilterT& filter,
+                   const bool /*inCoreOnly*/,
+                   const bool threaded)
+{
+    return pointCount(tree, filter, threaded);
+}
+
+
 template <typename PointDataTreeT, typename FilterT>
 Index64 pointOffsets(   std::vector<Index64>& pointOffsets,
                         const PointDataTreeT& tree,
                         const FilterT& filter,
-                        const bool inCoreOnly,
                         const bool threaded)
 {
     using LeafT = typename PointDataTreeT::LeafNodeType;
@@ -67,8 +75,7 @@ Index64 pointOffsets(   std::vector<Index64>& pointOffsets,
 
     LeafManagerT leafManager(tree);
     leafManager.foreach(
-        [&pointOffsets, &filter, &inCoreOnly](const LeafT& leaf, size_t pos) {
-            if (inCoreOnly && leaf.buffer().isOutOfCore())  return;
+        [&pointOffsets, &filter](const LeafT& leaf, size_t pos) {
             auto state = filter.state(leaf);
             if (state == index::ALL) {
                 pointOffsets[pos] = leaf.pointCount();
@@ -90,6 +97,18 @@ Index64 pointOffsets(   std::vector<Index64>& pointOffsets,
 }
 
 
+template <typename PointDataTreeT, typename FilterT>
+OPENVDB_DEPRECATED_MESSAGE("Use pointOffsets() without inCoreOnly parameter instead. This method is deprecated and will be removed. Delayed loading is no longer supported.")
+Index64 pointOffsets(   std::vector<Index64>& pointOffsets,
+                        const PointDataTreeT& tree,
+                        const FilterT& filter,
+                        const bool /*inCoreOnly*/,
+                        const bool threaded)
+{
+    return pointOffsets(pointOffsets, tree, filter, threaded);
+}
+
+
 template <typename PointDataGridT, typename GridT, typename FilterT>
 typename GridT::Ptr
 pointCountGrid( const PointDataGridT& points,

openvdb/openvdb/points/impl/PointMoveImpl.h

@@ -319,7 +319,6 @@ struct GlobalMovePointsOp
         // extract target array and ensure data is out-of-core and non-uniform
 
         auto& targetArray = leaf.attributeArray(mAttributeIndex);
-        targetArray.loadData();
         targetArray.expand();
 
         // perform the copy
@@ -344,7 +343,6 @@ struct GlobalMovePointsOp
 
                 const LeafT& sourceLeaf = mSourceLeafManager.leaf(sourceLeafIndex);
                 const auto& sourceArray = sourceLeaf.constAttributeArray(mAttributeIndex);
-                sourceArray.loadData();
 
                 targetArray.copyValuesUnsafe(sourceArray, copyIterator);
 
@@ -427,12 +425,10 @@ struct LocalMovePointsOp
         const Index sourceLeafOffset(mSourceIndices[idx]);
         LeafT& sourceLeaf = mSourceLeafManager.leaf(sourceLeafOffset);
         const auto& sourceArray = sourceLeaf.constAttributeArray(mAttributeIndex);
-        sourceArray.loadData();
 
         // extract target array and ensure data is out-of-core and non-uniform
 
         auto& targetArray = leaf.attributeArray(mAttributeIndex);
-        targetArray.loadData();
         targetArray.expand();
 
         // perform the copy

openvdb/openvdb/points/impl/PointRasterizeFrustumImpl.h

@@ -948,7 +948,7 @@ public:
         // generate leaf offsets (if necessary)
         if (mLeafOffsets.empty()) {
             openvdb::points::pointOffsets(mLeafOffsets, mGrid->constTree(), resolvedFilter,
-                /*inCoreOnly=*/false, mSettings.threaded);
+                mSettings.threaded);
         }
 
         // set streaming arbitrary attribute array flags

openvdb/openvdb/points/impl/PrincipalComponentAnalysisImpl.h

@@ -310,10 +310,7 @@ struct WeightPosSumsTransfer
         auto& leaf = this->mManager.leaf(idx);
 
         {
-            // @todo add API to get the array from the group handle. The handle
-            //   calls loadData but not expand.
             auto& array = leaf.attributeArray(this->mIndices.mEllipsesGroupIndex.first);
-            array.loadData(); // so we can call setUnsafe/getUnsafe
             array.expand();
         }
 

openvdb/openvdb/tools/Count.h

@@ -60,19 +60,13 @@ Index64 countActiveTiles(const TreeT& tree, bool threaded = true);
 
 
 /// @brief Return the total amount of memory in bytes occupied by this tree.
-/// @details  This method returns the total in-core memory usage which can be
-///   different to the maximum possible memory usage for trees which have not
-///   been fully deserialized (via delay-loading). Thus, this is the current
-///   true memory consumption.
 template <typename TreeT>
 Index64 memUsage(const TreeT& tree, bool threaded = true);
 
 
-/// @brief Return the deserialized memory usage of this tree. This is not
-///   necessarily equal to the current memory usage (returned by tools::memUsage)
-///   if delay-loading is enabled. See File::open.
 template <typename TreeT>
-Index64 memUsageIfLoaded(const TreeT& tree, bool threaded = true);
+OPENVDB_DEPRECATED_MESSAGE("Use tools::memUsage() instead. This method is deprecated and will be removed. Delayed loading is no longer supported.")
+Index64 memUsageIfLoaded(const TreeT& tree, bool threaded = true) { return tools::memUsage(tree, threaded); }
 
 
 /// @brief Return the minimum and maximum active values in this tree.
@@ -299,9 +293,9 @@ struct MemUsageOp
     using RootT = typename TreeType::RootNodeType;
     using LeafT = typename TreeType::LeafNodeType;
 
-    MemUsageOp(const bool inCoreOnly) : mInCoreOnly(inCoreOnly) {}
-    MemUsageOp(const MemUsageOp& other) : mCount(0), mInCoreOnly(other.mInCoreOnly) {}
-    MemUsageOp(const MemUsageOp& other, tbb::split) : MemUsageOp(other) {}
+    MemUsageOp() = default;
+    MemUsageOp(const MemUsageOp& other) = default;
+    MemUsageOp(const MemUsageOp&, tbb::split) : mCount(0) {}
 
     // accumulate size of the root node in bytes
     bool operator()(const RootT& root, size_t)
@@ -323,8 +317,7 @@ struct MemUsageOp
     // accumulate size of leaf node in bytes
     bool operator()(const LeafT& leaf, size_t)
     {
-        if (mInCoreOnly) mCount += leaf.memUsage();
-        else             mCount += leaf.memUsageIfLoaded();
+        mCount += leaf.memUsage();
         return false;
     }
 
@@ -334,7 +327,6 @@ struct MemUsageOp
     }
 
     openvdb::Index64 mCount{0};
-    const bool mInCoreOnly;
 }; // struct MemUsageOp
 
 /// @brief A DynamicNodeManager operator to find the minimum and maximum active values in this tree.
@@ -492,25 +484,12 @@ Index64 countActiveTiles(const TreeT& tree, bool threaded)
 template <typename TreeT>
 Index64 memUsage(const TreeT& tree, bool threaded)
 {
-    count_internal::MemUsageOp<TreeT> op(true);
+    count_internal::MemUsageOp<TreeT> op;
     tree::DynamicNodeManager<const TreeT> nodeManager(tree);
     nodeManager.reduceTopDown(op, threaded);
     return op.mCount + sizeof(tree);
 }
 
-template <typename TreeT>
-Index64 memUsageIfLoaded(const TreeT& tree, bool threaded)
-{
-    /// @note  For numeric (non-point) grids this really doesn't need to
-    ///   traverse the tree and could instead be computed from the node counts.
-    ///   We do so anyway as it ties this method into the tree data structure
-    ///   which makes sure that changes to the tree/nodes are reflected/kept in
-    ///   sync here.
-    count_internal::MemUsageOp<TreeT> op(false);
-    tree::DynamicNodeManager<const TreeT> nodeManager(tree);
-    nodeManager.reduceTopDown(op, threaded);
-    return op.mCount + sizeof(tree);
-}
 
 template <typename TreeT>
 math::MinMax<typename TreeT::ValueType> minMax(const TreeT& tree, bool threaded)

openvdb/openvdb/tools/Merge.h

@@ -581,16 +581,14 @@ struct UnallocatedBuffer
     static void allocateAndFill(BufferT& buffer, const ValueT& background)
     {
         if (buffer.empty()) {
-            if (!buffer.isOutOfCore()) {
-                buffer.allocate();
-                buffer.fill(background);
-            }
+            buffer.allocate();
+            buffer.fill(background);
         }
     }
 
     static bool isPartiallyConstructed(const BufferT& buffer)
     {
-        return !buffer.isOutOfCore() && buffer.empty();
+        return buffer.empty();
     }
 }; // struct AllocateAndFillBuffer
 

openvdb/openvdb/tools/PointIndexGrid.h

@@ -1491,7 +1491,8 @@ public:
 
 
     Index64 memUsage() const;
-    Index64 memUsageIfLoaded() const;
+    OPENVDB_DEPRECATED_MESSAGE("Use memUsage() instead. This method is deprecated and will be removed. Delayed loading is no longer supported.")
+    Index64 memUsageIfLoaded() const { return memUsage(); }
 
 
     ////////////////////////////////////////
@@ -1798,13 +1799,6 @@ PointIndexLeafNode<T, Log2Dim>::memUsage() const
     return BaseLeaf::memUsage() + Index64((sizeof(T)*mIndices.capacity()) + sizeof(mIndices));
 }
 
-template<typename T, Index Log2Dim>
-inline Index64
-PointIndexLeafNode<T, Log2Dim>::memUsageIfLoaded() const
-{
-    return BaseLeaf::memUsageIfLoaded() + Index64((sizeof(T)*mIndices.capacity()) + sizeof(mIndices));
-}
-
 } // namespace tools
 
 

openvdb/openvdb/tools/VolumeToMesh.h

@@ -616,8 +616,8 @@ template<>
 inline bool isInsideValue<bool>(bool value, bool /*isovalue*/) { return value; }
 
 
-/// @brief  Minor wrapper around the Leaf API to avoid atomic access with
-///   delayed loading.
+/// @brief  Minor wrapper around the Leaf API. This was originally added to avoid
+// atomic access with delayed loading, however delayed loading is no longer supported.
 template <typename LeafT,
     bool IsBool = std::is_same<typename LeafT::ValueType, bool>::value>
 struct LeafBufferAccessor

openvdb/openvdb/tree/LeafBuffer.h

@@ -8,9 +8,11 @@
 #include <openvdb/io/Compression.h> // for io::readCompressedValues(), etc
 #include <openvdb/util/NodeMasks.h>
 #include <openvdb/util/Assert.h>
+#if OPENVDB_ABI_VERSION_NUMBER < 14
 #include <tbb/spin_mutex.h>
-#include <algorithm> // for std::swap
 #include <atomic>
+#endif
+#include <algorithm> // for std::swap
 #include <cstddef> // for offsetof()
 #include <iostream>
 #include <memory>
@@ -36,58 +38,24 @@ public:
     using NodeMaskType = util::NodeMask<Log2Dim>;
     static const Index SIZE = 1 << 3 * Log2Dim;
 
-#ifdef OPENVDB_USE_DELAYED_LOADING
-    struct FileInfo
-    {
-        FileInfo(): bufpos(0) , maskpos(0) {}
-        std::streamoff bufpos;
-        std::streamoff maskpos;
-        io::MappedFile::Ptr mapping;
-        SharedPtr<io::StreamMetadata> meta;
-    };
-#endif
-
     /// Default constructor
-    inline LeafBuffer(): mData(new ValueType[SIZE])
-    {
-#ifdef OPENVDB_USE_DELAYED_LOADING
-        mOutOfCore = 0;
-#endif
-    }
+    inline LeafBuffer(): mData(new ValueType[SIZE]) {}
     /// Construct a buffer populated with the specified value.
     explicit inline LeafBuffer(const ValueType&);
     /// Copy constructor
     inline LeafBuffer(const LeafBuffer&);
     /// Construct a buffer but don't allocate memory for the full array of values.
-    LeafBuffer(PartialCreate, const ValueType&): mData(nullptr)
-    {
-#ifdef OPENVDB_USE_DELAYED_LOADING
-        mOutOfCore = 0;
-#endif
-    }
+    LeafBuffer(PartialCreate, const ValueType&): mData(nullptr) {}
     /// Destructor
     inline ~LeafBuffer();
 
-    /// Return @c true if this buffer's values have not yet been read from disk.
-    bool isOutOfCore() const
-    {
-#ifdef OPENVDB_USE_DELAYED_LOADING
-        return bool(mOutOfCore);
-#else
-        return false;
-#endif
-    }
+    OPENVDB_DEPRECATED_MESSAGE("Always returns false. This method is deprecated and will be removed. Delayed loading is no longer supported.")
+    bool isOutOfCore() const { return false; }
     /// Return @c true if memory for this buffer has not yet been allocated.
-    bool empty() const { return !mData || this->isOutOfCore(); }
+    bool empty() const { return !mData; }
     /// Allocate memory for this buffer if it has not already been allocated.
     bool allocate() { if (mData == nullptr) mData = new ValueType[SIZE]; return true; }
 
-#ifdef OPENVDB_USE_DELAYED_LOADING
-    /// Enable out-of-core in the LeafBuffer.
-    void enableOutOfCore(SharedPtr<io::StreamMetadata>& meta, const std::streamoff& bufpos,
-        io::MappedFile::Ptr& mappedFile, const std::streamoff& maskpos);
-#endif
-
     /// Populate this buffer with a constant value.
     inline void fill(const ValueType&);
 
@@ -113,7 +81,8 @@ public:
 
     /// Return the memory footprint of this buffer in bytes.
     inline Index memUsage() const;
-    inline Index memUsageIfLoaded() const;
+    OPENVDB_DEPRECATED_MESSAGE("Use memUsage() instead. This method is deprecated and will be removed. Delayed loading is no longer supported.")
+    inline Index memUsageIfLoaded() const { return memUsage(); }
     /// Return the number of values contained in this buffer.
     static Index size() { return SIZE; }
 
@@ -139,37 +108,12 @@ private:
 
     bool deallocate();
 
-    inline void setOutOfCore(bool b)
-    {
-        (void) b;
-#ifdef OPENVDB_USE_DELAYED_LOADING
-        mOutOfCore = b;
+    ValueType* mData = nullptr;
+#if OPENVDB_ABI_VERSION_NUMBER < 14
+    // Deprecated members kept for ABI compatibility
+    std::atomic<Index32> mDeprecatedAtomic{0};
+    tbb::spin_mutex mDeprecatedSpinMutex;
 #endif
-    }
-    // To facilitate inlining in the common case in which the buffer is in-core,
-    // the loading logic is split into a separate function, doLoad().
-    inline void loadValues() const
-    {
-#ifdef OPENVDB_USE_DELAYED_LOADING
-        if (this->isOutOfCore()) this->doLoad();
-#endif
-    }
-    inline void doLoad() const;
-    inline bool detachFromFile();
-
-    using FlagsType = std::atomic<Index32>;
-
-#ifdef OPENVDB_USE_DELAYED_LOADING
-    union {
-        ValueType* mData;
-        FileInfo*  mFileInfo;
-    };
-#else
-    ValueType* mData;
-#endif
-    FlagsType mOutOfCore; // interpreted as bool; extra bits reserved for future use
-    tbb::spin_mutex mMutex; // 1 byte
-    //int8_t mReserved[3]; // padding for alignment
 
     friend class ::TestLeaf;
     // Allow the parent LeafNode to access this buffer's data pointer.
@@ -185,9 +129,6 @@ inline
 LeafBuffer<T, Log2Dim>::LeafBuffer(const ValueType& val)
     : mData(new ValueType[SIZE])
 {
-#ifdef OPENVDB_USE_DELAYED_LOADING
-    mOutOfCore = 0;
-#endif
     this->fill(val);
 }
 
@@ -196,15 +137,7 @@ template<typename T, Index Log2Dim>
 inline
 LeafBuffer<T, Log2Dim>::~LeafBuffer()
 {
-#ifdef OPENVDB_USE_DELAYED_LOADING
-    if (this->isOutOfCore()) {
-        this->detachFromFile();
-    } else {
-        this->deallocate();
-    }
-#else
     this->deallocate();
-#endif
 }
 
 
@@ -212,25 +145,14 @@ template<typename T, Index Log2Dim>
 inline
 LeafBuffer<T, Log2Dim>::LeafBuffer(const LeafBuffer& other)
     : mData(nullptr)
-#ifdef OPENVDB_USE_DELAYED_LOADING
-    , mOutOfCore(other.mOutOfCore.load())
-#endif
 {
-#ifdef OPENVDB_USE_DELAYED_LOADING
-    if (other.isOutOfCore()) {
-        mFileInfo = new FileInfo(*other.mFileInfo);
-    } else {
-#endif
-        if (other.mData != nullptr) {
-            this->allocate();
-            ValueType* target = mData;
-            const ValueType* source = other.mData;
-            Index n = SIZE;
-            while (n--) *target++ = *source++;
-        }
-#ifdef OPENVDB_USE_DELAYED_LOADING
+    if (other.mData != nullptr) {
+        this->allocate();
+        ValueType* target = mData;
+        const ValueType* source = other.mData;
+        Index n = SIZE;
+        while (n--) *target++ = *source++;
     }
-#endif
 }
 
 
@@ -239,7 +161,6 @@ inline void
 LeafBuffer<T, Log2Dim>::setValue(Index i, const ValueType& val)
 {
     OPENVDB_ASSERT(i < SIZE);
-    this->loadValues();
     if (mData) mData[i] = val;
 }
 
@@ -249,52 +170,22 @@ inline LeafBuffer<T, Log2Dim>&
 LeafBuffer<T, Log2Dim>::operator=(const LeafBuffer& other)
 {
     if (&other != this) {
-#ifdef OPENVDB_USE_DELAYED_LOADING
-        if (this->isOutOfCore()) {
-            this->detachFromFile();
-        } else {
-            if (other.isOutOfCore()) this->deallocate();
+        if (other.mData != nullptr) {
+            this->allocate();
+            ValueType* target = mData;
+            const ValueType* source = other.mData;
+            Index n = SIZE;
+            while (n--) *target++ = *source++;
         }
-        if (other.isOutOfCore()) {
-            mOutOfCore.store(other.mOutOfCore.load(std::memory_order_acquire),
-                             std::memory_order_release);
-            mFileInfo = new FileInfo(*other.mFileInfo);
-        } else {
-#endif
-            if (other.mData != nullptr) {
-                this->allocate();
-                ValueType* target = mData;
-                const ValueType* source = other.mData;
-                Index n = SIZE;
-                while (n--) *target++ = *source++;
-            }
-#ifdef OPENVDB_USE_DELAYED_LOADING
-        }
-#endif
     }
     return *this;
 }
 
-#ifdef OPENVDB_USE_DELAYED_LOADING
-template<typename T, Index Log2Dim>
-void LeafBuffer<T, Log2Dim>::enableOutOfCore(
-    SharedPtr<io::StreamMetadata>& meta, const std::streamoff& bufpos,
-    io::MappedFile::Ptr& mappedFile, const std::streamoff& maskpos)
-{
-    this->setOutOfCore(true);
-    mFileInfo = new FileInfo;
-    mFileInfo->meta = meta;
-    mFileInfo->bufpos = bufpos;
-    mFileInfo->mapping = mappedFile;
-    mFileInfo->maskpos = maskpos;
-}
-#endif
 
 template<typename T, Index Log2Dim>
 inline void
 LeafBuffer<T, Log2Dim>::fill(const ValueType& val)
 {
-    this->detachFromFile();
     if (mData != nullptr) {
         ValueType* target = mData;
         Index n = SIZE;
@@ -307,8 +198,6 @@ template<typename T, Index Log2Dim>
 inline bool
 LeafBuffer<T, Log2Dim>::operator==(const LeafBuffer& other) const
 {
-    this->loadValues();
-    other.loadValues();
     const ValueType *target = mData, *source = other.mData;
     if (!target && !source) return true;
     if (!target || !source) return false;
@@ -323,15 +212,6 @@ inline void
 LeafBuffer<T, Log2Dim>::swap(LeafBuffer& other)
 {
     std::swap(mData, other.mData);
-#ifdef OPENVDB_USE_DELAYED_LOADING
-    // Two atomics can't be swapped because it would require hardware support:
-    // https://en.wikipedia.org/wiki/Double_compare-and-swap
-    // Note that there's a window in which other.mOutOfCore could be written
-    // between our load from it and our store to it.
-    auto tmp = other.mOutOfCore.load(std::memory_order_acquire);
-    tmp = mOutOfCore.exchange(std::move(tmp));
-    other.mOutOfCore.store(std::move(tmp), std::memory_order_release);
-#endif
 }
 
 
@@ -340,24 +220,7 @@ inline Index
 LeafBuffer<T, Log2Dim>::memUsage() const
 {
     size_t n = sizeof(*this);
-#ifdef OPENVDB_USE_DELAYED_LOADING
-    if (this->isOutOfCore()) n += sizeof(FileInfo);
-    else {
-#endif
-        if (mData) n += SIZE * sizeof(ValueType);
-#ifdef OPENVDB_USE_DELAYED_LOADING
-    }
-#endif
-    return static_cast<Index>(n);
-}
-
-
-template<typename T, Index Log2Dim>
-inline Index
-LeafBuffer<T, Log2Dim>::memUsageIfLoaded() const
-{
-    size_t n = sizeof(*this);
-    n += SIZE * sizeof(ValueType);
+    if (mData) n += SIZE * sizeof(ValueType);
     return static_cast<Index>(n);
 }
 
@@ -366,13 +229,8 @@ template<typename T, Index Log2Dim>
 inline const typename LeafBuffer<T, Log2Dim>::ValueType*
 LeafBuffer<T, Log2Dim>::data() const
 {
-    this->loadValues();
     if (mData == nullptr) {
         LeafBuffer* self = const_cast<LeafBuffer*>(this);
-#ifdef OPENVDB_USE_DELAYED_LOADING
-        // This lock will be contended at most once.
-        tbb::spin_mutex::scoped_lock lock(self->mMutex);
-#endif
         if (mData == nullptr) self->mData = new ValueType[SIZE];
     }
     return mData;
@@ -382,12 +240,7 @@ template<typename T, Index Log2Dim>
 inline typename LeafBuffer<T, Log2Dim>::ValueType*
 LeafBuffer<T, Log2Dim>::data()
 {
-    this->loadValues();
     if (mData == nullptr) {
-#ifdef OPENVDB_USE_DELAYED_LOADING
-        // This lock will be contended at most once.
-        tbb::spin_mutex::scoped_lock lock(mMutex);
-#endif
         if (mData == nullptr) mData = new ValueType[SIZE];
     }
     return mData;
@@ -400,7 +253,6 @@ LeafBuffer<T, Log2Dim>::at(Index i) const
 {
     static const ValueType sZero = zeroVal<T>();
     OPENVDB_ASSERT(i < SIZE);
-    this->loadValues();
     // We can't use the ternary operator here, otherwise Visual C++ returns
     // a reference to a temporary.
     if (mData) return mData[i]; else return sZero;
@@ -411,11 +263,7 @@ template<typename T, Index Log2Dim>
 inline bool
 LeafBuffer<T, Log2Dim>::deallocate()
 {
-
     if (mData != nullptr) {
-#ifdef OPENVDB_USE_DELAYED_LOADING
-        if (this->isOutOfCore())    return false;
-#endif
         delete[] mData;
         mData = nullptr;
         return true;
@@ -424,62 +272,6 @@ LeafBuffer<T, Log2Dim>::deallocate()
 }
 
 
-template<typename T, Index Log2Dim>
-inline void
-LeafBuffer<T, Log2Dim>::doLoad() const
-{
-#ifdef OPENVDB_USE_DELAYED_LOADING
-    if (!this->isOutOfCore()) return;
-
-    LeafBuffer<T, Log2Dim>* self = const_cast<LeafBuffer<T, Log2Dim>*>(this);
-
-    // This lock will be contended at most once, after which this buffer
-    // will no longer be out-of-core.
-    tbb::spin_mutex::scoped_lock lock(self->mMutex);
-    if (!this->isOutOfCore()) return;
-
-    std::unique_ptr<FileInfo> info(self->mFileInfo);
-    OPENVDB_ASSERT(info.get() != nullptr);
-    OPENVDB_ASSERT(info->mapping.get() != nullptr);
-    OPENVDB_ASSERT(info->meta.get() != nullptr);
-
-    /// @todo For now, we have to clear the mData pointer in order for allocate() to take effect.
-    self->mData = nullptr;
-    self->allocate();
-
-    SharedPtr<std::streambuf> buf = info->mapping->createBuffer();
-    std::istream is(buf.get());
-
-    io::setStreamMetadataPtr(is, info->meta, /*transfer=*/true);
-
-    NodeMaskType mask;
-    is.seekg(info->maskpos);
-    mask.load(is);
-
-    is.seekg(info->bufpos);
-    io::readCompressedValues(is, self->mData, SIZE, mask, io::getHalfFloat(is));
-
-    self->setOutOfCore(false);
-#endif
-}
-
-
-template<typename T, Index Log2Dim>
-inline bool
-LeafBuffer<T, Log2Dim>::detachFromFile()
-{
-#ifdef OPENVDB_USE_DELAYED_LOADING
-    if (this->isOutOfCore()) {
-        delete mFileInfo;
-        mFileInfo = nullptr;
-        this->setOutOfCore(false);
-        return true;
-    }
-#endif
-    return false;
-}
-
-
 ////////////////////////////////////////
 
 
@@ -524,7 +316,8 @@ public:
     void swap(LeafBuffer& other) { if (&other != this) std::swap(mData, other.mData); }
 
     Index memUsage() const { return sizeof(*this); }
-    Index memUsageIfLoaded() const { return sizeof(*this); }
+    OPENVDB_DEPRECATED_MESSAGE("Use memUsage() instead. This method is deprecated and will be removed. Delayed loading is no longer supported.")
+    Index memUsageIfLoaded() const { return memUsage(); }
     static Index size() { return SIZE; }
 
     /// @brief Return a pointer to the C-style array of words encoding the bits.

openvdb/openvdb/tree/LeafNode.h

@@ -152,13 +152,14 @@ public:
     /// Return @c true if this node contains only active voxels.
     bool isDense() const { return mValueMask.isOn(); }
     /// Return @c true if memory for this node's buffer has been allocated.
-    bool isAllocated() const { return !mBuffer.isOutOfCore() && !mBuffer.empty(); }
+    bool isAllocated() const { return !mBuffer.empty(); }
     /// Allocate memory for this node's buffer if it has not already been allocated.
     bool allocate() { return mBuffer.allocate(); }
 
     /// Return the memory in bytes occupied by this node.
     Index64 memUsage() const;
-    Index64 memUsageIfLoaded() const;
+    OPENVDB_DEPRECATED_MESSAGE("Use memUsage() instead. This method is deprecated and will be removed. Delayed loading is no longer supported.")
+    Index64 memUsageIfLoaded() const { return memUsage(); }
 
     /// Expand the given bounding box so that it includes this leaf node's active voxels.
     /// If visitVoxels is false this LeafNode will be approximated as dense, i.e. with all
@@ -466,7 +467,6 @@ public:
     template<typename ModifyOp>
     void modifyValue(Index offset, const ModifyOp& op)
     {
-        mBuffer.loadValues();
         if (!mBuffer.empty()) {
             // in-place modify value
             ValueType& val = const_cast<ValueType&>(mBuffer[offset]);
@@ -487,7 +487,6 @@ public:
     template<typename ModifyOp>
     void modifyValueAndActiveState(const Coord& xyz, const ModifyOp& op)
     {
-        mBuffer.loadValues();
         if (!mBuffer.empty()) {
             const Index offset = this->coordToOffset(xyz);
             bool state = mValueMask.isOn(offset);
@@ -1262,8 +1261,6 @@ template<typename DenseT>
 inline void
 LeafNode<T, Log2Dim>::copyToDense(const CoordBBox& bbox, DenseT& dense) const
 {
-    mBuffer.loadValues();
-
     using DenseValueType = typename DenseT::ValueType;
 
     const size_t xStride = dense.xStride(), yStride = dense.yStride(), zStride = dense.zStride();
@@ -1375,10 +1372,6 @@ LeafNode<T,Log2Dim>::readBuffers(std::istream& is, const CoordBBox& clipBBox, bo
     SharedPtr<io::StreamMetadata> meta = io::getStreamMetadataPtr(is);
     const bool seekable = meta && meta->seekable();
 
-#ifdef OPENVDB_USE_DELAYED_LOADING
-    std::streamoff maskpos = is.tellg();
-#endif
-
     if (seekable) {
         // Seek over the value mask.
         mValueMask.seek(is);
@@ -1401,37 +1394,16 @@ LeafNode<T,Log2Dim>::readBuffers(std::istream& is, const CoordBBox& clipBBox, bo
         // This node lies completely outside the clipping region.
         skipCompressedValues(seekable, is, fromHalf);
         mValueMask.setOff();
-        mBuffer.setOutOfCore(false);
     } else {
-#ifdef OPENVDB_USE_DELAYED_LOADING
-        // If this node lies completely inside the clipping region and it is being read
-        // from a memory-mapped file, delay loading of its buffer until the buffer
-        // is actually accessed.  (If this node requires clipping, its buffer
-        // must be accessed and therefore must be loaded.)
-        io::MappedFile::Ptr mappedFile = io::getMappedFilePtr(is);
-        const bool delayLoad = ((mappedFile.get() != nullptr) && clipBBox.isInside(nodeBBox));
+        mBuffer.allocate();
+        io::readCompressedValues(is, mBuffer.mData, SIZE, mValueMask, fromHalf);
 
-        if (delayLoad) {
-            // Save the offset to the value mask (maskpos), because the in-memory copy
-            // might change before the value buffer gets read.
-            mBuffer.enableOutOfCore(meta, is.tellg(), mappedFile, maskpos);
-            // Skip over voxel values.
-            skipCompressedValues(seekable, is, fromHalf);
-        } else {
-#endif
-            mBuffer.allocate();
-            io::readCompressedValues(is, mBuffer.mData, SIZE, mValueMask, fromHalf);
-            mBuffer.setOutOfCore(false);
-
-            // Get this tree's background value.
-            T background = zeroVal<T>();
-            if (const void* bgPtr = io::getGridBackgroundValuePtr(is)) {
-                background = *static_cast<const T*>(bgPtr);
-            }
-            this->clip(clipBBox, background);
-#ifdef OPENVDB_USE_DELAYED_LOADING
+        // Get this tree's background value.
+        T background = zeroVal<T>();
+        if (const void* bgPtr = io::getGridBackgroundValuePtr(is)) {
+            background = *static_cast<const T*>(bgPtr);
         }
-#endif
+        this->clip(clipBBox, background);
     }
 
     if (numBuffers > 1) {
@@ -1447,9 +1419,6 @@ LeafNode<T,Log2Dim>::readBuffers(std::istream& is, const CoordBBox& clipBBox, bo
             }
         }
     }
-
-    // increment the leaf number
-    if (meta)   meta->setLeaf(meta->leaf() + 1);
 }
 
 
@@ -1460,8 +1429,6 @@ LeafNode<T, Log2Dim>::writeBuffers(std::ostream& os, bool toHalf) const
     // Write out the value mask.
     mValueMask.save(os);
 
-    mBuffer.loadValues();
-
     io::writeCompressedValues(os, mBuffer.mData, SIZE,
         mValueMask, /*childMask=*/NodeMaskType(), toHalf);
 }
@@ -1490,16 +1457,6 @@ LeafNode<T, Log2Dim>::memUsage() const
 }
 
 
-template<typename T, Index Log2Dim>
-inline Index64
-LeafNode<T, Log2Dim>::memUsageIfLoaded() const
-{
-    // Use sizeof(*this) to capture alignment-related padding
-    // (but note that sizeof(*this) includes sizeof(mBuffer)).
-    return sizeof(*this) + mBuffer.memUsageIfLoaded() - sizeof(mBuffer);
-}
-
-
 template<typename T, Index Log2Dim>
 inline void
 LeafNode<T, Log2Dim>::evalActiveBoundingBox(CoordBBox& bbox, bool visitVoxels) const

openvdb/openvdb/tree/LeafNodeBool.h

@@ -145,7 +145,8 @@ public:
 
     /// Return the memory in bytes occupied by this node.
     Index64 memUsage() const;
-    Index64 memUsageIfLoaded() const;
+    OPENVDB_DEPRECATED_MESSAGE("Use memUsage() instead. This method is deprecated and will be removed. Delayed loading is no longer supported.")
+    Index64 memUsageIfLoaded() const { return memUsage(); }
 
     /// Expand the given bounding box so that it includes this leaf node's active voxels.
     /// If visitVoxels is false this LeafNode will be approximated as dense, i.e. with all
@@ -898,15 +899,6 @@ LeafNode<bool, Log2Dim>::memUsage() const
 }
 
 
-template<Index Log2Dim>
-inline Index64
-LeafNode<bool, Log2Dim>::memUsageIfLoaded() const
-{
-    // Use sizeof(*this) to capture alignment-related padding
-    return sizeof(*this);
-}
-
-
 template<Index Log2Dim>
 inline void
 LeafNode<bool, Log2Dim>::evalActiveBoundingBox(CoordBBox& bbox, bool visitVoxels) const

openvdb/openvdb/tree/LeafNodeMask.h

@@ -144,7 +144,8 @@ public:
 
     /// Return the memory in bytes occupied by this node.
     Index64 memUsage() const;
-    Index64 memUsageIfLoaded() const;
+    OPENVDB_DEPRECATED_MESSAGE("Use memUsage() instead. This method is deprecated and will be removed. Delayed loading is no longer supported.")
+    Index64 memUsageIfLoaded() const { return memUsage(); }
 
     /// Expand the given bounding box so that it includes this leaf node's active voxels.
     /// If visitVoxels is false this LeafNode will be approximated as dense, i.e. with all
@@ -884,15 +885,6 @@ LeafNode<ValueMask, Log2Dim>::memUsage() const
 }
 
 
-template<Index Log2Dim>
-inline Index64
-LeafNode<ValueMask, Log2Dim>::memUsageIfLoaded() const
-{
-    // Use sizeof(*this) to capture alignment-related padding
-    return sizeof(*this);
-}
-
-
 template<Index Log2Dim>
 inline void
 LeafNode<ValueMask, Log2Dim>::evalActiveBoundingBox(CoordBBox& bbox, bool visitVoxels) const

openvdb/openvdb/tree/Tree.h

@@ -95,13 +95,7 @@ public:
 
     virtual void getIndexRange(CoordBBox& bbox) const = 0;
 
-    /// @brief Replace with background tiles any nodes whose voxel buffers
-    /// have not yet been allocated.
-    /// @details Typically, unallocated nodes are leaf nodes whose voxel buffers
-    /// are not yet resident in memory because delayed loading is in effect.
-    /// @sa readNonresidentBuffers, io::File::open
     virtual void clipUnallocatedNodes() = 0;
-    /// Return the total number of unallocated leaf nodes residing in this tree.
 #if OPENVDB_ABI_VERSION_NUMBER >= 12
     virtual Index64 unallocatedLeafCount() const = 0;
 #else
@@ -157,22 +151,21 @@ public:
     /// @brief Read the tree topology from a stream.
     ///
     /// This will read the tree structure and tile values, but not voxel data.
-    virtual void readTopology(std::istream&, bool saveFloatAsHalf = false);
+    virtual void readTopology(std::istream&, bool saveFloatAsHalf = false) = 0;
     /// @brief Write the tree topology to a stream.
     ///
     /// This will write the tree structure and tile values, but not voxel data.
-    virtual void writeTopology(std::ostream&, bool saveFloatAsHalf = false) const;
+    virtual void writeTopology(std::ostream&, bool saveFloatAsHalf = false) const = 0;
 
     /// Read all data buffers for this tree.
     virtual void readBuffers(std::istream&, bool saveFloatAsHalf = false) = 0;
     /// Read all of this tree's data buffers that intersect the given bounding box.
     virtual void readBuffers(std::istream&, const CoordBBox&, bool saveFloatAsHalf = false) = 0;
-    /// @brief Read all of this tree's data buffers that are not yet resident in memory
-    /// (because delayed loading is in effect).
-    /// @details If this tree was read from a memory-mapped file, this operation
-    /// disconnects the tree from the file.
-    /// @sa clipUnallocatedNodes, io::File::open, io::MappedFile
+
+#if OPENVDB_ABI_VERSION_NUMBER < 14
+    OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.")
     virtual void readNonresidentBuffers() const = 0;
+#endif
     /// Write out all the data buffers for this tree.
     virtual void writeBuffers(std::ostream&, bool saveFloatAsHalf = false) const = 0;
 
@@ -339,12 +332,12 @@ public:
     void readBuffers(std::istream&, bool saveFloatAsHalf = false) override;
     /// Read all of this tree's data buffers that intersect the given bounding box.
     void readBuffers(std::istream&, const CoordBBox&, bool saveFloatAsHalf = false) override;
-    /// @brief Read all of this tree's data buffers that are not yet resident in memory
-    /// (because delayed loading is in effect).
-    /// @details If this tree was read from a memory-mapped file, this operation
-    /// disconnects the tree from the file.
-    /// @sa clipUnallocatedNodes, io::File::open, io::MappedFile
-    void readNonresidentBuffers() const override;
+
+#if OPENVDB_ABI_VERSION_NUMBER < 14
+    OPENVDB_DEPRECATED_MESSAGE("This method is deprecated and will be removed. Delayed loading is no longer supported.")
+    void readNonresidentBuffers() const override { }
+#endif
+
     /// Write out all data buffers for this tree.
     void writeBuffers(std::ostream&, bool saveFloatAsHalf = false) const override;
 
@@ -494,9 +487,6 @@ public:
     void clip(const CoordBBox&);
     /// @brief Replace with background tiles any nodes whose voxel buffers
     /// have not yet been allocated.
-    /// @details Typically, unallocated nodes are leaf nodes whose voxel buffers
-    /// are not yet resident in memory because delayed loading is in effect.
-    /// @sa readNonresidentBuffers, io::File::open
     void clipUnallocatedNodes() override;
 
     /// Return the total number of unallocated leaf nodes residing in this tree.
@@ -1140,23 +1130,6 @@ struct Tree5 {
 ////////////////////////////////////////
 
 
-inline void
-TreeBase::readTopology(std::istream& is, bool /*saveFloatAsHalf*/)
-{
-    int32_t bufferCount;
-    is.read(reinterpret_cast<char*>(&bufferCount), sizeof(int32_t));
-    if (bufferCount != 1) OPENVDB_LOG_WARN("multi-buffer trees are no longer supported");
-}
-
-
-inline void
-TreeBase::writeTopology(std::ostream& os, bool /*saveFloatAsHalf*/) const
-{
-    int32_t bufferCount = 1;
-    os.write(reinterpret_cast<char*>(&bufferCount), sizeof(int32_t));
-}
-
-
 inline void
 TreeBase::print(std::ostream& os, int /*verboseLevel*/) const
 {
@@ -1283,7 +1256,9 @@ void
 Tree<RootNodeType>::readTopology(std::istream& is, bool saveFloatAsHalf)
 {
     this->clearAllAccessors();
-    TreeBase::readTopology(is, saveFloatAsHalf);
+    int32_t bufferCount;
+    is.read(reinterpret_cast<char*>(&bufferCount), sizeof(int32_t));
+    if (bufferCount != 1) OPENVDB_LOG_WARN("multi-buffer trees are no longer supported");
     mRoot.readTopology(is, saveFloatAsHalf);
 }
 
@@ -1292,7 +1267,8 @@ template<typename RootNodeType>
 void
 Tree<RootNodeType>::writeTopology(std::ostream& os, bool saveFloatAsHalf) const
 {
-    TreeBase::writeTopology(os, saveFloatAsHalf);
+    int32_t bufferCount = 1;
+    os.write(reinterpret_cast<char*>(&bufferCount), sizeof(int32_t));
     mRoot.writeTopology(os, saveFloatAsHalf);
 }
 
@@ -1315,17 +1291,6 @@ Tree<RootNodeType>::readBuffers(std::istream &is, const CoordBBox& bbox, bool sa
 }
 
 
-template<typename RootNodeType>
-inline void
-Tree<RootNodeType>::readNonresidentBuffers() const
-{
-    for (LeafCIter it = this->cbeginLeaf(); it; ++it) {
-        // Retrieving the value of a leaf voxel forces loading of the leaf node's voxel buffer.
-        it->getValue(Index(0));
-    }
-}
-
-
 template<typename RootNodeType>
 inline void
 Tree<RootNodeType>::writeBuffers(std::ostream &os, bool saveFloatAsHalf) const

openvdb/openvdb/unittest/CMakeLists.txt

@@ -82,12 +82,12 @@ else()
     TestAttributeSet.cc
     TestBBox.cc
     TestClip.cc
+    TestCodec.cc
     TestConjGradient.cc
     TestCoord.cc
     TestCount.cc
     TestCpt.cc
     TestCurl.cc
-    TestDelayedLoadMetadata.cc
     TestDense.cc
     TestDenseSparseTools.cc
     TestDiagnostics.cc
@@ -146,6 +146,7 @@ else()
     TestParticlesToLevelSet.cc
     TestPointAdvect.cc
     TestPointAttribute.cc
+    TestPointCodec.cc
     TestPointConversion.cc
     TestPointCount.cc
     TestPointDataLeaf.cc

openvdb/openvdb/unittest/TestAttributeArray.cc

@@ -12,31 +12,6 @@
 
 #include <gtest/gtest.h>
 
-#ifdef OPENVDB_USE_DELAYED_LOADING
-#ifdef __clang__
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wunused-macros"
-#endif
-// Boost.Interprocess uses a header-only portion of Boost.DateTime
-#define BOOST_DATE_TIME_NO_LIB
-#ifdef __clang__
-#pragma GCC diagnostic pop
-#endif
-#include <boost/interprocess/file_mapping.hpp>
-#include <boost/interprocess/mapped_region.hpp>
-
-#ifdef _WIN32
-#include <boost/interprocess/detail/os_file_functions.hpp> // open_existing_file(), close_file()
-// boost::interprocess::detail was renamed to boost::interprocess::ipcdetail in Boost 1.48.
-// Ensure that both namespaces exist.
-namespace boost { namespace interprocess { namespace detail {} namespace ipcdetail {} } }
-#include <windows.h>
-#else
-#include <sys/types.h> // for struct stat
-#include <sys/stat.h> // for stat()
-#endif
-#endif // OPENVDB_USE_DELAYED_LOADING
-
 #include <tbb/tick_count.h>
 #include <atomic>
 
@@ -113,7 +88,6 @@ public:
 
     void testRegistry();
     void testAccessorEval();
-    void testDelayedLoad();
 }; // class TestAttributeArray
 
 
@@ -1400,801 +1374,6 @@ TEST_F(TestAttributeArray, testStrided)
     }
 }
 
-#ifdef OPENVDB_USE_DELAYED_LOADING
-void
-TestAttributeArray::testDelayedLoad()
-{
-    using AttributeArrayI = TypedAttributeArray<int>;
-    using AttributeArrayF = TypedAttributeArray<float>;
-
-    AttributeArrayI::registerType();
-    AttributeArrayF::registerType();
-
-    SharedPtr<io::MappedFile> mappedFile;
-
-    io::StreamMetadata::Ptr streamMetadata(new io::StreamMetadata);
-
-    std::string tempDir;
-    if (const char* dir = std::getenv("TMPDIR")) tempDir = dir;
-#ifdef _WIN32
-    if (tempDir.empty()) {
-        char tempDirBuffer[MAX_PATH+1];
-        int tempDirLen = GetTempPath(MAX_PATH+1, tempDirBuffer);
-        EXPECT_TRUE(tempDirLen > 0 && tempDirLen <= MAX_PATH);
-        tempDir = tempDirBuffer;
-    }
-#else
-    if (tempDir.empty()) tempDir = P_tmpdir;
-#endif
-
-    { // IO
-        const Index count = 50;
-        AttributeArrayI attrA(count);
-
-        for (unsigned i = 0; i < unsigned(count); ++i) {
-            attrA.set(i, int(i));
-        }
-
-        AttributeArrayF attrA2(count);
-
-        std::string filename;
-
-        // write out attribute array to a temp file
-        {
-            filename = tempDir + "/openvdb_delayed1";
-            std::ofstream fileout(filename.c_str(), std::ios_base::binary);
-            io::setStreamMetadataPtr(fileout, streamMetadata);
-            io::setDataCompression(fileout, io::COMPRESS_BLOSC);
-
-            attrA.writeMetadata(fileout, false, /*paged=*/true);
-            compression::PagedOutputStream outputStreamSize(fileout);
-            outputStreamSize.setSizeOnly(true);
-            attrA.writePagedBuffers(outputStreamSize, false);
-            outputStreamSize.flush();
-            compression::PagedOutputStream outputStream(fileout);
-            outputStream.setSizeOnly(false);
-            attrA.writePagedBuffers(outputStream, false);
-            outputStream.flush();
-
-            attrA2.writeMetadata(fileout, false, /*paged=*/true);
-            compression::PagedOutputStream outputStreamSize2(fileout);
-            outputStreamSize2.setSizeOnly(true);
-            attrA2.writePagedBuffers(outputStreamSize2, false);
-            outputStreamSize2.flush();
-            compression::PagedOutputStream outputStream2(fileout);
-            outputStream2.setSizeOnly(false);
-            attrA2.writePagedBuffers(outputStream2, false);
-            outputStream2.flush();
-
-            fileout.close();
-        }
-
-        mappedFile = TestMappedFile::create(filename);
-
-        // read in using delayed load and check manual loading of data
-        {
-            AttributeArrayI attrB;
-            AttributeArrayF attrB2;
-
-            std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary);
-            io::setStreamMetadataPtr(filein, streamMetadata);
-            io::setMappedFilePtr(filein, mappedFile);
-
-            attrB.readMetadata(filein);
-            compression::PagedInputStream inputStream(filein);
-            inputStream.setSizeOnly(true);
-            attrB.readPagedBuffers(inputStream);
-            inputStream.setSizeOnly(false);
-            attrB.readPagedBuffers(inputStream);
-
-            EXPECT_TRUE(matchingNamePairs(attrA.type(), attrB.type()));
-            EXPECT_EQ(attrA.size(), attrB.size());
-            EXPECT_EQ(attrA.isUniform(), attrB.isUniform());
-            EXPECT_EQ(attrA.isTransient(), attrB.isTransient());
-            EXPECT_EQ(attrA.isHidden(), attrB.isHidden());
-
-            AttributeArrayI attrBcopy(attrB);
-            AttributeArrayI attrBequal = attrB;
-
-            EXPECT_TRUE(attrB.isOutOfCore());
-            EXPECT_TRUE(attrBcopy.isOutOfCore());
-            EXPECT_TRUE(attrBequal.isOutOfCore());
-
-            EXPECT_TRUE(!static_cast<AttributeArray&>(attrB).isDataLoaded());
-            EXPECT_TRUE(!static_cast<AttributeArray&>(attrBcopy).isDataLoaded());
-            EXPECT_TRUE(!static_cast<AttributeArray&>(attrBequal).isDataLoaded());
-
-            attrB.loadData();
-            attrBcopy.loadData();
-            attrBequal.loadData();
-
-            EXPECT_TRUE(!attrB.isOutOfCore());
-            EXPECT_TRUE(!attrBcopy.isOutOfCore());
-            EXPECT_TRUE(!attrBequal.isOutOfCore());
-
-            EXPECT_TRUE(static_cast<AttributeArray&>(attrB).isDataLoaded());
-            EXPECT_TRUE(static_cast<AttributeArray&>(attrBcopy).isDataLoaded());
-            EXPECT_TRUE(static_cast<AttributeArray&>(attrBequal).isDataLoaded());
-
-            EXPECT_EQ(attrA.memUsage(), attrB.memUsage());
-            EXPECT_EQ(attrA.memUsage(), attrBcopy.memUsage());
-            EXPECT_EQ(attrA.memUsage(), attrBequal.memUsage());
-
-            for (unsigned i = 0; i < unsigned(count); ++i) {
-                EXPECT_EQ(attrA.get(i), attrB.get(i));
-                EXPECT_EQ(attrA.get(i), attrBcopy.get(i));
-                EXPECT_EQ(attrA.get(i), attrBequal.get(i));
-            }
-
-            attrB2.readMetadata(filein);
-            compression::PagedInputStream inputStream2(filein);
-            inputStream2.setSizeOnly(true);
-            attrB2.readPagedBuffers(inputStream2);
-            inputStream2.setSizeOnly(false);
-            attrB2.readPagedBuffers(inputStream2);
-
-            EXPECT_TRUE(matchingNamePairs(attrA2.type(), attrB2.type()));
-            EXPECT_EQ(attrA2.size(), attrB2.size());
-            EXPECT_EQ(attrA2.isUniform(), attrB2.isUniform());
-            EXPECT_EQ(attrA2.isTransient(), attrB2.isTransient());
-            EXPECT_EQ(attrA2.isHidden(), attrB2.isHidden());
-
-            AttributeArrayF attrB2copy(attrB2);
-            AttributeArrayF attrB2equal = attrB2;
-
-            EXPECT_TRUE(attrB2.isOutOfCore());
-            EXPECT_TRUE(attrB2copy.isOutOfCore());
-            EXPECT_TRUE(attrB2equal.isOutOfCore());
-            attrB2.loadData();
-            attrB2copy.loadData();
-            attrB2equal.loadData();
-
-            EXPECT_TRUE(!attrB2.isOutOfCore());
-            EXPECT_TRUE(!attrB2copy.isOutOfCore());
-            EXPECT_TRUE(!attrB2equal.isOutOfCore());
-
-            EXPECT_EQ(attrA2.memUsage(), attrB2.memUsage());
-            EXPECT_EQ(attrA2.memUsage(), attrB2copy.memUsage());
-            EXPECT_EQ(attrA2.memUsage(), attrB2equal.memUsage());
-
-            EXPECT_EQ(attrA2.get(0), attrB2.get(0));
-            EXPECT_EQ(attrA2.get(0), attrB2copy.get(0));
-            EXPECT_EQ(attrA2.get(0), attrB2equal.get(0));
-        }
-
-        // read in using delayed load and check fill()
-        {
-            AttributeArrayI attrB;
-
-            std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary);
-            io::setStreamMetadataPtr(filein, streamMetadata);
-            io::setMappedFilePtr(filein, mappedFile);
-
-            attrB.readMetadata(filein);
-            compression::PagedInputStream inputStream(filein);
-            inputStream.setSizeOnly(true);
-            attrB.readPagedBuffers(inputStream);
-            inputStream.setSizeOnly(false);
-            attrB.readPagedBuffers(inputStream);
-
-            EXPECT_TRUE(attrB.isOutOfCore());
-
-            EXPECT_TRUE(!attrB.isUniform());
-
-            attrB.fill(5);
-
-            EXPECT_TRUE(!attrB.isOutOfCore());
-
-            for (unsigned i = 0; i < unsigned(count); ++i) {
-                EXPECT_EQ(5, attrB.get(i));
-            }
-        }
-
-        // read in using delayed load and check streaming (write handle)
-        {
-            AttributeArrayI attrB;
-
-            std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary);
-            io::setStreamMetadataPtr(filein, streamMetadata);
-            io::setMappedFilePtr(filein, mappedFile);
-
-            attrB.readMetadata(filein);
-            compression::PagedInputStream inputStream(filein);
-            inputStream.setSizeOnly(true);
-            attrB.readPagedBuffers(inputStream);
-            inputStream.setSizeOnly(false);
-            attrB.readPagedBuffers(inputStream);
-
-            EXPECT_TRUE(attrB.isOutOfCore());
-
-            EXPECT_TRUE(!attrB.isUniform());
-
-            attrB.setStreaming(true);
-
-            {
-                AttributeWriteHandle<int> handle(attrB);
-                EXPECT_TRUE(!attrB.isOutOfCore());
-                EXPECT_TRUE(!attrB.isUniform());
-            }
-
-            EXPECT_TRUE(!attrB.isUniform());
-        }
-
-        // read in using delayed load and check streaming (read handle)
-        {
-            AttributeArrayI attrB;
-
-            std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary);
-            io::setStreamMetadataPtr(filein, streamMetadata);
-            io::setMappedFilePtr(filein, mappedFile);
-
-            attrB.readMetadata(filein);
-            compression::PagedInputStream inputStream(filein);
-            inputStream.setSizeOnly(true);
-            attrB.readPagedBuffers(inputStream);
-            inputStream.setSizeOnly(false);
-            attrB.readPagedBuffers(inputStream);
-
-            EXPECT_TRUE(attrB.isOutOfCore());
-
-            EXPECT_TRUE(!attrB.isUniform());
-
-            attrB.setStreaming(true);
-
-            {
-                AttributeHandle<int> handle(attrB);
-                EXPECT_TRUE(!attrB.isOutOfCore());
-                EXPECT_TRUE(!attrB.isUniform());
-            }
-
-            EXPECT_TRUE(attrB.isUniform());
-        }
-
-        // read in using delayed load and check implicit load through get()
-        {
-            AttributeArrayI attrB;
-
-            std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary);
-            io::setStreamMetadataPtr(filein, streamMetadata);
-            io::setMappedFilePtr(filein, mappedFile);
-
-            attrB.readMetadata(filein);
-            compression::PagedInputStream inputStream(filein);
-            inputStream.setSizeOnly(true);
-            attrB.readPagedBuffers(inputStream);
-            inputStream.setSizeOnly(false);
-            attrB.readPagedBuffers(inputStream);
-
-            EXPECT_TRUE(attrB.isOutOfCore());
-
-            attrB.get(0);
-
-            EXPECT_TRUE(!attrB.isOutOfCore());
-
-            for (unsigned i = 0; i < unsigned(count); ++i) {
-                EXPECT_EQ(attrA.get(i), attrB.get(i));
-            }
-        }
-
-        // read in using delayed load and check implicit load through compress()
-        {
-            AttributeArrayI attrB;
-
-            std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary);
-            io::setStreamMetadataPtr(filein, streamMetadata);
-            io::setMappedFilePtr(filein, mappedFile);
-
-            attrB.readMetadata(filein);
-            compression::PagedInputStream inputStream(filein);
-            inputStream.setSizeOnly(true);
-            attrB.readPagedBuffers(inputStream);
-            inputStream.setSizeOnly(false);
-            attrB.readPagedBuffers(inputStream);
-
-            EXPECT_TRUE(attrB.isOutOfCore());
-        }
-
-        // read in using delayed load and check copy and assignment constructors
-        {
-            AttributeArrayI attrB;
-
-            std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary);
-            io::setStreamMetadataPtr(filein, streamMetadata);
-            io::setMappedFilePtr(filein, mappedFile);
-
-            attrB.readMetadata(filein);
-            compression::PagedInputStream inputStream(filein);
-            inputStream.setSizeOnly(true);
-            attrB.readPagedBuffers(inputStream);
-            inputStream.setSizeOnly(false);
-            attrB.readPagedBuffers(inputStream);
-
-            EXPECT_TRUE(attrB.isOutOfCore());
-
-            AttributeArrayI attrC(attrB);
-            AttributeArrayI attrD = attrB;
-
-            EXPECT_TRUE(attrB.isOutOfCore());
-            EXPECT_TRUE(attrC.isOutOfCore());
-            EXPECT_TRUE(attrD.isOutOfCore());
-
-            attrB.loadData();
-            attrC.loadData();
-            attrD.loadData();
-
-            EXPECT_TRUE(!attrB.isOutOfCore());
-            EXPECT_TRUE(!attrC.isOutOfCore());
-            EXPECT_TRUE(!attrD.isOutOfCore());
-
-            for (unsigned i = 0; i < unsigned(count); ++i) {
-                EXPECT_EQ(attrA.get(i), attrB.get(i));
-                EXPECT_EQ(attrA.get(i), attrC.get(i));
-                EXPECT_EQ(attrA.get(i), attrD.get(i));
-            }
-        }
-
-        // read in using delayed load and check implicit load through AttributeHandle
-        {
-            AttributeArrayI attrB;
-
-            std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary);
-            io::setStreamMetadataPtr(filein, streamMetadata);
-            io::setMappedFilePtr(filein, mappedFile);
-
-            attrB.readMetadata(filein);
-            compression::PagedInputStream inputStream(filein);
-            inputStream.setSizeOnly(true);
-            attrB.readPagedBuffers(inputStream);
-            inputStream.setSizeOnly(false);
-            attrB.readPagedBuffers(inputStream);
-
-            EXPECT_TRUE(attrB.isOutOfCore());
-
-            AttributeHandle<int> handle(attrB);
-
-            EXPECT_TRUE(!attrB.isOutOfCore());
-        }
-
-        // read in using delayed load and check detaching of file (using collapse())
-        {
-            AttributeArrayI attrB;
-
-            std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary);
-            io::setStreamMetadataPtr(filein, streamMetadata);
-            io::setMappedFilePtr(filein, mappedFile);
-
-            attrB.readMetadata(filein);
-            compression::PagedInputStream inputStream(filein);
-            inputStream.setSizeOnly(true);
-            attrB.readPagedBuffers(inputStream);
-            inputStream.setSizeOnly(false);
-            attrB.readPagedBuffers(inputStream);
-
-            EXPECT_TRUE(attrB.isOutOfCore());
-
-            EXPECT_TRUE(!attrB.isUniform());
-
-            attrB.collapse();
-
-            EXPECT_TRUE(!attrB.isOutOfCore());
-            EXPECT_TRUE(attrB.isUniform());
-
-            EXPECT_EQ(0, attrB.get(0));
-        }
-
-        // read in and write out using delayed load to check writing out-of-core attributes
-        {
-            AttributeArrayI attrB;
-
-            std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary);
-            io::setStreamMetadataPtr(filein, streamMetadata);
-            io::setMappedFilePtr(filein, mappedFile);
-
-            attrB.readMetadata(filein);
-            compression::PagedInputStream inputStream(filein);
-            inputStream.setSizeOnly(true);
-            attrB.readPagedBuffers(inputStream);
-            inputStream.setSizeOnly(false);
-            attrB.readPagedBuffers(inputStream);
-
-            EXPECT_TRUE(attrB.isOutOfCore());
-
-            std::string filename2 = tempDir + "/openvdb_delayed5";
-            std::ofstream fileout2(filename2.c_str(), std::ios_base::binary);
-            io::setStreamMetadataPtr(fileout2, streamMetadata);
-            io::setDataCompression(fileout2, io::COMPRESS_BLOSC);
-
-            attrB.writeMetadata(fileout2, false, /*paged=*/true);
-            compression::PagedOutputStream outputStreamSize(fileout2);
-            outputStreamSize.setSizeOnly(true);
-            attrB.writePagedBuffers(outputStreamSize, false);
-            outputStreamSize.flush();
-            compression::PagedOutputStream outputStream(fileout2);
-            outputStream.setSizeOnly(false);
-            attrB.writePagedBuffers(outputStream, false);
-            outputStream.flush();
-
-            fileout2.close();
-
-            AttributeArrayI attrB2;
-
-            std::ifstream filein2(filename2.c_str(), std::ios_base::in | std::ios_base::binary);
-            io::setStreamMetadataPtr(filein2, streamMetadata);
-            io::setMappedFilePtr(filein2, mappedFile);
-
-            attrB2.readMetadata(filein2);
-            compression::PagedInputStream inputStream2(filein2);
-            inputStream2.setSizeOnly(true);
-            attrB2.readPagedBuffers(inputStream2);
-            inputStream2.setSizeOnly(false);
-            attrB2.readPagedBuffers(inputStream2);
-
-            EXPECT_TRUE(attrB2.isOutOfCore());
-
-            for (unsigned i = 0; i < unsigned(count); ++i) {
-                EXPECT_EQ(attrB.get(i), attrB2.get(i));
-            }
-
-            filein2.close();
-        }
-
-        // Clean up temp files.
-        std::remove(mappedFile->filename().c_str());
-        std::remove(filename.c_str());
-
-        AttributeArrayI attrUniform(count);
-
-        // write out uniform attribute array to a temp file
-        {
-            filename = tempDir + "/openvdb_delayed2";
-            std::ofstream fileout(filename.c_str(), std::ios_base::binary);
-            io::setStreamMetadataPtr(fileout, streamMetadata);
-            io::setDataCompression(fileout, io::COMPRESS_BLOSC);
-
-            attrUniform.writeMetadata(fileout, false, /*paged=*/true);
-
-            compression::PagedOutputStream outputStreamSize(fileout);
-            outputStreamSize.setSizeOnly(true);
-            attrUniform.writePagedBuffers(outputStreamSize, false);
-            outputStreamSize.flush();
-            compression::PagedOutputStream outputStream(fileout);
-            outputStream.setSizeOnly(false);
-            attrUniform.writePagedBuffers(outputStream, false);
-            outputStream.flush();
-
-            fileout.close();
-        }
-
-        mappedFile = TestMappedFile::create(filename);
-
-        // read in using delayed load and check fill()
-        {
-            AttributeArrayI attrB;
-
-            std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary);
-            io::setStreamMetadataPtr(filein, streamMetadata);
-            io::setMappedFilePtr(filein, mappedFile);
-
-            attrB.readMetadata(filein);
-            compression::PagedInputStream inputStream(filein);
-            inputStream.setSizeOnly(true);
-            attrB.readPagedBuffers(inputStream);
-            inputStream.setSizeOnly(false);
-            attrB.readPagedBuffers(inputStream);
-
-            EXPECT_TRUE(attrB.isUniform());
-
-            attrB.fill(5);
-
-            EXPECT_TRUE(attrB.isUniform());
-
-            for (unsigned i = 0; i < unsigned(count); ++i) {
-                EXPECT_EQ(5, attrB.get(i));
-            }
-        }
-
-        AttributeArrayI attrStrided(count, /*stride=*/3);
-
-        EXPECT_EQ(Index(3), attrStrided.stride());
-
-        // Clean up temp files.
-        std::remove(mappedFile->filename().c_str());
-        std::remove(filename.c_str());
-
-        // write out strided attribute array to a temp file
-        {
-            filename = tempDir + "/openvdb_delayed3";
-            std::ofstream fileout(filename.c_str(), std::ios_base::binary);
-            io::setStreamMetadataPtr(fileout, streamMetadata);
-            io::setDataCompression(fileout, io::COMPRESS_BLOSC);
-
-            attrStrided.writeMetadata(fileout, false, /*paged=*/true);
-
-            compression::PagedOutputStream outputStreamSize(fileout);
-            outputStreamSize.setSizeOnly(true);
-            attrStrided.writePagedBuffers(outputStreamSize, false);
-            outputStreamSize.flush();
-            compression::PagedOutputStream outputStream(fileout);
-            outputStream.setSizeOnly(false);
-            attrStrided.writePagedBuffers(outputStream, false);
-            outputStream.flush();
-
-            fileout.close();
-        }
-
-        mappedFile = TestMappedFile::create(filename);
-
-        // read in using delayed load and check fill()
-        {
-            AttributeArrayI attrB;
-
-            std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary);
-            io::setStreamMetadataPtr(filein, streamMetadata);
-            io::setMappedFilePtr(filein, mappedFile);
-
-            attrB.readMetadata(filein);
-            compression::PagedInputStream inputStream(filein);
-            inputStream.setSizeOnly(true);
-            attrB.readPagedBuffers(inputStream);
-            inputStream.setSizeOnly(false);
-            attrB.readPagedBuffers(inputStream);
-
-            EXPECT_EQ(Index(3), attrB.stride());
-        }
-
-        // Clean up temp files.
-        std::remove(mappedFile->filename().c_str());
-        std::remove(filename.c_str());
-
-        // write out compressed attribute array to a temp file
-        {
-            filename = tempDir + "/openvdb_delayed4";
-            std::ofstream fileout(filename.c_str(), std::ios_base::binary);
-            io::setStreamMetadataPtr(fileout, streamMetadata);
-            io::setDataCompression(fileout, io::COMPRESS_BLOSC);
-
-            attrA.writeMetadata(fileout, false, /*paged=*/true);
-
-            compression::PagedOutputStream outputStreamSize(fileout);
-            outputStreamSize.setSizeOnly(true);
-            attrA.writePagedBuffers(outputStreamSize, false);
-            outputStreamSize.flush();
-            compression::PagedOutputStream outputStream(fileout);
-            outputStream.setSizeOnly(false);
-            attrA.writePagedBuffers(outputStream, false);
-            outputStream.flush();
-
-            fileout.close();
-        }
-
-        mappedFile = TestMappedFile::create(filename);
-
-        // read in using delayed load and check manual loading of data
-        {
-            AttributeArrayI attrB;
-
-            std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary);
-            io::setStreamMetadataPtr(filein, streamMetadata);
-            io::setMappedFilePtr(filein, mappedFile);
-
-            attrB.readMetadata(filein);
-            compression::PagedInputStream inputStream(filein);
-            inputStream.setSizeOnly(true);
-            attrB.readPagedBuffers(inputStream);
-            inputStream.setSizeOnly(false);
-            attrB.readPagedBuffers(inputStream);
-
-            EXPECT_TRUE(attrB.isOutOfCore());
-            attrB.loadData();
-            EXPECT_TRUE(!attrB.isOutOfCore());
-
-            EXPECT_EQ(attrA.memUsage(), attrB.memUsage());
-
-            for (unsigned i = 0; i < unsigned(count); ++i) {
-                EXPECT_EQ(attrA.get(i), attrB.get(i));
-            }
-        }
-
-        // read in using delayed load and check partial read state
-        {
-            std::unique_ptr<AttributeArrayI> attrB(new AttributeArrayI);
-
-            EXPECT_TRUE(!(attrB->flags() & AttributeArray::PARTIALREAD));
-
-            std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary);
-            io::setStreamMetadataPtr(filein, streamMetadata);
-            io::setMappedFilePtr(filein, mappedFile);
-
-            attrB->readMetadata(filein);
-
-            // PARTIALREAD flag should now be set
-            EXPECT_TRUE(attrB->flags() & AttributeArray::PARTIALREAD);
-
-            // copy-construct and assign AttributeArray
-            AttributeArrayI attrC(*attrB);
-            EXPECT_TRUE(attrC.flags() & AttributeArray::PARTIALREAD);
-            AttributeArrayI attrD = *attrB;
-            EXPECT_TRUE(attrD.flags() & AttributeArray::PARTIALREAD);
-
-            // verify deleting attrB is safe
-            attrB.reset();
-
-            // verify data is not valid
-            EXPECT_TRUE(!attrC.validData());
-
-            { // attempting to write a partially-read AttributeArray throws
-                std::string filename = tempDir + "/openvdb_partial1";
-                ScopedFile f(filename);
-                std::ofstream fileout(filename.c_str(), std::ios_base::binary);
-                io::setStreamMetadataPtr(fileout, streamMetadata);
-                io::setDataCompression(fileout, io::COMPRESS_BLOSC);
-
-                EXPECT_THROW(attrC.writeMetadata(fileout, false, /*paged=*/true), IoError);
-            }
-
-            // continue loading with copy-constructed AttributeArray
-
-            compression::PagedInputStream inputStream(filein);
-            inputStream.setSizeOnly(true);
-            attrC.readPagedBuffers(inputStream);
-            inputStream.setSizeOnly(false);
-            attrC.readPagedBuffers(inputStream);
-
-            EXPECT_TRUE(attrC.isOutOfCore());
-            attrC.loadData();
-            EXPECT_TRUE(!attrC.isOutOfCore());
-
-            // verify data is now valid
-            EXPECT_TRUE(attrC.validData());
-
-            EXPECT_EQ(attrA.memUsage(), attrC.memUsage());
-
-            for (unsigned i = 0; i < unsigned(count); ++i) {
-                EXPECT_EQ(attrA.get(i), attrC.get(i));
-            }
-        }
-
-        // read in using delayed load and check implicit load through get()
-        {
-            AttributeArrayI attrB;
-
-            std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary);
-            io::setStreamMetadataPtr(filein, streamMetadata);
-            io::setMappedFilePtr(filein, mappedFile);
-
-            attrB.readMetadata(filein);
-            compression::PagedInputStream inputStream(filein);
-            inputStream.setSizeOnly(true);
-            attrB.readPagedBuffers(inputStream);
-            inputStream.setSizeOnly(false);
-            attrB.readPagedBuffers(inputStream);
-
-            EXPECT_TRUE(attrB.isOutOfCore());
-
-            attrB.get(0);
-
-            EXPECT_TRUE(!attrB.isOutOfCore());
-
-            for (unsigned i = 0; i < unsigned(count); ++i) {
-                EXPECT_EQ(attrA.get(i), attrB.get(i));
-            }
-        }
-
-#ifdef OPENVDB_USE_BLOSC
-        // read in using delayed load and check copy and assignment constructors
-        {
-            AttributeArrayI attrB;
-
-            std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary);
-            io::setStreamMetadataPtr(filein, streamMetadata);
-            io::setMappedFilePtr(filein, mappedFile);
-
-            attrB.readMetadata(filein);
-            compression::PagedInputStream inputStream(filein);
-            inputStream.setSizeOnly(true);
-            attrB.readPagedBuffers(inputStream);
-            inputStream.setSizeOnly(false);
-            attrB.readPagedBuffers(inputStream);
-
-            EXPECT_TRUE(attrB.isOutOfCore());
-
-            AttributeArrayI attrC(attrB);
-            AttributeArrayI attrD = attrB;
-
-            EXPECT_TRUE(attrB.isOutOfCore());
-            EXPECT_TRUE(attrC.isOutOfCore());
-            EXPECT_TRUE(attrD.isOutOfCore());
-
-            attrB.loadData();
-            attrC.loadData();
-            attrD.loadData();
-
-            EXPECT_TRUE(!attrB.isOutOfCore());
-            EXPECT_TRUE(!attrC.isOutOfCore());
-            EXPECT_TRUE(!attrD.isOutOfCore());
-
-            for (unsigned i = 0; i < unsigned(count); ++i) {
-                EXPECT_EQ(attrA.get(i), attrB.get(i));
-                EXPECT_EQ(attrA.get(i), attrC.get(i));
-                EXPECT_EQ(attrA.get(i), attrD.get(i));
-            }
-        }
-
-        // read in using delayed load and check implicit load through AttributeHandle
-        {
-            AttributeArrayI attrB;
-
-            std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary);
-            io::setStreamMetadataPtr(filein, streamMetadata);
-            io::setMappedFilePtr(filein, mappedFile);
-
-            attrB.readMetadata(filein);
-            compression::PagedInputStream inputStream(filein);
-            inputStream.setSizeOnly(true);
-            attrB.readPagedBuffers(inputStream);
-            inputStream.setSizeOnly(false);
-            attrB.readPagedBuffers(inputStream);
-
-            EXPECT_TRUE(attrB.isOutOfCore());
-
-            AttributeHandle<int> handle(attrB);
-
-            EXPECT_TRUE(!attrB.isOutOfCore());
-
-            for (unsigned i = 0; i < unsigned(count); ++i) {
-                EXPECT_EQ(attrA.get(i), handle.get(i));
-            }
-        }
-#endif
-
-        // Clean up temp files.
-        std::remove(mappedFile->filename().c_str());
-        std::remove(filename.c_str());
-
-        // write out invalid serialization flags as metadata to a temp file
-        {
-            filename = tempDir + "/openvdb_delayed5";
-            std::ofstream fileout(filename.c_str(), std::ios_base::binary);
-            io::setStreamMetadataPtr(fileout, streamMetadata);
-            io::setDataCompression(fileout, io::COMPRESS_BLOSC);
-
-            // write out unknown serialization flags to check forwards-compatibility
-
-            Index64 bytes(0);
-            uint8_t flags(0);
-            uint8_t serializationFlags(Int16(0x10));
-            Index size(0);
-
-            fileout.write(reinterpret_cast<const char*>(&bytes), sizeof(Index64));
-            fileout.write(reinterpret_cast<const char*>(&flags), sizeof(uint8_t));
-            fileout.write(reinterpret_cast<const char*>(&serializationFlags), sizeof(uint8_t));
-            fileout.write(reinterpret_cast<const char*>(&size), sizeof(Index));
-
-            fileout.close();
-        }
-
-        mappedFile = TestMappedFile::create(filename);
-
-        // read in using delayed load and check metadata fail due to serialization flags
-        {
-            AttributeArrayI attrB;
-
-            std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary);
-            io::setStreamMetadataPtr(filein, streamMetadata);
-            io::setMappedFilePtr(filein, mappedFile);
-
-            EXPECT_THROW(attrB.readMetadata(filein), openvdb::IoError);
-        }
-
-        // cleanup temp files
-
-        std::remove(mappedFile->filename().c_str());
-        std::remove(filename.c_str());
-    }
-}
-TEST_F(TestAttributeArray, testDelayedLoad) { testDelayedLoad(); }
-#endif
-
 
 TEST_F(TestAttributeArray, testDefaultValue)
 {

openvdb/openvdb/unittest/TestCodec.cc

@@ -0,0 +1,405 @@
+// Copyright Contributors to the OpenVDB Project
+// SPDX-License-Identifier: Apache-2.0
+
+#include <openvdb/io/Codec.h>
+#include <openvdb/io/File.h>
+#include <openvdb/openvdb.h>
+#include <openvdb/codecs/ScalarCodec.h>
+#include <openvdb/Exceptions.h>
+#include <openvdb/tools/Clip.h>
+#include <gtest/gtest.h>
+
+class TestCodec: public ::testing::Test
+{
+public:
+    void SetUp() override { openvdb::initialize(); }
+    void TearDown() override { openvdb::uninitialize(); }
+};
+
+struct MockCodec : public openvdb::io::Codec
+{
+    static std::string name() { return "mock"; }
+
+    openvdb::io::CodecData::Ptr createData() final { return nullptr; }
+};
+
+TEST_F(TestCodec, testCodecRegistry)
+{
+    using namespace openvdb::io;
+
+    // Start clean
+    CodecRegistry::clear();
+
+    // Test isRegistered on empty registry
+    EXPECT_FALSE(CodecRegistry::isRegistered("mock"));
+
+    // Test registerCodecByName
+    EXPECT_NO_THROW(
+        CodecRegistry::registerCodecByName("mock", std::make_unique<MockCodec>())
+    );
+
+    EXPECT_TRUE(CodecRegistry::isRegistered("mock"));
+    EXPECT_FALSE(CodecRegistry::isRegistered("nonexistent"));
+
+    // Test duplicate registration throws KeyError
+    EXPECT_THROW(
+        CodecRegistry::registerCodecByName("mock", std::make_unique<MockCodec>()),
+        openvdb::KeyError
+    );
+
+    // Test registerCodec template form also throws on duplicate
+    EXPECT_THROW(
+        CodecRegistry::registerCodec<MockCodec>(),
+        openvdb::KeyError
+    );
+
+    // Test get
+    EXPECT_NE(CodecRegistry::get("mock"), nullptr);
+    EXPECT_EQ(CodecRegistry::get("nonexistent"), nullptr);
+
+    // Test clear
+    CodecRegistry::clear();
+    EXPECT_FALSE(CodecRegistry::isRegistered("mock"));
+    EXPECT_NO_THROW(CodecRegistry::clear());  // Clear on empty registry
+
+    // Test registerCodec template form on fresh registry
+    EXPECT_NO_THROW(CodecRegistry::registerCodec<MockCodec>());
+    EXPECT_TRUE(CodecRegistry::isRegistered("mock"));
+
+    // Test io::initialize and io::uninitialize
+    CodecRegistry::clear();
+    EXPECT_FALSE(CodecRegistry::isRegistered(openvdb::BoolGrid::gridType()));
+
+    EXPECT_NO_THROW(internal::initialize());
+    EXPECT_TRUE(CodecRegistry::isRegistered(openvdb::BoolGrid::gridType()));
+
+    EXPECT_NO_THROW(internal::uninitialize());
+    EXPECT_FALSE(CodecRegistry::isRegistered(openvdb::BoolGrid::gridType()));
+}
+
+
+TEST_F(TestCodec, testReadDiagnostics)
+{
+    using namespace openvdb;
+    using namespace openvdb::io;
+
+    // ReadDiagnostics struct: disabled by default, addWarning is a no-op until enabled
+    {
+        ReadDiagnostics diags;
+        EXPECT_FALSE(diags.enabled());
+        diags.addWarning("grid_a", "something went wrong");
+        EXPECT_TRUE(diags.diagnostics().empty());
+
+        diags.enable();
+        diags.addWarning("grid_a", "something went wrong");
+        ASSERT_EQ(diags.diagnostics().size(), size_t(1));
+        EXPECT_EQ(diags.diagnostics()[0].severity, DiagnosticSeverity::Warning);
+
+        diags.clear();
+        EXPECT_TRUE(diags.diagnostics().empty());
+    }
+
+    CodecRegistry::clear();
+    openvdb::io::internal::initialize();
+
+    // Archive API and getGrids() with diagnostics
+
+    BoolGrid::Ptr srcGrid = BoolGrid::create(false);
+    srcGrid->setName("bool_grid");
+    srcGrid->fill(CoordBBox(Coord(-5), Coord(5)), true, true);
+
+    const std::string codecPath = "testReadDiagnostics.vdb";
+    {
+        io::File f(codecPath);
+        f.write(GridPtrVec{srcGrid});
+    }
+
+    // Disabled by default; enabling produces no warnings on a clean read
+    {
+        io::File f(codecPath);
+        f.open();
+        EXPECT_FALSE(f.readDiagnostics().enabled());
+        f.enableReadDiagnostics();
+        EXPECT_TRUE(f.readDiagnostics().enabled());
+        f.readGrid("bool_grid");
+        EXPECT_TRUE(f.readDiagnostics().diagnostics().empty());
+        f.close();
+    }
+
+    // clearReadDiagnostics() resets entries but keeps diagnostics enabled
+    {
+        io::File f(codecPath);
+        f.open();
+        f.enableReadDiagnostics();
+        GridPtrVecPtr grids = f.getGrids();
+        ASSERT_TRUE(grids && !grids->empty());
+        f.clearReadDiagnostics();
+        EXPECT_TRUE(f.readDiagnostics().enabled());
+        EXPECT_TRUE(f.readDiagnostics().diagnostics().empty());
+        f.close();
+    }
+
+    std::remove(codecPath.c_str());
+}
+
+
+template <typename GridT>
+void testIOImpl(
+    const std::string& gridName,
+    const typename GridT::ValueType& bgValue,
+    const typename GridT::ValueType& fillValue)
+{
+    using namespace openvdb;
+    using namespace openvdb::io;
+
+    typename GridT::Ptr srcGrid = GridT::create(bgValue);
+    srcGrid->setName(gridName);
+    srcGrid->fill(CoordBBox(Coord(-5), Coord(5)), fillValue, true);
+
+    std::stringstream ss("test");
+    if (CodecRegistry::isRegistered(GridT::gridType())) {
+        ss << "_codec";
+    } else {
+        ss << "_tree";
+    }
+    ss << "_" << GridT::gridType() << ".vdb";
+    const std::string path = ss.str();
+    {
+        io::File f(path);
+        f.write(GridPtrVec{srcGrid});
+    }
+
+    typename GridT::Ptr readGrid;
+    {
+        io::File f(path);
+        f.open();
+        readGrid = gridPtrCast<GridT>(f.readGrid(gridName));
+        f.close();
+    }
+    ASSERT_TRUE(readGrid);
+    EXPECT_TRUE(srcGrid->tree().hasSameTopology(readGrid->tree()));
+    {
+        auto readAcc = readGrid->getConstAccessor();
+        for (typename GridT::ValueOnCIter it = srcGrid->cbeginValueOn(); it; ++it) {
+            EXPECT_EQ(*it, readAcc.getValue(it.getCoord()));
+        }
+    }
+
+    // clip read
+    const BBoxd clipBBox(Vec3d(0.0), Vec3d(3.5));
+    auto srcClipped = tools::clip(*srcGrid, clipBBox);
+
+    typename GridT::Ptr readClipped;
+    {
+        io::File f(path);
+        f.open();
+        readClipped = gridPtrCast<GridT>(f.readGrid(gridName, clipBBox));
+        f.close();
+    }
+    ASSERT_TRUE(readClipped);
+    EXPECT_TRUE(srcClipped->tree().hasSameTopology(readClipped->tree()));
+    {
+        auto readAcc = readClipped->getConstAccessor();
+        for (typename GridT::ValueOnCIter it = srcClipped->cbeginValueOn(); it; ++it) {
+            EXPECT_EQ(*it, readAcc.getValue(it.getCoord()));
+        }
+    }
+
+    // topology-only read
+    ReadOptions topoOpts;
+    topoOpts.readMode = ReadMode::TopologyOnly;
+
+    typename GridT::Ptr readTopo;
+    {
+        io::File f(path);
+        f.open();
+        GridBase::Ptr base;
+        EXPECT_NO_THROW(base = f.readGrid(gridName, topoOpts));
+        readTopo = gridPtrCast<GridT>(base);
+        f.close();
+    }
+    ASSERT_TRUE(readTopo);
+    EXPECT_EQ(readTopo->activeVoxelCount(), Index64(0));
+    EXPECT_TRUE(readTopo->tree().leafCount() == 0);
+    EXPECT_EQ(readTopo->getName(), gridName);
+
+    // Cleanup
+    std::remove(path.c_str());
+}
+
+template <typename GridT>
+void testCodecIOImpl(
+    const std::string& gridName,
+    const typename GridT::ValueType& bgValue,
+    const typename GridT::ValueType& fillValue)
+{
+    // initialize to register all the codecs
+    openvdb::io::CodecRegistry::clear();
+    openvdb::io::internal::initialize();
+    // ensure the codec is registered
+    ASSERT_TRUE(openvdb::io::CodecRegistry::isRegistered(GridT::gridType()));
+    // test the io implementation (codec)
+    testIOImpl<GridT>(gridName, bgValue, fillValue);
+    // clear the codec registry (now read/write falls back to Tree I/O)
+    openvdb::io::CodecRegistry::clear();
+    // ensure the codec is not registered
+    ASSERT_FALSE(openvdb::io::CodecRegistry::isRegistered(GridT::gridType()));
+    // test the io implementation (tree I/O)
+    testIOImpl<GridT>(gridName, bgValue, fillValue);
+}
+
+TEST_F(TestCodec, testFloatCodecIO) { testCodecIOImpl<openvdb::FloatGrid>("float_grid", 0.0f, 1.0f); }
+TEST_F(TestCodec, testDoubleCodecIO) { testCodecIOImpl<openvdb::DoubleGrid>("double_grid", 0.0, 1.0); }
+TEST_F(TestCodec, testInt32CodecIO) { testCodecIOImpl<openvdb::Int32Grid>("int32_grid", 0, 1); }
+TEST_F(TestCodec, testInt64CodecIO) { testCodecIOImpl<openvdb::Int64Grid>("int64_grid", openvdb::Int64(0), openvdb::Int64(1)); }
+TEST_F(TestCodec, testHalfCodecIO) { testCodecIOImpl<openvdb::HalfGrid>("half_grid", openvdb::Half(0.0), openvdb::Half(1.5)); }
+TEST_F(TestCodec, testVec3ICodecIO) { testCodecIOImpl<openvdb::Vec3IGrid>("vec3i_grid", openvdb::Vec3i(0), openvdb::Vec3i(1, 2, 3)); }
+TEST_F(TestCodec, testVec3SCodecIO) { testCodecIOImpl<openvdb::Vec3SGrid>("vec3s_grid", openvdb::Vec3s(0.0f), openvdb::Vec3s(1.0f, 2.0f, 3.0f)); }
+TEST_F(TestCodec, testVec3DCodecIO) { testCodecIOImpl<openvdb::Vec3DGrid>("vec3d_grid", openvdb::Vec3d(0.0), openvdb::Vec3d(1.0, 2.0, 3.0)); }
+TEST_F(TestCodec, testBoolCodecIO) { testCodecIOImpl<openvdb::BoolGrid>("bool_grid", false, true); }
+TEST_F(TestCodec, testMaskCodecIO) { testCodecIOImpl<openvdb::MaskGrid>("mask_grid", false, true); }
+
+TEST_F(TestCodec, testFloatToHalfCodecConversion)
+{
+    using namespace openvdb;
+    using namespace openvdb::io;
+
+    openvdb::initialize();
+    CodecRegistry::clear();
+
+    // Verify the conversion codec name
+    const std::string expectedName = FloatGrid::gridType() + "_to_half";
+    EXPECT_EQ((codecs::ScalarCodec<HalfGrid, FloatGrid, CodecMode::ReadOnly>::name()),
+              expectedName);
+
+    // Verify the codec is registered after initialize()
+    io::internal::initialize();
+    EXPECT_TRUE(CodecRegistry::isRegistered(expectedName));
+
+    // Write a FloatGrid with a known fill value (1.5f is exactly representable in half)
+    const std::string floatPath = "test_float_to_half.vdb";
+    const std::string gridName = "float_grid";
+    FloatGrid::Ptr srcGrid = FloatGrid::create(0.0f);
+    srcGrid->setName(gridName);
+    srcGrid->fill(CoordBBox(Coord(-5), Coord(5)), 1.5f, true);
+
+    {
+        io::File f(floatPath);
+        f.write(GridPtrVec{srcGrid});
+    }
+
+    // Read back with ReadMode::Half — triggers the float-to-half conversion codec
+    ReadOptions halfOpts;
+    halfOpts.readMode = ReadMode::Half;
+
+    GridBase::Ptr base;
+    {
+        io::File f(floatPath);
+        f.open();
+        base = f.readGrid(gridName, halfOpts);
+        f.close();
+    }
+    ASSERT_TRUE(base);
+
+    // The returned grid must be a HalfGrid
+    EXPECT_TRUE(base->isType<HalfGrid>());
+    HalfGrid::Ptr halfGrid = gridPtrCast<HalfGrid>(base);
+    ASSERT_TRUE(halfGrid);
+
+    // Topology must match the source FloatGrid
+    EXPECT_TRUE(srcGrid->tree().hasSameTopology(halfGrid->tree()));
+
+    // All active voxel values must equal Half(1.5f)
+    for (HalfGrid::ValueOnCIter it = halfGrid->cbeginValueOn(); it; ++it) {
+        EXPECT_EQ(*it, Half(1.5f));
+    }
+
+    // Background value must equal Half(0.0f)
+    EXPECT_EQ(halfGrid->background(), Half(0.0f));
+
+    // Cleanup
+    std::remove(floatPath.c_str());
+}
+
+template <typename SrcGridT, typename DstGridT, openvdb::io::ReadMode mode>
+void testConvertCodecImpl()
+{
+    using namespace openvdb;
+    using namespace openvdb::io;
+
+    openvdb::io::CodecRegistry::clear();
+    openvdb::io::internal::initialize();
+
+    // Verify the conversion codec name
+    const std::string expectedName =
+        SrcGridT::gridType() + "_to_" + typeNameAsString<typename DstGridT::BuildType>();
+    EXPECT_EQ((codecs::ScalarCodec<DstGridT, SrcGridT, CodecMode::ReadOnly>::name()),
+              expectedName);
+
+    // Verify the codec is registered after initialize()
+    EXPECT_TRUE(CodecRegistry::isRegistered(expectedName));
+
+    // Write a SrcGridT with a non-zero fill value
+    const std::string testPath = "test_" + expectedName + ".vdb";
+    const std::string gridName = SrcGridT::gridType();
+    typename SrcGridT::Ptr srcGrid =
+        SrcGridT::create(typename SrcGridT::ValueType(0));
+    srcGrid->setName(gridName);
+    srcGrid->fill(CoordBBox(Coord(-5), Coord(5)),
+                  typename SrcGridT::ValueType(1), true);
+
+    {
+        io::File f(testPath);
+        f.write(GridPtrVec{srcGrid});
+    }
+
+    // Read back with the specified ReadMode — triggers the conversion codec
+    ReadOptions opts;
+    opts.readMode = mode;
+
+    GridBase::Ptr base;
+    {
+        io::File f(testPath);
+        f.open();
+        base = f.readGrid(gridName, opts);
+        f.close();
+    }
+    ASSERT_TRUE(base);
+
+    // The returned grid must be a DstGridT
+    EXPECT_TRUE(base->isType<DstGridT>());
+    typename DstGridT::Ptr dstGrid = gridPtrCast<DstGridT>(base);
+    ASSERT_TRUE(dstGrid);
+
+    // Topology must match the source grid
+    EXPECT_TRUE(srcGrid->tree().hasSameTopology(dstGrid->tree()));
+
+    // All active voxel values must be true
+    for (typename DstGridT::ValueOnCIter it = dstGrid->cbeginValueOn(); it; ++it) {
+        EXPECT_EQ(*it, typename DstGridT::ValueType(true));
+    }
+
+    // Background value must be false
+    EXPECT_EQ(dstGrid->background(), typename DstGridT::ValueType(false));
+
+    // Cleanup
+    std::remove(testPath.c_str());
+}
+
+TEST_F(TestCodec, testNumericToBoolCodecConversion)
+{
+    testConvertCodecImpl<openvdb::FloatGrid,  openvdb::BoolGrid, openvdb::io::ReadMode::Bool>();
+    testConvertCodecImpl<openvdb::DoubleGrid, openvdb::BoolGrid, openvdb::io::ReadMode::Bool>();
+    testConvertCodecImpl<openvdb::Int32Grid,  openvdb::BoolGrid, openvdb::io::ReadMode::Bool>();
+    testConvertCodecImpl<openvdb::Int64Grid,  openvdb::BoolGrid, openvdb::io::ReadMode::Bool>();
+    testConvertCodecImpl<openvdb::HalfGrid,   openvdb::BoolGrid, openvdb::io::ReadMode::Bool>();
+}
+
+TEST_F(TestCodec, testNumericToMaskCodecConversion)
+{
+    testConvertCodecImpl<openvdb::FloatGrid,  openvdb::MaskGrid, openvdb::io::ReadMode::Mask>();
+    testConvertCodecImpl<openvdb::DoubleGrid, openvdb::MaskGrid, openvdb::io::ReadMode::Mask>();
+    testConvertCodecImpl<openvdb::Int32Grid,  openvdb::MaskGrid, openvdb::io::ReadMode::Mask>();
+    testConvertCodecImpl<openvdb::Int64Grid,  openvdb::MaskGrid, openvdb::io::ReadMode::Mask>();
+    testConvertCodecImpl<openvdb::HalfGrid,   openvdb::MaskGrid, openvdb::io::ReadMode::Mask>();
+}

openvdb/openvdb/unittest/TestCount.cc

@@ -8,7 +8,6 @@
 #include <openvdb/tools/LevelSetSphere.h> // tools::createLevelSetSphere
 #include <openvdb/tools/LevelSetUtil.h> // tools::sdfToFogVolume
 #include <openvdb/tree/ValueAccessor.h>
-#include <openvdb/io/TempFile.h>
 
 
 class TestCount: public ::testing::Test
@@ -224,8 +223,7 @@ TEST_F(TestCount, testMemUsage)
             internalNodeMemUsage += sizeof(Coord);
 
             for (auto leafIter = internal2Iter->cbeginChildOn(); leafIter; ++leafIter) {
-                EXPECT_EQ(leafIter->memUsage(), leafIter->memUsageIfLoaded());
-                expectedMaxMem += leafIter->memUsageIfLoaded();
+                expectedMaxMem += leafIter->memUsage();
                 ++leafCount;
             }
         }
@@ -233,54 +231,9 @@ TEST_F(TestCount, testMemUsage)
 
     expectedMaxMem += internalNodeMemUsage;
 
-    Index64 inCoreMemUsage = tools::memUsage(grid->tree());
-    Index64 memUsageIfLoaded = tools::memUsageIfLoaded(grid->tree());
+    Index64 memUsage = tools::memUsage(grid->tree());
 
-    EXPECT_EQ(expectedMaxMem, inCoreMemUsage);
-    EXPECT_EQ(expectedMaxMem, memUsageIfLoaded);
-
-#ifdef OPENVDB_USE_DELAYED_LOADING
-    // Write out the grid and read it in with delay-loading. Check the
-    // expected memory usage values.]
-
-    openvdb::initialize();
-
-    std::string filename;
-
-    // write out grid to a temp file
-    {
-        io::TempFile file;
-        filename = file.filename();
-        io::File fileOut(filename);
-        fileOut.write({grid});
-    }
-
-    io::File fileIn(filename);
-    fileIn.open(true); // delay-load
-    auto grids = fileIn.getGrids();
-    fileIn.close();
-
-    grid = GridBase::grid<FloatGrid>((*grids)[0]);
-    EXPECT_TRUE(grid);
-
-    inCoreMemUsage = tools::memUsage(grid->tree());
-    memUsageIfLoaded = tools::memUsageIfLoaded(grid->tree());
-
-    EXPECT_EQ(expectedMaxMem, memUsageIfLoaded);
-    EXPECT_TRUE(inCoreMemUsage < expectedMaxMem);
-
-    // in core memory should be the max memory without the leaf buffers but
-    // with the FileInfo
-
-    const Index64 leafBuffers = sizeof(FloatGrid::ValueType) * FloatTree::LeafNodeType::SIZE;
-    const Index64 fileInfo = sizeof(FloatTree::LeafNodeType::Buffer::FileInfo);
-    const Index64 expectedInCoreMemUsage = expectedMaxMem + (leafCount * (-leafBuffers + fileInfo));
-    EXPECT_EQ(expectedInCoreMemUsage, inCoreMemUsage);
-
-    std::remove(filename.c_str());
-
-    openvdb::uninitialize();
-#endif
+    EXPECT_EQ(expectedMaxMem, memUsage);
 }
 
 

openvdb/openvdb/unittest/TestDelayedLoadMetadata.cc

@@ -1,206 +0,0 @@
-// Copyright Contributors to the OpenVDB Project
-// SPDX-License-Identifier: Apache-2.0
-
-#include <openvdb/Exceptions.h>
-#include <openvdb/openvdb.h>
-#include <openvdb/io/DelayedLoadMetadata.h>
-
-#include <gtest/gtest.h>
-
-class TestDelayedLoadMetadata : public ::testing::Test
-{
-};
-
-
-TEST_F(TestDelayedLoadMetadata, test)
-{
-    using namespace openvdb::io;
-
-    // registration
-
-    EXPECT_TRUE(!DelayedLoadMetadata::isRegisteredType());
-
-    DelayedLoadMetadata::registerType();
-
-    EXPECT_TRUE(DelayedLoadMetadata::isRegisteredType());
-
-    DelayedLoadMetadata::unregisterType();
-
-    EXPECT_TRUE(!DelayedLoadMetadata::isRegisteredType());
-
-    openvdb::initialize();
-
-    EXPECT_TRUE(DelayedLoadMetadata::isRegisteredType());
-
-    // construction
-
-    DelayedLoadMetadata metadata;
-
-    EXPECT_TRUE(metadata.empty());
-
-    metadata.resizeMask(size_t(2));
-
-    EXPECT_TRUE(!metadata.empty());
-
-    metadata.setMask(0, DelayedLoadMetadata::MaskType(5));
-    metadata.setMask(1, DelayedLoadMetadata::MaskType(-3));
-
-    EXPECT_EQ(metadata.getMask(0), DelayedLoadMetadata::MaskType(5));
-    EXPECT_EQ(metadata.getMask(1), DelayedLoadMetadata::MaskType(-3));
-
-    metadata.resizeCompressedSize(size_t(3));
-
-    metadata.setCompressedSize(0, DelayedLoadMetadata::CompressedSizeType(6));
-    metadata.setCompressedSize(1, DelayedLoadMetadata::CompressedSizeType(101));
-    metadata.setCompressedSize(2, DelayedLoadMetadata::CompressedSizeType(-13522));
-
-    EXPECT_EQ(metadata.getCompressedSize(0), DelayedLoadMetadata::CompressedSizeType(6));
-    EXPECT_EQ(metadata.getCompressedSize(1), DelayedLoadMetadata::CompressedSizeType(101));
-    EXPECT_EQ(metadata.getCompressedSize(2), DelayedLoadMetadata::CompressedSizeType(-13522));
-
-    // copy construction
-
-    DelayedLoadMetadata metadataCopy1(metadata);
-
-    EXPECT_TRUE(!metadataCopy1.empty());
-
-    EXPECT_EQ(metadataCopy1.getMask(0), DelayedLoadMetadata::MaskType(5));
-    EXPECT_EQ(metadataCopy1.getCompressedSize(2), DelayedLoadMetadata::CompressedSizeType(-13522));
-
-    openvdb::Metadata::Ptr baseMetadataCopy2 = metadata.copy();
-    DelayedLoadMetadata::Ptr metadataCopy2 =
-        openvdb::StaticPtrCast<DelayedLoadMetadata>(baseMetadataCopy2);
-
-    EXPECT_EQ(metadataCopy2->getMask(0), DelayedLoadMetadata::MaskType(5));
-    EXPECT_EQ(metadataCopy2->getCompressedSize(2), DelayedLoadMetadata::CompressedSizeType(-13522));
-
-    // I/O
-
-    metadata.clear();
-    EXPECT_TRUE(metadata.empty());
-
-    const size_t headerInitialSize(sizeof(openvdb::Index32));
-    const size_t headerCountSize(sizeof(openvdb::Index32));
-    const size_t headerMaskSize(sizeof(openvdb::Index32));
-    const size_t headerCompressedSize(sizeof(openvdb::Index32));
-    const size_t headerTotalSize(headerInitialSize + headerCountSize + headerMaskSize + headerCompressedSize);
-
-    { // empty buffer
-        std::stringstream ss(std::ios_base::out | std::ios_base::in | std::ios_base::binary);
-        metadata.write(ss);
-        EXPECT_EQ(ss.tellp(), std::streampos(headerInitialSize));
-
-        DelayedLoadMetadata newMetadata;
-        newMetadata.read(ss);
-        EXPECT_TRUE(newMetadata.empty());
-    }
-
-    { // single value, no compressed sizes
-        metadata.clear();
-        metadata.resizeMask(size_t(1));
-        metadata.setMask(0, DelayedLoadMetadata::MaskType(5));
-
-        std::stringstream ss(std::ios_base::out | std::ios_base::in | std::ios_base::binary);
-        metadata.write(ss);
-        std::streampos expectedPos(headerTotalSize + sizeof(int8_t));
-        EXPECT_EQ(ss.tellp(), expectedPos);
-        EXPECT_EQ(static_cast<size_t>(expectedPos)-headerInitialSize, size_t(metadata.size()));
-
-        DelayedLoadMetadata newMetadata;
-        newMetadata.read(ss);
-        EXPECT_TRUE(!newMetadata.empty());
-        EXPECT_EQ(newMetadata.getMask(0), DelayedLoadMetadata::MaskType(5));
-    }
-
-    { // single value, with compressed sizes
-        metadata.clear();
-        metadata.resizeMask(size_t(1));
-        metadata.setMask(0, DelayedLoadMetadata::MaskType(5));
-
-        metadata.resizeCompressedSize(size_t(1));
-        metadata.setCompressedSize(0, DelayedLoadMetadata::CompressedSizeType(-10322));
-
-        std::stringstream ss(std::ios_base::out | std::ios_base::in | std::ios_base::binary);
-        metadata.write(ss);
-        std::streampos expectedPos(headerTotalSize + sizeof(int8_t) + sizeof(int64_t));
-
-        EXPECT_EQ(expectedPos, ss.tellp());
-        EXPECT_EQ(static_cast<size_t>(ss.tellp())-headerInitialSize, size_t(metadata.size()));
-
-        DelayedLoadMetadata newMetadata;
-        newMetadata.read(ss);
-        EXPECT_TRUE(!newMetadata.empty());
-        EXPECT_EQ(newMetadata.getMask(0), DelayedLoadMetadata::MaskType(5));
-        EXPECT_EQ(newMetadata.getCompressedSize(0), DelayedLoadMetadata::CompressedSizeType(-10322));
-    }
-
-    { // larger, but compressible buffer
-        metadata.clear();
-
-        const size_t size = 1000;
-
-        const size_t uncompressedBufferSize = (sizeof(int8_t)+sizeof(int64_t))*size;
-
-        metadata.resizeMask(size);
-        metadata.resizeCompressedSize(size);
-        for (size_t i = 0; i < size; i++) {
-            metadata.setMask(i,
-                DelayedLoadMetadata::MaskType(static_cast<int8_t>((i%32)*2)));
-            metadata.setCompressedSize(i,
-                DelayedLoadMetadata::CompressedSizeType(static_cast<int64_t>((i%64)*200)));
-        }
-
-        std::stringstream ss(std::ios_base::out | std::ios_base::in | std::ios_base::binary);
-        metadata.write(ss);
-
-        EXPECT_EQ(static_cast<size_t>(ss.tellp())-headerInitialSize, size_t(metadata.size()));
-
-        std::streampos uncompressedSize(uncompressedBufferSize + headerTotalSize);
-#ifdef OPENVDB_USE_BLOSC
-        // expect a compression ratio of more than 10x
-        EXPECT_TRUE(ss.tellp() * 10 < uncompressedSize);
-#else
-        EXPECT_TRUE(ss.tellp() == uncompressedSize);
-#endif
-
-        DelayedLoadMetadata newMetadata;
-        newMetadata.read(ss);
-        EXPECT_EQ(metadata.size(), newMetadata.size());
-        for (size_t i = 0; i < size; i++) {
-            EXPECT_EQ(metadata.getMask(i), newMetadata.getMask(i));
-        }
-    }
-
-    // when read as unknown metadata should be treated as temporary metadata
-
-    {
-        metadata.clear();
-        metadata.resizeMask(size_t(1));
-        metadata.setMask(0, DelayedLoadMetadata::MaskType(5));
-
-        std::stringstream ss(std::ios_base::out | std::ios_base::in | std::ios_base::binary);
-
-        openvdb::MetaMap metamap;
-        metamap.insertMeta("delayload", metadata);
-
-        EXPECT_EQ(size_t(1), metamap.metaCount());
-
-        metamap.writeMeta(ss);
-
-        {
-            openvdb::MetaMap newMetamap;
-            newMetamap.readMeta(ss);
-
-            EXPECT_EQ(size_t(1), newMetamap.metaCount());
-        }
-
-        {
-            DelayedLoadMetadata::unregisterType();
-
-            openvdb::MetaMap newMetamap;
-            newMetamap.readMeta(ss);
-
-            EXPECT_EQ(size_t(0), newMetamap.metaCount());
-        }
-    }
-}

openvdb/openvdb/unittest/TestFastSweeping.cc

@@ -252,7 +252,7 @@ TEST_F(TestFastSweeping, testMaskSdf)
          openvdb::initialize();//required whenever I/O of OpenVDB files is performed!
          const std::string path(TestFastSweeping_DATA_PATH);
          io::File file( path + "bunny.vdb" );
-         file.open(false);//disable delayed loading
+         file.open();
          FloatGrid::Ptr mask = openvdb::gridPtrCast<openvdb::FloatGrid>(file.getGrids()->at(0));
 
          //this->writeFile("/tmp/bunny_mask_input.vdb", grid);
@@ -595,7 +595,7 @@ TEST_F(TestFastSweeping, velocityExtensionOfFogBunny)
   openvdb::initialize();//required whenever I/O of OpenVDB files is performed!
   const std::string path(TestFastSweeping_DATA_PATH);
   io::File file( path + "bunny.vdb" );
-  file.open(false);//disable delayed loading
+  file.open();
   auto grid = openvdb::gridPtrCast<openvdb::FloatGrid>(file.getGrids()->at(0));
   tools::sdfToFogVolume(*grid);
   writeFile("/tmp/bunny1_fog_in.vdb", grid);
@@ -619,7 +619,7 @@ TEST_F(TestFastSweeping, velocityExtensionOfSdfBunny)
   using namespace openvdb;
   const std::string path(TestFastSweeping_DATA_PATH);
   io::File file( path + "bunny.vdb" );
-  file.open(false);//disable delayed loading
+  file.open();
   auto grid = openvdb::gridPtrCast<openvdb::FloatGrid>(file.getGrids()->at(0));
   writeFile("/tmp/bunny2_sdf_in.vdb", grid);
   auto bbox = grid->evalActiveVoxelBoundingBox();

openvdb/openvdb/unittest/TestGrid.cc

@@ -65,7 +65,9 @@ public:
 
     void readBuffers(std::istream& is,
         const openvdb::CoordBBox&, bool /*saveFloatAsHalf*/=false) override { is.seekg(0); }
+#if OPENVDB_ABI_VERSION_NUMBER < 14
     void readNonresidentBuffers() const override {}
+#endif
     void readBuffers(std::istream& is, bool /*saveFloatAsHalf*/=false) override { is.seekg(0); }
     void writeBuffers(std::ostream& os, bool /*saveFloatAsHalf*/=false) const override
         { os.seekp(0, std::ios::beg); }

openvdb/openvdb/unittest/TestGridDescriptor.cc

@@ -42,16 +42,15 @@ TEST_F(TestGridDescriptor, testIO)
 
     GridDescriptor gd2;
 
-    EXPECT_THROW(gd2.read(istr), openvdb::LookupError);
-
     // Register the grid.
     GridBase::clearRegistry();
     GridType::registerGrid();
 
     // seek back and read again.
     istr.seekg(0, std::ios_base::beg);
-    GridBase::Ptr grid;
-    EXPECT_NO_THROW(grid = gd2.read(istr));
+    gd2.readHeader(istr);
+    gd2.readStreamPos(istr);
+    GridBase::Ptr grid = GridBase::createGrid(gd2.gridType());
 
     EXPECT_EQ(gd.gridName(), gd2.gridName());
     EXPECT_EQ(gd.uniqueName(), gd2.uniqueName());

openvdb/openvdb/unittest/TestGridIO.cc

@@ -3,6 +3,7 @@
 
 #include <openvdb/Exceptions.h>
 #include <openvdb/openvdb.h>
+#include <openvdb/codecs/ScalarCodec.h>
 #include <gtest/gtest.h>
 
 #include <cstdio> // for remove()
@@ -278,5 +279,10 @@ TEST_F(TestGridIO, testReadAllBool) { readAllTest<openvdb::BoolGrid>(); }
 TEST_F(TestGridIO, testReadAllFloat) { readAllTest<openvdb::FloatGrid>(); }
 TEST_F(TestGridIO, testReadAllHalf) { readAllTest<openvdb::HalfGrid>(); }
 TEST_F(TestGridIO, testReadAllVec3S) { readAllTest<openvdb::Vec3SGrid>(); }
-TEST_F(TestGridIO, testReadAllFloat5432) { Float5432Grid::registerGrid(); readAllTest<Float5432Grid>(); }
+TEST_F(TestGridIO, testReadAllFloat5432)
+{
+    Float5432Grid::registerGrid();
+    openvdb::io::CodecRegistry::registerCodec<openvdb::codecs::ScalarCodec<Float5432Grid>>();
+    readAllTest<Float5432Grid>();
+}
 TEST_F(TestGridIO, testCreateWriteReadHalf) { testCreateWriteReadHalf(); }

openvdb/openvdb/unittest/TestLeafBool.cc

@@ -5,6 +5,7 @@
 #include <openvdb/Types.h>
 #include <openvdb/tools/Filter.h>
 #include <openvdb/util/logging.h>
+#include <openvdb/io/io.h>
 #include "util.h" // for unittest_util::makeSphere()
 #include <gtest/gtest.h>
 #include <set>
@@ -293,6 +294,60 @@ TEST_F(TestLeafBool, testIO)
     leaf.setValueOn(openvdb::Coord(0, 1, 0));
     leaf.setValueOn(openvdb::Coord(1, 0, 0));
 
+    // read and write topology to disk
+
+    {
+        // create a grid with the leaf for topology testing
+        typedef openvdb::Grid<openvdb::tree::Tree<openvdb::tree::RootNode<
+            openvdb::tree::InternalNode<openvdb::tree::InternalNode<LeafType, 4>, 5>>>> BoolGrid;
+        BoolGrid::Ptr grid = BoolGrid::create();
+        grid->setName("bool_leaf");
+        grid->tree().addLeaf(new LeafType(leaf));
+
+        openvdb::GridCPtrVec grids;
+        grids.push_back(grid);
+
+        // write to file
+        {
+            openvdb::io::File file("leaf_bool.vdb");
+            file.write(grids);
+            file.close();
+        }
+
+        // read grid from file
+        BoolGrid::Ptr gridFromDisk;
+        {
+            openvdb::io::File file("leaf_bool.vdb");
+            file.open();
+            openvdb::GridBase::Ptr baseGrid = file.readGrid("bool_leaf");
+            file.close();
+
+            gridFromDisk = openvdb::gridPtrCast<BoolGrid>(baseGrid);
+        }
+
+        LeafType* leaf2 = gridFromDisk->tree().probeLeaf(origin);
+        EXPECT_TRUE(leaf2);
+
+        // check topology and values match
+
+        EXPECT_EQ(origin, leaf2->origin());
+        EXPECT_TRUE(leaf2->isValueOn(openvdb::Coord(0, 1, 0)));
+        EXPECT_TRUE(leaf2->isValueOn(openvdb::Coord(1, 0, 0)));
+        EXPECT_TRUE(leaf2->onVoxelCount() == 2);
+
+        remove("leaf_bool.vdb");
+    }
+}
+
+
+TEST_F(TestLeafBool, testTreeIO)
+{
+    LeafType leaf(openvdb::Coord(1, 3, 5));
+    const openvdb::Coord origin = leaf.origin();
+
+    leaf.setValueOn(openvdb::Coord(0, 1, 0));
+    leaf.setValueOn(openvdb::Coord(1, 0, 0));
+
     std::ostringstream ostr(std::ios_base::binary);
 
     leaf.writeBuffers(ostr);
@@ -301,8 +356,6 @@ TEST_F(TestLeafBool, testIO)
     leaf.setValueOn(openvdb::Coord(0, 1, 1));
 
     std::istringstream istr(ostr.str(), std::ios_base::binary);
-    // Since the input stream doesn't include a VDB header with file format version info,
-    // tag the input stream explicitly with the current version number.
     openvdb::io::setCurrentVersion(istr);
 
     leaf.readBuffers(istr);

openvdb/openvdb/unittest/TestLeafIO.cc

@@ -2,8 +2,10 @@
 // SPDX-License-Identifier: Apache-2.0
 
 #include <openvdb/Exceptions.h>
+#include <openvdb/openvdb.h>
 #include <openvdb/tree/LeafNode.h>
 #include <openvdb/Types.h>
+#include <openvdb/io/io.h>
 #include <gtest/gtest.h>
 
 #include <cctype> // for toupper()
@@ -15,13 +17,73 @@ class TestLeafIO
 {
 public:
     static void testBuffer();
+    static void testTreeIO();
 };
 
 template<typename T>
 void
 TestLeafIO<T>::testBuffer()
 {
-    openvdb::tree::LeafNode<T, 3> leaf(openvdb::Coord(0, 0, 0));
+    using LeafT = openvdb::tree::LeafNode<T, 3>;
+    LeafT leaf(openvdb::Coord(0, 0, 0));
+    const openvdb::Coord origin = leaf.origin();
+
+    leaf.setValueOn(openvdb::Coord(0, 1, 0), T(1));
+    leaf.setValueOn(openvdb::Coord(1, 0, 0), T(1));
+
+    // read and write topology to disk
+
+    {
+        // create a grid with the leaf for topology testing
+        typedef openvdb::Grid<openvdb::tree::Tree<openvdb::tree::RootNode<
+            openvdb::tree::InternalNode<openvdb::tree::InternalNode<LeafT, 4>, 5>>>> GridType;
+        if (!GridType::isRegistered()) GridType::registerGrid();
+
+        typename GridType::Ptr grid = GridType::create();
+        grid->setName("leaf_io");
+        grid->tree().addLeaf(new LeafT(leaf));
+
+        openvdb::GridCPtrVec grids;
+        grids.push_back(grid);
+
+        // write to file
+        {
+            openvdb::io::File file("leaf_io.vdb");
+            file.write(grids);
+            file.close();
+        }
+
+        // read grid from file
+        typename GridType::Ptr gridFromDisk;
+        {
+            openvdb::io::File file("leaf_io.vdb");
+            file.open();
+            openvdb::GridBase::Ptr baseGrid = file.readGrid("leaf_io");
+            file.close();
+
+            gridFromDisk = openvdb::gridPtrCast<GridType>(baseGrid);
+        }
+
+        LeafT* leaf2 = gridFromDisk->tree().probeLeaf(origin);
+        EXPECT_TRUE(leaf2);
+
+        // check topology and values match
+
+        EXPECT_NEAR(T(1), leaf2->getValue(openvdb::Coord(0, 1, 0)), /*tolerance=*/0);
+        EXPECT_NEAR(T(1), leaf2->getValue(openvdb::Coord(1, 0, 0)), /*tolerance=*/0);
+        EXPECT_TRUE(leaf2->onVoxelCount() == 2);
+
+        remove("leaf_io.vdb");
+    }
+}
+
+
+template<typename T>
+void
+TestLeafIO<T>::testTreeIO()
+{
+    using LeafT = openvdb::tree::LeafNode<T, 3>;
+    LeafT leaf(openvdb::Coord(0, 0, 0));
 
     leaf.setValueOn(openvdb::Coord(0, 1, 0), T(1));
     leaf.setValueOn(openvdb::Coord(1, 0, 0), T(1));
@@ -34,9 +96,6 @@ TestLeafIO<T>::testBuffer()
     leaf.setValueOn(openvdb::Coord(0, 1, 1), T(1));
 
     std::istringstream istr(ostr.str(), std::ios_base::binary);
-
-    // Since the input stream doesn't include a VDB header with file format version info,
-    // tag the input stream explicitly with the current version number.
     openvdb::io::setCurrentVersion(istr);
 
     leaf.readBuffers(istr);
@@ -50,6 +109,9 @@ TestLeafIO<T>::testBuffer()
 
 class TestLeafIOTest: public ::testing::Test
 {
+public:
+    void SetUp() override { openvdb::initialize(); }
+    void TearDown() override { openvdb::uninitialize(); }
 };
 
 
@@ -62,7 +124,68 @@ TEST_F(TestLeafIOTest, testBufferByte) { TestLeafIO<openvdb::Byte>::testBuffer()
 
 TEST_F(TestLeafIOTest, testBufferVec3R)
 {
-    openvdb::tree::LeafNode<openvdb::Vec3R, 3> leaf(openvdb::Coord(0, 0, 0));
+    using LeafT = openvdb::tree::LeafNode<openvdb::Vec3R, 3>;
+    LeafT leaf(openvdb::Coord(0, 0, 0));
+    const openvdb::Coord origin = leaf.origin();
+
+    leaf.setValueOn(openvdb::Coord(0, 1, 0), openvdb::Vec3R(1, 1, 1));
+    leaf.setValueOn(openvdb::Coord(1, 0, 0), openvdb::Vec3R(1, 1, 1));
+
+    // read and write topology to disk
+
+    {
+        // create a grid with the leaf for topology testing
+        typedef openvdb::Grid<openvdb::tree::Tree<openvdb::tree::RootNode<
+            openvdb::tree::InternalNode<openvdb::tree::InternalNode<LeafT, 4>, 5>>>> GridType;
+        GridType::Ptr grid = GridType::create();
+        grid->setName("leaf_vec3r");
+        grid->tree().addLeaf(new LeafT(leaf));
+
+        openvdb::GridCPtrVec grids;
+        grids.push_back(grid);
+
+        // write to file
+        {
+            openvdb::io::File file("leaf_vec3r.vdb");
+            file.write(grids);
+            file.close();
+        }
+
+        // read grid from file
+        GridType::Ptr gridFromDisk;
+        {
+            openvdb::io::File file("leaf_vec3r.vdb");
+            file.open();
+            openvdb::GridBase::Ptr baseGrid = file.readGrid("leaf_vec3r");
+            file.close();
+
+            gridFromDisk = openvdb::gridPtrCast<GridType>(baseGrid);
+        }
+
+        LeafT* leaf2 = gridFromDisk->tree().probeLeaf(origin);
+        EXPECT_TRUE(leaf2);
+
+        // check topology and values match
+
+        EXPECT_TRUE(leaf2->getValue(openvdb::Coord(0, 1, 0)) == openvdb::Vec3R(1, 1, 1));
+        EXPECT_TRUE(leaf2->getValue(openvdb::Coord(1, 0, 0)) == openvdb::Vec3R(1, 1, 1));
+        EXPECT_TRUE(leaf2->onVoxelCount() == 2);
+
+        remove("leaf_vec3r.vdb");
+    }
+}
+
+TEST_F(TestLeafIOTest, testTreeIOInt) { TestLeafIO<int>::testTreeIO(); }
+TEST_F(TestLeafIOTest, testTreeIOFloat) { TestLeafIO<float>::testTreeIO(); }
+TEST_F(TestLeafIOTest, testTreeIODouble) { TestLeafIO<double>::testTreeIO(); }
+TEST_F(TestLeafIOTest, testTreeIOBool) { TestLeafIO<bool>::testTreeIO(); }
+TEST_F(TestLeafIOTest, testTreeIOByte) { TestLeafIO<openvdb::Byte>::testTreeIO(); }
+
+
+TEST_F(TestLeafIOTest, testTreeIOVec3R)
+{
+    using LeafT = openvdb::tree::LeafNode<openvdb::Vec3R, 3>;
+    LeafT leaf(openvdb::Coord(0, 0, 0));
 
     leaf.setValueOn(openvdb::Coord(0, 1, 0), openvdb::Vec3R(1, 1, 1));
     leaf.setValueOn(openvdb::Coord(1, 0, 0), openvdb::Vec3R(1, 1, 1));
@@ -75,9 +198,6 @@ TEST_F(TestLeafIOTest, testBufferVec3R)
     leaf.setValueOn(openvdb::Coord(0, 1, 1), openvdb::Vec3R(1, 1, 1));
 
     std::istringstream istr(ostr.str(), std::ios_base::binary);
-
-    // Since the input stream doesn't include a VDB header with file format version info,
-    // tag the input stream explicitly with the current version number.
     openvdb::io::setCurrentVersion(istr);
 
     leaf.readBuffers(istr);

openvdb/openvdb/unittest/TestLeafMask.cc

@@ -7,6 +7,7 @@
 #include <openvdb/tools/Prune.h>
 #include <openvdb/tree/LeafNode.h>
 #include <openvdb/util/logging.h>
+#include <openvdb/io/io.h>
 #include "util.h" // for unittest_util::makeSphere()
 #include <gtest/gtest.h>
 #include <set>
@@ -291,6 +292,60 @@ TEST_F(TestLeafMask, testIO)
     leaf.setValueOn(openvdb::Coord(0, 1, 0));
     leaf.setValueOn(openvdb::Coord(1, 0, 0));
 
+    // read and write topology to disk
+
+    {
+        // create a grid with the leaf for topology testing
+        typedef openvdb::Grid<openvdb::tree::Tree<openvdb::tree::RootNode<
+            openvdb::tree::InternalNode<openvdb::tree::InternalNode<LeafType, 4>, 5>>>> MaskGrid;
+        MaskGrid::Ptr grid = MaskGrid::create();
+        grid->setName("leaf_mask");
+        grid->tree().addLeaf(new LeafType(leaf));
+
+        openvdb::GridCPtrVec grids;
+        grids.push_back(grid);
+
+        // write to file
+        {
+            openvdb::io::File file("leaf_mask.vdb");
+            file.write(grids);
+            file.close();
+        }
+
+        // read grid from file
+        MaskGrid::Ptr gridFromDisk;
+        {
+            openvdb::io::File file("leaf_mask.vdb");
+            file.open();
+            openvdb::GridBase::Ptr baseGrid = file.readGrid("leaf_mask");
+            file.close();
+
+            gridFromDisk = openvdb::gridPtrCast<MaskGrid>(baseGrid);
+        }
+
+        LeafType* leaf2 = gridFromDisk->tree().probeLeaf(origin);
+        EXPECT_TRUE(leaf2);
+
+        // check topology and values match
+
+        EXPECT_EQ(origin, leaf2->origin());
+        EXPECT_TRUE(leaf2->isValueOn(openvdb::Coord(0, 1, 0)));
+        EXPECT_TRUE(leaf2->isValueOn(openvdb::Coord(1, 0, 0)));
+        EXPECT_TRUE(leaf2->onVoxelCount() == 2);
+
+        remove("leaf_mask.vdb");
+    }
+}
+
+
+TEST_F(TestLeafMask, testTreeIO)
+{
+    LeafType leaf(openvdb::Coord(1, 3, 5));
+    const openvdb::Coord origin = leaf.origin();
+
+    leaf.setValueOn(openvdb::Coord(0, 1, 0));
+    leaf.setValueOn(openvdb::Coord(1, 0, 0));
+
     std::ostringstream ostr(std::ios_base::binary);
 
     leaf.writeBuffers(ostr);
@@ -299,8 +354,6 @@ TEST_F(TestLeafMask, testIO)
     leaf.setValueOn(openvdb::Coord(0, 1, 1));
 
     std::istringstream istr(ostr.str(), std::ios_base::binary);
-    // Since the input stream doesn't include a VDB header with file format version info,
-    // tag the input stream explicitly with the current version number.
     openvdb::io::setCurrentVersion(istr);
 
     leaf.readBuffers(istr);

openvdb/openvdb/unittest/TestMeshToVolume.cc

@@ -211,8 +211,8 @@ TEST_F(TestMeshToVolume, testInterrupt)
     // Should have returned _something_
     EXPECT_TRUE(grid);
 
-    // Expect to interrupt in under a second
+    // Expect to interrupt in under two seconds
     const auto duration = std::chrono::duration_cast<std::chrono::milliseconds>(end - start);
-    EXPECT_LT(duration.count(), 1000);
+    EXPECT_LT(duration.count(), 2000);
 }
 

openvdb/openvdb/unittest/TestMultiResGrid.cc

@@ -208,6 +208,8 @@ TEST_F(TestMultiResGrid, testIO)
 {
     using namespace openvdb;
 
+    openvdb::initialize();
+
     const float radius = 1.0f;
     const Vec3f center(0.0f, 0.0f, 0.0f);
     const float voxelSize = 0.01f;
@@ -243,7 +245,6 @@ TEST_F(TestMultiResGrid, testIO)
     outputFile.close();
 
     // Read grids
-    openvdb::initialize();
     openvdb::io::File file( filename );
     file.open();
     GridPtrVecPtr grids = file.getGrids();
@@ -280,7 +281,7 @@ TEST_F(TestMultiResGrid, testModels)
             << "\" =====================" << std::endl;
         std::cerr << "Reading \"" << filenames[i] << "\" ...";
         io::File file( path + filenames[i] );
-        file.open(false);//disable delayed loading
+        file.open();
         FloatGrid::Ptr model = gridPtrCast<FloatGrid>(file.getGrids()->at(0));
         std::cerr << " done\nProcessing \"" << filenames[i] << "\" ...";
         timer.start("\nMultiResGrid processing");

openvdb/openvdb/unittest/TestPointCodec.cc

@@ -0,0 +1,547 @@
+// Copyright Contributors to the OpenVDB Project
+// SPDX-License-Identifier: Apache-2.0
+
+#include <openvdb/io/Codec.h>
+#include <openvdb/io/File.h>
+#include <openvdb/openvdb.h>
+#include <openvdb/tools/PointIndexGrid.h>
+#include <openvdb/points/PointConversion.h>
+#include <gtest/gtest.h>
+#include "util.h" // for unittest_util::genPoints
+
+namespace {
+
+class PointList {
+public:
+    using PosType = openvdb::Vec3R;
+    PointList(const std::vector<PosType>& points) : mPoints(&points) {}
+    size_t size() const { return mPoints->size(); }
+    void getPos(size_t n, PosType& xyz) const { xyz = (*mPoints)[n]; }
+private:
+    std::vector<PosType> const * const mPoints;
+};
+
+} // namespace
+
+class TestPointCodec: public ::testing::Test
+{
+};
+
+TEST_F(TestPointCodec, testPointIndexCodecIO)
+{
+    using namespace openvdb;
+    using namespace openvdb::io;
+    using PointIndexGrid = tools::PointIndexGrid;
+
+    openvdb::initialize();
+    CodecRegistry::clear();
+
+    // Generate points on a unit sphere and build a PointIndexGrid
+    std::vector<Vec3R> points;
+    unittest_util::genPoints(100, points);
+    PointList pointList(points);
+
+    const double voxelSize = 0.1;
+    math::Transform::Ptr transform = math::Transform::createLinearTransform(voxelSize);
+
+    PointIndexGrid::Ptr srcGrid =
+        tools::createPointIndexGrid<PointIndexGrid>(pointList, *transform);
+    srcGrid->setName("point_index_grid");
+
+    const std::string rawPath = "testPointIndexCodec_raw.vdb";
+
+    // Phase 1: write/read without codec
+    {
+        io::File f(rawPath);
+        f.write(GridPtrVec{srcGrid});
+    }
+
+    PointIndexGrid::Ptr rawGrid;
+    {
+        io::File f(rawPath);
+        f.open();
+        rawGrid = gridPtrCast<PointIndexGrid>(f.readGrid("point_index_grid"));
+        f.close();
+    }
+    ASSERT_TRUE(rawGrid);
+
+    PointIndexGrid::Ptr rawTopo;
+    {
+        io::File f(rawPath);
+        f.open();
+        GridBase::Ptr base;
+        EXPECT_NO_THROW(base = f.readGrid("point_index_grid"));
+        rawTopo = gridPtrCast<PointIndexGrid>(base);
+        f.close();
+    }
+    ASSERT_TRUE(rawTopo);
+    EXPECT_EQ(rawTopo->activeVoxelCount(), Index64(97));
+    EXPECT_EQ(rawTopo->getName(), std::string("point_index_grid"));
+
+    std::remove(rawPath.c_str());
+
+    const std::string codecPath = "testPointIndexCodec_codec.vdb";
+
+    // Phase 2: register codec, write/read with codec
+    io::internal::initialize();
+    ASSERT_TRUE(CodecRegistry::isRegistered(PointIndexGrid::gridType()));
+
+    {
+        io::File f(codecPath);
+        f.write(GridPtrVec{srcGrid});
+    }
+
+    PointIndexGrid::Ptr codecGrid;
+    {
+        io::File f(codecPath);
+        f.open();
+        codecGrid = gridPtrCast<PointIndexGrid>(f.readGrid("point_index_grid"));
+        f.close();
+    }
+    ASSERT_TRUE(codecGrid);
+
+    // Phase 3: full read comparison
+    EXPECT_TRUE(srcGrid->tree().hasSameTopology(srcGrid->tree()));
+    EXPECT_TRUE(srcGrid->tree().hasSameTopology(codecGrid->tree()));
+    {
+        auto codecAcc = codecGrid->getConstAccessor();
+        for (PointIndexGrid::ValueOnCIter it = srcGrid->cbeginValueOn(); it; ++it) {
+            EXPECT_EQ(*it, codecAcc.getValue(it.getCoord()));
+        }
+    }
+
+    // Compare leaf indices arrays
+    {
+        auto srcLeafIt = srcGrid->tree().cbeginLeaf();
+        auto codecLeafIt = codecGrid->tree().cbeginLeaf();
+        for (; srcLeafIt; ++srcLeafIt, ++codecLeafIt) {
+            ASSERT_TRUE(codecLeafIt);
+            EXPECT_EQ(srcLeafIt->indices().size(), codecLeafIt->indices().size());
+            for (size_t i = 0; i < srcLeafIt->indices().size(); ++i) {
+                EXPECT_EQ(srcLeafIt->indices()[i], codecLeafIt->indices()[i]);
+            }
+        }
+        EXPECT_TRUE(!codecLeafIt);
+    }
+
+    // Phase 4: TopologyOnly read
+    ReadOptions topoOpts;
+    topoOpts.readMode = ReadMode::TopologyOnly;
+
+    PointIndexGrid::Ptr codecTopo;
+    {
+        io::File f(codecPath);
+        f.open();
+        GridBase::Ptr base;
+        EXPECT_NO_THROW(base = f.readGrid("point_index_grid", topoOpts));
+        codecTopo = gridPtrCast<PointIndexGrid>(base);
+        f.close();
+    }
+    ASSERT_TRUE(codecTopo);
+    EXPECT_EQ(codecTopo->activeVoxelCount(), Index64(0));
+    EXPECT_TRUE(codecTopo->tree().leafCount() == 0);
+    EXPECT_EQ(codecTopo->getName(), std::string("point_index_grid"));
+
+    // Cleanup
+    CodecRegistry::clear();
+    std::remove(codecPath.c_str());
+}
+
+TEST_F(TestPointCodec, testPointDataCodecIO)
+{
+    using namespace openvdb;
+    using namespace openvdb::io;
+    using namespace openvdb::points;
+    using PointDataTree = PointDataGrid::TreeType;
+
+    openvdb::initialize();
+    CodecRegistry::clear();
+
+    // Helper: compare P attribute values leaf-by-leaf between two PointDataGrids
+    auto comparePositions = [](const PointDataGrid& a, const PointDataGrid& b) {
+        auto aIt = a.tree().cbeginLeaf();
+        auto bIt = b.tree().cbeginLeaf();
+        for (; aIt && bIt; ++aIt, ++bIt) {
+            EXPECT_EQ(aIt->pointCount(), bIt->pointCount());
+            AttributeHandle<Vec3f> aH(aIt->constAttributeArray("P"));
+            AttributeHandle<Vec3f> bH(bIt->constAttributeArray("P"));
+            for (Index i = 0; i < aIt->pointCount(); ++i) {
+                const Vec3f av = aH.get(i);
+                const Vec3f bv = bH.get(i);
+                EXPECT_NEAR(av.x(), bv.x(), 1e-6f);
+                EXPECT_NEAR(av.y(), bv.y(), 1e-6f);
+                EXPECT_NEAR(av.z(), bv.z(), 1e-6f);
+            }
+        }
+        EXPECT_TRUE(!aIt && !bIt);
+    };
+
+    // -----------------------------------------------------------------------
+    // Section A: Positions only
+    // -----------------------------------------------------------------------
+    {
+        const std::vector<Vec3f> positions = {
+            Vec3f(0.0f, 1.0f, 0.0f),
+            Vec3f(1.5f, 3.5f, 1.0f),
+            Vec3f(-1.0f, 6.0f, -2.0f),
+            Vec3f(1.1f, 1.25f, 0.06f)
+        };
+
+        const float voxelSize = 0.5f;
+        math::Transform::Ptr transform = math::Transform::createLinearTransform(voxelSize);
+
+        PointDataGrid::Ptr srcGrid =
+            createPointDataGrid<NullCodec, PointDataGrid>(positions, *transform);
+        srcGrid->setName("pdg_positions");
+
+        const std::string rawPath = "testPDG_A_raw.vdb";
+
+        // Phase 1: write/read without codec
+        {
+            io::File f(rawPath);
+            f.write(GridPtrVec{srcGrid});
+        }
+
+        PointDataGrid::Ptr rawGrid;
+        {
+            io::File f(rawPath);
+            f.open();
+            rawGrid = gridPtrCast<PointDataGrid>(f.readGrid("pdg_positions"));
+            f.close();
+        }
+        ASSERT_TRUE(rawGrid);
+        EXPECT_TRUE(srcGrid->tree().hasSameTopology(rawGrid->tree()));
+
+        PointDataGrid::Ptr rawTopo;
+        {
+            io::File f(rawPath);
+            f.open();
+            GridBase::Ptr base;
+            EXPECT_NO_THROW(base = f.readGrid("pdg_positions"));
+            rawTopo = gridPtrCast<PointDataGrid>(base);
+            f.close();
+        }
+        ASSERT_TRUE(rawTopo);
+        EXPECT_EQ(rawTopo->activeVoxelCount(), Index64(4));
+
+        std::remove(rawPath.c_str());
+
+        const std::string codecPath = "testPDG_A_codec.vdb";
+
+        // Phase 2: register codec, write/read with codec
+        io::internal::initialize();
+        ASSERT_TRUE(CodecRegistry::isRegistered(PointDataGrid::gridType()));
+
+        {
+            io::File f(codecPath);
+            f.write(GridPtrVec{srcGrid});
+        }
+
+        PointDataGrid::Ptr codecGrid;
+        {
+            io::File f(codecPath);
+            f.open();
+            codecGrid = gridPtrCast<PointDataGrid>(f.readGrid("pdg_positions"));
+            f.close();
+        }
+        ASSERT_TRUE(codecGrid);
+
+        // Phase 3: compare src vs codec
+        EXPECT_TRUE(srcGrid->tree().hasSameTopology(codecGrid->tree()));
+        EXPECT_EQ(pointCount(srcGrid->tree()), pointCount(codecGrid->tree()));
+        comparePositions(*srcGrid, *codecGrid);
+
+        // Phase 4: TopologyOnly read
+        ReadOptions topoOpts;
+        topoOpts.readMode = ReadMode::TopologyOnly;
+
+        PointDataGrid::Ptr codecTopo;
+        {
+            io::File f(codecPath);
+            f.open();
+            GridBase::Ptr base;
+            EXPECT_NO_THROW(base = f.readGrid("pdg_positions", topoOpts));
+            codecTopo = gridPtrCast<PointDataGrid>(base);
+            f.close();
+        }
+        ASSERT_TRUE(codecTopo);
+        EXPECT_EQ(codecTopo->activeVoxelCount(), Index64(0));
+        EXPECT_TRUE(codecTopo->tree().leafCount() == 0);
+
+        std::remove(codecPath.c_str());
+    }
+
+    // -----------------------------------------------------------------------
+    // Section B: Multiple attributes
+    // -----------------------------------------------------------------------
+    {
+        const std::vector<Vec3f> positions = {
+            Vec3f(0.0f, 1.0f, 0.0f),
+            Vec3f(1.5f, 3.5f, 1.0f),
+            Vec3f(-1.0f, 6.0f, -2.0f),
+            Vec3f(1.1f, 1.25f, 0.06f)
+        };
+        const std::vector<Vec3f> velocities = {
+            Vec3f(1.0f, 0.0f, 0.0f),
+            Vec3f(0.0f, 1.0f, 0.0f),
+            Vec3f(0.0f, 0.0f, 1.0f),
+            Vec3f(1.0f, 1.0f, 0.5f)
+        };
+        const std::vector<int> ids = {0, 1, 2, 3};
+
+        const float voxelSize = 0.5f;
+        math::Transform::Ptr transform = math::Transform::createLinearTransform(voxelSize);
+
+        PointAttributeVector<Vec3f> posWrapper(positions);
+        tools::PointIndexGrid::Ptr pointIndexGrid =
+            tools::createPointIndexGrid<tools::PointIndexGrid>(posWrapper, *transform);
+
+        PointDataGrid::Ptr srcGrid =
+            createPointDataGrid<NullCodec, PointDataGrid>(
+                *pointIndexGrid, posWrapper, *transform);
+        srcGrid->setName("pdg_multi");
+
+        PointDataTree& tree = srcGrid->tree();
+        tools::PointIndexTree& indexTree = pointIndexGrid->tree();
+
+        appendAttribute<Vec3f>(tree, "velocity");
+        populateAttribute<PointDataTree, tools::PointIndexTree,
+            PointAttributeVector<Vec3f>>(
+                tree, indexTree, "velocity",
+                PointAttributeVector<Vec3f>(velocities));
+
+        appendAttribute<int>(tree, "id");
+        populateAttribute<PointDataTree, tools::PointIndexTree,
+            PointAttributeVector<int>>(
+                tree, indexTree, "id",
+                PointAttributeVector<int>(ids));
+
+        // Verify attribute count on src grid (P, velocity, id)
+        {
+            auto leafIt = srcGrid->tree().cbeginLeaf();
+            ASSERT_TRUE(leafIt);
+            EXPECT_EQ(leafIt->attributeSet().size(), size_t(3));
+        }
+
+        CodecRegistry::clear();
+
+        const std::string rawPath = "testPDG_B_raw.vdb";
+
+        // Phase 1: write/read without codec
+        {
+            io::File f(rawPath);
+            f.write(GridPtrVec{srcGrid});
+        }
+
+        PointDataGrid::Ptr rawGrid;
+        {
+            io::File f(rawPath);
+            f.open();
+            rawGrid = gridPtrCast<PointDataGrid>(f.readGrid("pdg_multi"));
+            f.close();
+        }
+        ASSERT_TRUE(rawGrid);
+
+        std::remove(rawPath.c_str());
+
+        const std::string codecPath = "testPDG_B_codec.vdb";
+
+        // Phase 2: register codec, write/read with codec
+        io::internal::initialize();
+
+        {
+            io::File f(codecPath);
+            f.write(GridPtrVec{srcGrid});
+        }
+
+        PointDataGrid::Ptr codecGrid;
+        {
+            io::File f(codecPath);
+            f.open();
+            codecGrid = gridPtrCast<PointDataGrid>(f.readGrid("pdg_multi"));
+            f.close();
+        }
+        ASSERT_TRUE(codecGrid);
+
+        EXPECT_TRUE(srcGrid->tree().hasSameTopology(codecGrid->tree()));
+        EXPECT_EQ(pointCount(srcGrid->tree()), pointCount(codecGrid->tree()));
+
+        // Verify attribute count on codec grid
+        {
+            auto leafIt = codecGrid->tree().cbeginLeaf();
+            ASSERT_TRUE(leafIt);
+            EXPECT_EQ(leafIt->attributeSet().size(), size_t(3));
+        }
+
+        // Compare all three attributes leaf-by-leaf
+        {
+            auto srcIt = srcGrid->tree().cbeginLeaf();
+            auto codecIt = codecGrid->tree().cbeginLeaf();
+            for (; srcIt && codecIt; ++srcIt, ++codecIt) {
+                EXPECT_EQ(srcIt->pointCount(), codecIt->pointCount());
+                AttributeHandle<Vec3f> srcP(srcIt->constAttributeArray("P"));
+                AttributeHandle<Vec3f> codecP(codecIt->constAttributeArray("P"));
+                AttributeHandle<Vec3f> srcVel(srcIt->constAttributeArray("velocity"));
+                AttributeHandle<Vec3f> codecVel(codecIt->constAttributeArray("velocity"));
+                AttributeHandle<int> srcId(srcIt->constAttributeArray("id"));
+                AttributeHandle<int> codecId(codecIt->constAttributeArray("id"));
+                for (Index i = 0; i < srcIt->pointCount(); ++i) {
+                    const Vec3f rp = srcP.get(i);
+                    const Vec3f cp = codecP.get(i);
+                    EXPECT_NEAR(rp.x(), cp.x(), 1e-6f);
+                    EXPECT_NEAR(rp.y(), cp.y(), 1e-6f);
+                    EXPECT_NEAR(rp.z(), cp.z(), 1e-6f);
+                    const Vec3f rv = srcVel.get(i);
+                    const Vec3f cv = codecVel.get(i);
+                    EXPECT_NEAR(rv.x(), cv.x(), 1e-6f);
+                    EXPECT_NEAR(rv.y(), cv.y(), 1e-6f);
+                    EXPECT_NEAR(rv.z(), cv.z(), 1e-6f);
+                    EXPECT_EQ(srcId.get(i), codecId.get(i));
+                }
+            }
+            EXPECT_TRUE(!srcIt && !codecIt);
+        }
+
+        CodecRegistry::clear();
+        std::remove(codecPath.c_str());
+    }
+
+    // -----------------------------------------------------------------------
+    // Section C: Shared vs non-shared descriptors
+    // -----------------------------------------------------------------------
+    {
+        std::vector<Vec3R> pts;
+        unittest_util::genPoints(100, pts);
+
+        std::vector<Vec3f> positions;
+        positions.reserve(pts.size());
+        for (const auto& p : pts) {
+            positions.emplace_back(float(p.x()), float(p.y()), float(p.z()));
+        }
+
+        const double voxelSize = 0.1;
+        math::Transform::Ptr transform = math::Transform::createLinearTransform(voxelSize);
+
+        PointDataGrid::Ptr srcGrid =
+            createPointDataGrid<NullCodec, PointDataGrid>(positions, *transform);
+        srcGrid->setName("pdg_desc");
+
+        // All leaves should share one Descriptor::Ptr initially
+        ASSERT_GT(srcGrid->tree().leafCount(), Index32(1));
+        {
+            auto leafIt = srcGrid->tree().cbeginLeaf();
+            auto firstDescPtr = leafIt->attributeSet().descriptorPtr();
+            for (; leafIt; ++leafIt) {
+                EXPECT_EQ(leafIt->attributeSet().descriptorPtr(), firstDescPtr);
+            }
+        }
+
+        io::internal::initialize();
+
+        // -- C1: Shared descriptors (default) --
+        // All leaves already share one pointer; this exercises the header=1 write path.
+        const std::string sharedPath = "testPDG_C_shared.vdb";
+        {
+            io::File f(sharedPath);
+            f.write(GridPtrVec{srcGrid});
+        }
+
+        PointDataGrid::Ptr sharedGrid;
+        {
+            io::File f(sharedPath);
+            f.open();
+            sharedGrid = gridPtrCast<PointDataGrid>(f.readGrid("pdg_desc"));
+            f.close();
+        }
+        ASSERT_TRUE(sharedGrid);
+        EXPECT_TRUE(srcGrid->tree().hasSameTopology(sharedGrid->tree()));
+        EXPECT_EQ(pointCount(srcGrid->tree()), pointCount(sharedGrid->tree()));
+        comparePositions(*srcGrid, *sharedGrid);
+
+        // -- C2: After makeDescriptorUnique --
+        // makeDescriptorUnique() creates ONE new descriptor and assigns it to every
+        // leaf, so all leaves still share a single pointer (the new copy).
+        // The codec still detects matching descriptors and writes header=1;
+        // this is a regression check that the round-trip remains correct.
+        makeDescriptorUnique(srcGrid->tree());
+        {
+            auto leafIt = srcGrid->tree().cbeginLeaf();
+            auto firstDescPtr = leafIt->attributeSet().descriptorPtr();
+            ++leafIt;
+            if (leafIt) {
+                // All leaves share the same new pointer
+                EXPECT_EQ(leafIt->attributeSet().descriptorPtr(), firstDescPtr);
+            }
+        }
+
+        const std::string nonSharedPath = "testPDG_C_nonshared.vdb";
+        {
+            io::File f(nonSharedPath);
+            f.write(GridPtrVec{srcGrid});
+        }
+
+        PointDataGrid::Ptr nonSharedGrid;
+        {
+            io::File f(nonSharedPath);
+            f.open();
+            nonSharedGrid = gridPtrCast<PointDataGrid>(f.readGrid("pdg_desc"));
+            f.close();
+        }
+        ASSERT_TRUE(nonSharedGrid);
+        EXPECT_TRUE(srcGrid->tree().hasSameTopology(nonSharedGrid->tree()));
+        EXPECT_EQ(pointCount(srcGrid->tree()), pointCount(nonSharedGrid->tree()));
+        comparePositions(*sharedGrid, *nonSharedGrid);
+
+        // -- C3: Genuinely different descriptors (exercises the header=0 write path) --
+        // Add "extra" to all leaves, then drop it from only the first leaf so that
+        // leaf descriptors differ by value, triggering matching=false in the codec.
+        appendAttribute<float>(srcGrid->tree(), "extra");
+        makeDescriptorUnique(srcGrid->tree());
+        srcGrid->setName("pdg_desc_diff");
+
+        {
+            auto leafIt = srcGrid->tree().beginLeaf();
+            ASSERT_TRUE(leafIt);
+            const size_t extraIdx =
+                leafIt->attributeSet().descriptor().find("extra");
+            ASSERT_NE(extraIdx, AttributeSet::INVALID_POS);
+            const std::vector<size_t> dropIndices = {extraIdx};
+            AttributeSet::Descriptor::Ptr newDesc =
+                leafIt->attributeSet().descriptor().duplicateDrop(dropIndices);
+            leafIt->dropAttributes(
+                dropIndices, leafIt->attributeSet().descriptor(), newDesc);
+        }
+
+        const std::string diffPath = "testPDG_C_diff.vdb";
+        {
+            io::File f(diffPath);
+            f.write(GridPtrVec{srcGrid});
+        }
+
+        PointDataGrid::Ptr diffGrid;
+        {
+            io::File f(diffPath);
+            f.open();
+            diffGrid = gridPtrCast<PointDataGrid>(f.readGrid("pdg_desc_diff"));
+            f.close();
+        }
+        ASSERT_TRUE(diffGrid);
+        EXPECT_TRUE(srcGrid->tree().hasSameTopology(diffGrid->tree()));
+        EXPECT_EQ(pointCount(srcGrid->tree()), pointCount(diffGrid->tree()));
+
+        // First leaf has {P} only; remaining leaves have {P, extra}
+        {
+            auto diffIt = diffGrid->tree().cbeginLeaf();
+            ASSERT_TRUE(diffIt);
+            EXPECT_EQ(diffIt->attributeSet().size(), size_t(1));
+            ++diffIt;
+            if (diffIt) {
+                EXPECT_EQ(diffIt->attributeSet().size(), size_t(2));
+            }
+        }
+
+        std::remove(sharedPath.c_str());
+        std::remove(nonSharedPath.c_str());
+        std::remove(diffPath.c_str());
+    }
+}

openvdb/openvdb/unittest/TestPointConversion.cc

@@ -1,7 +1,6 @@
 // Copyright Contributors to the OpenVDB Project
 // SPDX-License-Identifier: Apache-2.0
 
-#include <openvdb/io/TempFile.h>
 #include <openvdb/math/Math.h>
 #include <openvdb/points/PointDataGrid.h>
 #include <openvdb/points/PointAttribute.h>

openvdb/openvdb/unittest/TestPointCount.cc

@@ -3,7 +3,6 @@
 
 #include <openvdb/points/PointDataGrid.h>
 #include <openvdb/openvdb.h>
-#include <openvdb/io/TempFile.h>
 #include <openvdb/math/Math.h>
 
 #include <openvdb/points/PointGroup.h>
@@ -314,30 +313,14 @@ TEST_F(TestPointCount, testGroup)
 
             GroupFilter groupFilter("test", attributeSet);
 
-            bool inCoreOnly;
-#ifdef OPENVDB_USE_DELAYED_LOADING
-            inCoreOnly = true;
-
-            EXPECT_EQ(pointCount(inputTree, NullFilter(), inCoreOnly), Index64(0));
-            EXPECT_EQ(pointCount(inputTree, ActiveFilter(), inCoreOnly), Index64(0));
-            EXPECT_EQ(pointCount(inputTree, InactiveFilter(), inCoreOnly), Index64(0));
-            EXPECT_EQ(pointCount(inputTree, groupFilter, inCoreOnly), Index64(0));
+            EXPECT_EQ(pointCount(inputTree, NullFilter()), Index64(4));
+            EXPECT_EQ(pointCount(inputTree, ActiveFilter()), Index64(3));
+            EXPECT_EQ(pointCount(inputTree, InactiveFilter()), Index64(1));
+            EXPECT_EQ(pointCount(inputTree, groupFilter), Index64(2));
             EXPECT_EQ(pointCount(inputTree, BinaryFilter<GroupFilter, ActiveFilter>(
-                groupFilter, ActiveFilter()), inCoreOnly), Index64(0));
+                groupFilter, ActiveFilter())), Index64(1));
             EXPECT_EQ(pointCount(inputTree, BinaryFilter<GroupFilter, InactiveFilter>(
-                groupFilter, InactiveFilter()), inCoreOnly), Index64(0));
-#endif
-
-            inCoreOnly = false;
-
-            EXPECT_EQ(pointCount(inputTree, NullFilter(), inCoreOnly), Index64(4));
-            EXPECT_EQ(pointCount(inputTree, ActiveFilter(), inCoreOnly), Index64(3));
-            EXPECT_EQ(pointCount(inputTree, InactiveFilter(), inCoreOnly), Index64(1));
-            EXPECT_EQ(pointCount(inputTree, groupFilter, inCoreOnly), Index64(2));
-            EXPECT_EQ(pointCount(inputTree, BinaryFilter<GroupFilter, ActiveFilter>(
-                groupFilter, ActiveFilter()), inCoreOnly), Index64(1));
-            EXPECT_EQ(pointCount(inputTree, BinaryFilter<GroupFilter, InactiveFilter>(
-                groupFilter, InactiveFilter()), inCoreOnly), Index64(1));
+                groupFilter, InactiveFilter())), Index64(1));
         }
 
         std::remove(filename.c_str());
@@ -529,51 +512,6 @@ TEST_F(TestPointCount, testOffsets)
         fileOut.write(grids);
     }
 
-#ifdef OPENVDB_USE_DELAYED_LOADING
-    // test point offsets for a delay-loaded grid
-    {
-        io::File fileIn(filename);
-        fileIn.open();
-
-        GridPtrVecPtr grids = fileIn.getGrids();
-
-        fileIn.close();
-
-        EXPECT_EQ(grids->size(), size_t(1));
-
-        PointDataGrid::Ptr inputGrid = GridBase::grid<PointDataGrid>((*grids)[0]);
-
-        EXPECT_TRUE(inputGrid);
-
-        PointDataTree& inputTree = inputGrid->tree();
-
-        std::vector<Index64> offsets;
-        std::vector<Name> includeGroups;
-        std::vector<Name> excludeGroups;
-
-        MultiGroupFilter filter(includeGroups, excludeGroups, inputTree.cbeginLeaf()->attributeSet());
-        Index64 total = pointOffsets(offsets, inputTree, filter, /*inCoreOnly=*/true);
-
-        EXPECT_EQ(offsets.size(), size_t(4));
-        EXPECT_EQ(offsets[0], Index64(0));
-        EXPECT_EQ(offsets[1], Index64(0));
-        EXPECT_EQ(offsets[2], Index64(0));
-        EXPECT_EQ(offsets[3], Index64(0));
-        EXPECT_EQ(total, Index64(0));
-
-        offsets.clear();
-
-        total = pointOffsets(offsets, inputTree, filter, /*inCoreOnly=*/false);
-
-        EXPECT_EQ(offsets.size(), size_t(4));
-        EXPECT_EQ(offsets[0], Index64(1));
-        EXPECT_EQ(offsets[1], Index64(3));
-        EXPECT_EQ(offsets[2], Index64(4));
-        EXPECT_EQ(offsets[3], Index64(5));
-        EXPECT_EQ(total, Index64(5));
-    }
-#endif
-
     std::remove(filename.c_str());
 }
 

openvdb/openvdb/unittest/TestPointDataLeaf.cc

@@ -1058,76 +1058,93 @@ TEST_F(TestPointDataLeaf, testIO)
     // read and write topology to disk
 
     {
-        LeafType leaf2(openvdb::Coord(0, 0, 0));
+        // create a grid with the leaf for topology testing
+        PointDataGrid::Ptr grid = PointDataGrid::create();
+        grid->setName("points");
+        grid->tree().addLeaf(new LeafType(leaf));
 
-        std::ostringstream ostr(std::ios_base::binary);
-        leaf.writeTopology(ostr);
+        openvdb::GridCPtrVec grids;
+        grids.push_back(grid);
 
-        std::istringstream istr(ostr.str(), std::ios_base::binary);
-        leaf2.readTopology(istr);
+        // write to file
+        {
+            io::File file("leaf_topology.vdb");
+            file.write(grids);
+            file.close();
+        }
+
+        // read grid from file
+        PointDataGrid::Ptr gridFromDisk;
+        {
+            io::File file("leaf_topology.vdb");
+            file.open();
+            openvdb::GridBase::Ptr baseGrid = file.readGrid("points");
+            file.close();
+
+            gridFromDisk = openvdb::gridPtrCast<PointDataGrid>(baseGrid);
+        }
+
+        LeafType* leaf2 = gridFromDisk->tree().probeLeaf(openvdb::Coord(0, 0, 0));
+        EXPECT_TRUE(leaf2);
 
         // check topology matches
 
-        EXPECT_EQ(leaf.onVoxelCount(), leaf2.onVoxelCount());
-        EXPECT_TRUE(leaf2.isValueOn(4));
-        EXPECT_TRUE(!leaf2.isValueOn(5));
+        EXPECT_EQ(leaf.onVoxelCount(), leaf2->onVoxelCount());
+        EXPECT_TRUE(leaf2->isValueOn(4));
+        EXPECT_TRUE(!leaf2->isValueOn(5));
 
-        // check only topology (values and attributes still empty)
+        // check that values and attributes are correctly read
 
-        EXPECT_EQ(leaf2.getValue(4), ValueType(0));
-        EXPECT_EQ(leaf2.attributeSet().size(), size_t(0));
+        EXPECT_EQ(leaf2->getValue(4), ValueType(20));
+        EXPECT_EQ(leaf2->attributeSet().size(), size_t(2));
+
+        remove("leaf_topology.vdb");
     }
 
     // read and write buffers to disk
 
     {
-        LeafType leaf2(openvdb::Coord(0, 0, 0));
+        // create a grid with the leaf for buffer testing
+        PointDataGrid::Ptr grid = PointDataGrid::create();
+        grid->setName("points");
+        grid->tree().addLeaf(new LeafType(leaf));
 
-        io::StreamMetadata::Ptr streamMetadata(new io::StreamMetadata);
+        openvdb::GridCPtrVec grids;
+        grids.push_back(grid);
 
-        std::ostringstream ostr(std::ios_base::binary);
-        io::setStreamMetadataPtr(ostr, streamMetadata);
-        io::setDataCompression(ostr, io::COMPRESS_BLOSC);
-        leaf.writeTopology(ostr);
-        for (Index b = 0; b < leaf.buffers(); b++) {
-            uint32_t pass = (uint32_t(leaf.buffers()) << 16) | uint32_t(b);
-            streamMetadata->setPass(pass);
-            leaf.writeBuffers(ostr);
-        }
-        { // error checking
-            streamMetadata->setPass(1000);
-            leaf.writeBuffers(ostr);
-
-            io::StreamMetadata::Ptr meta;
-            io::setStreamMetadataPtr(ostr, meta);
-            EXPECT_THROW(leaf.writeBuffers(ostr), openvdb::IoError);
+        // write to file
+        {
+            io::File file("leaf_buffers.vdb");
+            file.write(grids);
+            file.close();
         }
 
-        std::istringstream istr(ostr.str(), std::ios_base::binary);
-        io::setStreamMetadataPtr(istr, streamMetadata);
-        io::setDataCompression(istr, io::COMPRESS_BLOSC);
+        // read grid from file
+        PointDataGrid::Ptr gridFromDisk;
+        {
+            io::File file("leaf_buffers.vdb");
+            file.open();
+            openvdb::GridBase::Ptr baseGrid = file.readGrid("points");
+            file.close();
 
-        // Since the input stream doesn't include a VDB header with file format version info,
-        // tag the input stream explicitly with the current version number.
-        io::setCurrentVersion(istr);
-
-        leaf2.readTopology(istr);
-        for (Index b = 0; b < leaf.buffers(); b++) {
-            uint32_t pass = (uint32_t(leaf.buffers()) << 16) | uint32_t(b);
-            streamMetadata->setPass(pass);
-            leaf2.readBuffers(istr);
+            gridFromDisk = openvdb::gridPtrCast<PointDataGrid>(baseGrid);
         }
 
+        LeafType* leaf2 = gridFromDisk->tree().probeLeaf(openvdb::Coord(0, 0, 0));
+        EXPECT_TRUE(leaf2);
+
         // check topology matches
 
-        EXPECT_EQ(leaf.onVoxelCount(), leaf2.onVoxelCount());
-        EXPECT_TRUE(leaf2.isValueOn(4));
-        EXPECT_TRUE(!leaf2.isValueOn(5));
+        EXPECT_EQ(leaf.onVoxelCount(), leaf2->onVoxelCount());
+        EXPECT_TRUE(leaf2->isValueOn(4));
+        EXPECT_TRUE(!leaf2->isValueOn(5));
 
-        // check only topology (values and attributes still empty)
+        // check values and attributes are correctly read
 
-        EXPECT_EQ(leaf2.getValue(4), ValueType(20));
-        EXPECT_EQ(leaf2.attributeSet().size(), size_t(2));
+        EXPECT_EQ(leaf2->getValue(4), ValueType(20));
+        EXPECT_EQ(leaf2->attributeSet().size(), size_t(2));
+
+        remove("leaf_buffers.vdb");
     }
 
     { // test multi-buffer IO
@@ -1165,111 +1182,6 @@ TEST_F(TestPointDataLeaf, testIO)
 
             EXPECT_TRUE(leaf == *leafFromDisk);
         }
-
-#ifdef OPENVDB_USE_DELAYED_LOADING
-        { // read grids from file and pre-fetch
-            PointDataGrid::Ptr gridFromDisk;
-
-            {
-                io::File file("leaf.vdb");
-                file.open();
-                openvdb::GridBase::Ptr baseGrid = file.readGrid("points");
-                file.close();
-
-                gridFromDisk = openvdb::gridPtrCast<PointDataGrid>(baseGrid);
-            }
-
-            LeafType* leafFromDisk = gridFromDisk->tree().probeLeaf(openvdb::Coord(0, 0, 0));
-            EXPECT_TRUE(leafFromDisk);
-
-            const AttributeVec3s& position(
-                AttributeVec3s::cast(leafFromDisk->constAttributeArray("P")));
-            const AttributeF& density(
-                AttributeF::cast(leafFromDisk->constAttributeArray("density")));
-
-            EXPECT_TRUE(leafFromDisk->buffer().isOutOfCore());
-#ifdef OPENVDB_USE_BLOSC
-            EXPECT_TRUE(position.isOutOfCore());
-            EXPECT_TRUE(density.isOutOfCore());
-#else
-            // delayed-loading is only available on attribute arrays when using Blosc
-            EXPECT_TRUE(!position.isOutOfCore());
-            EXPECT_TRUE(!density.isOutOfCore());
-#endif
-
-            // prefetch voxel data only
-            prefetch(gridFromDisk->tree(), /*position=*/false, /*attributes=*/false);
-
-            // ensure out-of-core data is now in-core after pre-fetching
-
-            EXPECT_TRUE(!leafFromDisk->buffer().isOutOfCore());
-#ifdef OPENVDB_USE_BLOSC
-            EXPECT_TRUE(position.isOutOfCore());
-            EXPECT_TRUE(density.isOutOfCore());
-#else
-            EXPECT_TRUE(!position.isOutOfCore());
-            EXPECT_TRUE(!density.isOutOfCore());
-#endif
-
-            { // re-open
-                io::File file("leaf.vdb");
-                file.open();
-                openvdb::GridBase::Ptr baseGrid = file.readGrid("points");
-                file.close();
-
-                gridFromDisk = openvdb::gridPtrCast<PointDataGrid>(baseGrid);
-            }
-
-            leafFromDisk = gridFromDisk->tree().probeLeaf(openvdb::Coord(0, 0, 0));
-            EXPECT_TRUE(leafFromDisk);
-
-            const AttributeVec3s& position2(
-                AttributeVec3s::cast(leafFromDisk->constAttributeArray("P")));
-            const AttributeF& density2(
-                AttributeF::cast(leafFromDisk->constAttributeArray("density")));
-
-            // prefetch voxel and position attribute data
-            prefetch(gridFromDisk->tree(), /*position=*/true, /*attribute=*/false);
-
-            // ensure out-of-core voxel and position data is now in-core after pre-fetching
-
-            EXPECT_TRUE(!leafFromDisk->buffer().isOutOfCore());
-            EXPECT_TRUE(!position2.isOutOfCore());
-#ifdef OPENVDB_USE_BLOSC
-            EXPECT_TRUE(density2.isOutOfCore());
-#else
-            EXPECT_TRUE(!density2.isOutOfCore());
-#endif
-
-            { // re-open
-                io::File file("leaf.vdb");
-                file.open();
-                openvdb::GridBase::Ptr baseGrid = file.readGrid("points");
-                file.close();
-
-                gridFromDisk = openvdb::gridPtrCast<PointDataGrid>(baseGrid);
-            }
-
-            leafFromDisk = gridFromDisk->tree().probeLeaf(openvdb::Coord(0, 0, 0));
-            EXPECT_TRUE(leafFromDisk);
-
-            const AttributeVec3s& position3(
-                AttributeVec3s::cast(leafFromDisk->constAttributeArray("P")));
-            const AttributeF& density3(
-                AttributeF::cast(leafFromDisk->constAttributeArray("density")));
-
-            // prefetch all data
-            prefetch(gridFromDisk->tree());
-
-            // ensure out-of-core voxel and position data is now in-core after pre-fetching
-
-            EXPECT_TRUE(!leafFromDisk->buffer().isOutOfCore());
-            EXPECT_TRUE(!position3.isOutOfCore());
-            EXPECT_TRUE(!density3.isOutOfCore());
-        }
-
-        remove("leaf.vdb");
-#endif // OPENVDB_USE_DELAYED_LOADING
     }
 
     { // test multi-buffer IO with varying attribute storage per-leaf
@@ -1364,6 +1276,97 @@ TEST_F(TestPointDataLeaf, testIO)
 }
 
 
+TEST_F(TestPointDataLeaf, testTreeIO)
+{
+    using AttributeVec3s    = TypedAttributeArray<openvdb::Vec3s>;
+    using AttributeF        = TypedAttributeArray<float>;
+
+    using Descriptor = AttributeSet::Descriptor;
+
+    Descriptor::Ptr descrA = Descriptor::create(AttributeVec3s::attributeType());
+
+    const size_t size = LeafType::NUM_VOXELS;
+
+    LeafType leaf(openvdb::Coord(0, 0, 0));
+    leaf.initializeAttributes(descrA, /*arrayLength=*/size/2);
+
+    descrA = descrA->duplicateAppend("density", AttributeF::attributeType());
+    leaf.appendAttribute(leaf.attributeSet().descriptor(), descrA, descrA->find("density"));
+
+    leaf.setOffsetOn(1, 10);
+    leaf.setOffsetOn(4, 20);
+    leaf.setOffsetOn(7, 5);
+
+    TypedAttributeArray<float>& attr =
+        TypedAttributeArray<float>::cast(leaf.attributeArray("density"));
+
+    attr.set(0, 5.0f);
+    attr.set(50, 2.0f);
+    attr.set(51, 8.1f);
+
+    {
+        LeafType leaf2(openvdb::Coord(0, 0, 0));
+
+        std::ostringstream ostr(std::ios_base::binary);
+        leaf.writeTopology(ostr);
+
+        std::istringstream istr(ostr.str(), std::ios_base::binary);
+        leaf2.readTopology(istr);
+
+        EXPECT_EQ(leaf.onVoxelCount(), leaf2.onVoxelCount());
+        EXPECT_TRUE(leaf2.isValueOn(4));
+        EXPECT_TRUE(!leaf2.isValueOn(5));
+
+        EXPECT_EQ(leaf2.getValue(4), ValueType(0));
+        EXPECT_EQ(leaf2.attributeSet().size(), size_t(0));
+    }
+
+    {
+        LeafType leaf2(openvdb::Coord(0, 0, 0));
+
+        io::StreamMetadata::Ptr streamMetadata(new io::StreamMetadata);
+
+        std::ostringstream ostr(std::ios_base::binary);
+        io::setStreamMetadataPtr(ostr, streamMetadata);
+        io::setDataCompression(ostr, io::COMPRESS_BLOSC);
+        leaf.writeTopology(ostr);
+        for (Index b = 0; b < leaf.buffers(); b++) {
+            uint32_t pass = (uint32_t(leaf.buffers()) << 16) | uint32_t(b);
+            streamMetadata->setPass(pass);
+            leaf.writeBuffers(ostr);
+        }
+        {
+            streamMetadata->setPass(1000);
+            leaf.writeBuffers(ostr);
+
+            io::StreamMetadata::Ptr meta;
+            io::setStreamMetadataPtr(ostr, meta);
+            EXPECT_THROW(leaf.writeBuffers(ostr), openvdb::IoError);
+        }
+
+        std::istringstream istr(ostr.str(), std::ios_base::binary);
+        io::setStreamMetadataPtr(istr, streamMetadata);
+        io::setDataCompression(istr, io::COMPRESS_BLOSC);
+
+        io::setCurrentVersion(istr);
+
+        leaf2.readTopology(istr);
+        for (Index b = 0; b < leaf.buffers(); b++) {
+            uint32_t pass = (uint32_t(leaf.buffers()) << 16) | uint32_t(b);
+            streamMetadata->setPass(pass);
+            leaf2.readBuffers(istr);
+        }
+
+        EXPECT_EQ(leaf.onVoxelCount(), leaf2.onVoxelCount());
+        EXPECT_TRUE(leaf2.isValueOn(4));
+        EXPECT_TRUE(!leaf2.isValueOn(5));
+
+        EXPECT_EQ(leaf2.getValue(4), ValueType(20));
+        EXPECT_EQ(leaf2.attributeSet().size(), size_t(2));
+    }
+}
+
+
 TEST_F(TestPointDataLeaf, testSwap)
 {
     using AttributeVec3s    = TypedAttributeArray<openvdb::Vec3s>;

openvdb/openvdb/unittest/TestPointRasterizeFrustum.cc

@@ -2154,9 +2154,6 @@ TEST_F(TestPointRasterizeFrustum, testStreaming)
 
     auto leaf = points->tree().cbeginLeaf();
     EXPECT_TRUE(leaf);
-#ifdef OPENVDB_USE_DELAYED_LOADING
-    EXPECT_TRUE(leaf->buffer().isOutOfCore());
-#endif
 
     using Rasterizer = FrustumRasterizer<PointDataGrid>;
     using Settings = FrustumRasterizerSettings;
@@ -2327,25 +2324,15 @@ TEST_F(TestPointRasterizeFrustum, testStreaming)
     auto points2 = points->deepCopy();
     points2->setTransform(transform);
 
-#ifdef OPENVDB_USE_DELAYED_LOADING
-    // verify both grids are out-of-core
-
-    EXPECT_TRUE(points->tree().cbeginLeaf()->buffer().isOutOfCore());
-    EXPECT_TRUE(points2->tree().cbeginLeaf()->buffer().isOutOfCore());
-#endif
-
 #ifndef ONLY_RASTER_FLOAT
     // memory tests
 
-    if (io::Archive::isDelayedLoadingEnabled() && io::Archive::hasBloscCompression()) {
+    if (io::Archive::hasBloscCompression()) {
 
         FloatGrid::Ptr density1, density2, density3;
         Vec3SGrid::Ptr velocity1, velocity2, velocity3;
 
         const size_t mb = 1024*1024;
-        const size_t tinyMemory = static_cast<size_t>(0.1*mb);
-
-        size_t initialMemory;
 
         { // memory test 1 - retain caches and streaming disabled
             Rasterizer rasterizer(settings);
@@ -2353,114 +2340,25 @@ TEST_F(TestPointRasterizeFrustum, testStreaming)
             rasterizer.addPoints(points, /*stream=*/false);
             rasterizer.addPoints(points2, /*stream=*/false);
 
-            initialMemory = rasterizer.memUsage();
+            // memory usage should be around 80-100MB throughout rasterization
 
-            EXPECT_TRUE(initialMemory > size_t(4*mb) && initialMemory < size_t(16*mb));
+            EXPECT_TRUE(rasterizer.memUsage() > size_t(80*mb) &&
+                rasterizer.memUsage() < size_t(100*mb));
 
             EXPECT_EQ(size_t(2), rasterizer.size());
 
             velocity1 = rasterizer.rasterizeAttribute<Vec3SGrid, Vec3s>("v");
             EXPECT_EQ(Index64(219780), velocity1->activeVoxelCount());
 
-            EXPECT_TRUE(rasterizer.memUsage() > size_t(71*mb) &&
-                rasterizer.memUsage() < size_t(91*mb));
+            EXPECT_TRUE(rasterizer.memUsage() > size_t(80*mb) &&
+                rasterizer.memUsage() < size_t(100*mb));
 
             density1 = rasterizer.rasterizeDensity("density");
             EXPECT_EQ(Index64(219780), density1->activeVoxelCount());
 
-            // no data is discarded so expect a fairly high memory footprint
-
             EXPECT_TRUE(rasterizer.memUsage() > size_t(80*mb) &&
                 rasterizer.memUsage() < size_t(100*mb));
         }
-
-        { // memory test 2 - retain caches and streaming enabled
-
-            { // reopen file and deep copy while setting transform
-                io::File file(filename);
-                file.open();
-                openvdb::GridBase::Ptr baseGrid = file.readGrid("points");
-                file.close();
-
-                points = openvdb::gridPtrCast<PointDataGrid>(baseGrid);
-                points2 = points->deepCopy();
-                points2->setTransform(transform);
-            }
-
-            Rasterizer rasterizer(settings);
-
-            rasterizer.addPoints(points, /*stream=*/true);
-            rasterizer.addPoints(points2, /*stream=*/true);
-
-            EXPECT_EQ(initialMemory, rasterizer.memUsage());
-
-            EXPECT_EQ(size_t(2), rasterizer.size());
-
-            velocity2 = rasterizer.rasterizeAttribute<Vec3SGrid, Vec3s>("v");
-            EXPECT_EQ(Index64(219780), velocity2->activeVoxelCount());
-
-            size_t postRasterMemory = rasterizer.memUsage();
-
-            EXPECT_TRUE(postRasterMemory > size_t(70*mb) && postRasterMemory < size_t(85*mb));
-
-            density2 = rasterizer.rasterizeDensity("density");
-            EXPECT_EQ(Index64(219780), density2->activeVoxelCount());
-
-            // as data is being streamed, second attribute shouldn't change memory usage very much
-
-            EXPECT_TRUE(rasterizer.memUsage() < (postRasterMemory + tinyMemory));
-        }
-
-        { // memory test 3 - release caches and streaming enabled
-
-            { // reopen file and deep copy while setting transform
-                io::File file(filename);
-                file.open();
-                openvdb::GridBase::Ptr baseGrid = file.readGrid("points");
-                file.close();
-
-                points = openvdb::gridPtrCast<PointDataGrid>(baseGrid);
-                points2 = points->deepCopy();
-                points2->setTransform(transform);
-            }
-
-            auto points3 = points->deepCopy();
-            auto points4 = points2->deepCopy();
-
-            Settings settings2(*frustum);
-            settings2.threshold = 0.0f;
-
-            Mask mask2(*frustum, nullptr, BBoxd(), /*clipToFrustum=*/false);
-
-            Rasterizer rasterizer(settings2, mask2);
-
-            rasterizer.addPoints(points, /*stream=*/true);
-            rasterizer.addPoints(points2, /*stream=*/true);
-
-            EXPECT_EQ(initialMemory, rasterizer.memUsage());
-            EXPECT_EQ(size_t(2), rasterizer.size());
-
-            density3 = rasterizer.rasterizeDensity("density", RasterMode::ACCUMULATE, true);
-            EXPECT_EQ(Index64(219780), density3->activeVoxelCount());
-
-            // all voxel data, attribute data and caches are being discarded,
-            // so memory after rasterizing shouldn't change very much
-
-            EXPECT_TRUE(rasterizer.memUsage() < (initialMemory + tinyMemory));
-
-            // deep-copies of delay-loaded point grids need to be used for repeat rasterization
-
-            rasterizer.clear();
-            rasterizer.addPoints(points3, /*stream=*/true);
-            rasterizer.addPoints(points4, /*stream=*/true);
-
-            EXPECT_EQ(size_t(2), rasterizer.size());
-
-            EXPECT_TRUE(rasterizer.memUsage() < (initialMemory + tinyMemory));
-
-            velocity3 = rasterizer.rasterizeAttribute<Vec3SGrid, Vec3s>("v", RasterMode::ACCUMULATE, true);
-            EXPECT_EQ(Index64(219780), velocity3->activeVoxelCount());
-        }
     }
 #endif
 

openvdb/openvdb/unittest/TestStream.cc

@@ -36,26 +36,7 @@ protected:
 void
 TestStream::SetUp()
 {
-    openvdb::uninitialize();
-
-    openvdb::Int32Grid::registerGrid();
-    openvdb::FloatGrid::registerGrid();
-
-    openvdb::StringMetadata::registerType();
-    openvdb::Int32Metadata::registerType();
-    openvdb::Int64Metadata::registerType();
-    openvdb::Vec3IMetadata::registerType();
-    openvdb::io::DelayedLoadMetadata::registerType();
-
-    // Register maps
-    openvdb::math::MapRegistry::clear();
-    openvdb::math::AffineMap::registerMap();
-    openvdb::math::ScaleMap::registerMap();
-    openvdb::math::UniformScaleMap::registerMap();
-    openvdb::math::TranslationMap::registerMap();
-    openvdb::math::ScaleTranslateMap::registerMap();
-    openvdb::math::UniformScaleTranslateMap::registerMap();
-    openvdb::math::NonlinearFrustumMap::registerMap();
+    openvdb::initialize();
 }
 
 

openvdb/openvdb/unittest/TestStreamCompression.cc

@@ -8,31 +8,6 @@
 
 #include <gtest/gtest.h>
 
-#ifdef OPENVDB_USE_DELAYED_LOADING
-#ifdef __clang__
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wunused-macros"
-#endif
-// Boost.Interprocess uses a header-only portion of Boost.DateTime
-#define BOOST_DATE_TIME_NO_LIB
-#ifdef __clang__
-#pragma GCC diagnostic pop
-#endif
-#include <boost/interprocess/file_mapping.hpp>
-#include <boost/interprocess/mapped_region.hpp>
-#include <boost/iostreams/device/array.hpp>
-#include <boost/iostreams/stream.hpp>
-
-#ifdef _WIN32
-#include <boost/interprocess/detail/os_file_functions.hpp> // open_existing_file(), close_file()
-#include <windows.h>
-#else
-#include <sys/types.h> // for struct stat
-#include <sys/stat.h> // for stat()
-#include <unistd.h> // for unlink()
-#endif
-#endif // OPENVDB_USE_DELAYED_LOADING
-
 #include <atomic>
 #include <fstream>
 #include <numeric> // for std::iota()
@@ -41,6 +16,10 @@
 #include <blosc.h>
 #endif
 
+#ifdef _WIN32
+#include <windows.h>
+#endif
+
 using namespace openvdb;
 using namespace openvdb::compression;
 
@@ -467,98 +446,6 @@ TestStreamCompression::testPagedStreams()
             EXPECT_EQ(fileout.tellp(), std::streampos(values.size()+sizeof(int)*pages));
 #endif
 
-
-#ifdef OPENVDB_USE_DELAYED_LOADING
-            auto mappedFile = TestMappedFile::create(filename);
-
-            // read
-            std::ifstream filein(filename.c_str(), std::ios_base::in | std::ios_base::binary);
-            io::setStreamMetadataPtr(filein, streamMetadata);
-            io::setMappedFilePtr(filein, mappedFile);
-
-            EXPECT_EQ(filein.tellg(), std::streampos(0));
-
-            PagedInputStream istreamSizeOnly(filein);
-            istreamSizeOnly.setSizeOnly(true);
-
-            std::vector<PageHandle::Ptr> handles;
-
-            for (size_t i = 0; i < values.size(); i += increment) {
-                if (size_t(i+increment) > values.size()) {
-                    handles.push_back(istreamSizeOnly.createHandle(values.size() - i));
-                }
-                else {
-                    handles.push_back(istreamSizeOnly.createHandle(increment));
-                }
-            }
-
-#ifdef OPENVDB_USE_BLOSC
-            // two integers - compressed size and uncompressed size
-            EXPECT_EQ(filein.tellg(), std::streampos(pages*sizeof(int)*2));
-#else
-            // one integer - uncompressed size
-            EXPECT_EQ(filein.tellg(), std::streampos(pages*sizeof(int)));
-#endif
-
-            PagedInputStream istream(filein);
-
-            int pageHandle = 0;
-
-            for (size_t i = 0; i < values.size(); i += increment) {
-                if (size_t(i+increment) > values.size()) {
-                    istream.read(handles[pageHandle++], values.size() - i);
-                }
-                else {
-                    istream.read(handles[pageHandle++], increment);
-                }
-            }
-
-            // first three handles live in the same page
-
-            Page& page0 = handles[0]->page();
-            Page& page1 = handles[1]->page();
-            Page& page2 = handles[2]->page();
-            Page& page3 = handles[3]->page();
-
-            EXPECT_TRUE(page0.isOutOfCore());
-            EXPECT_TRUE(page1.isOutOfCore());
-            EXPECT_TRUE(page2.isOutOfCore());
-            EXPECT_TRUE(page3.isOutOfCore());
-
-            handles[0]->read();
-
-            // store the Page shared_ptr
-
-            Page::Ptr page = handles[0]->mPage;
-
-            // verify use count is four (one plus three handles)
-
-            EXPECT_EQ(page.use_count(), long(4));
-
-            // on reading from the first handle, all pages referenced
-            // in the first three handles are in-core
-
-            EXPECT_TRUE(!page0.isOutOfCore());
-            EXPECT_TRUE(!page1.isOutOfCore());
-            EXPECT_TRUE(!page2.isOutOfCore());
-            EXPECT_TRUE(page3.isOutOfCore());
-
-            handles[1]->read();
-
-            EXPECT_TRUE(handles[0]->mPage);
-
-            handles[2]->read();
-
-            handles.erase(handles.begin());
-            handles.erase(handles.begin());
-            handles.erase(handles.begin());
-
-            // after all three handles have been read,
-            // page should have just one use count (itself)
-
-            EXPECT_EQ(page.use_count(), long(1));
-
-#endif // OPENVDB_USE_DELAYED_LOADING
         }
         std::remove(filename.c_str());
     }

openvdb/openvdb/unittest/TestTools.cc

@@ -1762,7 +1762,7 @@ TEST_F(TestTools, testPrune)
         util::CpuTimer timer;
         initialize();//required whenever I/O of OpenVDB files is performed!
         io::File sourceFile("/usr/pic1/Data/OpenVDB/LevelSetModels/crawler.vdb");
-        sourceFile.open(false);//disable delayed loading
+        sourcefile.open();
         FloatGrid::Ptr grid = gridPtrCast<FloatGrid>(sourceFile.getGrids()->at(0));
         const Index64 leafCount = grid->tree().leafCount();
 
@@ -1775,7 +1775,7 @@ TEST_F(TestTools, testPrune)
         util::CpuTimer timer;
         initialize();//required whenever I/O of OpenVDB files is performed!
         io::File sourceFile("/usr/pic1/Data/OpenVDB/LevelSetModels/crawler.vdb");
-        sourceFile.open(false);//disable delayed loading
+        sourcefile.open();
         FloatGrid::Ptr grid = gridPtrCast<FloatGrid>(sourceFile.getGrids()->at(0));
         const Index64 leafCount = grid->tree().leafCount();
 

openvdb/openvdb/unittest/TestTree.cc

@@ -3,10 +3,12 @@
 
 #include <openvdb/Exceptions.h>
 #include <openvdb/Types.h>
+#include <openvdb/codecs/ScalarCodec.h>
 #include <openvdb/math/Transform.h>
 #include <openvdb/tools/ValueTransformer.h> // for tools::setValueOnMin(), et al.
 #include <openvdb/tree/LeafNode.h>
 #include <openvdb/io/Compression.h> // for io::RealToHalf
+#include <openvdb/io/Stream.h>
 #include <openvdb/math/Math.h> // for Abs()
 #include <openvdb/openvdb.h>
 #include <openvdb/util/CpuTimer.h>
@@ -104,6 +106,14 @@ TEST_F(TestTree, testChangeBackground)
 
 TEST_F(TestTree, testHalf)
 {
+    // explicitly register these grid types as they are not registered by default
+    openvdb::Grid<openvdb::Vec2STree>::registerGrid();
+    openvdb::Grid<openvdb::Vec2DTree>::registerGrid();
+
+    // Register scalar codecs for all vec2 tree types which are not registered by default
+    openvdb::io::CodecRegistry::registerCodec<openvdb::codecs::ScalarCodec<openvdb::Grid<openvdb::Vec2STree>>>();
+    openvdb::io::CodecRegistry::registerCodec<openvdb::codecs::ScalarCodec<openvdb::Grid<openvdb::Vec2DTree>>>();
+
     testWriteHalf<openvdb::FloatTree>();
     testWriteHalf<openvdb::DoubleTree>();
     testWriteHalf<openvdb::Vec2STree>();
@@ -125,27 +135,54 @@ TestTree::testWriteHalf()
     using GridType = openvdb::Grid<TreeType>;
     using ValueT = typename TreeType::ValueType;
     ValueT background(5);
-    GridType grid(background);
+    typename GridType::Ptr grid = GridType::create(background);
+    grid->setName("density");
 
     unittest_util::makeSphere<GridType>(openvdb::Coord(64, 64, 64),
                                         openvdb::Vec3f(35, 30, 40),
-                                        /*radius=*/10, grid,
+                                        /*radius=*/10, *grid,
                                         /*dx=*/1.0f, unittest_util::SPHERE_DENSE);
-    EXPECT_TRUE(!grid.tree().empty());
+    EXPECT_TRUE(!grid->tree().empty());
+
+    // find stream and grid header position for save as float
+
+    size_t headerSize = 0;
+    {
+        std::ostringstream outFull(std::ios_base::binary);
+        openvdb::io::Stream streamFull(outFull);
+        streamFull.write({});
+        openvdb::io::GridDescriptor gd(grid->getName(), grid->type());
+        gd.writeHeader(outFull);
+        headerSize = outFull.str().size();
+    }
+
+    // find stream and grid header position for save as half
+
+    size_t halfHeaderSize = 0;
+    {
+        std::ostringstream outHalf(std::ios_base::binary);
+        openvdb::io::Stream streamHalf(outHalf);
+        streamHalf.write({});
+        openvdb::io::GridDescriptor gdHalf(grid->getName(), grid->type(), /*half=*/true);
+        gdHalf.writeHeader(outHalf);
+        halfHeaderSize = outHalf.str().size();
+    }
 
     // Write grid blocks in both float and half formats.
     std::ostringstream outFull(std::ios_base::binary);
-    grid.setSaveFloatAsHalf(false);
-    grid.writeBuffers(outFull);
+    openvdb::io::Stream streamFull(outFull);
+    grid->setSaveFloatAsHalf(false);
+    streamFull.write({grid});
     outFull.flush();
-    const size_t fullBytes = outFull.str().size();
+    const size_t fullBytes = outFull.str().size() - headerSize;
     if (fullBytes == 0) FAIL() << "wrote empty full float buffers";
 
     std::ostringstream outHalf(std::ios_base::binary);
-    grid.setSaveFloatAsHalf(true);
-    grid.writeBuffers(outHalf);
+    openvdb::io::Stream streamHalf(outHalf);
+    grid->setSaveFloatAsHalf(true);
+    streamHalf.write({grid});
     outHalf.flush();
-    const size_t halfBytes = outHalf.str().size();
+    const size_t halfBytes = outHalf.str().size() - halfHeaderSize;
     if (halfBytes == 0) FAIL() << "wrote empty half float buffers";
 
     if (openvdb::io::RealToHalf<ValueT>::isReal) {
@@ -166,22 +203,26 @@ TestTree::testWriteHalf()
     // then write it out again in half float format.  Verify that the resulting file
     // is identical to the original half float file.
     {
-        openvdb::Grid<TreeType> gridCopy(grid);
-        gridCopy.setSaveFloatAsHalf(true);
         std::istringstream is(outHalf.str(), std::ios_base::binary);
+        openvdb::io::Stream streamHalf2(is);
 
-        // Since the input stream doesn't include a VDB header with file format version info,
-        // tag the input stream explicitly with the current version number.
-        openvdb::io::setCurrentVersion(is);
-
-        gridCopy.readBuffers(is);
+        openvdb::GridPtrVecPtr grids = streamHalf2.getGrids();
+        openvdb::GridBase::Ptr gridCopy = (*grids)[0];
+        gridCopy->setSaveFloatAsHalf(true);
 
         std::ostringstream outDiff(std::ios_base::binary);
-        gridCopy.writeBuffers(outDiff);
+        openvdb::io::Stream streamDiff(outDiff);
+        streamDiff.write({gridCopy});
         outDiff.flush();
 
-        if (outHalf.str() != outDiff.str()) {
-            FAIL() << "half-from-full and half-from-half buffers differ";
+        // Compare the two buffers, skipping the header region
+        {
+            const std::string s1 = outHalf.str().substr(halfHeaderSize);
+            const std::string s2 = outDiff.str().substr(halfHeaderSize);
+            if (s1 != s2)
+            {
+                FAIL() << "half-from-full and half-from-half buffers differ";
+            }
         }
     }
 }
@@ -731,7 +772,68 @@ TEST_F(TestTree, testIterators)
 
 TEST_F(TestTree, testIO)
 {
-    const char* filename = "testIO.dbg";
+    using TreeType = openvdb::tree::Tree<RootNodeType>;
+    using GridType = openvdb::Grid<TreeType>;
+
+    const char* filename = "testIO.vdb";
+    openvdb::SharedPtr<const char> scopedFile(filename, ::remove);
+    {
+        ValueType background=5.0f;
+        GridType::Ptr grid = GridType::create(background);
+        grid->setName("test_grid");
+        grid->tree().setValueOn(openvdb::Coord(5,10,20),0.234f);
+        grid->tree().setValueOn(openvdb::Coord(50000,20000,30000),4.5678f);
+
+        openvdb::GridCPtrVec grids;
+        grids.push_back(grid);
+
+        openvdb::io::File file(filename);
+        file.write(grids);
+        file.close();
+    }
+    {
+        ValueType background=2.0f;
+        GridType::Ptr grid = GridType::create(background);
+        ASSERT_DOUBLES_EXACTLY_EQUAL(background, grid->tree().getValue(openvdb::Coord(5,10,20)));
+
+        {
+            openvdb::io::File file(filename);
+            file.open();
+            openvdb::GridBase::Ptr baseGrid = file.readGrid("test_grid");
+            file.close();
+
+            grid = openvdb::gridPtrCast<GridType>(baseGrid);
+            EXPECT_TRUE(grid.get() != nullptr);
+        }
+
+        ASSERT_DOUBLES_EXACTLY_EQUAL(0.234f, grid->tree().getValue(openvdb::Coord(5,10,20)));
+        ASSERT_DOUBLES_EXACTLY_EQUAL(5.0f, grid->tree().getValue(openvdb::Coord(5,11,20)));
+        ValueType sum=0.0f;
+        for (RootNodeType::ChildOnIter root_iter = grid->tree().root().beginChildOn();
+            root_iter.test(); ++root_iter)
+        {
+            for (InternalNodeType2::ChildOnIter internal_iter2 = root_iter->beginChildOn();
+                internal_iter2.test(); ++internal_iter2)
+            {
+                for (InternalNodeType1::ChildOnIter internal_iter1 =
+                    internal_iter2->beginChildOn(); internal_iter1.test(); ++internal_iter1)
+                {
+                    for (LeafNodeType::ValueOnIter block_iter =
+                        internal_iter1->beginValueOn(); block_iter.test(); ++block_iter)
+                    {
+                        sum += *block_iter;
+                    }
+                }
+            }
+        }
+        ASSERT_DOUBLES_EXACTLY_EQUAL(sum, (0.234f + 4.5678f));
+    }
+}
+
+
+TEST_F(TestTree, testTreeIO)
+{
+    const char* filename = "testTreeIO.dbg";
     openvdb::SharedPtr<const char> scopedFile(filename, ::remove);
     {
         ValueType background=5.0f;
@@ -750,8 +852,6 @@ TEST_F(TestTree, testIO)
         ASSERT_DOUBLES_EXACTLY_EQUAL(background, root_node.getValue(openvdb::Coord(5,10,20)));
         {
             std::ifstream is(filename, std::ios_base::binary);
-            // Since the test file doesn't include a VDB header with file format version info,
-            // tag the input stream explicitly with the current version number.
             openvdb::io::setCurrentVersion(is);
             root_node.readTopology(is);
             root_node.readBuffers(is);

openvdb/openvdb/unittest/util.h

@@ -10,18 +10,6 @@
 #include <openvdb/tools/Prune.h>// for pruneLevelSet
 #include <sstream>
 
-#ifdef OPENVDB_USE_DELAYED_LOADING
-/// @brief io::MappedFile has a private constructor, so declare a class that acts as the friend
-class TestMappedFile
-{
-public:
-    static openvdb::io::MappedFile::Ptr create(const std::string& filename)
-    {
-        return openvdb::SharedPtr<openvdb::io::MappedFile>(new openvdb::io::MappedFile(filename));
-    }
-};
-#endif
-
 namespace unittest_util
 {
 

openvdb/openvdb/version.h.in

@@ -138,11 +138,6 @@
 #cmakedefine OPENVDB_USE_ZLIB
 #endif
 
-/* Denotes whether VDB was built with Delayed Loading support */
-#ifndef OPENVDB_USE_DELAYED_LOADING
-#cmakedefine OPENVDB_USE_DELAYED_LOADING
-#endif
-
 /* Denotes whether VDB was built asserts enabled in VDB code */
 #ifndef OPENVDB_ENABLE_ASSERTS
 #cmakedefine OPENVDB_ENABLE_ASSERTS

openvdb_ax/openvdb_ax/compiler/PointExecutable.cc

@@ -362,7 +362,6 @@ struct PointFunctionArguments
         // @todo  if the array is shared we should probably make it unique?
 
         if (mData.mUseBufferKernel) {
-            const_cast<points::AttributeArray&>(array).loadData();
             const char* data = array.constDataAsByteArray();
             void* ptr = static_cast<void*>(const_cast<char*>(data));
             mHandlesOrBuffers.emplace_back(ptr);
@@ -387,14 +386,13 @@ struct PointFunctionArguments
         array.expand();
 
         if (mData.mUseBufferKernel) {
-            array.loadData();
             const char* data = array.constDataAsByteArray();
             void* ptr = static_cast<void*>(const_cast<char*>(data));
             mHandlesOrBuffers.emplace_back(ptr);
             const codegen::Codec* codec =
                 codegen::getCodec(ast::tokens::tokenFromTypeString(array.valueType()), array.codecType());
             if (codec) flag |= codec->flag();
-            OPENVDB_ASSERT(array.isDataLoaded() && !array.isUniform());
+            OPENVDB_ASSERT(!array.isUniform());
         }
         else {
             typename WriteHandle<ValueT>::UniquePtr handle(new WriteHandle<ValueT>(leaf, Index(pos)));

openvdb_cmd/vdb_ax/main.cc

@@ -791,9 +791,7 @@ main(int argc, char *argv[])
 
     if (opts.mMode.get() == openvdb::ax::VDB_AX_MODE::Execute) {
         // read vdb file data for
-        axlog("[INFO] Reading VDB data"
-            << (openvdb::io::Archive::isDelayedLoadingEnabled() ?
-                " (delay-load)" : "") << '\n');
+        axlog("[INFO] Reading VDB data" << '\n');
         for (const auto& filename : opts.mInputVDBFiles.get()) {
             openvdb::io::File file(filename);
             try {

openvdb_cmd/vdb_print/main.cc

@@ -209,14 +209,9 @@ printShortListing(const StringVec& filenames, bool metadata)
 
             // Print the grid's size, in bytes
 
-            // no support for memUsageIfLoaded until ABI >= 10 for points::PointDataGrid types
             using ListT = openvdb::GridTypes;
             grid->apply<ListT>([&](const auto& typed){
-                // @todo combine these methods to avoid iterating across the tree twice
-                const openvdb::Index64 incore = openvdb::tools::memUsage(typed.tree());
-                const openvdb::Index64 total = openvdb::tools::memUsageIfLoaded(typed.tree());
-
-                std::cout << " " << std::right << std::setw(6) << bytesAsString(incore) << " (In Core)";
+                const openvdb::Index64 total = openvdb::tools::memUsage(typed.tree());
                 std::cout << " " << std::right << std::setw(6) << bytesAsString(total) << " (Total)";
             });
 

openvdb_cmd/vdb_render/main.cc

@@ -768,7 +768,7 @@ main(int argc, char *argv[])
                 }
             } else {
                 // If no grid was specified by name, retrieve the first float grid from the file.
-                file.open(/*delayLoad=*/false);
+                file.open();
                 openvdb::io::File::NameIterator it = file.beginName();
                 openvdb::GridPtrVecPtr grids = file.readAllGridMetadata();
                 for (size_t i = 0; i < grids->size(); ++i, ++it) {

openvdb_cmd/vdb_tool/include/Geometry.h

@@ -1098,7 +1098,7 @@ void Geometry::readVDB(const std::string &fileName)
 {
     initialize();
     io::File file(fileName);
-    file.open();// enables delayed loading by default
+    file.open();
     GridPtrVecPtr meta = file.readAllGridMetadata();
     for (auto m : *meta) {
         if (m->isType<points::PointDataGrid>()) {

openvdb_houdini/openvdb_houdini/CMakeLists.txt

@@ -123,6 +123,8 @@ set(OPENVDB_HOUDINI_ICON_INSTALL_PREFIX ${OPENVDB_HOUDINI_INSTALL_PREFIX}/config
   CACHE PATH "Install path for the OpenVDB Houdini node icons")
 set(OPENVDB_HOUDINI_PYTHON_INSTALL_PREFIX ${OPENVDB_HOUDINI_INSTALL_PREFIX}/python2.7libs
   CACHE PATH "Install path for the OpenVDB Houdini startup script")
+set(OPENVDB_HOUDINI_GLSL_INSTALL_PREFIX ${OPENVDB_HOUDINI_INSTALL_PREFIX}/glsl
+  CACHE PATH "Install path for the OpenVDB Houdini GLSL shaders")
 
 #########################################################################
 
@@ -395,6 +397,9 @@ install(FILES
 install(DIRECTORY help/
   DESTINATION ${OPENVDB_HOUDINI_HELP_INSTALL_PREFIX})
 
+install(DIRECTORY glsl/
+  DESTINATION ${OPENVDB_HOUDINI_GLSL_INSTALL_PREFIX})
+
 if(OPENVDB_INSTALL_HOUDINI_PYTHONRC)
   install(FILES
     pythonrc.py

openvdb_houdini/openvdb_houdini/GEO_VDBTranslator.cc

@@ -108,7 +108,7 @@ GEO_VDBTranslator::fileStat(const char *filename, GA_Stat &stat, uint /*level*/)
     try {
         openvdb::io::File file(filename);
 
-        file.open(/*delayLoad*/false);
+        file.open();
 
         int             nprim = 0;
         UT_BoundingBox  bbox;
@@ -204,7 +204,7 @@ GEO_VDBTranslator::fileLoad(GEO_Detail *geogdp, UT_IStream &is, bool /*ate_magic
 
     try {
         // Create and open a VDB file, but don't read any grids yet.
-        openvdb::io::Stream file(*stdstream, /*delayLoad*/false);
+        openvdb::io::Stream file(*stdstream);
 
         // Read the file-level metadata into global attributes.
         openvdb::MetaMap::Ptr fileMetadata = file.getMetadata();

openvdb_houdini/openvdb_houdini/GR_PrimVDBPoints.cc

@@ -7,8 +7,6 @@
 ///
 /// @brief GR Render Hook and Primitive for VDB PointDataGrid
 
-#include <UT/UT_Version.h>
-
 #include <openvdb/Grid.h>
 #include <openvdb/Platform.h>
 #include <openvdb/Types.h>
@@ -30,6 +28,18 @@
 #include <RE/RE_VertexArray.h>
 #include <UT/UT_DSOVersion.h>
 #include <UT/UT_UniquePtr.h>
+#include <UT/UT_Version.h>
+
+#if UT_VERSION_INT >= 0x15000000 && defined(USE_VULKAN) // 21.0 or later - Vulkan support
+#include <GR/GR_Uniforms.h>
+#include <RV/RV_Geometry.h>
+#include <RV/RV_Render.h>
+#include <RV/RV_ShaderProgram.h>
+#include <RV/RV_ShaderBlock.h>
+#include <RV/RV_ShaderVariableSet.h>
+#include <RV/RV_VKBuffer.h>
+#include <RV/RV_VKDescriptorSet.h>
+#endif
 
 #include <iostream>
 #include <limits>
@@ -39,19 +49,19 @@
 #include <utility>
 #include <vector>
 
-#if UT_VERSION_INT < 0x14000000 // Below 20.0, there is no RE_RenderContext
-#define RE_RenderContext  RE_Render *
-#endif
-
 ////////////////////////////////////////
 
 static RE_ShaderHandle theMarkerDecorShader("decor/GL32/point_marker.prog");
 static RE_ShaderHandle theNormalDecorShader("decor/GL32/point_normal.prog");
 static RE_ShaderHandle theVelocityDecorShader("decor/GL32/user_point_vector3.prog");
-static RE_ShaderHandle theLineShader("basic/GL32/wire_color.prog");
 static RE_ShaderHandle thePixelShader("particle/GL32/pixel.prog");
 static RE_ShaderHandle thePointShader("particle/GL32/point.prog");
 
+#if UT_VERSION_INT >= 0x15000000 && defined(USE_VULKAN) // 21.0 or later - Vulkan support
+static RV_ShaderProgram* theVkPointShader = nullptr;
+static RV_ShaderProgram* theVkVelocityShader = nullptr;
+#endif
+
 /// @note  An additional scale for velocity trails to accurately match
 ///        the visualization of velocity for Houdini points
 #define VELOCITY_DECOR_SCALE -0.041f;
@@ -120,11 +130,8 @@ public:
 
     /// return true if the primitive is in or overlaps the view frustum.
     /// always returning true will effectively disable frustum culling.
-    bool inViewFrustum(const UT_Matrix4D &objviewproj
-#if (UT_VERSION_INT >= 0x1105014e) // 17.5.334 or later
-                       , const UT_BoundingBoxD *bbox
-#endif
-                       ) override;
+    bool inViewFrustum(const UT_Matrix4D &objviewproj,
+                       const UT_BoundingBoxD *bbox) override;
 
     /// Called whenever the primitive is required to render, which may be more
     /// than one time per viewport redraw (beauty, shadow passes, wireframe-over)
@@ -145,10 +152,6 @@ protected:
                          const openvdb::points::PointDataGrid& grid,
                          const RE_CacheVersion& version);
 
-    void updateWireBuffer(RE_Render* r,
-                          const openvdb::points::PointDataGrid& grid,
-                          const RE_CacheVersion& version);
-
     bool updateVec3Buffer(RE_Render* r,
                           const openvdb::points::PointDataGrid& grid,
                           const std::string& attributeName,
@@ -162,12 +165,35 @@ protected:
 
     void removeBuffer(const std::string& name);
 
+#if UT_VERSION_INT >= 0x15000000 && defined(USE_VULKAN) // 21.0 or later - Vulkan support
+    void updatePosBufferVk(RV_Render* r,
+                           const openvdb::points::PointDataGrid& grid,
+                           const RE_CacheVersion& version);
+
+    bool updateVec3BufferVk(RV_Render* r,
+                            const openvdb::points::PointDataGrid& grid,
+                            const std::string& attributeName,
+                            const std::string& bufferName,
+                            const RE_CacheVersion& version);
+#endif
+
 private:
     UT_UniquePtr<RE_Geometry> myGeo;
-    UT_UniquePtr<RE_Geometry> myWire;
     bool mDefaultPointColor = true;
     openvdb::Vec3f mCentroid{0, 0, 0};
     openvdb::BBoxd mBbox;
+#if UT_VERSION_INT >= 0x15000000 && defined(USE_VULKAN) // 21.0 or later - Vulkan support
+    UT_UniquePtr<RV_Geometry> myGeoVk;
+    UT_UniquePtr<RV_ShaderVariableSet> myVkObjectSet;
+    UT_UniquePtr<RV_ShaderBlock> myVkObjectBlock;
+    UT_UniquePtr<RV_ShaderVariableSet> myVkDrawingSet;
+    UT_UniquePtr<RV_ShaderBlock> myVkGeoBlock;
+    // velocity shader uniform state (separate to avoid clobbering point shader)
+    UT_UniquePtr<RV_ShaderVariableSet> myVkVelObjectSet;
+    UT_UniquePtr<RV_ShaderBlock> myVkVelObjectBlock;
+    bool mHasNormals = false;
+    bool mHasVelocity = false;
+#endif
 };
 
 
@@ -252,10 +278,15 @@ bool patchShader(RE_Render* r, RE_ShaderHandle& shader, RE_ShaderType type,
     shader->getShaderSource(r, source, type);
     const int version = shader->getCodeVersion();
 
+    // normalize whitespace (collapse tabs and runs of spaces to single space)
+
+    source.substitute("\t", " ");
+    while (source.substitute("  ", " ")) {}
+
     // patch the shader to replace the strings
 
     for (const auto& stringPair : stringReplacements) {
-        source.substitute(stringPair.first.c_str(), stringPair.second.c_str(), /*all=*/true);
+        source.substitute(stringPair.first.c_str(), stringPair.second.c_str());
     }
 
     // patch the shader to insert the strings
@@ -312,9 +343,6 @@ void patchShaderNoRedeclarations(RE_Render* r, RE_ShaderHandle& shader)
 {
     static const std::vector<StringPair> stringReplacements
     {
-        StringPair("\t", " "),
-        StringPair("  ", " "),
-        StringPair("  ", " "),
         StringPair("uniform vec2 glH_DepthProject;", "//uniform vec2 glH_DepthProject;"),
         StringPair("uniform vec2 glH_ScreenSize", "//uniform vec2 glH_ScreenSize")
     };
@@ -354,84 +382,6 @@ GR_PrimVDBPoints::acceptPrimitive(GT_PrimitiveType,
     return GR_NOT_PROCESSED;
 }
 
-namespace gr_primitive_internal
-{
-
-struct FillGPUBuffersLeafBoxes
-{
-    FillGPUBuffersLeafBoxes(UT_Vector3H* buffer,
-                         const std::vector<openvdb::Coord>& coords,
-                         const openvdb::math::Transform& transform,
-                         const openvdb::Vec3f& positionOffset)
-        : mBuffer(buffer)
-        , mCoords(coords)
-        , mTransform(transform)
-        , mPositionOffset(positionOffset) { }
-
-    void operator()(const tbb::blocked_range<size_t>& range) const
-    {
-        std::vector<UT_Vector3H> corners;
-        corners.reserve(8);
-
-        for (size_t n = range.begin(), N = range.end(); n != N; ++n) {
-            const openvdb::Coord& origin = mCoords[n];
-
-            // define 8 corners
-
-            corners.clear();
-
-            const openvdb::Vec3f pos000 = mTransform.indexToWorld(origin.asVec3d() + openvdb::Vec3f(0.0, 0.0, 0.0)) - mPositionOffset;
-            corners.emplace_back(pos000.x(), pos000.y(), pos000.z());
-            const openvdb::Vec3f pos001 = mTransform.indexToWorld(origin.asVec3d() + openvdb::Vec3f(0.0, 0.0, 8.0)) - mPositionOffset;
-            corners.emplace_back(pos001.x(), pos001.y(), pos001.z());
-            const openvdb::Vec3f pos010 = mTransform.indexToWorld(origin.asVec3d() + openvdb::Vec3f(0.0, 8.0, 0.0)) - mPositionOffset;
-            corners.emplace_back(pos010.x(), pos010.y(), pos010.z());
-            const openvdb::Vec3f pos011 = mTransform.indexToWorld(origin.asVec3d() + openvdb::Vec3f(0.0, 8.0, 8.0)) - mPositionOffset;
-            corners.emplace_back(pos011.x(), pos011.y(), pos011.z());
-            const openvdb::Vec3f pos100 = mTransform.indexToWorld(origin.asVec3d() + openvdb::Vec3f(8.0, 0.0, 0.0)) - mPositionOffset;
-            corners.emplace_back(pos100.x(), pos100.y(), pos100.z());
-            const openvdb::Vec3f pos101 = mTransform.indexToWorld(origin.asVec3d() + openvdb::Vec3f(8.0, 0.0, 8.0)) - mPositionOffset;
-            corners.emplace_back(pos101.x(), pos101.y(), pos101.z());
-            const openvdb::Vec3f pos110 = mTransform.indexToWorld(origin.asVec3d() + openvdb::Vec3f(8.0, 8.0, 0.0)) - mPositionOffset;
-            corners.emplace_back(pos110.x(), pos110.y(), pos110.z());
-            const openvdb::Vec3f pos111 = mTransform.indexToWorld(origin.asVec3d() + openvdb::Vec3f(8.0, 8.0, 8.0)) - mPositionOffset;
-            corners.emplace_back(pos111.x(), pos111.y(), pos111.z());
-
-            openvdb::Index64 offset = n*8*3;
-
-            // Z axis
-
-            mBuffer[offset++] = corners[0]; mBuffer[offset++] = corners[1];
-            mBuffer[offset++] = corners[2]; mBuffer[offset++] = corners[3];
-            mBuffer[offset++] = corners[4]; mBuffer[offset++] = corners[5];
-            mBuffer[offset++] = corners[6]; mBuffer[offset++] = corners[7];
-
-            // Y axis
-
-            mBuffer[offset++] = corners[0]; mBuffer[offset++] = corners[2];
-            mBuffer[offset++] = corners[1]; mBuffer[offset++] = corners[3];
-            mBuffer[offset++] = corners[4]; mBuffer[offset++] = corners[6];
-            mBuffer[offset++] = corners[5]; mBuffer[offset++] = corners[7];
-
-            // X axis
-
-            mBuffer[offset++] = corners[0]; mBuffer[offset++] = corners[4];
-            mBuffer[offset++] = corners[1]; mBuffer[offset++] = corners[5];
-            mBuffer[offset++] = corners[2]; mBuffer[offset++] = corners[6];
-            mBuffer[offset++] = corners[3]; mBuffer[offset++] = corners[7];
-        }
-    }
-
-    //////////
-
-    UT_Vector3H*                        mBuffer;
-    const std::vector<openvdb::Coord>&  mCoords;
-    const openvdb::math::Transform&     mTransform;
-    const openvdb::Vec3f                mPositionOffset;
-}; // class FillGPUBuffersLeafBoxes
-
-} // namespace gr_primitive_internal
-
 void
 GR_PrimVDBPoints::computeCentroid(const openvdb::points::PointDataGrid& grid)
 {
@@ -530,12 +480,49 @@ private:
     UT_Vector3H* mBuffer;
 }; // struct VectorAttribute
 
+struct VectorAttribute4H
+{
+    using ValueType = Vec3f;
+
+    struct Handle
+    {
+        Handle(VectorAttribute4H& attribute)
+            : mBuffer(attribute.mBuffer) { }
+
+        template <typename ValueType>
+        void set(openvdb::Index offset,
+                 openvdb::Index /*stride*/,
+                 const openvdb::math::Vec3<ValueType>& value)
+        {
+            mBuffer[offset] = UT_Vector4H(
+                fpreal16(float(value.x())),
+                fpreal16(float(value.y())),
+                fpreal16(float(value.z())),
+                fpreal16(0));
+        }
+
+    private:
+        UT_Vector4H* mBuffer;
+    }; // struct Handle
+
+    VectorAttribute4H(UT_Vector4H* buffer)
+        : mBuffer(buffer) { }
+
+    void expand() { }
+    void compact() { }
+
+private:
+    UT_Vector4H* mBuffer;
+}; // struct VectorAttribute4H
+
 void
 GR_PrimVDBPoints::updatePosBuffer(RE_Render* r,
              const openvdb::points::PointDataGrid& grid,
              const RE_CacheVersion& version)
 {
-    const bool gl3 = (getRenderVersion() >= GR_RENDER_GL3);
+    const GR_RenderVersion renderVersion = getRenderVersion();
+    const bool gl3 = (renderVersion == GR_RENDER_GL3
+                      || renderVersion == GR_RENDER_GL4);
 
     // Initialize the geometry with the proper name for the GL cache
     if (!myGeo) myGeo.reset(new RE_Geometry);
@@ -563,10 +550,10 @@ GR_PrimVDBPoints::updatePosBuffer(RE_Render* r,
     int numPoints = 0;
     if (useGroup) {
         GroupFilter filter(groupName, iter->attributeSet());
-        numPoints = static_cast<int>(pointCount(tree, filter, /*inCoreOnly=*/true));
+        numPoints = static_cast<int>(pointCount(tree, filter));
     } else {
         NullFilter filter;
-        numPoints = static_cast<int>(pointCount(tree, filter, /*inCoreOnly=*/true));
+        numPoints = static_cast<int>(pointCount(tree, filter));
     }
 
     if (numPoints == 0) return;
@@ -596,13 +583,13 @@ GR_PrimVDBPoints::updatePosBuffer(RE_Render* r,
         MultiGroupFilter filter(includeGroups, excludeGroups, iter->attributeSet());
 
         std::vector<Index64> offsets;
-        pointOffsets(offsets, grid.tree(), filter, /*inCoreOnly=*/true);
+        pointOffsets(offsets, grid.tree(), filter);
 
         UT_UniquePtr<UT_Vector3F[]> pdata(new UT_Vector3F[numPoints]);
 
         PositionAttribute positionAttribute(pdata.get(), mCentroid);
         convertPointDataGridPosition(positionAttribute, grid, offsets,
-                                    /*startOffset=*/ 0, filter, /*inCoreOnly=*/true);
+                                    /*startOffset=*/ 0, filter);
 
         posGeo->setArray(r, pdata.get());
         posGeo->setCacheVersion(version);
@@ -631,97 +618,43 @@ GR_PrimVDBPoints::updatePosBuffer(RE_Render* r,
     myGeo->connectAllPrims(r, RE_GEO_WIRE_IDX, primType, nullptr, true);
 }
 
-void
-GR_PrimVDBPoints::updateWireBuffer(RE_Render *r,
-             const openvdb::points::PointDataGrid& grid,
-             const RE_CacheVersion& version)
-{
-    const bool gl3 = (getRenderVersion() >= GR_RENDER_GL3);
-
-    // Initialize the geometry with the proper name for the GL cache
-    if (!myWire) myWire.reset(new RE_Geometry);
-    myWire->cacheBuffers(getCacheName());
-
-    using GridType = openvdb::points::PointDataGrid;
-    using TreeType = GridType::TreeType;
-    using LeafNode = TreeType::LeafNodeType;
-
-    const TreeType& tree = grid.tree();
-    if (tree.leafCount() == 0)  return;
-
-    // count up total points ignoring any leaf nodes that are out of core
-
-    size_t outOfCoreLeaves = 0;
-    for (auto iter = tree.cbeginLeaf(); iter; ++iter) {
-        if (iter->buffer().isOutOfCore()) outOfCoreLeaves++;
-    }
-
-    if (outOfCoreLeaves == 0) return;
-
-    // Initialize the number of points for the wireframe box per leaf.
-
-    int numPoints = static_cast<int>(outOfCoreLeaves*8*3);
-    myWire->setNumPoints(numPoints);
-
-    // fetch wireframe position, if its cache version matches, no upload is required.
-
-    RE_VertexArray* posWire = myWire->findCachedAttrib(r, "P", RE_GPU_FLOAT16, 3, RE_ARRAY_POINT, true);
-
-    if (posWire->getCacheVersion() != version)
-    {
-        using gr_primitive_internal::FillGPUBuffersLeafBoxes;
-
-        // fill the wire data
-
-        UT_UniquePtr<UT_Vector3H[]> data(new UT_Vector3H[numPoints]);
-
-        std::vector<openvdb::Coord> coords;
-
-        for (auto iter = tree.cbeginLeaf(); iter; ++iter) {
-            const LeafNode& leaf = *iter;
-
-            // skip in-core leaf nodes (for use when delay loading VDBs)
-            if (!leaf.buffer().isOutOfCore()) continue;
-
-            coords.push_back(leaf.origin());
-        }
-
-        FillGPUBuffersLeafBoxes fill(data.get(), coords, grid.transform(), mCentroid);
-        const tbb::blocked_range<size_t> range(0, coords.size());
-        tbb::parallel_for(range, fill);
-
-        posWire->setArray(r, data.get());
-        posWire->setCacheVersion(version);
-    }
-
-    if (gl3)
-    {
-        // Extra constant inputs for the GL3 default shaders we are using.
-
-        fpreal32 uv[2]  = { 0.0, 0.0 };
-        fpreal32 alpha  = 1.0;
-        fpreal32 pnt    = 0.0;
-        UT_Matrix4F instance;
-        instance.identity();
-
-        myWire->createConstAttribute(r, "uv",    RE_GPU_FLOAT32, 2, uv);
-        myWire->createConstAttribute(r, "Alpha", RE_GPU_FLOAT32, 1, &alpha);
-        myWire->createConstAttribute(r, "pointSelection", RE_GPU_FLOAT32, 1,&pnt);
-        myWire->createConstAttribute(r, "instmat", RE_GPU_MATRIX4, 1,
-                    instance.data());
-    }
-
-    myWire->connectAllPrims(r, RE_GEO_WIRE_IDX, RE_PRIM_LINES, nullptr, true);
-}
-
 void
 GR_PrimVDBPoints::update(RE_RenderContext r,
              const GT_PrimitiveHandle &primh,
              const GR_UpdateParms &p)
 {
+#if UT_VERSION_INT >= 0x15000000 && defined(USE_VULKAN) // 21.0 or later - Vulkan support
+    if (r.isVulkan())
+    {
+        if (p.reason & (GR_GEO_CHANGED | GR_GEO_TOPOLOGY_CHANGED))
+        {
+            const GT_PrimVDB& gt_primVDB =
+                static_cast<const GT_PrimVDB&>(*primh);
+
+            const openvdb::GridBase* grid =
+                const_cast<GT_PrimVDB&>(gt_primVDB).getGrid();
+
+            const openvdb::points::PointDataGrid& pointDataGrid =
+                static_cast<const openvdb::points::PointDataGrid&>(*grid);
+
+            computeCentroid(pointDataGrid);
+            computeBbox(pointDataGrid);
+
+            RV_Render* rv = r.vkRender();
+            updatePosBufferVk(rv, pointDataGrid, p.geo_version);
+            mDefaultPointColor = !updateVec3BufferVk(
+                rv, pointDataGrid, "Cd", "Cd", p.geo_version);
+            updateVec3BufferVk(
+                rv, pointDataGrid, "N", "N", p.geo_version);
+            updateVec3BufferVk(
+                rv, pointDataGrid, "v", "V", p.geo_version);
+        }
+        return;
+    }
+#endif
+
     // patch the point shaders at run-time to add an offset (does nothing if already patched)
 
-    patchShaderVertexOffset(r, theLineShader);
     patchShaderVertexOffset(r, thePixelShader);
     patchShaderVertexOffset(r, thePointShader);
 
@@ -749,27 +682,19 @@ GR_PrimVDBPoints::update(RE_RenderContext r,
         computeCentroid(pointDataGrid);
         computeBbox(pointDataGrid);
         updatePosBuffer(r, pointDataGrid, p.geo_version);
-        updateWireBuffer(r, pointDataGrid, p.geo_version);
 
         mDefaultPointColor = !updateVec3Buffer(r, pointDataGrid, "Cd", "Cd", p.geo_version);
     }
 }
 
 bool
-GR_PrimVDBPoints::inViewFrustum(const UT_Matrix4D& objviewproj
-#if (UT_VERSION_INT >= 0x1105014e) // 17.5.334 or later
-                                , const UT_BoundingBoxD *passed_bbox
-#endif
-                                )
+GR_PrimVDBPoints::inViewFrustum(const UT_Matrix4D& objviewproj,
+                                const UT_BoundingBoxD *passed_bbox)
 {
     const UT_BoundingBoxD bbox( mBbox.min().x(), mBbox.min().y(), mBbox.min().z(),
                                 mBbox.max().x(), mBbox.max().y(), mBbox.max().z());
-#if (UT_VERSION_INT >= 0x1105014e) // 17.5.334 or later
-    return GR_Utils::inViewFrustum(passed_bbox ? *passed_bbox :bbox,
+    return GR_Utils::inViewFrustum(passed_bbox ? *passed_bbox : bbox,
                                    objviewproj);
-#else
-    return GR_Utils::inViewFrustum(bbox, objviewproj);
-#endif
 }
 
 bool
@@ -826,12 +751,12 @@ GR_PrimVDBPoints::updateVec3Buffer(RE_Render* r,
             MultiGroupFilter filter(includeGroups, excludeGroups, iter->attributeSet());
 
             std::vector<Index64> offsets;
-            pointOffsets(offsets, grid.tree(), filter, /*inCoreOnly=*/true);
+            pointOffsets(offsets, grid.tree(), filter);
 
             VectorAttribute<Vec3f> typedAttribute(data.get());
             convertPointDataGridAttribute(typedAttribute, grid.tree(), offsets,
                 /*startOffset=*/ 0, static_cast<unsigned>(index), /*stride=*/1,
-                filter, /*inCoreOnly=*/true);
+                filter);
         }
 
         bufferGeo->setArray(r, data.get());
@@ -864,14 +789,294 @@ GR_PrimVDBPoints::removeBuffer(const std::string& name)
     myGeo->clearAttribute(name.c_str());
 }
 
+
+#if UT_VERSION_INT >= 0x15000000 && defined(USE_VULKAN) // 21.0 or later - Vulkan support
+
+void
+GR_PrimVDBPoints::updatePosBufferVk(RV_Render* r,
+             const openvdb::points::PointDataGrid& grid,
+             const RE_CacheVersion& version)
+{
+    using GridType = openvdb::points::PointDataGrid;
+    using TreeType = GridType::TreeType;
+    using AttributeSet = openvdb::points::AttributeSet;
+
+    const TreeType& tree = grid.tree();
+    auto iter = tree.cbeginLeaf();
+    if (!iter) return;
+
+    const AttributeSet::Descriptor& descriptor =
+        iter->attributeSet().descriptor();
+
+    // check if group viewport is in use
+
+    const openvdb::StringMetadata::ConstPtr s =
+        grid.getMetadata<openvdb::StringMetadata>(
+            openvdb_houdini::META_GROUP_VIEWPORT);
+
+    const std::string groupName = s ? s->value() : "";
+    const bool useGroup =
+        !groupName.empty() && descriptor.hasGroup(groupName);
+
+    // count up total points
+
+    int numPoints = 0;
+    if (useGroup) {
+        GroupFilter filter(groupName, iter->attributeSet());
+        numPoints = static_cast<int>(pointCount(tree, filter));
+    } else {
+        NullFilter filter;
+        numPoints = static_cast<int>(pointCount(tree, filter));
+    }
+
+    if (numPoints == 0) return;
+
+    const size_t positionIndex = descriptor.find("P");
+    if (positionIndex == AttributeSet::INVALID_POS) return;
+
+    // check for decoration attributes
+
+    mHasNormals = (descriptor.find("N") != AttributeSet::INVALID_POS);
+    mHasVelocity = (descriptor.find("v") != AttributeSet::INVALID_POS);
+
+    // (re)create the VK geometry
+
+    myGeoVk.reset(new RV_Geometry);
+    myGeoVk->setName("vdb_points");
+    myGeoVk->setNumPoints(numPoints);
+    myGeoVk->createAttribute("P", RV_GPU_FLOAT32, 3);
+    myGeoVk->createAttribute("Cd", RV_GPU_FLOAT16, 4);
+    if (mHasNormals)
+        myGeoVk->createAttribute("N", RV_GPU_FLOAT16, 4);
+    if (mHasVelocity)
+        myGeoVk->createAttribute("V", RV_GPU_FLOAT16, 4);
+    myGeoVk->createConstant("pointSelection", RV_GPU_UINT32, 1);
+    myGeoVk->connectAllPrims(0, RV_PRIM_POINTS);
+    if (mHasVelocity)
+        myGeoVk->connectAllPrims(1, RV_PRIM_PATCHES, /*patch_size=*/1);
+    myGeoVk->populateBuffers(r);
+
+    // set pointSelection to zero (no selection)
+
+    myGeoVk->setAttributeConstValue("pointSelection", 0.0);
+
+    // fill the position buffer (world-space, no offset subtraction)
+
+    {
+        std::vector<Name> includeGroups, excludeGroups;
+        if (useGroup) includeGroups.emplace_back(groupName);
+
+        MultiGroupFilter filter(
+            includeGroups, excludeGroups, iter->attributeSet());
+
+        std::vector<Index64> offsets;
+        pointOffsets(offsets, grid.tree(), filter);
+
+        UT_UniquePtr<UT_Vector3F[]> pdata(new UT_Vector3F[numPoints]);
+
+        // use zero offset so positions are stored in world space
+        PositionAttribute positionAttribute(
+            pdata.get(), Vec3f(0, 0, 0));
+        convertPointDataGridPosition(
+            positionAttribute, grid, offsets,
+            /*startOffset=*/ 0, filter);
+
+        const exint byteSize =
+            static_cast<exint>(numPoints) * sizeof(UT_Vector3F);
+        myGeoVk->getAttribute("P")->uploadData(r, pdata.get(), byteSize);
+    }
+
+    // set a default Cd (will be overwritten if Cd attribute exists)
+
+    myGeoVk->setAttributeConstVecValue(
+        "Cd", UT_Vector4F(0.6f, 0.6f, 0.5f, 1.0f));
+
+}
+
+
+bool
+GR_PrimVDBPoints::updateVec3BufferVk(RV_Render* r,
+             const openvdb::points::PointDataGrid& grid,
+             const std::string& attributeName,
+             const std::string& bufferName,
+             const RE_CacheVersion& version)
+{
+    if (!myGeoVk) return false;
+
+    using GridType = openvdb::points::PointDataGrid;
+    using TreeType = GridType::TreeType;
+    using AttributeSet = openvdb::points::AttributeSet;
+
+    const TreeType& tree = grid.tree();
+    auto iter = tree.cbeginLeaf();
+    if (!iter) return false;
+
+    const int numPoints = static_cast<int>(myGeoVk->getNumPoints());
+    if (numPoints == 0) return false;
+
+    const AttributeSet::Descriptor& descriptor =
+        iter->attributeSet().descriptor();
+    const size_t index = descriptor.find(attributeName);
+
+    if (index == AttributeSet::INVALID_POS) return false;
+
+    const openvdb::Name& type = descriptor.type(index).first;
+    if (type != "vec3s") return false;
+
+    // check if group viewport is in use
+
+    const openvdb::StringMetadata::ConstPtr s =
+        grid.getMetadata<openvdb::StringMetadata>(
+            openvdb_houdini::META_GROUP_VIEWPORT);
+
+    const std::string groupName = s ? s->value() : "";
+    const bool useGroup =
+        !groupName.empty() && descriptor.hasGroup(groupName);
+
+    std::vector<Name> includeGroups, excludeGroups;
+    if (useGroup) includeGroups.emplace_back(groupName);
+
+    MultiGroupFilter filter(
+        includeGroups, excludeGroups, iter->attributeSet());
+
+    std::vector<Index64> offsets;
+    pointOffsets(offsets, grid.tree(), filter);
+
+    // read vec3s attribute data directly into 4-component float16
+    // (VK_FORMAT_R16G16B16A16_SFLOAT is mandatory; 3-component is not)
+
+    UT_UniquePtr<UT_Vector4H[]> data(new UT_Vector4H[numPoints]);
+
+    VectorAttribute4H typedAttribute(data.get());
+    convertPointDataGridAttribute(typedAttribute, grid.tree(), offsets,
+        /*startOffset=*/ 0, static_cast<unsigned>(index),
+        /*stride=*/1, filter);
+
+    const exint byteSize =
+        static_cast<exint>(numPoints) * sizeof(UT_Vector4H);
+    RV_VKBuffer* buf = myGeoVk->getAttribute(bufferName.c_str());
+    if (buf) {
+        buf->uploadData(r, data.get(), byteSize);
+    }
+
+    return true;
+}
+
+#endif // UT_VERSION_INT >= 0x15000000 && defined(USE_VULKAN)
+
+
 void
 GR_PrimVDBPoints::render(RE_RenderContext r, GR_RenderMode, GR_RenderFlags, GR_DrawParms dp)
 {
-    if (!myGeo && !myWire)  return;
+#if UT_VERSION_INT >= 0x15000000 && defined(USE_VULKAN) // 21.0 or later - Vulkan support
+    if (r.isVulkan())
+    {
+        if (!myGeoVk)  return;
 
-    const bool gl3 = (getRenderVersion() >= GR_RENDER_GL3);
+        RV_Render* rv = r.vkRender();
+        GR_Uniforms* uniforms = r.uniforms();
 
-    if (!gl3)   return;
+        // initialize the VK point shader on first use
+
+        if (!theVkPointShader) {
+            theVkPointShader = RV_ShaderProgram::loadShaderProgram(
+                rv->instance(), "openvdb/VK/points.prog");
+            if (!theVkPointShader) return;
+        }
+
+        // set the shader
+
+        rv->setShader(theVkPointShader);
+
+        // bind global uniform blocks (pass info, object transforms)
+
+        if (uniforms) {
+            bool globalBound = uniforms->bindRVGlobalBlock(rv, theVkPointShader);
+
+            // bind set 1 (glH_Object) for per-object transform uniforms
+            if (theVkPointShader->hasSet(1)) {
+                const auto* objBinding =
+                    theVkPointShader->getBinding(1, 0);
+                if (objBinding) {
+                    if (!myVkObjectBlock) {
+                        myVkObjectBlock.reset(
+                            RV_ShaderBlock::create(
+                                rv->instance(), *objBinding));
+                    }
+                    if (!myVkObjectSet ||
+                        !theVkPointShader->isSetCompatible(
+                            *myVkObjectSet)) {
+                        myVkObjectSet =
+                            theVkPointShader->createSet(
+                                rv->instance(), 1);
+                    }
+                    uniforms->assignRVBlock(
+                        rv, myVkObjectBlock.get(), theVkPointShader);
+                    // override DecorationScale with the user's
+                    // point size from display options
+                    const GR_CommonDispOption& commonOpts =
+                        dp.opts->common();
+                    myVkObjectBlock->bindFloat(
+                        "DecorationScale",
+                        static_cast<float>(commonOpts.pointSize()));
+                    myVkObjectBlock->uploadBuffer(rv);
+                    myVkObjectSet->attachBufferBlock(
+                        rv->instance(), "Object",
+                        myVkObjectBlock.get());
+                    rv->bindSet(
+                        myVkObjectSet.get(), theVkPointShader);
+                }
+            }
+
+            // bind set 2 (auto-injected Geometry uniform block)
+            if (theVkPointShader->hasSet(2)) {
+                const auto* geoBinding =
+                    theVkPointShader->getBinding(2, 2);
+                if (geoBinding) {
+                    if (!myVkGeoBlock) {
+                        myVkGeoBlock.reset(
+                            RV_ShaderBlock::create(
+                                rv->instance(), *geoBinding));
+                    }
+                    if (!myVkDrawingSet ||
+                        !theVkPointShader->isSetCompatible(
+                            *myVkDrawingSet)) {
+                        myVkDrawingSet =
+                            theVkPointShader->createSet(
+                                rv->instance(), 2);
+                    }
+                    myVkGeoBlock->uploadBuffer(rv);
+                    myVkDrawingSet->attachBufferBlock(
+                        rv->instance(), "Geometry",
+                        myVkGeoBlock.get());
+                    rv->bindSet(
+                        myVkDrawingSet.get(), theVkPointShader);
+                }
+            }
+
+        }
+
+        // draw - must be inside beginRendering/endRendering block
+
+        bool beganRendering = false;
+        if (!rv->isRendering()) {
+            beganRendering = rv->beginRendering();
+        }
+
+        rv->draw(myGeoVk.get(), 0);
+
+        if (beganRendering) {
+            rv->endRendering();
+        }
+        return;
+    }
+#endif
+
+    if (!myGeo)  return;
+
+    const GR_RenderVersion renderVersion = getRenderVersion();
+    if (renderVersion != GR_RENDER_GL3 && renderVersion != GR_RENDER_GL4)
+        return;
 
     const GR_CommonDispOption& commonOpts = dp.opts->common();
 
@@ -911,35 +1116,133 @@ GR_PrimVDBPoints::render(RE_RenderContext r, GR_RenderMode, GR_RenderFlags, GR_D
 
         r->popShader();
     }
-
-    // draw leaf bboxes
-
-    if (myWire && myWire->getNumPoints() > 0) {
-
-        // bind the shader
-
-        r->pushShader();
-        r->bindShader(theLineShader);
-
-        // bind the position offset
-
-        UT_Vector3F positionOffset(mCentroid.x(), mCentroid.y(), mCentroid.z());
-        theLineShader->bindVector(r, "offset", positionOffset);
-
-        fpreal32 constcol[3] = { 0.6f, 0.6f, 0.6f };
-        myWire->createConstAttribute(r, "Cd", RE_GPU_FLOAT32, 3, constcol);
-
-        r->pushLineWidth(commonOpts.wireWidth());
-        myWire->draw(r, RE_GEO_WIRE_IDX);
-        r->popLineWidth();
-        r->popShader();
-    }
 }
 
 
 void
 GR_PrimVDBPoints::renderDecoration(RE_RenderContext r, GR_Decoration decor, const GR_DecorationParms& p)
 {
+#if UT_VERSION_INT >= 0x15000000 && defined(USE_VULKAN) // 21.0 or later - Vulkan support
+    if (r.isVulkan())
+    {
+        if (!myGeoVk) return;
+
+        if (decor == GR_POINT_MARKER)
+        {
+            drawDecorationForGeo(r, myGeoVk.get(), decor, p.opts,
+                GR_DECOR_RENDER_FLAG_NONE,
+                /*overlay=*/false, /*override_vis=*/false,
+                /*instance_group=*/-1, GR_SELECT_NONE,
+                GR_DecorationRender::PRIM_POINT);
+        }
+        else if (decor == GR_POINT_NORMAL && mHasNormals)
+        {
+            drawDecorationForGeo(r, myGeoVk.get(), decor, p.opts,
+                GR_DECOR_RENDER_FLAG_NONE,
+                /*overlay=*/false, /*override_vis=*/false,
+                /*instance_group=*/-1, GR_SELECT_NONE,
+                GR_DecorationRender::PRIM_POINT);
+        }
+        else if (decor == GR_POINT_VELOCITY && mHasVelocity)
+        {
+            // GPU-based velocity trail rendering using tessellation.
+            // The VK decoration system requires GR_GeoRenderVK for its
+            // built-in tessellation shaders, which custom GR_Primitive
+            // subclasses don't have. Instead, we use a custom velocity
+            // shader that reads P and V as vertex inputs and generates
+            // isolines via tessellation on the GPU.
+
+            RV_Render* rv = r.vkRender();
+            GR_Uniforms* uniforms = r.uniforms();
+
+            // load the velocity tessellation shader on first use
+
+            if (!theVkVelocityShader) {
+                theVkVelocityShader =
+                    RV_ShaderProgram::loadShaderProgram(
+                        rv->instance(),
+                        "openvdb/VK/velocity.prog");
+                if (!theVkVelocityShader) return;
+            }
+
+            rv->setShader(theVkVelocityShader);
+
+            // compute velocity scale and trail color
+
+            const GR_CommonDispOption& commonOpts =
+                p.opts->common();
+            const float velocityScale =
+                static_cast<float>(commonOpts.vectorScale())
+                * -0.041f;
+
+            UT_Color trailColor =
+                commonOpts.getColor(GR_POINT_TRAIL_COLOR);
+            float cr, cg, cb;
+            trailColor.getRGB(&cr, &cg, &cb);
+
+            // bind uniform blocks
+
+            if (uniforms) {
+                uniforms->bindRVGlobalBlock(
+                    rv, theVkVelocityShader);
+
+                // bind set 1 (glH_Object) with velocity overrides
+                if (theVkVelocityShader->hasSet(1)) {
+                    const auto* objBinding =
+                        theVkVelocityShader->getBinding(1, 0);
+                    if (objBinding) {
+                        if (!myVkVelObjectBlock) {
+                            myVkVelObjectBlock.reset(
+                                RV_ShaderBlock::create(
+                                    rv->instance(),
+                                    *objBinding));
+                        }
+                        if (!myVkVelObjectSet ||
+                            !theVkVelocityShader->isSetCompatible(
+                                *myVkVelObjectSet)) {
+                            myVkVelObjectSet =
+                                theVkVelocityShader->createSet(
+                                    rv->instance(), 1);
+                        }
+                        // fill with viewport transforms
+                        uniforms->assignRVBlock(rv,
+                            myVkVelObjectBlock.get(),
+                            theVkVelocityShader);
+                        // override decoration scale and wire color
+                        myVkVelObjectBlock->bindFloat(
+                            "DecorationScale", velocityScale);
+                        myVkVelObjectBlock->bindVector(
+                            "WireColor",
+                            UT_Vector4F(cr, cg, cb, 1.0f));
+                        myVkVelObjectBlock->uploadBuffer(rv);
+                        myVkVelObjectSet->attachBufferBlock(
+                            rv->instance(), "Object",
+                            myVkVelObjectBlock.get());
+                        rv->bindSet(myVkVelObjectSet.get(),
+                            theVkVelocityShader);
+                    }
+                }
+            }
+
+            // draw using PATCHES connection group (index 1)
+
+            bool beganRendering = false;
+            if (!rv->isRendering()) {
+                beganRendering = rv->beginRendering();
+            }
+
+            rv->draw(myGeoVk.get(), 1);
+
+            if (beganRendering) {
+                rv->endRendering();
+            }
+        }
+        return;
+    }
+#endif
+
+    if (!myGeo) return;
+
     // just render native GR_Primitive decorations if position not available
 
     const RE_VertexArray* const position = myGeo->getAttribute("P");

openvdb_houdini/openvdb_houdini/PointUtils.cc

@@ -1073,8 +1073,7 @@ convertPointDataGridToHoudini(
     const PointDataGrid& grid,
     const std::vector<std::string>& attributes,
     const std::vector<std::string>& includeGroups,
-    const std::vector<std::string>& excludeGroups,
-    const bool inCoreOnly)
+    const std::vector<std::string>& excludeGroups)
 {
     using namespace openvdb::math;
 
@@ -1096,14 +1095,14 @@ convertPointDataGridToHoudini(
     // obtain cumulative point offsets and total points
     std::vector<Index64> offsets;
     MultiGroupFilter filter(includeGroups, excludeGroups, leafIter->attributeSet());
-    const Index64 total = pointOffsets(offsets, tree, filter, inCoreOnly);
+    const Index64 total = pointOffsets(offsets, tree, filter);
 
     // a block's global offset is needed to transform its point offsets to global offsets
     const Index64 startOffset = detail.appendPointBlock(total);
 
     HoudiniWriteAttribute<Vec3f> positionAttribute(*detail.getP());
     convertPointDataGridPosition(positionAttribute, grid, offsets, startOffset,
-        filter, inCoreOnly);
+        filter);
 
     // add other point attributes to the hdk detail
     const AttributeSet::Descriptor::NameToPosMap& nameToPosMap = descriptor.map();
@@ -1186,87 +1185,87 @@ convertPointDataGridToHoudini(
         if (valueType == "string") {
             HoudiniWriteAttribute<Name> attribute(*attributeRef.getAttribute());
             convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride,
-                filter, inCoreOnly);
+                filter);
         }
         else if (valueType == "bool") {
             HoudiniWriteAttribute<bool> attribute(*attributeRef.getAttribute());
             convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride,
-                filter, inCoreOnly);
+                filter);
         }
         else if (valueType == "int8") {
             HoudiniWriteAttribute<int8_t> attribute(*attributeRef.getAttribute());
             convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride,
-                filter, inCoreOnly);
+                filter);
         }
         else if (valueType == "int16") {
             HoudiniWriteAttribute<int16_t> attribute(*attributeRef.getAttribute());
             convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride,
-                filter, inCoreOnly);
+                filter);
         }
         else if (valueType == "int32") {
             HoudiniWriteAttribute<int32_t> attribute(*attributeRef.getAttribute());
             convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride,
-                filter, inCoreOnly);
+                filter);
         }
         else if (valueType == "int64") {
             HoudiniWriteAttribute<int64_t> attribute(*attributeRef.getAttribute());
             convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride,
-                filter, inCoreOnly);
+                filter);
         }
         else if (valueType == "float") {
             HoudiniWriteAttribute<float> attribute(*attributeRef.getAttribute());
             convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride,
-                filter, inCoreOnly);
+                filter);
         }
         else if (valueType == "double") {
             HoudiniWriteAttribute<double> attribute(*attributeRef.getAttribute());
             convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride,
-                filter, inCoreOnly);
+                filter);
         }
         else if (valueType == "vec3i") {
             HoudiniWriteAttribute<Vec3<int> > attribute(*attributeRef.getAttribute());
             convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride,
-                filter, inCoreOnly);
+                filter);
         }
         else if (valueType == "vec3s") {
             HoudiniWriteAttribute<Vec3<float> > attribute(*attributeRef.getAttribute());
             convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride,
-                filter, inCoreOnly);
+                filter);
         }
         else if (valueType == "vec3d") {
             HoudiniWriteAttribute<Vec3<double> > attribute(*attributeRef.getAttribute());
             convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride,
-                filter, inCoreOnly);
+                filter);
         }
         else if (valueType == "quats") {
             HoudiniWriteAttribute<Quat<float> > attribute(*attributeRef.getAttribute());
             convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride,
-                filter, inCoreOnly);
+                filter);
         }
         else if (valueType == "quatd") {
             HoudiniWriteAttribute<Quat<double> > attribute(*attributeRef.getAttribute());
             convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride,
-                filter, inCoreOnly);
+                filter);
         }
         else if (valueType == "mat3s") {
             HoudiniWriteAttribute<Mat3<float> > attribute(*attributeRef.getAttribute());
             convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride,
-                filter, inCoreOnly);
+                filter);
         }
         else if (valueType == "mat3d") {
             HoudiniWriteAttribute<Mat3<double> > attribute(*attributeRef.getAttribute());
             convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride,
-                filter, inCoreOnly);
+                filter);
         }
         else if (valueType == "mat4s") {
             HoudiniWriteAttribute<Mat4<float> > attribute(*attributeRef.getAttribute());
             convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride,
-                filter, inCoreOnly);
+                filter);
         }
         else if (valueType == "mat4d") {
             HoudiniWriteAttribute<Mat4<double> > attribute(*attributeRef.getAttribute());
             convertPointDataGridAttribute(attribute, tree, offsets, startOffset, index, stride,
-                filter, inCoreOnly);
+                filter);
         }
         else {
             throw std::runtime_error("Unknown Attribute Type for Conversion: " + valueType);
@@ -1288,7 +1287,7 @@ convertPointDataGridToHoudini(
             attributeSet.groupIndex(name);
 
         HoudiniGroup group(*pointGroup, startOffset, total);
-        convertPointDataGridGroup(group, tree, offsets, startOffset, index, filter, inCoreOnly);
+        convertPointDataGridGroup(group, tree, offsets, startOffset, index, filter);
     }
 }
 
@@ -1615,32 +1614,7 @@ collectPointInfo(const PointDataGrid& grid,
 
     const PointDataTree& tree = grid.constTree();
 
-    // iterate through all leaf nodes to find out if all are out-of-core
-    bool allOutOfCore = true;
-    for (auto iter = tree.cbeginLeaf(); iter; ++iter) {
-        if (!iter->buffer().isOutOfCore()) {
-            allOutOfCore = false;
-            break;
-        }
-    }
-
-    openvdb::Index64 totalPointCount = 0;
-
-    // it is more technically correct to rely on the voxel count as this may be
-    // out of sync with the attribute size, however for faster node preview when
-    // the voxel buffers are all out-of-core, count up the sizes of the first
-    // attribute array instead
-
-    if (allOutOfCore) {
-        for (auto iter = tree.cbeginLeaf(); iter; ++iter) {
-            if (iter->attributeSet().size() > 0) {
-                totalPointCount += iter->constAttributeArray(0).size();
-            }
-        }
-    }
-    else {
-        totalPointCount = openvdb::points::pointCount(tree);
-    }
+    openvdb::Index64 totalPointCount = openvdb::points::pointCount(tree);
 
     std::ostringstream os;
     os << openvdb::util::formattedInt(totalPointCount);
@@ -1674,16 +1648,8 @@ collectPointInfo(const PointDataGrid& grid,
 
         os << it.first << "(";
 
-        // for faster node preview when all the voxel buffers are out-of-core,
-        // don't load the group arrays to display the group sizes, just print
-        // "out-of-core" instead @todo - put the group sizes into the grid
-        // metadata on write for this use case
-
-        if (allOutOfCore) os << "out-of-core";
-        else {
-            const openvdb::points::GroupFilter filter(it.first, attributeSet);
-            os << openvdb::util::formattedInt(pointCount(tree, filter));
-        }
+        const openvdb::points::GroupFilter filter(it.first, attributeSet);
+        os << openvdb::util::formattedInt(pointCount(tree, filter));
         os << ")";
     }
 

openvdb_houdini/openvdb_houdini/PointUtils.h

@@ -109,7 +109,6 @@ convertHoudiniToPointDataGrid(
 ///                        (empty vector defaults to all)
 /// @param  excludeGroups  a vector of VDB Points groups to be excluded
 ///                        (empty vector defaults to none)
-/// @param inCoreOnly      true if out-of-core leaf nodes are to be ignored
 OPENVDB_HOUDINI_API
 void
 convertPointDataGridToHoudini(
@@ -117,8 +116,7 @@ convertPointDataGridToHoudini(
     const openvdb::points::PointDataGrid& grid,
     const std::vector<std::string>& attributes = {},
     const std::vector<std::string>& includeGroups = {},
-    const std::vector<std::string>& excludeGroups = {},
-    const bool inCoreOnly = false);
+    const std::vector<std::string>& excludeGroups = {});
 
 
 /// @brief Populate VDB Points grid metadata from Houdini detail attributes

openvdb_houdini/openvdb_houdini/SOP_OpenVDB_Points_Convert.cc

@@ -693,23 +693,10 @@ SOP_OpenVDB_Points_Convert::Cache::cookVDBSop(OP_Context& context)
                 // all attributes should be converted
                 const std::vector<std::string> emptyNameVector;
 
-                // if all point data is being converted, sequentially pre-fetch any out-of-core
-                // data for faster performance when using delayed-loading
-
-                const bool allData =    emptyNameVector.empty() &&
-                                        includeGroups.empty() &&
-                                        excludeGroups.empty();
-
                 for (const PointDataGrid::ConstPtr &grid : pointGrids) {
 
                     GU_Detail geo;
 
-                    // if all the data is being loaded, prefetch it for faster load performance
-
-                    if (allData) {
-                        prefetch(grid->tree());
-                    }
-
                     // perform conversion
 
                     hvdb::convertPointDataGridToHoudini(

openvdb_houdini/openvdb_houdini/SOP_OpenVDB_Rasterize_Frustum.cc

@@ -516,59 +516,24 @@ struct GridsToRasterize
 
     struct Grid
     {
-    private:
-        static bool isTreeOutOfCore(const TreeType& tree)
-        {
-            using LeafManagerT = openvdb::tree::LeafManager<const TreeType>;
-            using LeafRangeT = typename LeafManagerT::LeafRange;
-            LeafManagerT leafManager(tree);
-            return tbb::parallel_reduce(leafManager.leafRange(), true,
-                [] (const LeafRangeT& range, bool result) -> bool {
-                    for (const auto& leaf : range) {
-                        if (!leaf.buffer().isOutOfCore()) return false;
-                    }
-                    return result;
-                },
-                [] (bool n, bool m) -> bool { return n || m; });
-        }
-
     public:
         explicit Grid(const ConstPtr& grid)
-            : mGrid(grid)
-            , mOutOfCore(isTreeOutOfCore(grid->constTree())) { }
-
-        inline bool isOutOfCore() const { return mOutOfCore; }
+            : mGrid(grid) { }
 
         template <typename RasterizerT>
         void addToRasterizer(RasterizerT& rasterizer)
         {
-            if (mOutOfCore) {
-                auto newGrid = mGrid->deepCopy();
-                rasterizer.addPoints(newGrid, /*stream=*/true);
-            }
-            else {
-                rasterizer.addPoints(mGrid);
-            }
+            rasterizer.addPoints(mGrid);
         }
 
     private:
         ConstPtr mGrid;
-        const bool mOutOfCore;
     }; // struct Grid
 
     void push_back(ConstPtr& grid) { mGrids.emplace_back(grid); }
     size_t size() const { return mGrids.size(); }
     bool empty() const { return mGrids.empty(); }
 
-    // return true if any of the grids is out-of-core
-    bool streaming() const
-    {
-        for (auto& grid : mGrids) {
-            if (grid.isOutOfCore()) return true;
-        }
-        return false;
-    }
-
     template <typename RasterizerT>
     void addGridToRasterizer(RasterizerT& rasterizer, size_t index)
     {
@@ -885,8 +850,7 @@ Rasterizing can be performed into cartesian and frustum grids (otherwise referre
 grids). For frustum grids, approximations are used by default to accelerate the rasterization.\n\
 This can be disabled to achieve a more accurate result.\n\
 \n\
-This node supports streaming of VDB points and attributes if one or more delayed-load VDB is\n\
-provided as an input.\n\
+This node supports streaming of VDB points and attributes\n\
 \n\
 @related\n\
 - [OpenVDB Rasterize Points|Node:sop/DW_OpenVDBRasterizePoints]\n\
@@ -1257,8 +1221,6 @@ SOP_OpenVDB_Rasterize_Frustum::cookVDBSop(OP_Context& context)
                     iterations = 1;
                 }
 
-                bool streaming = pointGrids.streaming();
-
                 for (size_t i = 0; i < iterations; i++) {
 
                     if (!mergeVDBPoints) {
@@ -1274,12 +1236,6 @@ SOP_OpenVDB_Rasterize_Frustum::cookVDBSop(OP_Context& context)
                     if (std::find(vectorAttribNames.begin(), vectorAttribNames.end(), velocityAttribute) !=
                         vectorAttribNames.end()) {
                         velocity = rasterizer.rasterizeAttribute(velocityAttribute, vectorMode, reduceMemory, scale, rasterGroups);
-
-                        // need to deep-copy input grids again if caches are being discarded
-                        if (streaming && reduceMemory) {
-                            if (mergeVDBPoints)     pointGrids.addGridsToRasterizer(rasterizer);
-                            else                    pointGrids.addGridToRasterizer(rasterizer, i);
-                        }
                     }
 
                     // rasterize density
@@ -1287,12 +1243,6 @@ SOP_OpenVDB_Rasterize_Frustum::cookVDBSop(OP_Context& context)
                     if (createDensity) {
                         auto density = rasterizer.rasterizeDensity(densityAttribute, densityMode, reduceMemory, densityScale, rasterGroups);
                         outputGrids.push_back(density);
-
-                        // need to deep-copy input grids again if caches are being discarded
-                        if (streaming && reduceMemory) {
-                            if (mergeVDBPoints)     pointGrids.addGridsToRasterizer(rasterizer);
-                            else                    pointGrids.addGridToRasterizer(rasterizer, i);
-                        }
                     }
 
                     // rasterize mask
@@ -1307,12 +1257,6 @@ SOP_OpenVDB_Rasterize_Frustum::cookVDBSop(OP_Context& context)
                     for (const auto& name : scalarAttribNames) {
                         auto scalar = rasterizer.rasterizeAttribute(name, scalarMode, reduceMemory, scale, rasterGroups);
                         outputGrids.push_back(scalar);
-
-                        // need to deep-copy input grids again if caches are being discarded
-                        if (streaming && reduceMemory) {
-                            if (mergeVDBPoints)     pointGrids.addGridsToRasterizer(rasterizer);
-                            else                    pointGrids.addGridToRasterizer(rasterizer, i);
-                        }
                     }
 
                     // rasterize vector attributes
@@ -1327,12 +1271,6 @@ SOP_OpenVDB_Rasterize_Frustum::cookVDBSop(OP_Context& context)
                         else {
                             auto vector = rasterizer.rasterizeAttribute(name, vectorMode, reduceMemory, scale, rasterGroups);
                             outputGrids.push_back(vector);
-
-                            // need to deep-copy input grids again if caches are being discarded
-                            if (streaming && reduceMemory) {
-                                if (mergeVDBPoints)     pointGrids.addGridsToRasterizer(rasterizer);
-                                else                    pointGrids.addGridToRasterizer(rasterizer, i);
-                            }
                         }
                     }
                 }