Test only one Windows job

Signed-off-by: Dan Bailey <danbailey@ilm.com>
Remove numpy installation from Windows CI
2025-10-31 10:35:28 -07:00 · 2025-10-31 10:21:52 -07:00 · 2025-10-31 08:09:02 +13:00 · 2025-10-30 09:58:36 -07:00 · 2025-10-30 08:07:47 -07:00 · 2025-10-29 23:13:15 -07:00
430 changed files with 39319 additions and 14565 deletions
@@ -55,7 +55,7 @@ jobs:
      github.event_name != 'workflow_dispatch' ||
      github.event.inputs.type == 'all' ||
      github.event.inputs.type == 'linux'
-    runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'ubuntu-20.04-8c-32g-300h') || 'ubuntu-latest' }}
+    runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'ubuntu-22.04-8c-32g-300h') || 'ubuntu-latest' }}
    name: >
      linux-ax:${{ matrix.config.image }}-cxx:${{ matrix.config.cxx }}-${{ matrix.config.build }}
    container:
@@ -67,21 +67,22 @@ jobs:
      matrix:
        # Only test unified builds (core+ax). weekly CI should test standalone
        config:
-          - { image: '2023-clang15', cxx: 'clang++', build: 'Release', cmake: '' }
-          - { image: '2023-clang15', cxx: 'g++',     build: 'Release', cmake: '' }
-          - { image: '2023-clang15', cxx: 'clang++', build: 'Debug',   cmake: '' }
-          - { image: '2022-clang11', cxx: 'clang++', build: 'Release', cmake: '' }
-          - { image: '2022-clang11', cxx: 'g++',     build: 'Release', cmake: '' }
+          - { image: '2026',           cxx: 'clang++', build: 'Release', cmake: '' }
+          - { image: '2026',           cxx: 'g++',     build: 'Release', cmake: '' }
+          - { image: '2026',           cxx: 'clang++', build: 'Debug',   cmake: '' }
+          - { image: '2025-clang19.1', cxx: 'clang++', build: 'Release', cmake: '' }
+          - { image: '2024-clang16.2', cxx: 'clang++', build: 'Release', cmake: '' }
+          # Needed to test LLVM 15.X
+          - { image: '2023-clang15',   cxx: 'clang++', build: 'Release', cmake: '-DDISABLE_DEPENDENCY_VERSION_CHECKS=ON' }
      fail-fast: false
    steps:
-      - name: Enable Node 16
-        if: contains(matrix.config.image, '2022')
-        run: |
-          echo "ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION=true" >> $GITHUB_ENV
      - uses: actions/checkout@v3
      - name: nanobind
-        #if: contains(matrix.config.image, '2023') == false
-        run: ./ci/install_nanobind.sh 2.0.0
+        run: ./ci/install_nanobind.sh 2.5.0
+        # 2026 images are broken with gtest/glfw installs
+      - name: deps
+        if: contains(matrix.config.image, '2026')
+        run: ./ci/install_gtest.sh latest
      - name: timestamp
        id: timestamp
        run: echo "timestamp=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
@@ -98,12 +99,11 @@ jobs:
        run: >
          ./ci/build.sh -v
          --build-type=${{ matrix.config.build }}
-          --components='core,python,bin,axcore,axbin,axtest'
+          --components='core,python,axcore,axbin,axtest'
          --cargs=\"
          ${{ matrix.config.cmake }}
          -DOPENVDB_AX_TEST_CMD_DOWNLOADS=ON
          -DUSE_EXPLICIT_INSTANTIATION=OFF
-          -DOPENVDB_CXX_STRICT=ON
          \"
      - name: test
        run: cd build && ctest -V
@@ -126,10 +126,13 @@ jobs:
    strategy:
      matrix:
        config:
-          #@note llvm10 never got its own brew formula...
-          # Last macos runner befor M1 (macos-14)
-          - { runner: 'macos-13', cxx: 'clang++', build: 'Release', llvm: '12' }
-          - { runner: 'macos-13', cxx: 'clang++', build: 'Release', llvm: '13' }
+          - { runner: 'macos-15', cxx: 'clang++', build: 'Release', llvm: '21' }
+          - { runner: 'macos-14', cxx: 'clang++', build: 'Release', llvm: '21' }
+          - { runner: 'macos-14', cxx: 'clang++', build: 'Release', llvm: '20' }
+          - { runner: 'macos-14', cxx: 'clang++', build: 'Release', llvm: '19' }
+          - { runner: 'macos-14', cxx: 'clang++', build: 'Release', llvm: '18' }
+          - { runner: 'macos-14', cxx: 'clang++', build: 'Release', llvm: '17' }
+          - { runner: 'macos-14', cxx: 'clang++', build: 'Release', llvm: '16' }
      fail-fast: false
    steps:
      - uses: actions/checkout@v3
@@ -141,7 +144,7 @@ jobs:
        run: >
          ./ci/build.sh -v
          --build-type=${{ matrix.config.build }}
-          --components="core,python,bin,axcore,axbin,axtest"
+          --components="core,python,axcore,axbin,axtest"
          --cargs=\"
          -DOPENVDB_AX_TEST_CMD_DOWNLOADS=ON
          -DUSE_EXPLICIT_INSTANTIATION=OFF
@@ -157,15 +160,15 @@ jobs:
    if: |
      github.event_name == 'workflow_dispatch' &&
      github.event.inputs.type == 'grammar'
-    runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'ubuntu-20.04-8c-32g-300h') || 'ubuntu-latest' }}
+    runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'ubuntu-22.04-8c-32g-300h') || 'ubuntu-latest' }}
    container:
-      image: aswf/ci-openvdb:2022-clang11
+      image: aswf/ci-openvdb:2025-clang19.1
    steps:
    - uses: actions/checkout@v3
    - name: build
      run: ./ci/build.sh -v --components=axgr --target=openvdb_ax_grammar --cargs=\"-DOPENVDB_AX_GRAMMAR_NO_LINES=ON\"
    - name: upload grammar
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
      with:
        name: ax_grammar
        path: ./build/openvdb_ax/openvdb_ax/openvdb_ax/grammar
@@ -33,11 +33,8 @@ on:
      - 'pendingchanges/**'
      - '**.md'
  schedule:
-    # run this workflow every day at 7am UTC except Monday
-    - cron:  '0 7 * * 0,2-6'
-    # run this workflow Monday 7am UTC
-    # warning: This pattern is checked in various places below
-    - cron:  '0 7 * * 1'
+    # run this workflow every day 7am UTC
+    - cron:  '0 7 * * *'
  workflow_dispatch:
    inputs:
      type:
@@ -62,12 +59,9 @@ jobs:
      github.event_name != 'workflow_dispatch' ||
      github.event.inputs.type == 'all' ||
      github.event.inputs.type == 'linux'
-    runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'ubuntu-20.04-8c-32g-300h') || 'ubuntu-latest' }}
+    runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'ubuntu-22.04-8c-32g-300h') || 'ubuntu-latest' }}
    name: >
-      linux-vfx:${{ matrix.config.image }}-
-      abi:${{ matrix.config.abi }}-
-      cxx:${{ matrix.config.cxx }}-
-      type:${{ matrix.config.build }}
+      linux:${{ matrix.config.image }}-abi:${{ matrix.config.abi }}-cxx:${{ matrix.config.cxx }}-type:${{ matrix.config.build }}
    container:
      image: aswf/ci-openvdb:${{ matrix.config.image }}
    env:
@@ -76,27 +70,22 @@ jobs:
    strategy:
      matrix:
        config:
-          - { cxx: clang++, image: '2024', abi: '11', build: 'Release', cmake: '' }
-          - { cxx: g++,     image: '2024', abi: '11', build: 'Release', cmake: '' }
-          - { cxx: clang++, image: '2024', abi: '11', build: 'Debug',   cmake: '' }
-          - { cxx: clang++, image: '2023', abi: '11', build: 'Release', cmake: '' }
-          - { cxx: g++,     image: '2023', abi: '11', build: 'Release', cmake: '' }
-          - { cxx: clang++, image: '2022', abi: '10', build: 'Release', cmake: '-DDISABLE_DEPENDENCY_VERSION_CHECKS=ON' }
-          - { cxx: g++,     image: '2022', abi: '10', build: 'Release', cmake: '-DDISABLE_DEPENDENCY_VERSION_CHECKS=ON' }
+          - { cxx: clang++, image: '2026',           abi: '13', build: 'Release', cmake: '-DOPENVDB_USE_FUTURE_ABI_13=ON' }
+          - { cxx: g++,     image: '2026',           abi: '13', build: 'Release', cmake: '-DOPENVDB_USE_FUTURE_ABI_13=ON' }
+          - { cxx: clang++, image: '2026',           abi: '13', build: 'Debug',   cmake: '-DOPENVDB_USE_FUTURE_ABI_13=ON' }
+          - { cxx: clang++, image: '2025-clang19.1', abi: '12', build: 'Release', cmake: '' }
+          - { cxx: clang++, image: '2024-clang17.2', abi: '11', build: 'Release', cmake: '-DDISABLE_DEPENDENCY_VERSION_CHECKS=ON' }
      fail-fast: false
    steps:
-    - name: Enable Node 16
-      # Solution taken from https://github.blog/changelog/2024-03-07-github-actions-all-actions-will-run-on-node20-instead-of-node16-by-default
-      if: contains(matrix.config.image, '2022')
-      run: |
-        echo "ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION=true" >> $GITHUB_ENV
    - uses: actions/checkout@v3
    - name: nanobind
-      #if: contains(matrix.config.image, '2023') == false
-      run: ./ci/install_nanobind.sh 2.0.0
-    - name: glfw
-      if: contains(matrix.config.image, '2023') == true
-      run: ./ci/install_glfw.sh 3.3.10
+      run: ./ci/install_nanobind.sh 2.5.0
+      # 2026 images are broken with gtest/glfw installs
+    - name: deps
+      if: contains(matrix.config.image, '2026')
+      run: |
+        ./ci/install_gtest.sh latest
+        ./ci/install_glfw.sh latest
    - name: timestamp
      id: timestamp
      run: echo "timestamp=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
@@ -120,9 +109,8 @@ jobs:
        -DOPENVDB_ABI_VERSION_NUMBER=${{ matrix.config.abi }}
        \"
    - name: test
-      # Always run tests on weekly builds but skip Debug on commits as they take a while.
-      # https://github.community/t/distinct-job-for-each-schedule/17811/2
-      if: contains(github.event.schedule, '0 7 * * 1') || matrix.config.build == 'Release'
+      # Skip Debug on commits as they take a while.
+      if: github.event_name == 'schedule'|| matrix.config.build == 'Release'
      run: |
        cd build && ctest -V
        cd - && ./ci/test_install.sh
@@ -178,9 +166,15 @@ jobs:
      github.event_name != 'workflow_dispatch' ||
      github.event.inputs.type == 'all' ||
      github.event.inputs.type == 'mac'
-    runs-on: macos-13 # Last macos runner befor M1 (macos-14)
+    runs-on: ${{ matrix.config.image }}
+    name: ${{ matrix.config.image }}
    env:
      CXX: clang++
+    strategy:
+      matrix:
+        config:
+          - { image: macos-14 }
+          - { image: macos-15 }
    steps:
    - uses: actions/checkout@v3
    - name: install
@@ -197,3 +191,4 @@ jobs:
        \'
    - name: test
      run: cd build && ctest -V
+
@@ -38,28 +38,15 @@ jobs:
    env:
      CXX: g++
    container:
-      # @note We can't update this as epydoc doesn't support python3. We'll
-      # need to re-write the python docs to use sphinx
-      image: aswf/ci-openvdb:2022
+      image: aswf/ci-openvdb:2026
    steps:
-    - name: Enable Node 16
-      run: |
-        echo "ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION=true" >> $GITHUB_ENV
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
    - name: install_doxygen
      run: ./ci/install_doxygen.sh 1_8_11
    - name: nanobind
-      #if: contains(matrix.config.image, '2023') == false
-      run: ./ci/install_nanobind.sh 2.0.0
-    # - name: install_epydoc
-    #   run: pip install epydoc
+      run: ./ci/install_nanobind.sh 2.5.0
    - name: install_latex
-      run: |
-        # Fix error: Cannot prepare internal mirrorlist: No URLs in mirrorlist. CentOS 8 reached EOL means need to replace the official mirror to vault.centos.org
-        # Comment out mirrorlist and replace #baseurl=...mirror.centos.org with baseurl=...vault.centos.org in files starting with CentOS- in /etc/yum.repos.d folder
-        sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-*
-        sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*
-        yum -y install texlive-latex-bin texlive-dvips texlive-collection-fontsrecommended texlive-collection-latexrecommended
+      run: ./ci/install_texlive.sh
    - name: build
      run: >
        ./ci/build.sh -v
@@ -72,9 +59,6 @@ jobs:
        -DUSE_EXPLICIT_INSTANTIATION=OFF
        -DDISABLE_DEPENDENCY_VERSION_CHECKS=ON
        \"
-    # - name: epydoc
-    #   run: |
-    #     epydoc --html -o /usr/local/share/doc/OpenVDB/html/python pyopenvdb
    - name: pre_deploy
      # Overwrite global SSH configuration
      #  https://github.com/peaceiris/actions-gh-pages/issues/719
@@ -92,7 +76,7 @@ jobs:
        github.event.inputs.deploy == 'docs'
      uses: peaceiris/actions-gh-pages@v3
      with:
-        deploy_key: ${{ secrets.ACTIONS_DEPLOY_KEY }}
+        deploy_key: ${{ secrets.PRIVATE_KEY }}
        publish_dir: /usr/local/share/doc/OpenVDB/html
        destination_dir: documentation/doxygen
        external_repository: AcademySoftwareFoundation/openvdb-website
@@ -113,7 +97,7 @@ jobs:
      github.event.inputs.deploy == 'coverage'
    runs-on: ubuntu-latest
    container:
-      image: aswf/ci-openvdb:2023
+      image: aswf/ci-openvdb:2025-clang19.1
    env:
      CXX: g++
    steps:
@@ -65,8 +65,8 @@ jobs:
    if: >
      ${{ needs.checksecret.outputs.HOUDINI_SECRETS == 'true' ||
          github.repository_owner == 'AcademySoftwareFoundation' }}
-    runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'ubuntu-20.04-8c-32g-300h') || 'ubuntu-latest' }}
-    name: hou:${{ matrix.config.hou_hash }}-vfx:${{ matrix.config.image }}-cxx:${{ matrix.config.cxx }}
+    runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'ubuntu-22.04-8c-32g-300h') || 'ubuntu-latest' }}
+    name: hou:${{ matrix.config.hou_hash }}-vfx:${{ matrix.config.image }}-cxx:${{ matrix.config.cxx }}-build:${{ matrix.config.build }}
    container:
      image: aswf/ci-base:${{ matrix.config.image }}
    env:
@@ -75,32 +75,18 @@ jobs:
    strategy:
      matrix:
        config:
-          - { cxx: clang++, image: '2024', hou_hash: '20_5', build: 'Release', components: 'core,hou,bin,view,render,python,test' }
-          - { cxx: clang++, image: '2023', hou_hash: '20_5', build: 'Debug',   components: 'core,hou,bin,view,render,python,test,axcore,axbin,axtest' }
-          - { cxx: g++,     image: '2023', hou_hash: '20_5', build: 'Release', components: 'core,hou,bin,view,render,python,test,axcore,axbin,axtest' }
-          - { cxx: clang++, image: '2023', hou_hash: '20_0-newabi', build: 'Release', components: 'core,hou,bin,view,render,python,test,axcore,axbin,axtest' }
-          - { cxx: g++,     image: '2023', hou_hash: '20_0-newabi', build: 'Release', components: 'core,hou,bin,view,render,python,test,axcore,axbin,axtest' }
-          - { cxx: clang++, image: '2022',   hou_hash: '20_0-oldabi', build: 'Release', components: 'core,hou' }
-          - { cxx: g++,     image: '2022',   hou_hash: '20_0-oldabi', build: 'Release', components: 'core,hou' }
+          - { cxx: clang++, image: '2026', hou_hash: '21_0', build: 'Release', components: 'core,hou,bin,view,render,python,test,axcore,axbin,axtest' }
+          - { cxx: clang++, image: '2026', hou_hash: '21_0', build: 'Debug',   components: 'core,hou,bin,view,render,python,test,axcore,axbin,axtest' }
+          - { cxx: g++,     image: '2026', hou_hash: '21_0', build: 'Release', components: 'core,hou,bin,view,render,python,test,axcore,axbin,axtest' }
+          - { cxx: clang++, image: '2024', hou_hash: '20_5', build: 'Release', components: 'core,hou,bin,view,render,python,test,axcore,axbin,axtest' }
+          - { cxx: g++,     image: '2024', hou_hash: '20_5', build: 'Release', components: 'core,hou,bin,view,render,python,test,axcore,axbin,axtest' }
      fail-fast: false
    steps:
-      # See note on this step in the Houdini weekly.yml job
-      # We can remove this when we no longer use < 2023 images
-    - name: Enable Node 16
-      run: |
-        echo "ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION=true" >> $GITHUB_ENV
-    - name: remove zstd
-      run: yum -y remove zstd
    - uses: actions/checkout@v3
    - name: nanobind
-      #if: contains(matrix.config.image, '2023') == false
-      run: ./ci/install_nanobind.sh 2.0.0
-    - name: glfw
-      if: contains(matrix.config.image, '2023') == true
-      run: ./ci/install_glfw.sh 3.3.10
-    - name: cppunit
-      if: contains(matrix.config.image, '2022') == false
-      run: ./ci/install_cppunit.sh 1.15.1
+      run: ./ci/install_nanobind.sh 2.5.0
+    - name: install_gtest
+      run: ./ci/install_gtest.sh latest
    - name: timestamp
      id: timestamp
      run: echo "timestamp=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
@@ -131,7 +117,7 @@ jobs:
    - name: build
      run: |
        ./ci/build.sh -v \
-          --build-type=Release \
+          --build-type="${{ matrix.config.build }}" \
          --components="${{ matrix.config.components }}" \
          --cargs=\" \
            -DHOUDINI_ROOT=$HOME/houdini_install/hou \
@@ -141,6 +127,7 @@ jobs:
            -DDISABLE_DEPENDENCY_VERSION_CHECKS=ON \
            \"
    - name: test
+      if: matrix.config.build == 'Release'
      run: cd build && ctest -V
    # Keep ccache light by stripping out any caches not accessed in the last day
    - name: ccache_clean
@@ -171,7 +158,7 @@ jobs:
        cp hou/hou.tar.gz $HOME/houdini_install/hou.tar.gz
        cd $HOME/houdini_install && tar -xzf hou.tar.gz && cd -
    - name: install_deps
-      run: ./ci/install_macos.sh 15
+      run: ./ci/install_macos.sh 20
    - name: build
      run: |
        ./ci/build.sh -v \
@@ -184,8 +171,7 @@ jobs:
            -DDISABLE_CMAKE_SEARCH_PATHS=ON \
            -DDISABLE_DEPENDENCY_VERSION_CHECKS=ON \
            -DUSE_EXPLICIT_INSTANTIATION=OFF \
-            -DTbb_INCLUDE_DIR=$HOME/houdini_install/hou/Frameworks/Houdini.framework/Versions/Current/Resources/toolkit/include/tbb \
-            -DLLVM_DIR=/opt/homebrew/opt/llvm@15/lib/cmake/llvm \
+            -DLLVM_DIR=/opt/homebrew/opt/llvm@20/lib/cmake/llvm \
            -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/install \
            \"
    - name: test
@@ -51,7 +51,7 @@ jobs:
      github.event_name != 'workflow_dispatch' ||
      github.event.inputs.type == 'all' ||
      github.event.inputs.type == 'linux'
-    runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'ubuntu-20.04-8c-32g-300h') || 'ubuntu-latest' }}
+    runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'ubuntu-22.04-8c-32g-300h') || 'ubuntu-latest' }}
    name: >
      linux-nanovdb:cxx:${{ matrix.config.cxx }}-${{ matrix.config.build }}
    container:
@@ -61,20 +61,15 @@ jobs:
    strategy:
      matrix:
        config:
-          - { cxx: g++,     image: '2024', build: 'Release' }
-          - { cxx: g++,     image: '2024', build: 'Debug' }
-          - { cxx: clang++, image: '2024', build: 'Release' }
-          - { cxx: clang++, image: '2024', build: 'Debug' }
+          - { cxx: g++,     image: '2024-clang17.2', build: 'Release' }
+          - { cxx: g++,     image: '2024-clang17.2', build: 'Debug' }
+          - { cxx: clang++, image: '2024-clang17.2', build: 'Release' }
+          - { cxx: clang++, image: '2024-clang17.2', build: 'Debug' }
      fail-fast: false
    steps:
-      - name: Enable Node 16
-        if: contains(matrix.config.image, '2022')
-        run: |
-          echo "ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION=true" >> $GITHUB_ENV
      - uses: actions/checkout@v3
      - name: setup_cuda_12
        run: |
-          yum -y install yum-utils
          echo "/usr/local/cuda-12/bin" >> $GITHUB_PATH
          echo "LD_LIBRARY_PATH=/usr/local/cuda-12/lib64:$LD_LIBRARY_PATH" >> $GITHUB_ENV
      - name: build
@@ -87,13 +82,14 @@ jobs:
          --cargs=\'
          -DUSE_EXPLICIT_INSTANTIATION=OFF
          -DNANOVDB_USE_CUDA=ON
+          -DCMAKE_CUDA_ARCHITECTURES="80"
          -DNANOVDB_USE_OPENVDB=ON
          -DCMAKE_INSTALL_PREFIX=`pwd`
          -DUSE_BLOSC=OFF
          -DCMAKE_POSITION_INDEPENDENT_CODE=ON
          \'
      - name: test
-        run: cd build && sudo ctest -V -E ".*cuda.*"
+        run: cd build && sudo ctest -V -E ".*cuda.*|.*mgpu.*"

  windows-nanovdb:
    if: |
@@ -121,6 +117,7 @@ jobs:
      shell: powershell
      run: .\ci\install_windows.ps1
    - name: build
+    # nvcc doesn't set _WIN32 when run in bash so we need to set it manually
      shell: bash
      run: >
        ./ci/build.sh -v
@@ -131,13 +128,15 @@ jobs:
        -DMSVC_COMPRESS_PDB=ON
        -DUSE_EXPLICIT_INSTANTIATION=OFF
        -DNANOVDB_USE_CUDA=ON
+        -DCMAKE_CUDA_ARCHITECTURES="80"
+        -DCMAKE_CUDA_FLAGS="-D_WIN32"
        -DNANOVDB_USE_OPENVDB=ON
        -DVCPKG_TARGET_TRIPLET=${VCPKG_DEFAULT_TRIPLET}
        -DCMAKE_TOOLCHAIN_FILE=\"${VCPKG_INSTALLATION_ROOT}\\scripts\\buildsystems\\vcpkg.cmake\"
        \'
    - name: test
      shell: bash
-      run: cd build && ctest -V -E ".*cuda.*"
+      run: cd build && ctest -V -E ".*cuda.*|.*mgpu.*"

  macos-nanovdb:
    if: |
@@ -150,8 +149,8 @@ jobs:
    strategy:
      matrix:
        config:
-          - { runner: 'macos-12', cxx: 'clang++', build: 'Release' }
-          - { runner: 'macos-12', cxx: 'clang++', build: 'Debug'   }
+          - { runner: 'macos-14', cxx: 'clang++', build: 'Release' }
+          - { runner: 'macos-14', cxx: 'clang++', build: 'Debug'   }
      fail-fast: false
    steps:
      - uses: actions/checkout@v3
@@ -164,22 +163,22 @@ jobs:
          ./ci/build.sh -v
          --build-type=${{ matrix.config.build }}
          --components=core,nano,nanotest,nanoexam,nanobench,nanotool
-          --cargs=\'-DUSE_EXPLICIT_INSTANTIATION=OFF -DNANOVDB_USE_CUDA=OFF -DNANOVDB_USE_OPENVDB=ON\'
+          --cargs=\'-DUSE_EXPLICIT_INSTANTIATION=OFF -DNANOVDB_USE_CUDA=OFF -DNANOVDB_USE_OPENVDB=ON -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/install \'
      - name: test
-        run: cd build && ctest -V -E ".*cuda.*"
+        run: cd build && ctest -V -E ".*cuda.*|.*mgpu.*"

  nanovdb-lite:
    if: |
      github.event_name != 'workflow_dispatch' ||
      github.event.inputs.type == 'all' ||
      github.event.inputs.type == 'linux'
-    runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'ubuntu-20.04-8c-32g-300h') || 'ubuntu-latest' }}
+    runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'ubuntu-22.04-8c-32g-300h') || 'ubuntu-latest' }}
+    container:
+      image: aswf/ci-openvdb:2024-clang17.2
    steps:
      - uses: actions/checkout@v3
      - name: install_gtest
-        run: |
-          sudo apt-get update
-          sudo apt-get -q install -y libgtest-dev
+        run: ./ci/install_gtest.sh latest
      - name: build_and_test
        run: |
          cd nanovdb/nanovdb
@@ -61,7 +61,7 @@ jobs:
      (github.event_name != 'workflow_dispatch' ||
       github.event.inputs.type == 'all' ||
       github.event.inputs.type == 'houdini')
-    runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'ubuntu-20.04-8c-32g-300h') || 'ubuntu-latest' }}
+    runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'ubuntu-22.04-8c-32g-300h') || 'ubuntu-latest' }}
    name: linux-houdini:${{ matrix.config.hou_hash }}
    env:
      CXX: clang++
@@ -70,28 +70,13 @@ jobs:
    strategy:
      matrix:
        config:
-          - { houdini_version: '19.5', platform: 'linux_x86_64_gcc9.3',  hou_hash: '19_5' }
-          - { houdini_version: '20.0', platform: 'linux_x86_64_gcc9.3',  hou_hash: '20_0-oldabi' }
-          - { houdini_version: '20.0', platform: 'linux_x86_64_gcc11.2', hou_hash: '20_0-newabi' }
+          - { houdini_version: '21.0', platform: 'linux_x86_64_gcc11.2', hou_hash: '21_0' }
          - { houdini_version: '20.5', platform: 'linux_x86_64_gcc11.2', hou_hash: '20_5' }
      fail-fast: false
    container:
      image: aswf/ci-base:2024
    steps:
-    - name: Enable Node 16
-      run: |
-        echo "ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION=true" >> $GITHUB_ENV
    - uses: actions/checkout@v3
-    # We bumped from the 2021 CI image to 2023 here to fix some OpenSSL issues
-    # with the Houdini download script. In so doing we broke some of the caching
-    # between this job and the jobs in houdini.yml which _don't_ use the 2023
-    # image yet. The issue is that the cache action will use zstd if it's
-    # available to zip the cache and this causes it to be inserted with a unique
-    # hash which images without zstd (i.e. the 2021/2022 images don't have
-    # access to). For now, uninstall zstd here instead of installing it
-    # everywhere and ask the LF to add zstd to the older base images.
-    - name: remove zstd
-      run: yum -y remove zstd
    - name: timestamp
      id: timestamp
      run: echo "timestamp=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
@@ -118,7 +103,7 @@ jobs:
    # Note that macos-14 (current macos-latest) switches to M1. We could instead test
    # the arm build here instead of the x86 one.
    runs-on: macos-latest
-    name: macos-houdini-20
+    name: macos-houdini
    env:
      HOUDINI_CLIENT_ID: ${{ secrets.HOUDINI_CLIENT_ID }}
      HOUDINI_SECRET_KEY: ${{ secrets.HOUDINI_SECRET_KEY }}
@@ -128,7 +113,7 @@ jobs:
      id: timestamp
      run: echo "timestamp=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
    - name: download_houdini
-      run: ./ci/download_houdini.sh 20.0 macosx_arm64_clang14.0_13 --prod
+      run: ./ci/download_houdini.sh 21.0 macosx_arm64_clang15.0_14 --prod
    - name: install_houdini
      run: |
        mkdir $HOME/houdini_install
@@ -151,34 +136,38 @@ jobs:
      github.event_name != 'workflow_dispatch' ||
      github.event.inputs.type == 'all' ||
      github.event.inputs.type == 'extra'
-    runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'ubuntu-20.04-8c-32g-300h') || 'ubuntu-latest' }}
+    runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'ubuntu-22.04-8c-32g-300h') || 'ubuntu-latest' }}
    name: linux-extra:${{ matrix.config.name }}
    container:
-      image: aswf/ci-openvdb:2024
+      image: aswf/ci-openvdb:${{ matrix.config.image }}
    env:
      CXX: clang++
    strategy:
      matrix:
        config:
-          - { name: 'all',   build: 'Release', components: 'core,python,bin,view,render,test',               cmake: '-DUSE_BLOSC=ON  -DUSE_ZLIB=ON  -DUSE_EXR=ON  -DUSE_PNG=ON'  }
-          - { name: 'lite',  build: 'Release', components: 'core,python,bin,view,render,test',               cmake: '-DUSE_BLOSC=OFF -DUSE_ZLIB=OFF -DUSE_EXR=OFF -DUSE_PNG=OFF -DOPENVDB_USE_DELAYED_LOADING=OFF' }
-          - { name: 'half',  build: 'Release', components: 'core,python,bin,view,render,test',               cmake: '-DUSE_BLOSC=OFF -DUSE_IMATH_HALF=ON' }
-          - { name: 'sse',   build: 'Release', components: 'core,python,bin,view,render,test',               cmake: '-DOPENVDB_SIMD=SSE42' }
-          - { name: 'avx',   build: 'Release', components: 'core,python,bin,view,render,test',               cmake: '-DOPENVDB_SIMD=AVX' }
-          - { name: 'numpy', build: 'Release', components: 'core,python,bin,view,render,test',               cmake: '-DUSE_NUMPY=ON -DOPENVDB_PYTHON_WRAP_ALL_GRID_TYPES=ON' }
-          - { name: 'asan',  build: 'asan',    components: 'core,test',                                      cmake: '-DNANOVDB_USE_OPENVDB=ON -DOPENVDB_AX_STATIC=OFF -DOPENVDB_CORE_STATIC=OFF -DUSE_BLOSC=OFF' } # We never called blosc_destroy(), so disable blosc to silence these errors
-          - { name: 'ubsan', build: 'ubsan',   components: 'core,test',                                      cmake: '-DCMAKE_CXX_FLAGS="-Wno-deprecated-declarations" ' }
-          - { name: 'c++20', build: 'Release', components: 'core,test',                                      cmake: '-DCMAKE_CXX_STANDARD=20' }
-          - { name: 'conf',  build: 'Release', components: 'core,python,bin,view,render,test',               cmake: '-DCMAKE_FIND_PACKAGE_PREFER_CONFIG=ON' }
+          - { name: 'all',    image: '2026',           build: 'Release', components: 'core,python,bin,view,render,test', cmake: '-DUSE_BLOSC=ON  -DUSE_ZLIB=ON  -DUSE_EXR=ON  -DUSE_PNG=ON'  }
+          - { name: 'lite',   image: '2026',           build: 'Release', components: 'core,python,bin,view,render,test', cmake: '-DUSE_BLOSC=OFF -DUSE_ZLIB=OFF -DUSE_EXR=OFF -DUSE_PNG=OFF -DOPENVDB_USE_DELAYED_LOADING=OFF' }
+          - { name: 'half',   image: '2026',           build: 'Release', components: 'core,python,bin,view,render,test', cmake: '-DUSE_BLOSC=OFF -DUSE_IMATH_HALF=ON' }
+          - { name: 'sse',    image: '2026',           build: 'Release', components: 'core,python,bin,view,render,test', cmake: '-DOPENVDB_SIMD=SSE42' }
+          - { name: 'avx',    image: '2026',           build: 'Release', components: 'core,python,bin,view,render,test', cmake: '-DOPENVDB_SIMD=AVX' }
+          - { name: 'pygrid', image: '2026',           build: 'Release', components: 'core,python,bin,view,render,test', cmake: '-DOPENVDB_PYTHON_WRAP_ALL_GRID_TYPES=ON' }
+          - { name: 'asan',   image: '2026',           build: 'asan',    components: 'core,test',                        cmake: '-DDISABLE_DEPENDENCY_VERSION_CHECKS=ON -DNANOVDB_USE_OPENVDB=ON -DOPENVDB_AX_STATIC=OFF -DOPENVDB_CORE_STATIC=OFF -DUSE_BLOSC=OFF' } # We never called blosc_destroy(), so disable blosc to silence these errors
+          - { name: 'ubsan',  image: '2026',           build: 'ubsan',   components: 'core,test',                        cmake: '-DDISABLE_DEPENDENCY_VERSION_CHECKS=ON -DCMAKE_CXX_FLAGS="-Wno-deprecated-declarations" ' }
+          - { name: 'c++20',  image: '2026',           build: 'Release', components: 'core,test',                        cmake: '-DCMAKE_CXX_STANDARD=20' }
+          - { name: 'conf',   image: '2026',           build: 'Release', components: 'core,python,bin,view,render,test', cmake: '-DCMAKE_FIND_PACKAGE_PREFER_CONFIG=ON' }
      fail-fast: false
    steps:
-    - name: Enable Node 16
-      run: |
-        echo "ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION=true" >> $GITHUB_ENV
    - uses: actions/checkout@v3
    - name: nanobind
-      #if: contains(container.image, '2023') == false
-      run: ./ci/install_nanobind.sh 2.0.0
+      run: ./ci/install_nanobind.sh 2.5.0
+      # 2026 images are broken with gtest/glfw/exr installs
+    - name: deps
+      if: contains(matrix.config.image, '2026')
+      run: |
+        ./ci/install_gtest.sh latest
+        ./ci/install_glfw.sh latest
+        ./ci/install_imath.sh v3.2.2
+        ./ci/install_exr.sh v3.2.2
    - name: build
      run: >
        ./ci/build.sh -v
@@ -200,21 +189,22 @@ jobs:
    strategy:
      matrix:
        config:
-          - { runson: ubuntu-latest, cxx: g++,     cmake: '' }
+          # llvm-17 is currently the default which has symbol issues with the python module
+          - { runson: ubuntu-latest, cxx: g++, cmake: '-DOPENVDB_PYTHON_USE_AX=OFF' }
          # Disable the clang job for now. See https://github.com/actions/runner-images/issues/8659
          # - { runson: ubuntu-latest, cxx: clang++, cmake: '' }
          # @todo gcc on macos
-          - { runson: macos-latest,  cxx: '',      cmake: '-DCMAKE_CXX_COMPILER=/opt/homebrew/opt/llvm@15/bin/clang++ -DLLVM_DIR=/opt/homebrew/opt/llvm@15/lib/cmake/llvm' }
+          - { runson: macos-latest, cxx: '', cmake: '-DLLVM_DIR=/opt/homebrew/opt/llvm@20/lib/cmake/llvm' }
      fail-fast: false
    steps:
      - uses: actions/checkout@v3
      - name: install_deps
        run: |
          if [ "$RUNNER_OS" == "Linux" ]; then
-            sudo apt-get -q install -y libboost-dev libboost-iostreams-dev libtbb-dev libblosc-dev llvm-dev libgtest-dev libcppunit-dev
-            ./ci/install_nanobind.sh 2.0.0
+            sudo apt-get -q install -y libboost-dev libboost-iostreams-dev libtbb-dev libblosc-dev libgtest-dev libgmock-dev python3-numpy
+            ./ci/install_nanobind.sh 2.5.0
          elif [ "$RUNNER_OS" == "macOS" ]; then
-            ./ci/install_macos.sh 15
+            ./ci/install_macos.sh 20
            ./ci/install_tbb_macos.sh
          else
            echo "$RUNNER_OS not supported"; exit 1
@@ -224,7 +214,12 @@ jobs:
          ./ci/build.sh -v
          --build-type=Release
          --components=\"core,axcore,python,bin,render,test,axbin\"
-          --cargs=\"-DCMAKE_CXX_STANDARD=20 -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/install ${{ matrix.config.cmake }}\"
+          --cargs=\'
+          -DCMAKE_CXX_STANDARD=20
+          -DOPENVDB_USE_DELAYED_LOADING=OFF
+          -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/install
+          ${{ matrix.config.cmake }}
+          \'
      - name: test
        run: cd build && ctest -V

@@ -246,9 +241,9 @@ jobs:
          # Boost as both shared and static libs are installed.
          # USE_EXPLICIT_INSTANTIATION is disabled for debug static libraries
          # due to disk space constraints
-          - { vc: 'x64-windows-static', components: 'core,bin,view,render,test',       build: 'Release', cmake: '-A x64 -G \"Visual Studio 17 2022\" -DOPENVDB_CORE_SHARED=OFF -DUSE_STATIC_DEPENDENCIES=ON -DBLOSC_USE_EXTERNAL_SOURCES=ON' }
+          # - { vc: 'x64-windows-static', components: 'core,bin,view,render,test',       build: 'Release', cmake: '-A x64 -G \"Visual Studio 17 2022\" -DOPENVDB_CORE_SHARED=OFF -DUSE_STATIC_DEPENDENCIES=ON -DBLOSC_USE_EXTERNAL_SOURCES=ON' }
          - { vc: 'x64-windows',        components: 'core,bin,view,render,python,test', build: 'Release', cmake: '-A x64 -G \"Visual Studio 17 2022\" -DOPENVDB_CORE_STATIC=OFF' }
-          - { vc: 'x64-windows',        components: 'core,bin,view,render,python,test', build: 'Debug',   cmake: '-A x64 -G \"Visual Studio 17 2022\" -DOPENVDB_CORE_STATIC=OFF' }
+          # - { vc: 'x64-windows',        components: 'core,bin,view,render,python,test', build: 'Debug',   cmake: '-A x64 -G \"Visual Studio 17 2022\" -DOPENVDB_CORE_STATIC=OFF' }
      fail-fast: false
    steps:
    - uses: actions/checkout@v3
@@ -292,7 +287,7 @@ jobs:
      github.event_name != 'workflow_dispatch' ||
      github.event.inputs.type == 'all' ||
      github.event.inputs.type == 'ax'
-    runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'ubuntu-20.04-8c-32g-300h') || 'ubuntu-latest' }}
+    runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'ubuntu-22.04-8c-32g-300h') || 'ubuntu-latest' }}
    name: >
      linux-ax:${{ matrix.config.image }}-cxx:${{ matrix.config.cxx }}-${{ matrix.config.build }}
    container:
@@ -302,25 +297,14 @@ jobs:
    strategy:
      matrix:
        config:
-          # Unified
-          - { image: '2023-clang14', cxx: 'clang++', build: 'Release', components: 'core,bin,axcore,axbin,axtest', cmake: '' }
-          - { image: '2023-clang14', cxx: 'g++',     build: 'Release', components: 'core,bin,axcore,axbin,axtest', cmake: '' }
-          - { image: '2022-clang13', cxx: 'clang++', build: 'Release', components: 'core,bin,axcore,axbin,axtest', cmake: '' }
-          - { image: '2022-clang13', cxx: 'g++',     build: 'Release', components: 'core,bin,axcore,axbin,axtest', cmake: '' }
-          # Standalone
-          - { image: '2022-clang11', cxx: 'clang++', build: 'Debug',   components: 'core', cmake: '' }
-          - { image: '2022-clang11', cxx: 'clang++', build: 'Release', components: 'core', cmake: '' }
-          - { image: '2022-clang11', cxx: 'g++',     build: 'Release', components: 'core', cmake: '' }
+          - { image: '2025-clang18.1', cxx: 'clang++', build: 'Release', components: 'core,axcore,axbin,axtest', cmake: '' }
+          # @note  LLVM 17 builds are incompatible with the vdb python plugin on linux
+          - { image: '2024-clang17.2', cxx: 'clang++', build: 'Release', components: 'core,axcore,axbin,axtest', cmake: '-DOPENVDB_PYTHON_USE_AX=OFF' }
      fail-fast: false
    steps:
-      - name: Enable Node 16
-        if: contains(matrix.config.image, '2022')
-        run: |
-          echo "ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION=true" >> $GITHUB_ENV
      - uses: actions/checkout@v3
      - name: nanobind
-        #f: contains(matrix.config.image, '2023') == false
-        run: ./ci/install_nanobind.sh 2.0.0
+        run: ./ci/install_nanobind.sh 2.5.0
      - name: build
        run: >
          ./ci/build.sh -v
@@ -331,6 +315,7 @@ jobs:
          -DOPENVDB_AX_TEST_CMD_DOWNLOADS=ON
          -DUSE_EXPLICIT_INSTANTIATION=OFF
          -DOPENVDB_CXX_STRICT=ON
+          -DOPENVDB_BUILD_VDB_TOOL=OFF
          \"
      - name: clean
        if: matrix.config.components == 'core'
@@ -346,6 +331,7 @@ jobs:
          -DOPENVDB_AX_TEST_CMD_DOWNLOADS=ON
          -DUSE_EXPLICIT_INSTANTIATION=OFF
          -DOPENVDB_CXX_STRICT=ON
+          -DOPENVDB_BUILD_VDB_TOOL=OFF
          \"
      - name: test
        run: cd build && ctest -V
@@ -357,16 +343,14 @@ jobs:
      github.event_name != 'workflow_dispatch' ||
      github.event.inputs.type == 'all' ||
      github.event.inputs.type == 'ax'
-    runs-on: macos-13
+    runs-on: ${{ matrix.config.runner }}
    name: macos-cxx:${{ matrix.config.cxx }}-llvm:${{ matrix.config.llvm }}-${{ matrix.config.build }}
    env:
      CXX: ${{ matrix.config.cxx }}
    strategy:
      matrix:
        config:
-          - { cxx: 'clang++', build: 'Release', llvm: '14' }
-          - { cxx: 'clang++', build: 'Release', llvm: '15' }
-          #- { cxx: 'clang++', build: 'Release', llvm: '16' } - not supported yet
+          - { runner: 'macos-latest', cxx: 'clang++', build: 'Release', llvm: '20' }
      fail-fast: false
    steps:
      - uses: actions/checkout@v3
@@ -396,7 +380,7 @@ jobs:
      github.event.inputs.type == 'all' ||
      github.event.inputs.type == 'ax'
    runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'windows-2022-8c-32g-300h') || 'windows-latest' }}
-    name: windows-vc:${{ matrix.config.vc }}-type:${{ matrix.config.build }}
+    name: ${{ matrix.config.vc }}-${{ matrix.config.crt }}-${{ matrix.config.build }}-llvm${{ matrix.config.llvm }}
    env:
      VCPKG_DEFAULT_TRIPLET: ${{ matrix.config.vc }}
      # Export this with '' avoid bash treating \ as escape
@@ -410,17 +394,22 @@ jobs:
          # @todo  We don't currently run the axtests with shared builds of ax
          # due to symbol issues using LLVM as a static lib (which is the only
          # option on Windows).
-          - { vc: 'x64-windows',        crt: 'MD',  components: 'core,bin,axcore,axbin,python', build: 'Release', cmake: '-DOPENVDB_CORE_STATIC=OFF -DOPENVDB_AX_STATIC=OFF' }
-          - { vc: 'x64-windows-static', crt: 'MT',  components: 'core,bin,axcore,axbin,axtest', build: 'Release', cmake: '-DOPENVDB_CORE_SHARED=OFF -DOPENVDB_AX_SHARED=OFF -DUSE_STATIC_DEPENDENCIES=ON -DBLOSC_USE_EXTERNAL_SOURCES=ON' }
-          - { vc: 'x64-windows-static', crt: 'MTd', components: 'core,bin,axcore,axbin,axtest', build: 'Debug',   cmake: '-DOPENVDB_CORE_SHARED=OFF -DOPENVDB_AX_SHARED=OFF -DUSE_STATIC_DEPENDENCIES=ON -DBLOSC_USE_EXTERNAL_SOURCES=ON' }
+          # @note  currently only test static builds with MT as building LLVM as
+          # a shared lib on windows is a bit dicey
+          - { vc: 'x64-windows-static', llvm: '18.1.0', crt: 'MultiThreaded',      components: 'core,axcore,axbin,axtest', build: 'Release', cmake: '-DOPENVDB_CORE_SHARED=OFF -DOPENVDB_AX_SHARED=OFF -DUSE_STATIC_DEPENDENCIES=ON -DBLOSC_USE_EXTERNAL_SOURCES=ON' }
+          - { vc: 'x64-windows-static', llvm: '18.1.0', crt: 'MultiThreadedDebug', components: 'core,axcore,axbin,axtest', build: 'Debug',   cmake: '-DOPENVDB_CORE_SHARED=OFF -DOPENVDB_AX_SHARED=OFF -DUSE_STATIC_DEPENDENCIES=ON -DBLOSC_USE_EXTERNAL_SOURCES=ON' }
+          - { vc: 'x64-windows',        llvm: '18.1.0', crt: 'MultiThreadedDLL',   components: 'core,axcore,axbin',        build: 'Release', cmake: '-DOPENVDB_CORE_STATIC=OFF -DOPENVDB_AX_STATIC=OFF' }
+          - { vc: 'x64-windows-static', llvm: '15.0.0', crt: 'MultiThreaded',      components: 'core,axcore,axbin,axtest', build: 'Release', cmake: '-DOPENVDB_CORE_SHARED=OFF -DOPENVDB_AX_SHARED=OFF -DUSE_STATIC_DEPENDENCIES=ON -DBLOSC_USE_EXTERNAL_SOURCES=ON' }
      fail-fast: false
    steps:
    - uses: actions/checkout@v3
    - name: llvm
-      run: ./ci/install_llvm_windows.sh ${{ matrix.config.crt }}
+      run: ./ci/install_llvm_windows.sh ${{ matrix.config.llvm }} ${{ matrix.config.crt }}
    - name: install
      shell: powershell
-      run: .\ci\install_windows.ps1
+      run: |
+        vcpkg update
+        vcpkg install tbb gtest
    - name: build
      run: >
        ./ci/build.sh -v
@@ -428,10 +417,13 @@ jobs:
        --components="${{ matrix.config.components }}"
        --cargs=\'
        -A x64 -G \"Visual Studio 17 2022\"
+        -DOPENVDB_USE_DELAYED_LOADING=OFF
+        -DUSE_BLOSC=OFF \
+        -DUSE_ZLIB=OFF \
        -DVCPKG_TARGET_TRIPLET=${VCPKG_DEFAULT_TRIPLET}
        -DCMAKE_TOOLCHAIN_FILE=\"${VCPKG_INSTALLATION_ROOT}\\scripts\\buildsystems\\vcpkg.cmake\"
        -DMSVC_COMPRESS_PDB=ON
-        -DOPENVDB_AX_TEST_CMD_DOWNLOADS=ON
+        -DOPENVDB_AX_TEST_CMD_DOWNLOADS=OFF
        -DUSE_EXPLICIT_INSTANTIATION=OFF
        -DLLVM_DIR=\"${HOME}\\llvm_install\\lib\\cmake\\llvm\"
        -DCMAKE_INSTALL_PREFIX=\"${VDB_INSTALL_PREFIX}\"
@@ -454,7 +446,7 @@ jobs:
    if: |
      github.event_name != 'workflow_dispatch' ||
      github.event.inputs.type == 'all' ||
-      github.event.inputs.type == 'win'
+      github.event.inputs.type == 'winnano'
    runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'windows-2022-8c-32g-300h') || 'windows-latest' }}
    env:
      VCPKG_DEFAULT_TRIPLET: ${{ matrix.config.vc }}
@@ -496,6 +488,7 @@ jobs:
        -DMSVC_COMPRESS_PDB=ON
        -DUSE_EXPLICIT_INSTANTIATION=OFF
        -DNANOVDB_USE_CUDA=ON
+        -DCMAKE_CUDA_ARCHITECTURES="80"
        -DNANOVDB_USE_OPENVDB=ON
        -DVCPKG_TARGET_TRIPLET=${VCPKG_DEFAULT_TRIPLET}
        -DCMAKE_TOOLCHAIN_FILE=\"${VCPKG_INSTALLATION_ROOT}\\scripts\\buildsystems\\vcpkg.cmake\"
@@ -513,18 +506,15 @@ jobs:
      github.event_name != 'workflow_dispatch' ||
      github.event.inputs.type == 'all' ||
      github.event.inputs.type == 'blosc'
-    runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'ubuntu-20.04-8c-32g-300h') || 'ubuntu-latest' }}
+    runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'ubuntu-22.04-8c-32g-300h') || 'ubuntu-latest' }}
    name: linux-blosc:${{ matrix.blosc }}
    container:
-      image: aswf/ci-base:2023
+      image: aswf/ci-openvdb:2025-clang19.1
    strategy:
      matrix:
-        blosc: ['1.18.0','1.19.0','1.20.0','1.21.0']
+        blosc: ['1.21.0', 'latest']
      fail-fast: false
    steps:
-    - name: Enable Node 16
-      run: |
-        echo "ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION=true" >> $GITHUB_ENV
    - uses: actions/checkout@v3
    - name: install_blosc
      run: sudo ./ci/install_blosc.sh ${{ matrix.blosc }}
@@ -612,7 +602,7 @@ jobs:
        -strict
        -extended
    - name: upload_report
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
      if: always()
      with:
        name: abi_report
@@ -1,9 +1,199 @@
 OpenVDB Version History
 =======================

-Version 12.0.0 - In development
+Version 12.1.2 - In development
+
+
+Version 12.1.1 - September 30, 2025
+
+  OpenVDB AX:
+    Fixes:
+    - Fixed an issue which was causing host specific optimization passes to be
+      omitted when using LLVM 16 and higher.
+
+  Houdini:
+    - Fixed a compilation issue with the OpenVDB Activate SOP with Houdini 21
+    [Reported by @callen128]
+
+
+Version 12.1.0 - August 8, 2025

  OpenVDB:
+    New features:
+    - New level set constructor tools::createLevelSetCapsule creates a grid
+      containing a narrow-band level set representation of a capsule (tube
+      with constant radius and sphere caps).
+    - New level set constructor tools::createLevelSetTaperedCapsule creates a
+      grid containing a narrow-band level set representation of a tapered
+      capsule (tube with sphere caps and different radii at both ends, or
+      equivalently the convex hull of two spheres with possibly different
+      centers and radii).
+    - New level set constructor tools::createLevelSetTubeComplex creates a grid
+      containing a narrow-band level set representation of a tube complex (a
+      collection of capsules or tapered capsules defined by endpoint coordinates
+      and segment indices).
+    - New level set constructor tools::createLevelSetDilatedMesh creates a grid
+      containing a narrow-band level set representation of a dilated surface
+      mesh (dilated by a radius in all directions).
+    - New class tools::ConvexVoxelizer, used by the above methods, which can be
+      inherited to implement a convex level set constructor.
+    - Introduced anisotropic surfacing tools in points/PointRasterizeSDF.h and
+      principal component analysis (PCA) methods in points/PrincipalComponentAnalysis.h.
+      The latter tool analyses point neighborhoods and computes affine
+      transformations representing elliptical distributions. The prior tool
+      takes rotational and stretch point attributes (optionally computed from
+      the prior PCA methods) and generates anisotropic surfaces.
+
+    Improvements:
+    - Significantly improved the performance of all LeafNode ValueIterators,
+      up to 5x on some platforms and up to 10x when delay loading is enabled.
+      Construction of a ValueIterator from a leaf node now requests the leaf
+      buffers ahead of iteration to avoid potentially expensive API calls.
+    - Improved the performance of PointRasterizeSDF.h tools with larger radii
+      with better bound computation approximations.
+    - Significant performance improvement to delay loaded files - the warning
+      logic which checks the underlying mapped file for the last modified time
+      has been removed, resulting in performance gains upwards of 10x when
+      loading in node data buffers. Note that the check that was removed is
+      entirely redundant on Windows and would only print a warning on other
+      system.
+      [Contributed by jjtParadox]
+    - Added OPENVDB_HAS_INVOKABLE_MEMBER_FUNCTION and OPENVDB_HAS_MEMBER_FUNCTION
+      macros to help determine the presence and compatibility of interface
+      member functions at compile time.
+    - Added OPENVDB_ASSUME macros to mimic builtin assume and C++23 assume
+      attributes.
+    - Improved read and write of PLY files with vdb_tool so they support Vec3d
+      and 64 bit attributes. Changed vdb_tool version number from 10.6.1 to
+      10.7.0.
+
+    API Changes:
+    - Mat3 and Mat4 functions have been marked [[nodiscard]] where it may be
+      confusing if they are operating in place or returning a result, in
+      particular tranpose, invert, adjoint, and timesDiagonal.
+    - Re-worked the API in PointRasterizeSDF.h with the addition of anisotropic
+      surfacing kernels. The new API exposes more concise free functions which
+      take structs of parameters. The old API remains supported but is
+      deprecated.
+    - Deprecated providing a filter or interrupter as arguments to
+      points::rasterize. These can instead be implemented on user transfer
+      schemes.
+    - Transfer schemes provided to points::rasterize can now optionally
+      implement a rasterizePoints callback that takes a range of points per
+      voxel.
+
+    Fixes:
+    - Fixed a bug in points::rasterizeSpheres and points::rasterizeSmoothSpheres
+      which could cause sections of the generated surface to incorrectly be
+      marked as negative interior values.
+    - Fixed a bug in points::rasterizeSmoothSpheres which could result in
+      incomplete surface reconstruction with significantly larger search
+      distances.
+    - Fixed some precision issues in various Matrix methods.
+    - Fixed bugs in vdb_tool that prevented build with newer namespaces in
+      nanovdb.
+
+  OpenVDB AX:
+    Improvements:
+    - Added support for LLVM versions greater than 15. Upgraded CI to test with
+      LLVM versions 16, 17, 18, 19 and 20.
+    - As of this release, LLVM 15 is the minimum required version for OpenVDB
+      AX. Note that there is an issue with LLVM 17 symbols which makes it
+      incompatible with the OpenVDB Python plugin. AS such, LLVM version 18 and
+      greater are the recommended LLVM versions to use, with LLVM 18 being the
+      new future minimum version.
+    - Significant performance improvement to the compilation of AX kernels when
+      using LLVM 18 and greater.
+    - Significant backend refactoring to AX IR generation to support newer
+      versions of LLVM. In particular, parts of the codegen::Function and
+      codegen::FunctionBuilder API require new usage from LLVM 16 due to
+      the LLVM opaque pointer changes. If you're not using the OpenVDB AX
+      Function codegen API then no client changes should be necessary.
+    - Migrated OpenVDB AX unit tests from cppunit to gtest
+      [Contributed by Tim Straubinger]
+
+    Fixes:
+    - Fixed a case of undefined behaviour when explicitly casting variables to
+      the same type as their storage type e.g:
+        double a; double b = double(a);
+    - Fixed an AX compiler error which would produce invalid IR when using 64bit
+      integer variables for matrix array offsetting e.g:
+        int64 a = 0; mat4f b = 1; float c = b[a,a];
+
+  NanoVDB:
+    - Added nanovdb::openToIndexVDB, which can convert an OpenVDB grid into a
+      NanoVDB IndexGrid.
+    - Added new options "--index" and "--onIndex" to the command-line tool
+      nanovdb_convert so it can produce IndexGrids.
+    - Fixed memory leak in DeviceBuffer.
+    - Added constructor for DeviceBuffer from an existing HostBuffer.
+    - Added VoxelBlockManager, an acceleration structure for sequential access
+      and stencil ops over active voxels.
+    - Added helper functions for kernel launches with shared memory.
+    - Added support for CUDA 13.0.
+    - Fix warnings and compilation errors for GCC and Clang.
+    - Fixed inconsistent usage of util::cuda::mallocAsync/freeAsync that
+      affected vGPUs.
+    - Added support for custom memory resources in PointsToGrid.
+    - Fixed race condition in split/mergeGridHandles.
+    - Fixed offset bug in nanovdb::tools::cuda::IndexToGrid related to non-POD
+      build types.
+      [Contributed by Andreas Henne]
+
+  Houdini:
+    - Define OP_InputIdx and OP_OutputIdx for Houdini 21.0 compatibility
+    - Add Houdini 21 style camera interface to support LOP cameras
+    - Include env flag for avoiding stream writing (was backported to 19.5)
+
+
+Version 12.0.1 - April 3, 2025
+
+  OpenVDB:
+    New features:
+    - Added read and write support for OFF (Object File Format) files to
+      vdb_tool
+
+    Build:
+    - Fixed a build issue with vdb_render on Windows with newer versions of
+      glew not including glu.h by default
+    - Fixed issues with newer versions of clang refusing to accept older
+      ::template syntax
+
+  NanoVDB:
+    New features:
+    - Added support for multiple GPUs to DeviceBuffer
+    - Added a UnifiedBuffer class that wraps CUDA unified memory
+    - Added example for multiGPU sparse convolution
+    - Added CUDA utility functions for device queries
+    - Added functions to independently stop and compute the elapsed time for
+      timer classes
+
+    Improvements:
+    - Replaced CUB's CachingDeviceAllocator with the default asynchronous
+      stream ordered allocator in PointsToGrid for improved performance
+
+    Bug fix:
+    - Map is now properly transposed when converting NanoVDB to OpenVDB.
+    - Fixed ostream specializations being hidden within the nanovdb namespace
+    - Fixed a call to a non-existent method in GridBuild.h
+
+
+Version 12.0.0 - October 31, 2024
+
+      OpenVDB is now licensed under the Apache 2.0 license, instead of the MPL 2.0 license.
+
+      This version introduces ABI changes relative to older major releases,
+      so to preserve ABI compatibility it might be necessary to define the
+      macro OPENVDB_ABI_VERSION_NUMBER=N, where, for example, N is 10 for
+      Houdini 20.0 and 11 for Houdini 20.5.
+
+      GCC 9 is no longer supported.
+
+  OpenVDB:
+    New features:
+    - Added fillet() method in tools::LevelSetFilter to round off concave edges
+      to create smoother transition between surfaces.
+
    Improvements:
    - Added openvdb::assertAbort to replace cassert and a
      OPENVDB_ENABLE_ASSERTS cmake argument/compile define to toggle
@@ -19,11 +209,49 @@ Version 12.0.0 - In development
    - ValueAccessors are now defined and created in the Tree class
      instead of in the Grid class so that custom Tree implementations
      may define and create their own ValueAccessors if desired.
+    - Added support for PDAL to vdb_tool [Contributed by Tom Matterson]
+    - LeafManager and NodeManager now use Index64 for leaf counts internally.
+    - Added RootNode::probeChild() const.
+    - Added RootNode::probeChild() and RootNode::probeConstChild().
+    - Added RootNode::probe() and RootNode::probeConst() to query key presence,
+      child node, value and active state.
+    - Added InternalNode::probeChild() const.
+    - Added InternalNode::probeChild() and probeChildConst() with coord access
+      and optionally value and active state.
+    - Added InternalNode::probeChild() and probeChildConst() with index access
+      and optionally value and active state.
+    - Added InternalNode::isValueOff(), LeafNode::isValueOff(),
+      LeafNodeBool::isValueOff(), LeafNodeMask::isValueOff().
+    - Added LeafNodeMask::probeValue(Index,val), LeafNodeBool::probeValue(Index,val).
+    - Added RootNode::getValueUnsafe(), RootNode::getChildUnsafe(),
+      RootNode::getConstChildUnsafe().
+    - Added InternalNode::getValueUnsafe(), InternalNode::getChildUnsafe(),
+      InternalNode::getConstChildUnsafe().
+    - Added InternalNode::setActiveStateUnsafe(), InternalNode::setValueOnlyUnsafe(),
+      InternalNode::setValueOnUnsafe(), InternalNode::setValueOffUnsafe().
+    - Added InternalNode::setChildUnsafe(), InternalNode::resetChildUnsafe(),
+      InternalNode::stealChildUnsafe(), InternalNode::deleteChildUnsafe().
+    - For LeafNode, LeafNodeBool and LeafNodeMask - added
+      LeafNode::getValueUnsafe(), LeafNode::setActiveStateunsafe(),
+      LeafNode::setValueOnlyUnsafe(), LeafNode::setValueOnUnsafe(),
+      LeafNode::setValueOffUnsafe().
+
+    ABI changes:
+    - Tree::leafCount(), Tree::unallocatedLeafCount(),
+      Tree::nonLeafCount() and Tree::nodeCount() now use Index64 in their
+      return types instead of Index32.

    API Changes:
    - RootNode::tileCount(), RootNode::activeTileCount() and
      RootNode::inactiveTileCount() are now public.
    - RootNode::hasKey() and RootNode::coordToKey() are now public.
+    - RootNode::leafCount(), RootNode::nonLeafCount() and RootNode::nodeCount()
+      now use Index64 instead of Index32. The Index32 variant is deprecated.
+    - InternalNode::leafCount(), InternalNode::nonLeafCount() and
+      InternalNode::nodeCount() now use Index64 instead of Index32. The Index32
+      variant is deprecated.
+    - LeafNode::leafCount() and LeafNode::nonLeafCount() now use Index64 instead
+      of Index32. The Index32 variant is deprecated.

    Bug Fixes:
    - Fix potential crash reading corrupt .vdb files with invalid
@@ -549,7 +777,6 @@ Version 9.1.0 - June 9, 2022
      Positional arguments as input files are deprecated.
    - Added tools::minMax() which supports multithreaded evaluation of active
      minimum and maximum values. Grid::evalMinMax() has been deprecated.
-      [Contributed by Greg Hurst]
    - Significant performance improvements to AX point kernels, primarily due to
      providing AX access to attribute buffers for superior code generation.
    - vdb_print now prints both the in-core memory and total memory usage for
@@ -42,7 +42,7 @@

 # note: cmake_minimum_required must be called before project commands to
 #  ensure policy scope is set up correctly
-cmake_minimum_required(VERSION 3.18)
+cmake_minimum_required(VERSION 3.24)

 # CMP0091 allows for MSVC ABI targetting via CMAKE_MSVC_RUNTIME_LIBRARY
 # from CMake 3.15 and above. Must come before project().
@@ -52,9 +52,9 @@ endif()

 ###### Version

-set(OpenVDB_MAJOR_VERSION 11)
-set(OpenVDB_MINOR_VERSION 0)
-set(OpenVDB_PATCH_VERSION 1)
+set(OpenVDB_MAJOR_VERSION 12)
+set(OpenVDB_MINOR_VERSION 1)
+set(OpenVDB_PATCH_VERSION 2)
 set(OpenVDB_VERSION "${OpenVDB_MAJOR_VERSION}.${OpenVDB_MINOR_VERSION}.${OpenVDB_PATCH_VERSION}")

 project(OpenVDB LANGUAGES CXX VERSION ${OpenVDB_VERSION})
@@ -268,6 +268,13 @@ mark_as_advanced(

 include(cmake/config/OpenVDBVersions.cmake)

+if(OPENVDB_FUTURE_DEPRECATION AND FUTURE_MINIMUM_CMAKE_VERSION)
+  if(CMAKE_VERSION VERSION_LESS ${FUTURE_MINIMUM_CMAKE_VERSION})
+    message(DEPRECATION "Support for CMake versions < ${FUTURE_MINIMUM_CMAKE_VERSION} "
+      "is deprecated and will be removed.")
+  endif()
+endif()
+
 foreach(DEPRECATED_ABI ${OPENVDB_DEPRECATED_ABI_LIST})
  option(OPENVDB_USE_DEPRECATED_ABI_${DEPRECATED_ABI} "Bypass minimum ABI requirement checks" OFF)
  mark_as_advanced(OPENVDB_USE_DEPRECATED_ABI_${DEPRECATED_ABI})
@@ -478,10 +485,7 @@ endif()
 # Locate Python and nanobind if necessary
 if(OPENVDB_BUILD_PYTHON_MODULE OR (OPENVDB_BUILD_NANOVDB AND NANOVDB_BUILD_PYTHON_MODULE))

-  # Call find_package(Python ...)
-  find_package(Python 3.8 REQUIRED COMPONENTS Development Interpreter)
-  find_package(nanobind REQUIRED)
-
+  find_package(Python ${MINIMUM_PYTHON_VERSION} REQUIRED COMPONENTS Development Interpreter)
  if(OPENVDB_FUTURE_DEPRECATION AND FUTURE_MINIMUM_PYTHON_VERSION)
    if(Python_VERSION VERSION_LESS ${FUTURE_MINIMUM_PYTHON_VERSION})
      message(DEPRECATION "Support for Python versions < ${FUTURE_MINIMUM_GLFW_VERSION} "
@@ -489,6 +493,20 @@ if(OPENVDB_BUILD_PYTHON_MODULE OR (OPENVDB_BUILD_NANOVDB AND NANOVDB_BUILD_PYTHO
    endif()
  endif()

+  find_package(nanobind REQUIRED CONFIG)
+  include(FindPackageHandleStandardArgs)
+  find_package_handle_standard_args(nanobind
+    FOUND_VAR nanobind_FOUND
+    REQUIRED_VARS NB_DIR
+    VERSION_VAR nanobind_VERSION)
+
+  if(OPENVDB_FUTURE_DEPRECATION AND FUTURE_MINIMUM_NANOBIND_VERSION)
+    if(nanobind_VERSION VERSION_LESS ${FUTURE_MINIMUM_NANOBIND_VERSION})
+      message(DEPRECATION "Support for nanobind versions < ${FUTURE_MINIMUM_NANOBIND_VERSION} "
+        "is deprecated and will be removed.")
+    endif()
+  endif()
+
  if(NOT DEFINED VDB_PYTHON_INSTALL_DIRECTORY)
    if(DEFINED Python_SITELIB)
      get_filename_component(Python_PACKAGES_DIR ${Python_SITELIB} NAME)
@@ -1,4 +1,6 @@
 #!/usr/bin/env bash
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0

 set -e

@@ -31,8 +33,13 @@ PARMS[--target]=install
 PARMS[--build-dir]=build

 if [[ $RUNNER_NAME == *"8c-32g-300h"* ]]; then
-    # ASWF github actions runners have 8 threads
-    PARMS[-j]=8
+    if [[ $CXX == "g++" ]]; then
+        # GCC hits memory limits on runners, reduce threads
+        PARMS[-j]=6
+    else
+        # ASWF github actions runners have 8 threads
+        PARMS[-j]=8
+    fi
 else
    # Github actions runners have 2 threads
    # https://help.github.com/en/actions/reference/virtual-environments-for-github-hosted-runners
@@ -49,7 +56,8 @@ declare -A COMPONENTS
 COMPONENTS['core']='OPENVDB_BUILD_CORE'
 COMPONENTS['python']='OPENVDB_BUILD_PYTHON_MODULE'
 COMPONENTS['test']='OPENVDB_BUILD_UNITTESTS'
-COMPONENTS['bin']='OPENVDB_BUILD_BINARIES'
+# @todo split out vdb tool from this...
+COMPONENTS['bin']='OPENVDB_BUILD_VDB_PRINT,OPENVDB_BUILD_VDB_LOD,OPENVDB_BUILD_VDB_TOOL,OPENVDB_BUILD_VDB_TOOL_UNITTESTS'
 COMPONENTS['view']='OPENVDB_BUILD_VDB_VIEW'
 COMPONENTS['render']='OPENVDB_BUILD_VDB_RENDER'
 COMPONENTS['hou']='OPENVDB_BUILD_HOUDINI_PLUGIN'
@@ -148,7 +156,11 @@ for comp in "${!COMPONENTS[@]}"; do
            setting="ON"; break
        fi
    done
-    CMAKE_EXTRA+=("-D${COMPONENTS[$comp]}=$setting")
+    OPENVDB_TOGGLES="${COMPONENTS[$comp]}"
+    OPENVDB_TOGGLES=(${OPENVDB_TOGGLES//,/ })
+    for toggle in "${!OPENVDB_TOGGLES[@]}"; do
+        CMAKE_EXTRA+=("-D${OPENVDB_TOGGLES[$toggle]}=$setting")
+    done
 done

 ################################################
@@ -180,15 +192,14 @@ cd ${BUILD_DIR}
 set -x

 # Note:
-# - print and lod binary options are always on and can be toggles with: OPENVDB_BUILD_BINARIES=ON/OFF
+# - OPENVDB_BUILD_BINARIES is always ON, but each binary will be toggled depending on the provided comps
 # - always enabled the python tests with OPENVDB_BUILD_PYTHON_UNITTESTS if the python module is in use,
 #   regardless of the 'test' component being enabled or not (see the OPENVDB_BUILD_PYTHON_UNITTESTS option).
 cmake \
    -DOPENVDB_USE_DEPRECATED_ABI_10=ON \
-    -DOPENVDB_BUILD_VDB_PRINT=ON \
-    -DOPENVDB_BUILD_VDB_LOD=ON \
-    -DOPENVDB_BUILD_VDB_TOOL=ON \
+    -DOPENVDB_USE_DEPRECATED_ABI_11=ON \
    -DOPENVDB_TOOL_USE_NANO=OFF \
+    -DOPENVDB_BUILD_BINARIES=ON \
    -DOPENVDB_BUILD_PYTHON_UNITTESTS=ON \
    -DMSVC_MP_THREAD_COUNT=${PARMS[-j]} \
    "${CMAKE_EXTRA[@]}" \
@@ -1,4 +1,6 @@
 #!/usr/bin/env bash
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0

 set -ex

@@ -1,4 +1,6 @@
 #!/usr/bin/env bash
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0

 set -ex

@@ -6,8 +8,14 @@ HOUDINI_MAJOR="$1"
 PLATFORM="$2"
 OTHER_ARGS="$3"

-pip install --user requests
-python ci/download_houdini.py $HOUDINI_MAJOR $PLATFORM $OTHER_ARGS
+if [[ $PLATFORM =~ "linux" ]]; then
+    pip3 install --user requests
+elif [[ $PLATFORM =~ "macos" ]]; then
+    # Tell the homebrew install of pip3 that we don't care about installing directly through pip3
+    pip3 install --user requests --break-system-packages
+fi
+
+python3 ci/download_houdini.py $HOUDINI_MAJOR $PLATFORM $OTHER_ARGS

 if [[ $PLATFORM =~ "linux" ]]; then
    # create dir hierarchy
@@ -29,20 +37,20 @@ if [[ $PLATFORM =~ "linux" ]]; then
    ls -al dsolib/

    # copy required libraries
-    cp -r toolkit/cmake ../hou/toolkit/.
-    cp -r toolkit/include ../hou/toolkit/.
-    cp -r dsolib/libHoudini* ../hou/dsolib/.
-    cp -r dsolib/libopenvdb_sesi* ../hou/dsolib/.
-    cp -r dsolib/libblosc* ../hou/dsolib/.
-    cp -r dsolib/libhboost* ../hou/dsolib/.
-    cp -r dsolib/libz* ../hou/dsolib/.
-    cp -r dsolib/libbz2* ../hou/dsolib/.
-    cp -r dsolib/libtbb* ../hou/dsolib/.
-    cp -r dsolib/libjemalloc* ../hou/dsolib/.
-    cp -r dsolib/liblzma* ../hou/dsolib/.
-    cp -r dsolib/libIex* ../hou/dsolib/.
-    cp -r dsolib/libImath* ../hou/dsolib/.
-    cp -r dsolib/libIlmThread* ../hou/dsolib/.
+    mv toolkit/cmake ../hou/toolkit/.
+    mv toolkit/include ../hou/toolkit/.
+    mv dsolib/libHoudini* ../hou/dsolib/.
+    mv dsolib/libopenvdb_sesi* ../hou/dsolib/.
+    mv dsolib/libblosc* ../hou/dsolib/.
+    mv dsolib/libhboost* ../hou/dsolib/.
+    mv dsolib/libz* ../hou/dsolib/.
+    mv dsolib/libbz2* ../hou/dsolib/.
+    mv dsolib/libtbb* ../hou/dsolib/.
+    mv dsolib/libjemalloc* ../hou/dsolib/.
+    mv dsolib/liblzma* ../hou/dsolib/.
+    mv dsolib/libIex* ../hou/dsolib/.
+    mv dsolib/libImath* ../hou/dsolib/.
+    mv dsolib/libIlmThread* ../hou/dsolib/.
    cd ..

 elif [[ $PLATFORM =~ "macos" ]]; then
@@ -1,4 +1,6 @@
 #!/usr/bin/env python3
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0

 import os
 import sys
@@ -1,4 +1,6 @@
 #!/usr/bin/env bash
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
 #################################################################################
 # This script extracts all code blocks from AX documentation which are NOT      #
 # marked as cpp/sh/unparsed and attempts to parse or compile them through the   #
@@ -1,4 +1,6 @@
 #!/usr/bin/env bash
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0

 set -ex

@@ -1,4 +1,6 @@
 #!/usr/bin/env bash
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0

 set -ex
 CURL_VERSION="$1"
@@ -1,4 +1,6 @@
 #!/usr/bin/env bash
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0

 set -ex

@@ -13,6 +15,6 @@ fi

 mkdir build
 cd build
-cmake ../.
+cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 ../.
 make -j8
 make install
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+
+set -ex
+EXR_VERSION="$1"
+
+git clone https://github.com/AcademySoftwareFoundation/openexr.git
+cd openexr
+
+if [ "$EXR_VERSION" != "latest" ]; then
+    git checkout tags/${EXR_VERSION} -b ${EXR_VERSION}
+fi
+
+mkdir build
+cd build
+
+cmake ..
+
+make -j$(nproc)
+sudo make install
@@ -1,4 +1,6 @@
 #!/usr/bin/env bash
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0

 set -ex
 GLFW_VERSION="$1"
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+
+set -ex
+GTEST_VERSION="$1"
+
+git clone https://github.com/google/googletest.git
+cd googletest
+
+if [ "$GTEST_VERSION" != "latest" ]; then
+    git checkout tags/${GTEST_VERSION} -b ${GTEST_VERSION}
+fi
+
+mkdir build
+cd build
+
+cmake ..
+
+make -j$(nproc)
+sudo make install
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+
+set -ex
+IMATH_VERSION="$1"
+
+git clone https://github.com/AcademySoftwareFoundation/Imath.git
+cd Imath
+
+if [ "$IMATH_VERSION" != "latest" ]; then
+    git checkout tags/${IMATH_VERSION} -b ${IMATH_VERSION}
+fi
+
+mkdir build
+cd build
+
+cmake ..
+
+make -j$(nproc)
+sudo make install
@@ -1,27 +1,58 @@
 #!/usr/bin/env bash
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0

 set -ex

-LLVM_CRT="$1"
+LLVM_VER="$1"
+LLVM_CRT="$2"

-git clone -b llvmorg-12.0.0 --depth 1 https://github.com/llvm/llvm-project.git llvm
+# Legacy define for older versions of LLVM
+LLVM_LEGACY_CRT_DEFINE=""
+if [[ "${LLVM_CRT}" == "MultiThreaded" ]]; then
+    LLVM_LEGACY_CRT_DEFINE="MT"
+elif [[ "${LLVM_CRT}" == "MultiThreadedDebug" ]]; then
+    LLVM_LEGACY_CRT_DEFINE="MTd"
+elif [[ "${LLVM_CRT}" == "MultiThreadedDLL" ]]; then
+    LLVM_LEGACY_CRT_DEFINE="MD"
+elif [[ "${LLVM_CRT}" == "MultiThreadedDebugDLL" ]]; then
+    LLVM_LEGACY_CRT_DEFINE="MDd"
+fi
+
+git clone -b llvmorg-${LLVM_VER} --depth 1 https://github.com/llvm/llvm-project.git llvm
 cd llvm

 mkdir .build
 cd .build

-# @note  currently only test static builds with MT
 cmake -B . -S ../llvm -A x64 -G "Visual Studio 17 2022" -Thost=x64 \
    -DCMAKE_INSTALL_PREFIX="${HOME}/llvm_install" \
+    -DCMAKE_BUILD_TYPE=Release \
+    -DLLVM_ENABLE_ASSERTIONS=OFF \
+    -DLLVM_ENABLE_PROJECTS="" \
+    -DLLVM_ENABLE_RUNTIMES="" \
+    -DLLVM_ENABLE_UNWIND_TABLES=OFF \
    -DLLVM_TARGETS_TO_BUILD="host" \
-    -DLLVM_BUILD_TOOLS=OFF \
    -DLLVM_INCLUDE_BENCHMARKS=OFF \
    -DLLVM_INCLUDE_EXAMPLES=OFF \
    -DLLVM_INCLUDE_TESTS=OFF \
+    -DLLVM_INCLUDE_DOCS=OFF \
    -DLLVM_INCLUDE_TOOLS=OFF \
-    -DLLVM_USE_CRT_RELEASE=${LLVM_CRT}
+    -DLLVM_INCLUDE_UTILS=OFF \
+    -DLLVM_BUILD_BENCHMARKS=OFF \
+    -DLLVM_BUILD_EXAMPLES=OFF \
+    -DLLVM_BUILD_TESTS=OFF \
+    -DLLVM_BUILD_DOCS=OFF \
+    -DLLVM_BUILD_TOOLS=OFF \
+    -DLLVM_BUILD_UTILS=OFF \
+    -DLLVM_ENABLE_BINDINGS=OFF \
+    -DLLVM_ENABLE_OCAMLDOC=OFF \
+    -DLLVM_PARALLEL_COMPILE_JOBS=4 \
+    -DLLVM_PARALLEL_LINK_JOBS=4 \
+    -DCMAKE_MSVC_RUNTIME_LIBRARY=${LLVM_CRT} \
+    -DLLVM_USE_CRT_RELEASE=${LLVM_LEGACY_CRT_DEFINE}

-cmake --build . --config Release --target install
+cmake --build . --parallel 4 --config Release --target install

 cd ..

@@ -1,4 +1,6 @@
 #!/usr/bin/env bash
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0

 set -x

@@ -20,6 +22,7 @@ brew install jq # for trivial parsing of brew json
 brew install openexr
 brew install nanobind # also installs the dependent python version
 brew install robin-map # required for nanobind
+brew install numpy # only for python tests
 brew install zlib
 brew install jemalloc

@@ -43,9 +46,7 @@ LLVM_VERSION=$1
 if [ ! -z "$LLVM_VERSION" ]; then
    if [ "$LLVM_VERSION" == "latest" ]; then
        brew install llvm
-        brew install cppunit
    else
        brew install llvm@$LLVM_VERSION
-        brew install cppunit
    fi
 fi
@@ -1,4 +1,6 @@
 #!/usr/bin/env bash
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0

 set -ex

@@ -1,4 +1,6 @@
 #!/usr/bin/env bash
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0

 set -x

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+
+set -ex
+
+# Download the TeX Live installer
+wget https://mirror.ctan.org/systems/texlive/tlnet/install-tl-unx.tar.gz
+
+# Extract
+tar xzf install-tl-unx.tar.gz
+rm install-tl-unx.tar.gz
+cd install-tl-*
+
+# Run the installer with no docs or source installation
+./install-tl --no-interaction --no-doc-install --no-src-install --scheme=minimal
+
+# Export the texlive PATH to GITHUB_PATH
+TEXLIVE_PATH=/usr/local/texlive/2025/bin/x86_64-linux
+export PATH=${TEXLIVE_PATH}:${PATH}
+echo "${TEXLIVE_PATH}" >> $GITHUB_PATH
+
+# Install the required packages
+tlmgr install latex-bin dvips collection-fontsrecommended collection-latexrecommended
+
+# Validate that installing dvips and latex has been successful
+dvips --version
+latex --version
@@ -1,4 +1,6 @@
 #!/usr/bin/env bash
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
 set -e

 # Various tests to test the FindOpenVDB CMake modules and
@@ -8,7 +10,7 @@ set -e
 # the expected VDB installation

 cmakelists="
-cmake_minimum_required(VERSION 3.18)
+cmake_minimum_required(VERSION 3.24)
 project(TestInstall LANGUAGES CXX)
 find_package(OpenVDB REQUIRED COMPONENTS openvdb)
 add_executable(test_vdb_print \"../openvdb_cmd/vdb_print/main.cc\")
@@ -1,4 +1,6 @@
 #!/usr/bin/env bash
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0

 set -ex

@@ -0,0 +1,24 @@
+# SPDX-License-Identifier: MIT
+#
+# SPDX-FileCopyrightText: Copyright (c) 2019-2023 Lars Melchior and contributors
+
+set(CPM_DOWNLOAD_VERSION 0.40.2)
+set(CPM_HASH_SUM "c8cdc32c03816538ce22781ed72964dc864b2a34a310d3b7104812a5ca2d835d")
+
+if(CPM_SOURCE_CACHE)
+  set(CPM_DOWNLOAD_LOCATION "${CPM_SOURCE_CACHE}/cpm/CPM_${CPM_DOWNLOAD_VERSION}.cmake")
+elseif(DEFINED ENV{CPM_SOURCE_CACHE})
+  set(CPM_DOWNLOAD_LOCATION "$ENV{CPM_SOURCE_CACHE}/cpm/CPM_${CPM_DOWNLOAD_VERSION}.cmake")
+else()
+  set(CPM_DOWNLOAD_LOCATION "${CMAKE_BINARY_DIR}/cmake/CPM_${CPM_DOWNLOAD_VERSION}.cmake")
+endif()
+
+# Expand relative path. This is important if the provided path contains a tilde (~)
+get_filename_component(CPM_DOWNLOAD_LOCATION ${CPM_DOWNLOAD_LOCATION} ABSOLUTE)
+
+file(DOWNLOAD
+     https://github.com/cpm-cmake/CPM.cmake/releases/download/v${CPM_DOWNLOAD_VERSION}/CPM.cmake
+     ${CPM_DOWNLOAD_LOCATION} EXPECTED_HASH SHA256=${CPM_HASH_SUM}
+)
+
+include(${CPM_DOWNLOAD_LOCATION})
@@ -88,7 +88,7 @@ may be provided to tell this module where to look.

 #]=======================================================================]

-cmake_minimum_required(VERSION 3.18)
+cmake_minimum_required(VERSION 3.24)
 include(GNUInstallDirs)

 mark_as_advanced(
@@ -1,320 +0,0 @@
-# Copyright Contributors to the OpenVDB Project
-# SPDX-License-Identifier: Apache-2.0
-#
-#[=======================================================================[.rst:
-
-FindCppUnit
-----------
-
-Find CppUnit include dirs and libraries
-
-Use this module by invoking find_package with the form::
-
-  find_package(CppUnit
-    [version] [EXACT]      # Minimum or EXACT version
-    [REQUIRED]             # Fail with error if CppUnit is not found
-    )
-
-
-IMPORTED Targets
-^^^^^^^^^^^^^^^^
-
-``CppUnit::cppunit``
-  This module defines IMPORTED target CppUnit::cppunit, if CppUnit has been
-  found.
-
-Result Variables
-^^^^^^^^^^^^^^^^
-
-This will define the following variables:
-
-``CppUnit_FOUND``
-  True if the system has the CppUnit library.
-``CppUnit_VERSION``
-  The version of the CppUnit library which was found.
-``CppUnit_INCLUDE_DIRS``
-  Include directories needed to use CppUnit.
-``CppUnit_RELEASE_LIBRARIES``
-  Libraries needed to link to the release version of CppUnit.
-``CppUnit_RELEASE_LIBRARY_DIRS``
-  CppUnit release library directories.
-``CppUnit_DEBUG_LIBRARIES``
-  Libraries needed to link to the debug version of CppUnit.
-``CppUnit_DEBUG_LIBRARY_DIRS``
-  CppUnit debug library directories.
-
-Deprecated - use [RELEASE|DEBUG] variants:
-
-``CppUnit_LIBRARIES``
-  Libraries needed to link to CppUnit.
-``CppUnit_LIBRARY_DIRS``
-  CppUnit library directories.
-
-Cache Variables
-^^^^^^^^^^^^^^^
-
-The following cache variables may also be set:
-
-``CppUnit_INCLUDE_DIR``
-  The directory containing ``cppunit/Portability.h``.
-``CppUnit_LIBRARY``
-  The path to the CppUnit library.
-
-Hints
-^^^^^
-
-Instead of explicitly setting the cache variables, the following variables
-may be provided to tell this module where to look.
-
-``CppUnit_ROOT``
-  Preferred installation prefix.
-``CPPUNIT_INCLUDEDIR``
-  Preferred include directory e.g. <prefix>/include
-``CPPUNIT_LIBRARYDIR``
-  Preferred library directory e.g. <prefix>/lib
-``CPPUNIT_DEBUG_SUFFIX``
-  Suffix of the debug version of CppUnit libs. Defaults to "_d".
-``SYSTEM_LIBRARY_PATHS``
-  Global list of library paths intended to be searched by and find_xxx call
-``CPPUNIT_USE_STATIC_LIBS``
-  Only search for static cppunit libraries
-``DISABLE_CMAKE_SEARCH_PATHS``
-  Disable CMakes default search paths for find_xxx calls in this module
-
-#]=======================================================================]
-
-cmake_minimum_required(VERSION 3.18)
-include(GNUInstallDirs)
-
-
-mark_as_advanced(
-  CppUnit_INCLUDE_DIR
-  CppUnit_LIBRARY
-)
-
-set(_FIND_CPPUNIT_ADDITIONAL_OPTIONS "")
-if(DISABLE_CMAKE_SEARCH_PATHS)
-  set(_FIND_CPPUNIT_ADDITIONAL_OPTIONS NO_DEFAULT_PATH)
-endif()
-
-# Set _CPPUNIT_ROOT based on a user provided root var. Xxx_ROOT and ENV{Xxx_ROOT}
-# are prioritised over the legacy capitalized XXX_ROOT variables for matching
-# CMake 3.12 behaviour
-# @todo  deprecate -D and ENV CPPUNIT_ROOT from CMake 3.12
-if(CppUnit_ROOT)
-  set(_CPPUNIT_ROOT ${CppUnit_ROOT})
-elseif(DEFINED ENV{CppUnit_ROOT})
-  set(_CPPUNIT_ROOT $ENV{CppUnit_ROOT})
-elseif(CPPUNIT_ROOT)
-  set(_CPPUNIT_ROOT ${CPPUNIT_ROOT})
-elseif(DEFINED ENV{CPPUNIT_ROOT})
-  set(_CPPUNIT_ROOT $ENV{CPPUNIT_ROOT})
-endif()
-
-# Additionally try and use pkconfig to find cppunit
-if(USE_PKGCONFIG)
-  if(NOT DEFINED PKG_CONFIG_FOUND)
-    find_package(PkgConfig)
-  endif()
-  pkg_check_modules(PC_CppUnit QUIET cppunit)
-endif()
-
-# ------------------------------------------------------------------------
-#  Search for CppUnit include DIR
-# ------------------------------------------------------------------------
-
-set(_CPPUNIT_INCLUDE_SEARCH_DIRS "")
-list(APPEND _CPPUNIT_INCLUDE_SEARCH_DIRS
-  ${CPPUNIT_INCLUDEDIR}
-  ${_CPPUNIT_ROOT}
-  ${PC_CppUnit_INCLUDEDIR}
-  ${SYSTEM_LIBRARY_PATHS}
-)
-
-# Look for a standard cppunit header file.
-find_path(CppUnit_INCLUDE_DIR cppunit/Portability.h
-  ${_FIND_CPPUNIT_ADDITIONAL_OPTIONS}
-  PATHS ${_CPPUNIT_INCLUDE_SEARCH_DIRS}
-  PATH_SUFFIXES ${CMAKE_INSTALL_INCLUDEDIR} include
-)
-
-if(EXISTS "${CppUnit_INCLUDE_DIR}/cppunit/Portability.h")
-  file(STRINGS "${CppUnit_INCLUDE_DIR}/cppunit/Portability.h"
-    _cppunit_version_string REGEX "#define CPPUNIT_VERSION "
-  )
-  string(REGEX REPLACE "#define CPPUNIT_VERSION +\"(.+)\".*$" "\\1"
-    _cppunit_version_string "${_cppunit_version_string}"
-  )
-  string(STRIP "${_cppunit_version_string}" CppUnit_VERSION)
-  unset(_cppunit_version_string )
-endif()
-
-# ------------------------------------------------------------------------
-#  Search for CppUnit lib DIR
-# ------------------------------------------------------------------------
-
-if(NOT DEFINED CPPUNIT_DEBUG_SUFFIX)
-  set(CPPUNIT_DEBUG_SUFFIX "d")
-endif()
-
-set(_CPPUNIT_LIBRARYDIR_SEARCH_DIRS "")
-list(APPEND _CPPUNIT_LIBRARYDIR_SEARCH_DIRS
-  ${CPPUNIT_LIBRARYDIR}
-  ${_CPPUNIT_ROOT}
-  ${PC_CppUnit_LIBDIR}
-  ${SYSTEM_LIBRARY_PATHS}
-)
-
-# Library suffix handling
-
-set(_CPPUNIT_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES})
-
-if(MSVC)
-  if(CPPUNIT_USE_STATIC_LIBS)
-    set(CMAKE_FIND_LIBRARY_SUFFIXES ".lib")
-  else()
-    list(APPEND CMAKE_FIND_LIBRARY_SUFFIXES "_dll.lib")
-  endif()
-else()
-  if(CPPUNIT_USE_STATIC_LIBS)
-    set(CMAKE_FIND_LIBRARY_SUFFIXES ".a")
-  endif()
-endif()
-
-list(APPEND CPPUNIT_BUILD_TYPES RELEASE DEBUG)
-
-foreach(BUILD_TYPE ${CPPUNIT_BUILD_TYPES})
-  set(_CPPUNIT_LIB_NAME cppunit)
-  if(BUILD_TYPE STREQUAL DEBUG)
-    set(_CPPUNIT_LIB_NAME "${_CPPUNIT_LIB_NAME}${CPPUNIT_DEBUG_SUFFIX}")
-  endif()
-
-  # Find the lib
-  find_library(CppUnit_LIBRARY_${BUILD_TYPE} ${_CPPUNIT_LIB_NAME}
-    ${_FIND_CPPUNIT_ADDITIONAL_OPTIONS}
-    PATHS ${_CPPUNIT_LIBRARYDIR_SEARCH_DIRS}
-    PATH_SUFFIXES ${CMAKE_INSTALL_LIBDIR} lib64 lib
-  )
-endforeach()
-
-# Reset library suffix
-
-set(CMAKE_FIND_LIBRARY_SUFFIXES ${_CPPUNIT_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES})
-unset(_CPPUNIT_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES)
-
-if(CppUnit_LIBRARY_DEBUG AND CppUnit_LIBRARY_RELEASE)
-  # if the generator is multi-config or if CMAKE_BUILD_TYPE is set for
-  # single-config generators, set optimized and debug libraries
-  get_property(_isMultiConfig GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG)
-  if(_isMultiConfig OR CMAKE_BUILD_TYPE)
-    set(CppUnit_LIBRARY optimized ${CppUnit_LIBRARY_RELEASE} debug ${CppUnit_LIBRARY_DEBUG})
-  else()
-    # For single-config generators where CMAKE_BUILD_TYPE has no value,
-    # just use the release libraries
-    set(CppUnit_LIBRARY ${CppUnit_LIBRARY_RELEASE})
-  endif()
-  # FIXME: This probably should be set for both cases
-  set(CppUnit_LIBRARIES optimized ${CppUnit_LIBRARY_RELEASE} debug ${CppUnit_LIBRARY_DEBUG})
-endif()
-
-# if only the release version was found, set the debug variable also to the release version
-if(CppUnit_LIBRARY_RELEASE AND NOT CppUnit_LIBRARY_DEBUG)
-  set(CppUnit_LIBRARY_DEBUG ${CppUnit_LIBRARY_RELEASE})
-  set(CppUnit_LIBRARY       ${CppUnit_LIBRARY_RELEASE})
-  set(CppUnit_LIBRARIES     ${CppUnit_LIBRARY_RELEASE})
-endif()
-
-# if only the debug version was found, set the release variable also to the debug version
-if(CppUnit_LIBRARY_DEBUG AND NOT CppUnit_LIBRARY_RELEASE)
-  set(CppUnit_LIBRARY_RELEASE ${CppUnit_LIBRARY_DEBUG})
-  set(CppUnit_LIBRARY         ${CppUnit_LIBRARY_DEBUG})
-  set(CppUnit_LIBRARIES       ${CppUnit_LIBRARY_DEBUG})
-endif()
-
-# If the debug & release library ends up being the same, omit the keywords
-if("${CppUnit_LIBRARY_RELEASE}" STREQUAL "${CppUnit_LIBRARY_DEBUG}")
-  set(CppUnit_LIBRARY   ${CppUnit_LIBRARY_RELEASE} )
-  set(CppUnit_LIBRARIES ${CppUnit_LIBRARY_RELEASE} )
-endif()
-
-if(CppUnit_LIBRARY)
-  set(CppUnit_FOUND TRUE)
-else()
-  set(CppUnit_FOUND FALSE)
-endif()
-
-# ------------------------------------------------------------------------
-#  Cache and set CppUnit_FOUND
-# ------------------------------------------------------------------------
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(CppUnit
-  FOUND_VAR CppUnit_FOUND
-  REQUIRED_VARS
-    CppUnit_LIBRARY
-    CppUnit_INCLUDE_DIR
-  VERSION_VAR CppUnit_VERSION
-)
-
-if(NOT CppUnit_FOUND)
-  if(CppUnit_FIND_REQUIRED)
-    message(FATAL_ERROR "Unable to find CppUnit")
-  endif()
-  return()
-endif()
-
-# Partition release/debug lib vars
-
-set(CppUnit_RELEASE_LIBRARIES ${CppUnit_LIBRARY_RELEASE})
-get_filename_component(CppUnit_RELEASE_LIBRARY_DIRS ${CppUnit_RELEASE_LIBRARIES} DIRECTORY)
-set(CppUnit_DEBUG_LIBRARIES ${CppUnit_LIBRARY_DEBUG})
-get_filename_component(CppUnit_DEBUG_LIBRARY_DIRS ${CppUnit_DEBUG_LIBRARIES} DIRECTORY)
-set(CppUnit_LIBRARIES ${CppUnit_RELEASE_LIBRARIES})
-set(CppUnit_LIBRARY_DIRS ${CppUnit_RELEASE_LIBRARY_DIRS})
-set(CppUnit_INCLUDE_DIRS ${CppUnit_INCLUDE_DIR})
-
-# Configure lib type. If XXX_USE_STATIC_LIBS, we always assume a static
-# lib is in use. If win32, we can't mark the import .libs as shared, so
-# these are always marked as UNKNOWN. Otherwise, infer from extension.
-set(CPPUNIT_LIB_TYPE UNKNOWN)
-if(CPPUNIT_USE_STATIC_LIBS)
-  set(CPPUNIT_LIB_TYPE STATIC)
-elseif(UNIX)
-  get_filename_component(_CPPUNIT_EXT ${CppUnit_LIBRARY_RELEASE} EXT)
-  if(_CPPUNIT_EXT STREQUAL ".a")
-    set(CPPUNIT_LIB_TYPE STATIC)
-  elseif(_CPPUNIT_EXT STREQUAL ".so" OR
-         _CPPUNIT_EXT STREQUAL ".dylib")
-    set(CPPUNIT_LIB_TYPE SHARED)
-  endif()
-endif()
-
-if(NOT TARGET CppUnit::cppunit)
-  add_library(CppUnit::cppunit ${CPPUNIT_LIB_TYPE} IMPORTED)
-  set_target_properties(CppUnit::cppunit PROPERTIES
-    INTERFACE_COMPILE_OPTIONS "${PC_CppUnit_CFLAGS_OTHER}"
-    INTERFACE_INCLUDE_DIRECTORIES "${CppUnit_INCLUDE_DIRS}")
-
-  # Standard location
-  set_target_properties(CppUnit::cppunit PROPERTIES
-    IMPORTED_LINK_INTERFACE_LANGUAGES "CXX"
-    IMPORTED_LOCATION "${CppUnit_LIBRARY}")
-
-  # Release location
-  if(EXISTS "${CppUnit_LIBRARY_RELEASE}")
-    set_property(TARGET CppUnit::cppunit APPEND PROPERTY
-      IMPORTED_CONFIGURATIONS RELEASE)
-    set_target_properties(CppUnit::cppunit PROPERTIES
-      IMPORTED_LINK_INTERFACE_LANGUAGES_RELEASE "CXX"
-      IMPORTED_LOCATION_RELEASE "${CppUnit_LIBRARY_RELEASE}")
-  endif()
-
-  # Debug location
-  if(EXISTS "${CppUnit_LIBRARY_DEBUG}")
-    set_property(TARGET CppUnit::cppunit APPEND PROPERTY
-      IMPORTED_CONFIGURATIONS DEBUG)
-    set_target_properties(CppUnit::cppunit PROPERTIES
-      IMPORTED_LINK_INTERFACE_LANGUAGES_DEBUG "CXX"
-      IMPORTED_LOCATION_DEBUG "${CppUnit_LIBRARY_DEBUG}")
-  endif()
-endif()
@@ -64,7 +64,7 @@ may be provided to tell this module where to look.

 #]=======================================================================]

-cmake_minimum_required(VERSION 3.18)
+cmake_minimum_required(VERSION 3.24)
 include(GNUInstallDirs)


@@ -87,7 +87,7 @@ may be provided to tell this module where to look.

 #]=======================================================================]

-cmake_minimum_required(VERSION 3.18)
+cmake_minimum_required(VERSION 3.24)
 include(GNUInstallDirs)


@@ -0,0 +1,44 @@
+include (FindPackageHandleStandardArgs)
+
+if(NCCL_ROOT)
+  set(_NCCL_ROOT ${NCCL_ROOT})
+elseif(DEFINED ENV{NCCL_ROOT})
+  set(_NCCL_ROOT $ENV{NCCL_ROOT})
+endif()
+
+set(_NCCL_INCLUDE_SEARCH_DIRS "")
+list(APPEND _NCCL_INCLUDE_SEARCH_DIRS
+  ${_NCCL_ROOT}/include
+)
+find_path(NCCL_INCLUDE_DIR
+    NAMES nccl.h
+    PATHS ${_NCCL_INCLUDE_SEARCH_DIRS}
+)
+
+set(_NCCL_LIBRARY_SEARCH_DIRS "")
+list(APPEND _NCCL_LIBRARY_SEARCH_DIRS
+  ${_NCCL_ROOT}/bin
+)
+find_library(NCCL_LIBRARY
+    NAMES nccl
+    PATHS ${_NCCL_LIBRARY_SEARCH_DIRS}
+)
+
+find_package_handle_standard_args(NCCL REQUIRED_VARS NCCL_INCLUDE_DIR NCCL_LIBRARY)
+
+if (NCCL_FOUND)
+    mark_as_advanced(NCCL_INCLUDE_DIR)
+    mark_as_advanced(NCCL_LIBRARY)
+endif()
+
+if (NCCL_FOUND AND NOT TARGET NCCL::NCCL)
+    add_library(NCCL::NCCL SHARED IMPORTED)
+    if(MSVC)
+      string(REPLACE ".lib" ".dll" NCCL_DLL ${NCCL_LIBRARY})
+      set_property(TARGET NCCL::NCCL PROPERTY IMPORTED_LOCATION ${NCCL_DLL})
+      set_property(TARGET NCCL::NCCL PROPERTY IMPORTED_IMPLIB ${NCCL_LIBRARY})
+    else()
+      set_property(TARGET NCCL::NCCL PROPERTY IMPORTED_LOCATION ${NCCL_LIBRARY})
+    endif()
+    target_include_directories(NCCL::NCCL INTERFACE ${NCCL_INCLUDE_DIR})
+endif()
@@ -93,7 +93,7 @@ may be provided to tell this module where to look.

 #]=======================================================================]

-cmake_minimum_required(VERSION 3.18)
+cmake_minimum_required(VERSION 3.24)
 include(GNUInstallDirs)


@@ -109,7 +109,7 @@ may be provided to tell this module where to look.

 #]=======================================================================]

-cmake_minimum_required(VERSION 3.18)
+cmake_minimum_required(VERSION 3.24)
 include(GNUInstallDirs)


@@ -93,7 +93,7 @@ may be provided to tell this module where to look.

 #]=======================================================================]

-cmake_minimum_required(VERSION 3.18)
+cmake_minimum_required(VERSION 3.24)
 include(GNUInstallDirs)


@@ -47,7 +47,7 @@ The following variables may be provided to tell this module where to look.
 # Find the glfw3 installation and use glfw's CMake to initialize
 # the glfw lib

-cmake_minimum_required(VERSION 3.18)
+cmake_minimum_required(VERSION 3.24)

 set(_FIND_GLFW3_ADDITIONAL_OPTIONS "")
 if(DISABLE_CMAKE_SEARCH_PATHS)
@@ -109,18 +109,6 @@ endif()
 set(glfw3_FIND_VERSION ${MINIMUM_GLFW_VERSION})
 find_package(glfw3 ${MINIMUM_GLFW_VERSION} REQUIRED)

-# We only use find_package_handle_standard_args to verify and print
-# appropriate messages. This now explicitly errors in 3.19...
-# @todo Improve this entire GLFW3 search
-#  https://gitlab.kitware.com/cmake/cmake/-/issues/21505
-if(${CMAKE_VERSION} VERSION_LESS 3.19)
-  find_package(PackageHandleStandardArgs)
-  find_package_handle_standard_args(glfw3
-    REQUIRED_VARS glfw3_DIR glfw3_FOUND
-    VERSION_VAR glfw3_VERSION
-  )
-endif()
-
 if(OPENVDB_FUTURE_DEPRECATION AND FUTURE_MINIMUM_GLFW_VERSION)
  if(glfw3_VERSION VERSION_LESS ${FUTURE_MINIMUM_GLFW_VERSION})
    message(DEPRECATION "Support for GLFW versions < ${FUTURE_MINIMUM_GLFW_VERSION} "
@@ -73,7 +73,7 @@ may be provided to tell this module where to look.
 # Find the Houdini installation and use Houdini's CMake to initialize
 # the Houdini lib

-cmake_minimum_required(VERSION 3.18)
+cmake_minimum_required(VERSION 3.24)

 # Include utility functions for version information
 include(${CMAKE_CURRENT_LIST_DIR}/OpenVDBUtils.cmake)
@@ -51,7 +51,7 @@ variables may be provided to tell this module where to look.
 # Find the Maya installation and use Maya's CMake to initialize
 # the Maya lib

-cmake_minimum_required(VERSION 3.18)
+cmake_minimum_required(VERSION 3.24)


 set(_FIND_MAYA_ADDITIONAL_OPTIONS "")
@@ -57,7 +57,7 @@ The following functions are provided:

 #]=======================================================================]

-cmake_minimum_required(VERSION 3.18)
+cmake_minimum_required(VERSION 3.24)


 function(OPENVDB_GET_VERSION_DEFINE HEADER KEY VALUE)
@@ -16,7 +16,7 @@ existing from a previous run of cmake.

 #]=======================================================================]

-cmake_minimum_required(VERSION 3.18)
+cmake_minimum_required(VERSION 3.24)

 set(MANIFEST "${CMAKE_CURRENT_BINARY_DIR}/install_manifest.txt")

@@ -7,7 +7,7 @@

 #]=======================================================================]

-cmake_minimum_required(VERSION 3.18)
+cmake_minimum_required(VERSION 3.24)

 ###############################################################################

@@ -8,7 +8,7 @@

 #]=======================================================================]

-cmake_minimum_required(VERSION 3.18)
+cmake_minimum_required(VERSION 3.24)

 ###############################################################################

@@ -27,59 +27,59 @@ set(OPENVDB_DEPRECATED_ABI_LIST ${MINIMUM_OPENVDB_ABI_VERSION} ${_PREV_ABI})
 unset(_PREV_ABI)

 if(NOT DISABLE_DEPENDENCY_VERSION_CHECKS)
-  # @note  Currently tracking CY2022 of the VFX platform where available
+  # @note  Currently tracking CY2024 of the VFX platform where available
+  # (that is, our minimum requirements are for the current VFX platform)
+  # minus 2 years to be able to build with legacy ABI)

  # @Note  Compiler versions are not really a hard and fast rule, you just need
  # a compiler with complete support for our MINIMUM_CXX_STANDARD (currently 17).
  # What's usually important is that the version of libstdc++ and glibc in use
  # matches. Compilers other than GCC should provide options to ensure this
  # targetting e.g. --gcc-toolchain and -fgnuc-version with Clang.
-  set(MINIMUM_GCC_VERSION 9.3.1)
+  set(MINIMUM_GCC_VERSION 11.2.1)
  set(MINIMUM_CLANG_VERSION 5.0)
  set(MINIMUM_ICC_VERSION 19)
-  set(MINIMUM_MSVC_VERSION 19.28) # 1928 (Visual Studio 2019 Version 16.8 + 16.9)
+  set(MINIMUM_MSVC_VERSION 19.30) # 1928 (Visual Studio 2019 Version 16.8 + 16.9)

-  # Should be 1.76 for VFX 22, but only version in apt is 1.73
-  set(MINIMUM_BOOST_VERSION 1.73)
-  set(MINIMUM_PYBIND_VERSION 2.9.1)
-  set(MINIMUM_IMATH_VERSION 3.1)
-  set(MINIMUM_OPENEXR_VERSION 3.1)
+  set(MINIMUM_BOOST_VERSION 1.82)
+  set(MINIMUM_OPENEXR_VERSION 3.2)
+  set(MINIMUM_IMATH_VERSION 3.2)
  set(MINIMUM_ZLIB_VERSION 1.2.7)
  set(MINIMUM_TBB_VERSION 2020.3)
-  set(MINIMUM_LLVM_VERSION 10.0.0)
+  set(MINIMUM_LLVM_VERSION 15.0.0)
  set(MINIMUM_BLOSC_VERSION 1.17.0)
-  set(MINIMUM_GLFW_VERSION 3.1)
+  set(MINIMUM_GLFW_VERSION 3.3)

-  set(MINIMUM_PYTHON_VERSION 3.9.1)
-  set(MINIMUM_NUMPY_VERSION 1.20.0)
+  set(MINIMUM_PYTHON_VERSION 3.11)

  set(MINIMUM_GOOGLETEST_VERSION 1.10)
-  set(MINIMUM_LOG4CPLUS_VERSION 1.1.2)
-  set(MINIMUM_HOUDINI_VERSION 19.5)
+  set(MINIMUM_LOG4CPLUS_VERSION 2.0)
+  set(MINIMUM_HOUDINI_VERSION 20.5)

  # These always promote warnings rather than errors
  set(MINIMUM_MAYA_VERSION 2017)
  set(MINIMUM_DOXYGEN_VERSION 1.8.8)
 endif()

-# VFX 23 deprecations to transition to MINIMUM_* variables in OpenVDB 12.0.0
+# VFX 24 deprecations to transition to MINIMUM_* variables in OpenVDB 14.0.0
 # @note  At the time of writing, any variables that are commented out don't
 #   have target transitional versions.

-set(FUTURE_MINIMUM_GCC_VERSION 11.2.1)
-set(FUTURE_MINIMUM_MSVC_VERSION 19.30) # 1930 (Visual Studio 2022)
+# set(FUTURE_MINIMUM_GCC_VERSION 11.2.1)
+# set(FUTURE_MINIMUM_MSVC_VERSION 19.30)
 # set(FUTURE_MINIMUM_ICC_VERSION 19)

 # set(FUTURE_MINIMUM_CXX_STANDARD 20)
-set(FUTURE_MINIMUM_CMAKE_VERSION 3.20)
-# set(FUTURE_MINIMUM_OPENEXR_VERSION 3.1)
-set(FUTURE_MINIMUM_BOOST_VERSION 1.80)
-set(FUTURE_MINIMUM_GLFW_VERSION 3.3)
-set(FUTURE_MINIMUM_LOG4CPLUS_VERSION 2.0)
+set(FUTURE_MINIMUM_CMAKE_VERSION 4.0)
+set(FUTURE_MINIMUM_OPENEXR_VERSION 3.3)
+set(FUTURE_MINIMUM_IMATH_VERSION 3.3)
+set(FUTURE_MINIMUM_TBB_VERSION 2021)
+set(FUTURE_MINIMUM_BOOST_VERSION 1.85)
+#set(FUTURE_MINIMUM_GLFW_VERSION 3.3)
+#set(FUTURE_MINIMUM_LOG4CPLUS_VERSION 2.0)
+set(FUTURE_MINIMUM_NANOBIND_VERSION 2.5.0)

 # set(FUTURE_MINIMUM_BLOSC_VERSION 1.17.0)
-# set(FUTURE_MINIMUM_TBB_VERSION 2020.3)
-set(FUTURE_MINIMUM_PYTHON_VERSION 3.10)
-set(FUTURE_MINIMUM_NUMPY_VERSION 1.23.0)
+# set(FUTURE_MINIMUM_PYTHON_VERSION 3.11)
 # set(FUTURE_MINIMUM_HOUDINI_VERSION 20.0)
-set(FUTURE_MINIMUM_LLVM_VERSION 13.0.0)
+set(FUTURE_MINIMUM_LLVM_VERSION 18.0.0)
@@ -7,7 +7,7 @@

 #]=======================================================================]

-cmake_minimum_required(VERSION 3.18)
+cmake_minimum_required(VERSION 3.24)
 project(OpenVDBDocs LANGUAGES NONE)

 include(GNUInstallDirs)
@@ -44,6 +44,7 @@ set(DOXY_FILES
  openvdb/openvdb/points/impl
  openvdb/openvdb/python/pyopenvdb.h
  openvdb/openvdb/tools
+  openvdb/openvdb/tools/impl
  openvdb/openvdb/tree
  openvdb/openvdb/util
  doc/build.txt
@@ -238,10 +238,8 @@ it's a good idea to read the above section on

 DCC      | Supported Version | OpenVDB ABI |
 -------- | ----------------- | ----------- |
-Houdini  | 19.5              | 9           |
-Houdini  | 20.0              | 10          |
-Maya     | 2018              | Any         |
-Maya     | 2019              | Any         |
+Houdini  | 20.5              | 11          |
+Houdini  | 21.0              | 12          |

@subsection buildBuildHou Building Against Houdini

@@ -259,12 +257,10 @@ CMake          | Cross-platform family of tools designed to help build software
 C++17 Compiler | Matching Houdini compiler and version                           | All                |
 Boost          | Components: iostreams                                           | All                |
 GoogleTest     | A unit testing framework module for C++                         | Unit Tests         |
-CppUnit        | A unit testing framework module for C++                         | Unit Tests (AX)    |
 GLFW           | Simple API for OpenGL development                               | OpenVDB View       |
 Doxygen        | Documentation generation from C++                               | Documentation      |
 Log4cplus      | An optional dependency for improved OpenVDB Logging             | Optional (All)     |
 nanobind       | C++/python bindings                                             | Optional (Python)  |
-NumPy          | Scientific computing with Python                                | Optional (Python)  |
 LLVM           | Target-independent code generation                              | OpenVDB AX         |

 At a minimum a matching C++17 compiler and CMake will be required. See
@@ -351,14 +347,12 @@ Boost          | Components: iostreams
 ZLIB           | Compression library for disk serialization compression            | All                              |
 Blosc          | Recommended dependency for improved disk compression              | All*                             |
 GoogleTest     | A unit testing framework module for C++                           | Unit Tests                       |
-CppUnit        | A unit testing framework module for C++                           | Unit Tests (AX)                  |
 GLFW           | Simple API for OpenGL development                                 | OpenVDB View                     |
 Doxygen        | Documentation generation from C++                                 | Documentation                    |
 OpenEXR        | EXR serialization support                                         | OpenVDB Render                   |
 Imath          | Used half precision floating points and EXR serialization support | Optional (All)                   |
 Log4cplus      | An optional dependency for improved OpenVDB Logging               | Optional (All)                   |
 nanobind       | C++/python bindings                                               | Optional (Python)                |
-NumPy          | Scientific computing with Python                                  | Optional (Python)                |
 LLVM           | Target-independent code generation                                | OpenVDB AX                       |


@@ -514,7 +508,7 @@ You can add the below CMake snippet to your main `CMakeLists.txt` to bring in
 OpenVDB as a dependency:

@code{.cmake}
-cmake_minimum_required(VERSION 3.18)
+cmake_minimum_required(VERSION 3.24)
 list(APPEND CMAKE_MODULE_PATH "/location/of/openvdb/install/lib/cmake/OpenVDB")
 find_package(OpenVDB REQUIRED)
 target_link_libraries(myapp OpenVDB::openvdb)
@@ -607,17 +601,12 @@ OpenVDB uses [imported targets](https://cmake.org/cmake/help/latest/command/add_
 for all its dependencies. For imported Boost compatibility, the following
 versions of CMake are required:

-  - Boost 1.73 requires CMake 3.17.2 or newer.
-  - Boost 1.74 requires CMake 3.19 or newer.
-  - Boost 1.75 requires CMake 3.19.5 or newer.
-  - Boost 1.76 requires CMake 3.20.3 or newer.
-  - Boost 1.77 requires CMake 3.21.3 or newer.
-  - Boost 1.78 requires CMake 3.22.2 or newer.
-  - Boost 1.79 requires CMake 3.23.2 or newer.
-  - Boost 1.80 requires CMake 3.24.2 or newer.
-  - Boost 1.81 requires CMake 3.25.2 or newer.
  - Boost 1.82 requires CMake 3.27.0 or newer.
  - Boost 1.83 requires CMake 3.27.4 or newer.
+  - Boost 1.84 requires CMake 3.28.2 or newer.
+  - Boost 1.85 requires CMake 3.29.3 or newer.
+  - Boost 1.86 requires CMake 3.30.6, 3.31.1, or newer.
+  - Boost 1.87 requires CMake 3.31.4 or newer.

 Use the above list to identify the version of CMake you require for your version
 of Boost. Note that this may be implemented into the OpenVDB CMake in the future.
@@ -2,164 +2,413 @@

@page changes Release Notes

-@htmlonly <a name="v12_0_0_changes"></a>@endhtmlonly
+@htmlonly <a name="v12_1_2_changes"></a>@endhtmlonly
@par
-<B>Version 12.0.0</B> - <I>In Development</I>
+<B>Version 12.1.2</B> - <I>In development</I>
+
+
+@htmlonly <a name="v12_1_1_changes"></a>@endhtmlonly
+@par
+<B>Version 12.1.1</B> - <I>September 30, 2025</I>
+
+@par
+OpenVDB AX:
+- Fixes:
+  - Fixed an issue which was causing host specific optimization passes to be
+    omitted when using LLVM 16 and higher.
+
+@par
+Houdini:
+  - Fixed a compilation issue with the OpenVDB Activate SOP with Houdini 21
+  [Reported by @callen128]
+
+
+@htmlonly <a name="v12_1_0_changes"></a>@endhtmlonly
+@par
+<B>Version 12.1.0</B> - <I>August 8, 2025</I>

@par
 OpenVDB:
+- New features:
+  - New level set constructor tools::createLevelSetCapsule creates a grid
+    containing a narrow-band level set representation of a capsule (tube
+    with constant radius and sphere caps).
+  - New level set constructor tools::createLevelSetTaperedCapsule creates a
+    grid containing a narrow-band level set representation of a tapered
+    capsule (tube with sphere caps and different radii at both ends, or
+    equivalently the convex hull of two spheres with possibly different
+    centers and radii).
+  - New level set constructor tools::createLevelSetTubeComplex creates a grid
+    containing a narrow-band level set representation of a tube complex (a
+    collection of capsules or tapered capsules defined by endpoint coordinates
+    and segment indices).
+  - New level set constructor tools::createLevelSetDilatedMesh creates a grid
+    containing a narrow-band level set representation of a dilated surface
+    mesh (dilated by a radius in all directions).
+  - New class tools::ConvexVoxelizer, used by the above methods, which can be
+    inherited to implement a convex level set constructor.
+  - Introduced anisotropic surfacing tools in points/PointRasterizeSDF.h and
+    principal component analysis (PCA) methods in points/PrincipalComponentAnalysis.h.
+    The latter tool analyses point neighborhoods and computes affine
+    transformations representing elliptical distributions. The prior tool
+    takes rotational and stretch point attributes (optionally computed from
+      the prior PCA methods) and generates anisotropic surfaces.
+
 - Improvements:
-    - Added openvdb::assertAbort to replace cassert and a
-      OPENVDB_ENABLE_ASSERTS cmake argument/compile define to toggle
-      assertions in OpenVDB code, independantly of NDEBUG. Asserts are
-      no longer enabled by default in when NDEBUG is absent (e.g.
-      Debug builds).
-    - Removed last traces of Boost when OPENVDB_USE_DELAYED_LOADING is OFF
-      [Reported by Brian McKinnon]
-    - RootNode code cleanup to eliminate redundant key conversion and
-      to create map values in-place.
-    - Add RootNode::deleteChildOrTile() to delete a child or tile of
-      the root node.
-    - ValueAccessors are now defined and created in the Tree class
-      instead of in the Grid class so that custom Tree implementations
-      may define and create their own ValueAccessors if desired.
+  - Significantly improved the performance of all LeafNode ValueIterators,
+    up to 5x on some platforms and up to 10x when delay loading is enabled.
+    Construction of a ValueIterator from a leaf node now requests the leaf
+    buffers ahead of iteration to avoid potentially expensive API calls.
+  - Improved the performance of PointRasterizeSDF.h tools with larger radii
+    with better bound computation approximations.
+  - Significant performance improvement to delay loaded files - the warning
+    logic which checks the underlying mapped file for the last modified time
+    has been removed, resulting in performance gains upwards of 10x when
+    loading in node data buffers. Note that the check that was removed is
+    entirely redundant on Windows and would only print a warning on other
+    system.
+    [Contributed by jjtParadox]
+  - Added OPENVDB_HAS_INVOKABLE_MEMBER_FUNCTION and OPENVDB_HAS_MEMBER_FUNCTION
+    macros to help determine the presence and compatibility of interface
+    member functions at compile time.
+  - Added OPENVDB_ASSUME macros to mimic builtin assume and C++23 assume
+    attributes.
+  - Improved read and write of PLY files with vdb_tool so they support Vec3d
+    and 64 bit attributes. Changed vdb_tool version number from 10.6.1 to
+    10.7.0.

 - API Changes:
-    - RootNode::tileCount(), RootNode::activeTileCount() and
-      RootNode::inactiveTileCount() are now public.
-    - RootNode::hasKey() and RootNode::coordToKey() are now public.
+  - Mat3 and Mat4 functions have been marked [[nodiscard]] where it may be
+    confusing if they are operating in place or returning a result, in
+    particular tranpose, invert, adjoint, and timesDiagonal.
+  - Re-worked the API in PointRasterizeSDF.h with the addition of anisotropic
+    surfacing kernels. The new API exposes more concise free functions which
+    take structs of parameters. The old API remains supported but is
+    deprecated.
+  - Deprecated providing a filter or interrupter as arguments to
+    points::rasterize. These can instead be implemented on user transfer
+    schemes.
+  - Transfer schemes provided to points::rasterize can now optionally
+    implement a rasterizePoints callback that takes a range of points per
+    voxel.
+
+- Fixes:
+  - Fixed a bug in points::rasterizeSpheres and points::rasterizeSmoothSpheres
+    which could cause sections of the generated surface to incorrectly be
+    marked as negative interior values.
+  - Fixed a bug in points::rasterizeSmoothSpheres which could result in
+    incomplete surface reconstruction with significantly larger search
+    distances.
+  - Fixed some precision issues in various Matrix methods.
+  - Fixed bugs in vdb_tool that prevented build with newer namespaces in
+    nanovdb.
+
+@par
+OpenVDB AX:
+- Improvements:
+  - Added support for LLVM versions greater than 15. Upgraded CI to test with
+    LLVM versions 16, 17, 18, 19 and 20.
+  - As of this release, LLVM 15 is the minimum required version for OpenVDB
+    AX. Note that there is an issue with LLVM 17 symbols which makes it
+    incompatible with the OpenVDB Python plugin. AS such, LLVM version 18 and
+    greater are the recommended LLVM versions to use, with LLVM 18 being the
+    new future minimum version.
+  - Significant performance improvement to the compilation of AX kernels when
+    using LLVM 18 and greater.
+  - Significant backend refactoring to AX IR generation to support newer
+    versions of LLVM. In particular, parts of the codegen::Function and
+    codegen::FunctionBuilder API require new usage from LLVM 16 due to
+    the LLVM opaque pointer changes. If you're not using the OpenVDB AX
+    Function codegen API then no client changes should be necessary.
+  - Migrated OpenVDB AX unit tests from cppunit to gtest
+    [Contributed by Tim Straubinger]
+
+- Fixes:
+  - Fixed a case of undefined behaviour when explicitly casting variables to
+    the same type as their storage type e.g:
+      double a; double b = double(a);
+  - Fixed an AX compiler error which would produce invalid IR when using 64bit
+    integer variables for matrix array offsetting e.g:
+      int64 a = 0; mat4f b = 1; float c = b[a,a];
+
+@par
+NanoVDB:
+  - Added nanovdb::openToIndexVDB, which can convert an OpenVDB grid into a
+    NanoVDB IndexGrid.
+  - Added new options "--index" and "--onIndex" to the command-line tool
+    nanovdb_convert so it can produce IndexGrids.
+  - Fixed memory leak in DeviceBuffer.
+  - Added constructor for DeviceBuffer from an existing HostBuffer.
+  - Added VoxelBlockManager, an acceleration structure for sequential access
+    and stencil ops over active voxels.
+  - Added helper functions for kernel launches with shared memory.
+  - Added support for CUDA 13.0.
+  - Fix warnings and compilation errors for GCC and Clang.
+  - Fixed inconsistent usage of util::cuda::mallocAsync/freeAsync that
+    affected vGPUs.
+  - Added support for custom memory resources in PointsToGrid.
+  - Fixed race condition in split/mergeGridHandles.
+  - Fixed offset bug in nanovdb::tools::cuda::IndexToGrid related to non-POD
+    build types.
+    [Contributed by Andreas Henne]
+
+@par
+Houdini:
+  - Define OP_InputIdx and OP_OutputIdx for Houdini 21.0 compatibility
+  - Add Houdini 21 style camera interface to support LOP cameras
+  - Include env flag for avoiding stream writing (was backported to 19.5)
+
+
+@htmlonly <a name="v12_0_1_changes"></a>@endhtmlonly
+@par
+<B>Version 12.0.1</B> - <I>April 3, 2025</I>
+
+@par
+OpenVDB:
+- New features:
+  - Added read and write support for OFF (Object File Format) files to vdb_tool
+
+- Build:
+  - Fixed a build issue with vdb_render on Windows with newer versions of glew not including glu.h by default
+  - Fixed issues with newer versions of clang refusing to accept older ::template syntax
+
+@par
+NanoVDB:
+- New features:
+  - Added support for multiple GPUs to DeviceBuffer
+  - Added a UnifiedBuffer class that wraps CUDA unified memory
+  - Added example for multiGPU sparse convolution
+  - Added CUDA utility functions for device queries
+  - Added functions to independently stop and compute the elapsed time for timer classes
+
+- Improvements:
+  - Replaced CUB's CachingDeviceAllocator with the default asynchronous stream ordered allocator in PointsToGrid for improved performance
+
+- Bug fix:
+  - Map is now properly transposed when converting NanoVDB to OpenVDB.
+  - Fixed ostream specializations being hidden within the nanovdb namespace
+  - Fixed a call to a non-existent method in GridBuild.h
+
+
+@htmlonly <a name="v1_changes"></a>@endhtmlonly
+@par
+<B>Version 12.0.0</B> - <I>October 31, 2024</I>
+
+@par
+<BLOCKQUOTE>
+OpenVDB is now licensed under the Apache&nbsp;2.0 license, instead of the MPL&nbsp;2.0 license.
+</BLOCKQUOTE>
+
+@par
+<BLOCKQUOTE>
+This version introduces ABI changes relative to older major releases, so to
+preserve ABI compatibility it might be necessary to define the macro
+<TT>OPENVDB_ABI_VERSION_NUMBER=</TT><I>N</I>, where, for example,
+<I>N</I> is 10 for Houdini&nbsp;20.0 and 11 for Houdini&nbsp;20.5.
+</BLOCKQUOTE>
+
+@par
+<BLOCKQUOTE>
+GCC 9 is no longer supported.
+</BLOCKQUOTE>
+
+@par
+OpenVDB:
+- New features:
+  - Added tools::LevelSetFilter::fillet() method to round off concave edges
+    to create smoother transition between surfaces.
+
+- Improvements:
+  - Added openvdb::assertAbort to replace cassert and a
+    @c OPENVDB_ENABLE_ASSERTS cmake argument/compile define to toggle
+    assertions in OpenVDB code, independantly of NDEBUG. Asserts are
+    no longer enabled by default in when NDEBUG is absent (e.g.
+    Debug builds).
+  - Removed last traces of Boost when @c OPENVDB_USE_DELAYED_LOADING is OFF
+    <I>[Reported&nbsp;by&nbsp;Brian&nbsp;McKinnon]</I>
+  - RootNode code cleanup to eliminate redundant key conversion and
+    to create map values in-place.
+  - Add RootNode::deleteChildOrTile() to delete a child or tile of
+    the root node.
+  - ValueAccessors are now defined and created in the Tree class
+    instead of in the Grid class so that custom Tree implementations
+    may define and create their own ValueAccessors if desired.
+  - Added support for PDAL to vdb_tool [Contributed by Tom Matterson]
+  - LeafManager and NodeManager now use Index64 for leaf counts internally.
+  - Added RootNode::probeChild() const.
+  - Added RootNode::probeChild() and RootNode::probeConstChild().
+  - Added RootNode::probe() and RootNode::probeConst() to query key presence,
+    child node, value and active state.
+  - Added InternalNode::probeChild() const.
+  - Added InternalNode::probeChild() and probeChildConst() with coord access
+    and optionally value and active state.
+  - Added InternalNode::probeChild() and probeChildConst() with index access
+    and optionally value and active state.
+  - Added InternalNode::isValueOff(), LeafNode::isValueOff(),
+    LeafNodeBool::isValueOff(), LeafNodeMask::isValueOff().
+  - Added LeafNodeMask::probeValue(Index,val), LeafNodeBool::probeValue(Index,val).
+  - Added RootNode::getValueUnsafe(), RootNode::getChildUnsafe(),
+    RootNode::getConstChildUnsafe().
+  - Added InternalNode::getValueUnsafe(), InternalNode::getChildUnsafe(),
+    InternalNode::getConstChildUnsafe().
+  - Added InternalNode::setActiveStateUnsafe(), InternalNode::setValueOnlyUnsafe(),
+    InternalNode::setValueOnUnsafe(), InternalNode::setValueOffUnsafe().
+  - Added InternalNode::setChildUnsafe(), InternalNode::resetChildUnsafe(),
+    InternalNode::stealChildUnsafe(), InternalNode::deleteChildUnsafe().
+  - For LeafNode, LeafNodeBool and LeafNodeMask - added
+    LeafNode::getValueUnsafe(), LeafNode::setActiveStateunsafe(),
+    LeafNode::setValueOnlyUnsafe(), LeafNode::setValueOnUnsafe(),
+    LeafNode::setValueOffUnsafe().
+
+- ABI changes:
+  - Tree::leafCount(), Tree::unallocatedLeafCount(),
+    Tree::nonLeafCount() and Tree::nodeCount() now use Index64 in their
+    return types instead of Index32.
+
+- API Changes:
+  - RootNode::tileCount(), RootNode::activeTileCount() and
+    RootNode::inactiveTileCount() are now public.
+  - RootNode::hasKey() and RootNode::coordToKey() are now public.
+  - RootNode::leafCount(), RootNode::nonLeafCount() and RootNode::nodeCount()
+    now use Index64 instead of Index32. The Index32 variant is deprecated.
+  - InternalNode::leafCount(), InternalNode::nonLeafCount() and
+    InternalNode::nodeCount() now use Index64 instead of Index32. The Index32
+    variant is deprecated.
+  - LeafNode::leafCount() and LeafNode::nonLeafCount() now use Index64 instead
+    of Index32. The Index32 variant is deprecated.

 - Bug Fixes:
-    - Fix potential crash reading corrupt .vdb files with invalid
-      blosc or zip chunks.
-      [Fix thanks to Matthias Ueberheide]
-    - Fix a bug in RootNode::setOrigin() where the origin was updated
-      before the error was thrown potentially leaving the root in an
-      invalid state.
-    - Fixed a thread sanitizer issue which could cause undefined
-      behaviour in VolumeToSpheres::fillWithSpheres
-      [Reported by Jérémie Dumas]
-    - Fixed an occurance of undefined behaviour in tools::activate
-      (though this would typically not have manifested with any
-      unintended behaviour)
+  - Fix potential crash reading corrupt .vdb files with invalid
+    blosc or zip chunks.
+    <I>[Reported&nbsp;by&nbsp;Matthias&nbsp;Ueberheide]</I>
+
+  - Fix a bug in RootNode::setOrigin() where the origin was updated
+    before the error was thrown potentially leaving the root in an
+    invalid state.
+  - Fixed a thread sanitizer issue which could cause undefined
+    behaviour in VolumeToSpheres::fillWithSpheres
+    <I>[Reported&nbsp;by&nbsp;Jérémie&nbsp;Dumas]</I>
+  - Fixed an occurance of undefined behaviour in tools::activate
+    (though this would typically not have manifested with any
+    unintended behaviour)

@par
 NanoVDB:
 - Bug fix:
-    - nanovdb::readGrids works with raw grid buffer.
+  - nanovdb::readGrids works with raw grid buffer.

 - Improvements:
-    - Restructure files location and namespace to be more align with
-      OpenVDB. The namespaces touched by the restructuring are: io,
-      cuda, util, tools, and math.
-    - Add two scripts updateFiles.sh and updateFiles.py to update the
-      files using NanoVDB. The script updateFiles.py works on both
-      Windows and Linux.  For a more complete list of changes, see API
-      Changes (details).
+  - Restructure files location and namespace to be more align with
+    OpenVDB. The namespaces touched by the restructuring are: io,
+    cuda, util, tools, and math.
+  - Add two scripts updateFiles.sh and updateFiles.py to update the
+    files using NanoVDB. The script updateFiles.py works on both
+    Windows and Linux.  For a more complete list of changes, see API
+    Changes (details).

-    - cuda::PointsToGrid supports target density.
-    - Add support for NanoVDB Grid of type UInt8.
-    - Add ability to use externally managed CUDA buffer.
-    - Add create methods for CudaDeviceBuffer and exceptions.
-    - Improve GridValidator logic, e.g. include check for grid count.
-    - Add operator > and >= for class Coord according to lexicographical order.
-    - Add toCodec to convert string to Codec enumeration type.
-    - Add nanovdb::strlen<GridType>().
-    - Add strncpy util.
-    - Add NANOVDB_DISABLE_SYNC_CUDA_MALLOC that maps cudaMallocAsync
-      and cudaFreeAsync to cudaMalloc and cudaFree respectively.
-    - Add guard to UINT64_C.
-    - Remove use of cudaMallocAsync in PointsToGrid.cuh.
-    - Align PNanoVDB blind metadata to NanoVDB.
+  - cuda::PointsToGrid supports target density.
+  - Add support for NanoVDB Grid of type UInt8.
+  - Add ability to use externally managed CUDA buffer.
+  - Add create methods for CudaDeviceBuffer and exceptions.
+  - Improve GridValidator logic, e.g. include check for grid count.
+  - Add operator > and >= for class Coord according to lexicographical order.
+  - Add toCodec to convert string to Codec enumeration type.
+  - Add nanovdb::strlen<GridType>().
+  - Add strncpy util.
+  - Add @c NANOVDB_DISABLE_SYNC_CUDA_MALLOC that maps cudaMallocAsync
+    and cudaFreeAsync to cudaMalloc and cudaFree respectively.
+  - Add guard to @c UINT64_C.
+  - Remove use of cudaMallocAsync in PointsToGrid.cuh.
+  - Align PNanoVDB blind metadata to NanoVDB.

 - API Changes:
-    - Change mapToGridType to toGridType.
-    - Change mapToMagic to toMagic.
-    - Change CpuTimer.h to Timer.h.
+  - Change mapToGridType to toGridType.
+  - Change mapToMagic to toMagic.
+  - Change <TT>CpuTimer.h</TT> to <TT>Timer.h</TT>.

 - API Changes (details):
-    - These APIs are now under the math namespace: Ray, DDA, HDDA,
-      Vec3, Vec4, BBox, ZeroCrossing, TreeMarcher, PointTreeMarcher,
-      BoxStencil, CurvatureStencil, GradStencil, WenoStencil, AlignUp,
-      Min, Max, Abs, Clamp, Sqrt, Sign, Maximum, Delta, RoundDown, pi,
-      isApproxZero, Round, createSampler, SampleFromVoxels.
+  - These APIs are now under the math namespace: Ray, DDA, HDDA,
+    Vec3, Vec4, BBox, ZeroCrossing, TreeMarcher, PointTreeMarcher,
+    BoxStencil, CurvatureStencil, GradStencil, WenoStencil, AlignUp,
+    Min, Max, Abs, Clamp, Sqrt, Sign, Maximum, Delta, RoundDown, pi,
+    isApproxZero, Round, createSampler, SampleFromVoxels.

-    - These APIs are now under the tools namespace: createNanoGrid,
-      StatsMode, createLevelSetSphere, createFogVolumeSphere,
-      createFogVolumeSphere, createFogVolumeSphere,
-      createFogVolumeTorus, createLevelSetBox, CreateNanoGrid,
-      updateGridStats, evalChecksum, validateChecksum, checkGrid,
-      Extrema.
-    - These APIs are now under the util namespace: is_floating_point,
-      findLowestOn, findHighestOn, Range, streq, strcpy, strcat,
-      empty, Split, invoke, forEach, reduce, prefixSum, is_same,
-      is_specialization, PtrAdd, PtrDiff.
+  - These APIs are now under the tools namespace: createNanoGrid,
+    StatsMode, createLevelSetSphere, createFogVolumeSphere,
+    createFogVolumeSphere, createFogVolumeSphere,
+    createFogVolumeTorus, createLevelSetBox, CreateNanoGrid,
+    updateGridStats, evalChecksum, validateChecksum, checkGrid,
+    Extrema.
+  - These APIs are now under the util namespace: is_floating_point,
+    findLowestOn, findHighestOn, Range, streq, strcpy, strcat,
+    empty, Split, invoke, forEach, reduce, prefixSum, is_same,
+    is_specialization, PtrAdd, PtrDiff.

-    - Move nanovdb::build to nanovdb::tools::build.
-    - Rename nanovdb::BBoxR to nanovdb::Vec3dBBox.
-    - Rename nanovdb::BBox<nanovdb::Vec3d> to nanovdb::Vec3dBbox.
-    - Move nanovdb::cudaCreateNodeManager to nanovdb::cuda::createNodeManager.
-    - Move and rename nanovdb::cudaVoxelsToGrid to nanovdb::cuda::voxelsToGrid.
-    - Move and rename nanovdb::cudaPointsToGrid to nanovdb::cuda::pointsToGrid.
-    - Move nanovdb::DitherLUT to nanovdb::math::DitherLUT.
-    - Move and rename nanovdb::PackedRGBA8 to nanovdb::math::Rgba8.
-    - Move nanovdb::Rgba8 to nanovdb::math::Rgba8.
-    - Move and rename nanovdb::CpuTimer to nanovdb::util::Timer.
-    - Move nanovdb::GpuTimer to nanovdb::util::cuda::Timer.
-    - Move and rename nanovdb::CountOn to nanovdb::util::countOn.
+  - Move nanovdb::build to nanovdb::tools::build.
+  - Rename nanovdb::BBoxR to nanovdb::Vec3dBBox.
+  - Rename nanovdb::BBox<nanovdb::Vec3d> to nanovdb::Vec3dBbox.
+  - Move nanovdb::cudaCreateNodeManager to nanovdb::cuda::createNodeManager.
+  - Move and rename nanovdb::cudaVoxelsToGrid to nanovdb::cuda::voxelsToGrid.
+  - Move and rename nanovdb::cudaPointsToGrid to nanovdb::cuda::pointsToGrid.
+  - Move nanovdb::DitherLUT to nanovdb::math::DitherLUT.
+  - Move and rename nanovdb::PackedRGBA8 to nanovdb::math::Rgba8.
+  - Move nanovdb::Rgba8 to nanovdb::math::Rgba8.
+  - Move and rename nanovdb::CpuTimer to nanovdb::util::Timer.
+  - Move nanovdb::GpuTimer to nanovdb::util::cuda::Timer.
+  - Move and rename nanovdb::CountOn to nanovdb::util::countOn.

-    - Move util/GridHandle.h to GridHandle.h.
-    - Move util/BuildGrid.h to tools/GridBuilder.h.
-    - Move util/GridBuilder.h to tools/GridBuilder.h.
-    - Move util/IO.h to io/IO.h.
-    - Move util/CSampleFromVoxels.h to math/CSampleFromVoxels.h.
-    - Move util/DitherLUT.h to math/DitherLUT.h.
-    - Move util/HDDA.h to math/HDDA.h.
-    - Move util/Ray.h to math/Ray.h.
-    - Move util/SampleFromVoxels.h to math/SampleFromVoxels.h.
-    - Move util/Stencils.h to nanovdb/math/Stencils.h.
-    - Move util/CreateNanoGrid.h to tools/CreateNanoGrid.h.
-    - Move and rename util/Primitives.h to tools/CreatePrimitives.h.
-    - Move util/GridChecksum.h to tools/GridChecksum.h.
-    - Move util/GridStats.h to tools/GridStats.h.
-    - Move util/GridChecksum.h to tools/GridChecksum.h.
-    - Move util/GridValidator.h to tools/GridValidator.h.
-    - Move util/NanoToOpenVDB.h to tools/NanoToOpenVDB.h.
-    - Move util/cuda/CudaGridChecksum.cuh to tools/cuda/CudaGridChecksum.cuh.
-    - Move util/cuda/CudaGridStats.cuh to tools/cuda/CudaGridStats.cuh.
-    - Move util/cuda/CudaGridValidator.cuh to tools/cuda/CudaGridValidator.cuh.
-    - Move util/cuda/CudaIndexToGrid.cuh to tools/cuda/CudaIndexToGrid.cuh.
-    - Move and rename util/cuda/CudaPointsToGrid.cuh to tools/cuda/PointsToGrid.cuh.
-    - Move util/cuda/CudaSignedFloodFill.cuh to tools/cuda/CudaSignedFloodFill.cuh.
-    - Move and rename util/cuda/CudaDeviceBuffer.h to cuda/DeviceBuffer.h.
-    - Move and rename util/cuda/CudaGridHandle.cuh to cuda/GridHandle.cuh.
-    - Move and rename util/cuda/CudaUtils.h to util/cuda/Util.h.
-    - Move and consolidate util/cuda/GpuTimer.h to util/cuda/Timer.h.
+  - Move <TT>util/GridHandle.h</TT> to <TT>GridHandle.h</TT>.
+  - Move <TT>util/BuildGrid.h</TT> to <TT>tools/GridBuilder.h</TT>.
+  - Move <TT>util/GridBuilder.h</TT> to <TT>tools/GridBuilder.h</TT>.
+  - Move <TT>util/IO.h</TT> to <TT>io/IO.h</TT>.
+  - Move <TT>util/CSampleFromVoxels.h</TT> to <TT>math/CSampleFromVoxels.h</TT>.
+  - Move <TT>util/DitherLUT.h</TT> to <TT>math/DitherLUT.h</TT>.
+  - Move <TT>util/HDDA.h</TT> to <TT>math/HDDA.h</TT>.
+  - Move <TT>util/Ray.h</TT> to <TT>math/Ray.h</TT>.
+  - Move <TT>util/SampleFromVoxels.h</TT> to <TT>math/SampleFromVoxels.h</TT>.
+  - Move <TT>util/Stencils.h</TT> to <TT>nanovdb/math/Stencils.h</TT>.
+  - Move <TT>util/CreateNanoGrid.h</TT> to <TT>tools/CreateNanoGrid.h</TT>.
+  - Move and rename <TT>util/Primitives.h</TT> to <TT>tools/CreatePrimitives.h</TT>.
+  - Move <TT>util/GridChecksum.h</TT> to <TT>tools/GridChecksum.h</TT>.
+  - Move <TT>util/GridStats.h</TT> to <TT>tools/GridStats.h</TT>.
+  - Move <TT>util/GridChecksum.h</TT> to <TT>tools/GridChecksum.h</TT>.
+  - Move <TT>util/GridValidator.h</TT> to <TT>tools/GridValidator.h</TT>.
+  - Move <TT>util/NanoToOpenVDB.h</TT> to <TT>tools/NanoToOpenVDB.h</TT>.
+  - Move <TT>util/cuda/CudaGridChecksum.cuh</TT> to <TT>tools/cuda/CudaGridChecksum.cuh</TT>.
+  - Move <TT>util/cuda/CudaGridStats.cuh</TT> to <TT>tools/cuda/CudaGridStats.cuh</TT>.
+  - Move <TT>util/cuda/CudaGridValidator.cuh</TT> to <TT>tools/cuda/CudaGridValidator.cuh</TT>.
+  - Move <TT>util/cuda/CudaIndexToGrid.cuh</TT> to <TT>tools/cuda/CudaIndexToGrid.cuh</TT>.
+  - Move and rename <TT>util/cuda/CudaPointsToGrid.cuh</TT> to <TT>tools/cuda/PointsToGrid.cuh</TT>.
+  - Move <TT>util/cuda/CudaSignedFloodFill.cuh</TT> to <TT>tools/cuda/CudaSignedFloodFill.cuh</TT>.
+  - Move and rename <TT>util/cuda/CudaDeviceBuffer.h</TT> to <TT>cuda/DeviceBuffer.h</TT>.
+  - Move and rename <TT>util/cuda/CudaGridHandle.cuh</TT> to <TT>cuda/GridHandle.cuh</TT>.
+  - Move and rename <TT>util/cuda/CudaUtils.h</TT> to <TT>util/cuda/Util.h</TT>.
+  - Move and consolidate <TT>util/cuda/GpuTimer.h</TT> to <TT>util/cuda/Timer.h</TT>.

@par
 Python:
-    - OpenVDB Python bindings are now implemented using nanobind
-      instead of pybind11
-    - The OpenVDB Python module has been changed from pyopenvdb to openvdb
-    - Added Python bindings for NanoVDB
+  - OpenVDB Python bindings are now implemented using nanobind
+    instead of pybind11
+  - The OpenVDB Python module has been changed from pyopenvdb to openvdb
+  - Added Python bindings for NanoVDB

@par
 Houdini:
-    - When OPENVDB_ENABLE_RPATH is ON, the location of
-      libopenvdb_houdini is now added to the rpath of all Houdini
-      dsos.
+  - When @c OPENVDB_ENABLE_RPATH is @c ON, the location of
+    libopenvdb_houdini is now added to the rpath of all Houdini
+    dsos.

@par
 Build:
-    - Fixed an issue with OpenVDB AX's CMake on Windows where the
-      static and shared library targets would have the same name
-      [Reported by Nicholas Yue]
-    - USE_EXPLICIT_INSTANTIATION is now disabled on Windows by default
-      due to OOM linker issues.
-    - Jemalloc is now the preferred allocator of choice on all
-      platforms when CONCURRENT_MALLOC is set to Auto.
-    - Fixed an issue with the Blosc CMake FindPackage for the OpenVDB
-      Windows static library.
+  - Fixed an issue with OpenVDB AX's CMake on Windows where the
+    static and shared library targets would have the same name
+    <I>[Reported&nbsp;by&nbsp;Nicholas&nbsp;Yue]</I>
+  - @c USE_EXPLICIT_INSTANTIATION is now disabled on Windows by default
+    due to OOM linker issues.
+  - Jemalloc is now the preferred allocator of choice on all
+    platforms when @c CONCURRENT_MALLOC is set to Auto.
+  - Fixed an issue with the Blosc CMake FindPackage for the OpenVDB
+    Windows static library.


@htmlonly <a name="v11_0_0_changes"></a>@endhtmlonly
@@ -7,7 +7,7 @@
 This document details the coding practices that are used in the OpenVDB
 codebase.  Contributed code should conform to these guidelines to maintain
 consistency and maintainability.  If there is a rule that you would like
-clarified, changed, or added, please send a note to openvdb@gmail.com.
+clarified, changed, or added, please send a note to openvdb-dev@lists.aswf.io.


@section sStyleContents Contents
@@ -41,14 +41,14 @@ OpenVDB Print           | Core Library dependencies                       | -
 OpenVDB LOD             | Core Library dependencies                       | -
 OpenVDB Render          | Core Library dependencies                       | OpenEXR, Imath::Imath, libpng
 OpenVDB View            | Core Library dependencies, OpenGL, GLFW3, GLEW* | -
-OpenVDB Python          | Core Library dependencies, Python, nanobind     | NumPy
+OpenVDB Python          | Core Library dependencies, Python, nanobind     | -
 OpenVDB AX              | Core Library dependencies, LLVM                 | Bison, Flex
 NanoVDB                 | -                                               | Core Library, CUDA, TBB, Blosc, ZLib
-OpenVDB UnitTests       | Core Library dependencies, GoogleTest           | CppUnit**
+OpenVDB UnitTests       | Core Library dependencies, GoogleTest, numpy*   | -
 OpenVDB Documentation   | Doxygen                                         | -

 - @b * GLEW is only required for building the vdb_view binary on Windows.
- - @b ** CppUnit is only required for building the AX unit tests.
+ - @b * numpy only required for python binding unit tests
 - @b Note - Bison and Flex are only required for re-generation of the AX
 grammar which is not typically necessary for the majority of users.

@@ -56,41 +56,39 @@ OpenVDB Documentation   | Doxygen                                         | -

 Package        | Minimum | Recommended | Description                                                       | apt-get | Homebrew | Source
 -------------- | ------- | ----------- | ----------------------------------------------------------------- | ------- | -------- | ------
-CMake          | 3.18    | Latest      | Cross-platform family of tools designed to help build software    | Y       | Y        | https://cmake.org
-GCC            | 9.3.1   | 11.2.1      | C++ 17 Compiler: The GNU Compiler Collection                      | Y       | Y        | https://www.gnu.org/software/gcc
+CMake          | 3.24    | Latest      | Cross-platform family of tools designed to help build software    | Y       | Y        | https://cmake.org
+GCC            | 11.2.1  | 11.2.1      | C++ 17 Compiler: The GNU Compiler Collection                      | Y       | Y        | https://www.gnu.org/software/gcc
 Clang          | 5.0     | Latest      | C++ 17 Compiler: A C language family frontend for LLVM            | Y       | Y        | https://clang.llvm.org
 Intel ICC      | 19      | Latest      | C++ 17 Compiler: Intels C++ Compiler                              | Y       | Y        | https://software.intel.com/en-us/c-compilers
-MSVC           | 19.28   | 19.30       | C++ 17 Compiler: Microsoft Visual C++ Compiler                    | Y       | Y        | https://visualstudio.microsoft.com/vs
-Imath          | 3.1     | Latest      | Half precision floating points                                    | Y       | Y        | http://www.openexr.com
-OpenEXR        | 3.1     | Latest      | EXR serialization support                                         | Y       | Y        | http://www.openexr.com
-TBB            | 2020.2  | 2020.3      | Threading Building Blocks - template library for task parallelism | Y       | Y        | https://www.threadingbuildingblocks.org
+MSVC           | 19.30   | 19.30       | C++ 17 Compiler: Microsoft Visual C++ Compiler                    | Y       | Y        | https://visualstudio.microsoft.com/vs
+Imath          | 3.2     | >= 3.3      | Half precision floating points                                    | Y       | Y        | http://www.openexr.com
+OpenEXR        | 3.2     | >= 3.3      | EXR serialization support                                         | Y       | Y        | http://www.openexr.com
+TBB            | 2020.3  | >= 2021     | Threading Building Blocks - template library for task parallelism | Y       | Y        | https://www.threadingbuildingblocks.org
 ZLIB           | 1.2.7   | Latest      | Compression library for disk serialization compression            | Y       | Y        | https://www.zlib.net
-Boost          | 1.73    | 1.80        | Components: iostreams                                             | Y       | Y        | https://www.boost.org
-LLVM           | 10.0.0  | 13.0.0*     | Target-independent code generation                                | Y       | Y        | https://llvm.org/
-Bison          | 3.0.0   | 3.7.0       | General-purpose parser generator                                  | Y       | Y        | https://www.gnu.org/software/gcc
-Flex           | 2.6.0   | 2.6.4       | Fast lexical analyzer generator                                   | Y       | Y        | https://github.com/westes/flex
-Python         | 3.9.1   | 3.10        | The python interpreter and libraries                              | Y       | Y        | https://www.python.org
-nanobind       | 2.0.0   | Latest      | C++/python bindings                                               | Y       | Y        | https://nanobind.readthedocs.io
-NumPy          | 1.20.0  | 1.23.0      | Scientific computing with Python                                  | Y       | Y        | http://www.numpy.org
+Boost          | 1.82    | >= 1.85     | Components: iostreams                                             | Y       | Y        | https://www.boost.org
+LLVM           | 15.0.0* | 18.0.0      | Target-independent code generation                                | Y       | Y        | https://llvm.org/
+Bison          | 3.7.0   | 3.7.0       | General-purpose parser generator                                  | Y       | Y        | https://www.gnu.org/software/gcc
+Flex           | 2.6.4   | 2.6.4       | Fast lexical analyzer generator                                   | Y       | Y        | https://github.com/westes/flex
+Python         | 3.10    | 3.11        | The python interpreter and libraries                              | Y       | Y        | https://www.python.org
+nanobind**     | 2.0.0   | >= 2.5.0    | C++/python bindings                                               | Y       | Y        | https://nanobind.readthedocs.io
 GoogleTest     | 1.10    | Latest      | A unit testing framework module for C++                           | Y       | Y        | https://github.com/google/googletest
-CppUnit        | 1.10    | Latest      | A unit testing framework module for C++                           | N       | Y        | https://freedesktop.org/wiki/Software/cppunit
-Blosc          | 1.17.0* | 1.17.0      | Recommended dependency for improved disk compression              | Y       | Y        | https://github.com/Blosc/c-blosc/releases
+Blosc          | 1.17.0* | Latest      | Recommended dependency for improved disk compression              | Y       | Y        | https://github.com/Blosc/c-blosc/releases
 Log4cplus      | 1.1.2   | Latest      | An optional dependency for improved OpenVDB Logging               | Y       | Y        | https://github.com/log4cplus/log4cplus
 libpng         | -       | Latest      | Library for manipulating PNG images                               | Y       | Y        | http://www.libpng.org/pub/png/libpng.html
-GLFW           | 3.1     | > 3.3       | Simple API for OpenGL development                                 | Y       | Y        | https://www.glfw.org
+GLFW           | 3.3     | Latest      | Simple API for OpenGL development                                 | Y       | Y        | https://www.glfw.org
 OpenGL         | 3.2     | Latest      | Environment for developing portable graphics applications         | Y       | Y        | https://www.opengl.org
 GLEW           | 1.0.0   | Latest      | A cross-platform OpenGL extension loading library.                | Y       | Y        | http://glew.sourceforge.net
 CUDA           | -       | Latest      | Parallel computing platform for graphical processing units.       | Y       | N        | https://developer.nvidia.com/cuda-downloads
 Doxygen        | 1.8.8   | 1.8.11      | Documentation generation from C++                                 | Y       | Y        | http://www.doxygen.nl

- - @b * See the note in [known issues](@ref depKnownIssues) regarding supported blosc/llvm versions.
+ - @b *  See the note in [known issues](@ref depKnownIssues) regarding supported blosc/llvm versions.
+ - @b ** See the note in [known issues](@ref depKnownIssues) regarding supported nanobind build issues.

@subsection depKnownIssues Known Issues

- - @b LLVM >= 15. LLVM has recently changed their IR pointer data types to be
- completely opaque. Some support is provided in LLVM 15 but tests with AX show
- that this is extremely unstable. AX currently only supports LLVM versions up
- until 14.X.X. See https://llvm.org/docs/OpaquePointers.html for more details.
+ - @b On Linux, LLVM == 17 has symbol issues which makes it incompatible when
+ used with the OpenVDB Python plugin. LLVM versions greater than 18 should be
+ preferred.

 - @b GLFW: Crash with GLFW 3.2.1 when used with EGL on Linux. OpenVDB View may
 segfault on closure. Fixed in GLFW 3.3
@@ -112,6 +110,11 @@ Doxygen        | 1.8.8   | 1.8.11      | Documentation generation from C++
 - https://github.com/Blosc/c-blosc/issues/260
 - https://github.com/Blosc/c-blosc/issues/261

+ - @b nanobind: Nanobind requires a fast-hash-map implementation library called
+ tsl-robin-map. If you're installing nanobind through homebrew, this library
+ may not be installed correctly and OpenVDB's CMake will only produce a warning.
+ Make sure to also: `brew install robin-map`.
+
@section depInstallingDependencies Installing Dependencies

 This section details a number of ways to get you up and running with the
@@ -145,17 +148,15 @@ apt-get install zlibc                   # zlib
 apt-get install libboost-iostreams-dev  # Boost::iostream
 apt-get install libblosc-dev            # Blosc
 # AX
-apt-get install llvm-10-dev             # LLVM
+apt-get install llvm-15-dev             # LLVM
 # Python
 apt-get install python-dev              # Python
-apt-get install python-numpy            # NumPy
 pip install nanobind                    # nanobind
 # Optional
 apt-get install libpng-dev              # libpng
 apt-get install libopenexr-dev          # OpenEXR
 apt-get install liblog4cplus-dev        # Log4cplus
 apt-get install googletest              # GoogleTest
-apt-get install libcppunit-dev          # CppUnit
 # vdb_view
 apt-get install libglfw3-dev            # GLFW
 # Documentation
@@ -181,11 +182,9 @@ brew install c-blosc                   # blosc
 brew install llvm                      # LLVM
 # Python
 brew install python                    # Python
-brew install numpy                     # NumPy
-brew install nanobind                  # nanobind
+brew install nanobind robin-map        # nanobind
 # Unit Tests
 brew install gtest                     # GoogleTest
-brew install cppunit                   # CppUnit
 # vdb_view
 brew install glfw                      # GLFW
 # vdb_render
@@ -441,11 +441,9 @@ not on their active states or other attributes.

 Large data sets are often handled in Python using
 <a href="http://www.numpy.org/">NumPy</a>.
-The OpenVDB Python module can optionally be compiled with NumPy support.
-With NumPy enabled, the @b copyFromArray and @b copyToArray grid methods
-can be used to exchange data efficiently between scalar-valued grids
-and three-dimensional NumPy arrays and between vector-valued grids and
-four-dimensional NumPy arrays.
+The @b copyFromArray and @b copyToArray grid methods can be used to exchange
+data efficiently between scalar-valued grids and three-dimensional NumPy
+arrays and between vector-valued grids and four-dimensional NumPy arrays.
@code{.py}
 >>> import pyopenvdb as vdb
 >>> import numpy
@@ -7,7 +7,7 @@

 #]=======================================================================]

-cmake_minimum_required(VERSION 3.18)
+cmake_minimum_required(VERSION 3.24)
 project(NanoVDB LANGUAGES C CXX)

 include(GNUInstallDirs)
@@ -74,18 +74,16 @@ endif()

 #if(NANOVDB_BUILD_UNITTESTS OR NANOVDB_BUILD_BENCHMARK)
 if(NANOVDB_BUILD_UNITTESTS)
-  find_package(GTest REQUIRED)
+  find_package(GTest ${MINIMUM_GOOGLETEST_VERSION} REQUIRED)
 endif()

 if(NANOVDB_USE_CUDA)
  set(CMAKE_CUDA_STANDARD 17)
  set(CMAKE_CUDA_STANDARD_REQUIRED ON)

-  if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.18)
-    # Allow the user to provide CMAKE_CUDA_ARCHITECTURES
-    if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES)
-      set(CMAKE_CUDA_ARCHITECTURES 75)
-    endif()
+  # Allow the user to provide CMAKE_CUDA_ARCHITECTURES
+  if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES)
+    set(CMAKE_CUDA_ARCHITECTURES 75)
  endif()

  enable_language(CUDA)
@@ -115,6 +113,20 @@ if(NANOVDB_USE_CUDA)
      get_target_property(VDB_MSVC_RUNTIME_SELECTION openvdb MSVC_RUNTIME_LIBRARY)
    endif()
  endif()
+
+  find_package(CUDAToolkit)
+
+
+  include(../../cmake/CPM.cmake)
+
+  # This will automatically clone CCCL from GitHub and make the exported cmake targets available
+  CPMAddPackage(
+      NAME CCCL
+      GITHUB_REPOSITORY nvidia/cccl
+      GIT_TAG main # Fetches the latest commit on the main branch
+  )
+
+  find_package(NCCL)
 endif()

 if(NANOVDB_USE_OPENVDB)
@@ -124,15 +136,15 @@ if(NANOVDB_USE_OPENVDB)
 endif()

 if(NANOVDB_USE_TBB AND NOT TARGET TBB::tbb)
-  find_package(TBB REQUIRED)
+  find_package(TBB ${MINIMUM_TBB_VERSION} REQUIRED)
 endif()

 if(NANOVDB_USE_BLOSC AND NOT TARGET Blosc::blosc)
-  find_package(Blosc REQUIRED)
+  find_package(Blosc ${MINIMUM_BLOSC_VERSION} REQUIRED)
 endif()

 if(NANOVDB_USE_ZLIB AND NOT TARGET ZLIB::ZLIB)
-  find_package(ZLIB REQUIRED)
+  find_package(ZLIB ${MINIMUM_ZLIB_VERSION} REQUIRED)
 endif()

 if(NANOVDB_USE_MAGICAVOXEL)
@@ -171,8 +183,13 @@ set(NANOVDB_INCLUDE_FILES
 # NanoVDB cuda header files
 set(NANOVDB_INCLUDE_CUDA_FILES
  cuda/DeviceBuffer.h
+  cuda/DeviceMesh.h
+  cuda/DeviceResource.h
+  cuda/DeviceStreamMap.h
  cuda/GridHandle.cuh
  cuda/NodeManager.cuh
+  cuda/TempPool.h
+  cuda/UnifiedBuffer.h
 )

 # NanoVDB io header files
@@ -200,6 +217,7 @@ set(NANOVDB_INCLUDE_TOOLS_FILES
  tools/GridStats.h
  tools/GridValidator.h
  tools/NanoToOpenVDB.h
+  tools/VoxelBlockManager.h
 )

 # NanoVDB tools/cuda header files
@@ -211,6 +229,14 @@ set(NANOVDB_INCLUDE_TOOLS_CUDA_FILES
  tools/cuda/IndexToGrid.cuh
  tools/cuda/PointsToGrid.cuh
  tools/cuda/SignedFloodFill.cuh
+  tools/cuda/MergeGrids.cuh
+  tools/cuda/TopologyBuilder.cuh
+  tools/cuda/RefineGrid.cuh
+  tools/cuda/PruneGrid.cuh
+  tools/cuda/DistributedPointsToGrid.cuh
+  tools/cuda/DilateGrid.cuh
+  tools/cuda/CoarsenGrid.cuh
+  tools/cuda/VoxelBlockManager.cuh
 )

 # NanoVDB util header files
@@ -239,6 +265,7 @@ set(NANOVDB_INCLUDE_UTIL_FILES
  util/Stencils.h
  util/Timer.h
  util/Util.h
+  util/MorphologyHelpers.h
 )

 # NanoVDB util/cuda header files
@@ -257,6 +284,10 @@ set(NANOVDB_INCLUDE_UTIL_CUDA_FILES
  util/cuda/CudaGridValidator.cuh
  util/cuda/CudaPointsToGrid.cuh
  util/cuda/GpuTimer.h
+  util/cuda/Morphology.cuh
+  util/cuda/MorphologyHelpers.cuh
+  util/cuda/DeviceGridTraits.cuh
+  util/cuda/Injection.cuh
 )

 add_library(nanovdb INTERFACE)
@@ -309,8 +340,12 @@ if(NANOVDB_USE_CUDA)
  endif()

  # Add CUDA includes to any C++ units which use NanoVDB
-  target_include_directories(nanovdb INTERFACE
-    ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
+  target_link_libraries(nanovdb INTERFACE CCCL::CCCL CUDA::cudart)
+
+  if(NCCL_FOUND)
+    target_compile_definitions(nanovdb INTERFACE -DNANOVDB_USE_NCCL)
+    target_link_libraries(nanovdb INTERFACE NCCL::NCCL)
+  endif()
 endif()

 if(NANOVDB_USE_MAGICAVOXEL)
@@ -5,6 +5,9 @@
 // Simple C-wrapper for the nanovdb structure
 // Meant for systems where you lack a C++ compiler.
 //
+
+/// @warning This header file will soon be deprecated in favor of PNanoVDB.h
+
 #ifndef __CNANOVDB__
 #define __CNANOVDB__

@@ -45,13 +45,14 @@ class GridHandle
 public:
    using BufferType = BufferT;

-    /// @brief  Move constructor from a host buffer
+    /// @brief  Move constructor from a dual host-device buffer
    /// @param buffer buffer containing one or more NanoGrids that will be moved into this GridHandle
    /// @throw Will throw and error with the buffer does not contain a valid NanoGrid!
+    /// @note The implementation of this template specialization is in nanovdb/cuda/GridHandle.cuh since it requires CUDA
    template<typename T = BufferT, typename util::enable_if<BufferTraits<T>::hasDeviceDual, int>::type = 0>
    GridHandle(T&& buffer);

-    /// @brief  Move constructor from a dual host-device buffer
+    /// @brief  Move constructor from a host buffer
    /// @param buffer buffer containing one or more NanoGrids that will be moved into this GridHandle
    /// @throw Will throw and error with the buffer does not contain a valid NanoGrid!
    template<typename T = BufferT, typename util::disable_if<BufferTraits<T>::hasDeviceDual, int>::type = 0>
@@ -110,19 +111,28 @@ public:
    typename util::enable_if<BufferTraits<U>::hasDeviceDual, const void*>::type
    deviceData() const { return mBuffer.deviceData(); }
    template<typename U = BufferT>
+    typename util::enable_if<BufferTraits<U>::hasDeviceDual, const void*>::type
+    deviceData(int device) const { return mBuffer.deviceData(device); }
+    template<typename U = BufferT>
    typename util::enable_if<BufferTraits<U>::hasDeviceDual, void*>::type
    deviceData() { return mBuffer.deviceData(); }
+    template<typename U = BufferT>
+    typename util::enable_if<BufferTraits<U>::hasDeviceDual, void*>::type
+    deviceData(int device) { return mBuffer.deviceData(device); }

+    //@{
    /// @brief Returns the size in bytes of the raw memory buffer managed by this GridHandle.
-    uint64_t size() const { return mBuffer.size(); }
+    [[deprecated("Use GridHandle::bufferSize instead.")]] uint64_t size() const { return mBuffer.size(); }
+    uint64_t bufferSize() const { return mBuffer.size(); }
+    //@}

    //@{
    /// @brief Return true if this handle is empty, i.e. has no allocated memory
-    bool empty()   const { return this->size() == 0; }
-    bool isEmpty() const { return this->size() == 0; }
+    bool empty()   const { return mBuffer.size() == 0; }
+    bool isEmpty() const { return mBuffer.size() == 0; }
    //@}

-    /// @brief Return true if this handle contains any grids
+    /// @brief Return true if this handle is not empty, i.e. contains at least one grid
    operator bool() const { return !this->empty(); }

    /// @brief Returns a const host pointer to the @a n'th NanoVDB grid encoded in this GridHandle.
@@ -152,7 +162,7 @@ public:

    /// @brief Return a const pointer to the @a n'th grid encoded in this GridHandle on the device, e.g. GPU
    /// @tparam ValueT Value type of the grid point to be returned
-    /// @param n Index if of the grid pointer to be returned
+    /// @param n Index of the grid pointer to be returned
    /// @param verbose if non-zero error messages will be printed in case something failed
    /// @warning Note that the return pointer can be NULL if the GridHandle was not initialized, @a n is invalid,
    ///          or if the template parameter does not match the specified grid.
@@ -164,13 +174,25 @@ public:
    /// @note This method is only available if the buffer supports devices
    template<typename U = BufferT>
    typename util::enable_if<BufferTraits<U>::hasDeviceDual, void>::type
-    deviceUpload(void* stream = nullptr, bool sync = true) { mBuffer.deviceUpload(stream, sync); }
+    deviceUpload(void* stream, bool sync = true) { mBuffer.deviceUpload(stream, sync); }
+
+    /// @brief Upload the host buffer to a specific device buffer. It device buffer doesn't exist it's created first
+    /// @param device Device to upload host data to
+    /// @param stream cuda stream
+    /// @param sync if false the memory copy is asynchronous
+    template<typename U = BufferT>
+    typename util::enable_if<BufferTraits<U>::hasDeviceDual, void>::type
+    deviceUpload(int device = 0, void* stream = nullptr, bool sync = true) { mBuffer.deviceUpload(device, stream, sync); }

    /// @brief Download the grid to from the device, e.g. from GPU to CPU
    /// @note This method is only available if the buffer supports devices
    template<typename U = BufferT>
    typename util::enable_if<BufferTraits<U>::hasDeviceDual, void>::type
-    deviceDownload(void* stream = nullptr, bool sync = true) { mBuffer.deviceDownload(stream, sync); }
+    deviceDownload(void* stream, bool sync = true) { mBuffer.deviceDownload(stream, sync); }
+
+    template<typename U = BufferT>
+    typename util::enable_if<BufferTraits<U>::hasDeviceDual, void>::type
+    deviceDownload(int device = 0, void* stream = nullptr, bool sync = true) { mBuffer.deviceDownload(device, stream, sync); }

    /// @brief Check if the buffer is this handle has any padding, i.e. if the buffer is larger than the combined size of all its grids
    /// @return true is the combined size of all grid is smaller than the buffer size
@@ -184,6 +206,23 @@ public:
    /// @return Return the byte size of the specified grid
    uint64_t gridSize(uint32_t n = 0) const {return mMetaData[n].size; }

+    /// @brief compute the total sum of memory footprints of all the grids in this buffer
+    /// @return the number of bytes occupied by all grids associated with this buffer
+    uint64_t totalGridSize() const {
+        uint64_t sum = 0;
+        for (auto &m : mMetaData) sum += m.size;
+        NANOVDB_ASSERT(sum <= mBuffer.size());
+        return sum;
+    }
+
+    /// @brief compute the size of unused storage in this buffer
+    /// @return the number of unused bytes in this buffer.
+    uint64_t freeSize() const {return mBuffer.size() - this->totalGridSize();}
+
+    /// @brief Test if this buffer has any unused storage left, i.e. memory not occupied by grids
+    /// @return true if there is no extra storage left in this buffer, i.e. empty or fully occupied with grids
+    bool isFull() const { return this->totalGridSize() == mBuffer.size(); }
+
    /// @brief Return the GridType of the @a n'th grid in this GridHandle
    /// @param n index of the grid (assumed to be less than gridCount())
    /// @return Return the GridType of the specified grid
@@ -250,12 +289,12 @@ public:
    /// @param is input stream containing a raw grid buffer
    /// @param gridName string name of the grid to be read
    /// @param pool optional pool from which to allocate the new grid buffer
-    /// @throw Will throw a std::logic_error if the stream does not contain a valid raw grid with the speficied name
+    /// @throw Will throw a std::logic_error if the stream does not contain a valid raw grid with the specified name
    void read(std::istream& is, const std::string &gridName, const BufferT& pool = BufferT());

    /// @brief Read a raw grid buffer from a file
    /// @param filename string name of the input file containing a raw grid buffer
-    /// @param pool optional pool from which to allocate the new grid buffe
+    /// @param pool optional pool from which to allocate the new grid buffer
    void read(const std::string &fileName, const BufferT& pool = BufferT()) {
        std::ifstream is(fileName, std::ios::in | std::ios::binary);
        if (!is.is_open()) throw std::ios_base::failure("Unable to open file named \"" + fileName + "\" for input");
@@ -315,6 +354,7 @@ inline __hostdev__ void cpyGridHandleMeta(const GridData *data, GridHandleMetaDa
    }
 }// void cpyGridHandleMeta(const GridData *data, GridHandleMetaData *meta)

+// template specialization of move constructor from a host buffer
 template<typename BufferT>
 template<typename T, typename util::disable_if<BufferTraits<T>::hasDeviceDual, int>::type>
 GridHandle<BufferT>::GridHandle(T&& buffer)
@@ -450,7 +490,7 @@ splitGrids(const GridHandle<BufferT> &handle, const BufferT* other = nullptr)
        h = HandleT(std::move(buffer));
        ptr = util::PtrAdd(ptr, src->mGridSize);
    }
-    return std::move(handles);
+    return handles;
 }// splitGrids

 /// @brief Combines (or merges) multiple GridHandles into a single GridHandle containing all grids
@@ -192,6 +192,13 @@ public:
    void* data() { return mData; }
    //@}

+    /// @brief Returns an offset pointer of a specific type from the allocated host memory
+    /// @tparam T Type of the pointer returned
+    /// @param count Numbers of elements of @c parameter type T to skip
+    /// @warning might return NULL
+    template <typename T>
+    T* data(ptrdiff_t count = 0) const {return mData ? reinterpret_cast<T*>(mData) + count : nullptr;}
+
    //@{
    /// @brief Returns the size in bytes associated with this buffer.
    uint64_t bufferSize() const { return mSize; }
@@ -316,7 +323,6 @@ struct HostBuffer::Pool
               << mSize << " bytes of which\n\t" << (util::PtrDiff(alignedFree, mData) - mPadding)
               << " bytes are used by " << mRegister.size() << " other buffer(s). "
               << "Pool is " << (mManaged ? "internally" : "externally") << " managed.\n";
-            //std::cerr << ss.str();
            throw std::runtime_error(ss.str());
        }
        buffer->mSize = size;
@@ -389,7 +395,6 @@ struct HostBuffer::Pool
            }

            for (HostBuffer* buffer : mRegister) { // update registered buffers
-                //buffer->mData = paddedData + ptrdiff_t(buffer->mData - (mData + mPadding));
                buffer->mData = util::PtrAdd(paddedData, util::PtrDiff(buffer->mData, util::PtrAdd(mData, mPadding)));
            }
            mFree = util::PtrAdd(paddedData, memUsage); // update the free pointer
@@ -414,12 +419,8 @@ private:

    static void* alloc(uint64_t size)
    {
-//#if (__cplusplus >= 201703L)
-//    return std::aligned_alloc(NANOVDB_DATA_ALIGNMENT, size);//C++17 or newer
-//#else
    // make sure we alloc enough space to align the result
    return std::malloc(size + NANOVDB_DATA_ALIGNMENT);
-//#endif
    }

    static void* realloc(void* const origData,
@@ -42,8 +42,8 @@ NodeManagerHandle<BufferT> createNodeManager(const NanoGrid<BuildT> &grid,
                                             const BufferT& buffer = BufferT());

 struct NodeManagerData
-{// 48B = 6*8B
-    uint64_t        mMagic;// 8B
+{// 40B = 5*8B
+    __hostdev__ NodeManagerData(void *grid) : mPadding{0}, mGrid(grid), mPtr{0,0,0}{}
    union {int64_t  mPadding; uint8_t mLinear;};// 8B of which 1B is used for a binary flag
    void           *mGrid;//  8B pointer to either host or device grid
    union {int64_t *mPtr[3], mOff[3];};// 24B, use mOff if mLinear!=0
@@ -288,11 +288,7 @@ NodeManagerHandle<BufferT> createNodeManager(const NanoGrid<BuildT> &grid,
    auto *data = reinterpret_cast<NodeManagerData*>(handle.data());
    NANOVDB_ASSERT(data && isAligned(data));
    NANOVDB_ASSERT(toGridType<BuildT>() == grid.gridType());
-#ifdef NANOVDB_USE_NEW_MAGIC_NUMBERS
-    *data = NodeManagerData{NANOVDB_MAGIC_NODE, {0u}, (void*)&grid, {{0u,0u,0u}}};
-#else
-    *data = NodeManagerData{NANOVDB_MAGIC_NUMB, {0u}, (void*)&grid, {{0u,0u,0u}}};
-#endif
+    *data = NodeManagerData((void*)&grid);

    if (NodeManager<BuildT>::isLinear(grid)) {
        data->mLinear = uint8_t(1u);
@@ -316,7 +312,7 @@ NodeManagerHandle<BufferT> createNodeManager(const NanoGrid<BuildT> &grid,
    }

    return handle;// // is converted to r-value so return value is move constructed!
-}
+}// createNodeManager

 } // namespace nanovdb

@@ -198,47 +198,110 @@ PNANOVDB_BUF_FORCE_INLINE void pnanovdb_buf_write_uint64(pnanovdb_buf_t buf, uin
 typedef uint32_t pnanovdb_grid_type_t;
 #define PNANOVDB_GRID_TYPE_GET(grid_typeIn, nameIn) pnanovdb_grid_type_constants[grid_typeIn].nameIn
 #elif defined(PNANOVDB_BUF_HLSL)
-#if defined(PNANOVDB_ADDRESS_32)
+#if defined(PNANOVDB_BUF_HLSL_RW)
+#if defined(PNANOVDB_BUF_HLSL_64)
+#define pnanovdb_buf_t RWStructuredBuffer<uint2>
+#else
+#define pnanovdb_buf_t RWStructuredBuffer<uint>
+#endif
+#else
+#if defined(PNANOVDB_BUF_HLSL_64)
+#define pnanovdb_buf_t StructuredBuffer<uint2>
+#else
 #define pnanovdb_buf_t StructuredBuffer<uint>
+#endif
+#endif
+#if defined(PNANOVDB_ADDRESS_32)
 uint pnanovdb_buf_read_uint32(pnanovdb_buf_t buf, uint byte_offset)
 {
+#if defined(PNANOVDB_BUF_HLSL_64)
+    uint2 val = buf[(byte_offset >> 3u)];
+    return ((byte_offset & 4u) == 0u) ? val.x : val.y;
+#else
    return buf[(byte_offset >> 2u)];
+#endif
 }
 uint2 pnanovdb_buf_read_uint64(pnanovdb_buf_t buf, uint byte_offset)
 {
    uint2 ret;
+#if defined(PNANOVDB_BUF_HLSL_64)
+    ret = buf[(byte_offset >> 3u)];
+#else
    ret.x = pnanovdb_buf_read_uint32(buf, byte_offset + 0u);
    ret.y = pnanovdb_buf_read_uint32(buf, byte_offset + 4u);
+#endif
    return ret;
 }
 void pnanovdb_buf_write_uint32(pnanovdb_buf_t buf, uint byte_offset, uint value)
 {
    // NOP, by default no write in HLSL
+#if defined(PNANOVDB_BUF_HLSL_RW)
+#if defined(PNANOVDB_BUF_HLSL_64)
+    if ((byte_offset & 4u) == 0u) {buf[(byte_offset >> 3u)].x = value;}
+    else {buf[(byte_offset >> 3u)].y = value;}
+#else
+    buf[(byte_offset >> 2u)] = value;
+#endif
+#endif
 }
 void pnanovdb_buf_write_uint64(pnanovdb_buf_t buf, uint byte_offset, uint2 value)
 {
    // NOP, by default no write in HLSL
+#if defined(PNANOVDB_BUF_HLSL_RW)
+#if defined(PNANOVDB_BUF_HLSL_64)
+    buf[(byte_offset >> 3u)] = value;
+#else
+    pnanovdb_buf_write_uint32(buf, byte_offset + 0u, value.x);
+    pnanovdb_buf_write_uint32(buf, byte_offset + 4u, value.y);
+#endif
+#endif
 }
 #elif defined(PNANOVDB_ADDRESS_64)
-#define pnanovdb_buf_t StructuredBuffer<uint>
 uint pnanovdb_buf_read_uint32(pnanovdb_buf_t buf, uint64_t byte_offset)
 {
+#if defined(PNANOVDB_BUF_HLSL_64)
+    uint2 val = buf[uint(byte_offset >> 3u)];
+    return ((uint(byte_offset) & 4u) == 0u) ? val.x : val.y;
+#else
    return buf[uint(byte_offset >> 2u)];
+#endif
 }
 uint64_t pnanovdb_buf_read_uint64(pnanovdb_buf_t buf, uint64_t byte_offset)
 {
    uint64_t ret;
+#if defined(PNANOVDB_BUF_HLSL_64)
+    uint2 raw = buf[uint(byte_offset >> 3u)];
+    ret = uint64_t(raw.x) | (uint64_t(raw.y) << 32u);
+#else
    ret = pnanovdb_buf_read_uint32(buf, byte_offset + 0u);
-    ret = ret + (uint64_t(pnanovdb_buf_read_uint32(buf, byte_offset + 4u)) << 32u);
+    ret = ret | (uint64_t(pnanovdb_buf_read_uint32(buf, byte_offset + 4u)) << 32u);
+#endif
    return ret;
 }
 void pnanovdb_buf_write_uint32(pnanovdb_buf_t buf, uint64_t byte_offset, uint value)
 {
    // NOP, by default no write in HLSL
+#if defined(PNANOVDB_BUF_HLSL_RW)
+#if defined(PNANOVDB_BUF_HLSL_64)
+    if ((byte_offset & 4u) == 0u) {buf[uint(byte_offset >> 3u)].x = value;}
+    else {buf[uint(byte_offset >> 3u)].y = value;}
+#else
+    buf[uint(byte_offset >> 2u)] = value;
+#endif
+#endif
 }
 void pnanovdb_buf_write_uint64(pnanovdb_buf_t buf, uint64_t byte_offset, uint64_t value)
 {
    // NOP, by default no write in HLSL
+#if defined(PNANOVDB_BUF_HLSL_RW)
+#if defined(PNANOVDB_BUF_HLSL_64)
+    uint2 raw = uint2(uint(value), uint(value >> 32u));
+    buf[uint(byte_offset >> 3u)] = raw;
+#else
+    pnanovdb_buf_write_uint32(buf, byte_offset + 0u, uint(value));
+    pnanovdb_buf_write_uint32(buf, byte_offset + 4u, uint(value >> 32u));
+#endif
+#endif
 }
 #endif
 #define pnanovdb_grid_type_t uint
@@ -343,14 +406,26 @@ typedef struct pnanovdb_vec3_t
 {
    float x, y, z;
 }pnanovdb_vec3_t;
+typedef struct pnanovdb_uvec4_t
+{
+    pnanovdb_uint32_t x, y, z, w;
+}pnanovdb_uvec4_t;
+typedef struct pnanovdb_ivec4_t
+{
+    pnanovdb_int32_t x, y, z, w;
+}pnanovdb_ivec4_t;
+typedef struct pnanovdb_vec4_t
+{
+    float x, y, z, w;
+}pnanovdb_vec4_t;
 PNANOVDB_FORCE_INLINE pnanovdb_int32_t pnanovdb_uint32_as_int32(pnanovdb_uint32_t v) { return (pnanovdb_int32_t)v; }
 PNANOVDB_FORCE_INLINE pnanovdb_int64_t pnanovdb_uint64_as_int64(pnanovdb_uint64_t v) { return (pnanovdb_int64_t)v; }
 PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_int64_as_uint64(pnanovdb_int64_t v) { return (pnanovdb_uint64_t)v; }
 PNANOVDB_FORCE_INLINE pnanovdb_uint32_t pnanovdb_int32_as_uint32(pnanovdb_int32_t v) { return (pnanovdb_uint32_t)v; }
 PNANOVDB_FORCE_INLINE float pnanovdb_uint32_as_float(pnanovdb_uint32_t v) { float vf; pnanovdb_memcpy(&vf, &v, sizeof(vf)); return vf; }
-PNANOVDB_FORCE_INLINE pnanovdb_uint32_t pnanovdb_float_as_uint32(float v) { return *((pnanovdb_uint32_t*)(&v)); }
+PNANOVDB_FORCE_INLINE pnanovdb_uint32_t pnanovdb_float_as_uint32(float v) { pnanovdb_uint32_t vu; pnanovdb_memcpy(&vu, &v, sizeof(vu)); return vu; }
 PNANOVDB_FORCE_INLINE double pnanovdb_uint64_as_double(pnanovdb_uint64_t v) { double vf; pnanovdb_memcpy(&vf, &v, sizeof(vf)); return vf; }
-PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_double_as_uint64(double v) { return *((pnanovdb_uint64_t*)(&v)); }
+PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_double_as_uint64(double v) { pnanovdb_uint64_t vu; pnanovdb_memcpy(&vu, &v, sizeof(vu)); return vu; }
 PNANOVDB_FORCE_INLINE pnanovdb_uint32_t pnanovdb_uint64_low(pnanovdb_uint64_t v) { return (pnanovdb_uint32_t)v; }
 PNANOVDB_FORCE_INLINE pnanovdb_uint32_t pnanovdb_uint64_high(pnanovdb_uint64_t v) { return (pnanovdb_uint32_t)(v >> 32u); }
 PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_uint32_as_uint64(pnanovdb_uint32_t x, pnanovdb_uint32_t y) { return ((pnanovdb_uint64_t)x) | (((pnanovdb_uint64_t)y) << 32u); }
@@ -373,6 +448,9 @@ typedef bool pnanovdb_bool_t;
 #define PNANOVDB_TRUE true
 typedef int3 pnanovdb_coord_t;
 typedef float3 pnanovdb_vec3_t;
+typedef uint4 pnanovdb_uvec4_t;
+typedef int4 pnanovdb_ivec4_t;
+typedef float4 pnanovdb_vec4_t;
 pnanovdb_int32_t pnanovdb_uint32_as_int32(pnanovdb_uint32_t v) { return int(v); }
 pnanovdb_uint32_t pnanovdb_int32_as_uint32(pnanovdb_int32_t v) { return uint(v); }
 float pnanovdb_uint32_as_float(pnanovdb_uint32_t v) { return asfloat(v); }
@@ -420,6 +498,9 @@ bool pnanovdb_int64_is_zero(pnanovdb_int64_t a) { return a == 0; }
 #define pnanovdb_int64_t ivec2
 #define pnanovdb_coord_t ivec3
 #define pnanovdb_vec3_t vec3
+#define pnanovdb_uvec4_t uvec4
+#define pnanovdb_ivec4_t ivec4
+#define pnanovdb_vec4_t vec4
 pnanovdb_int32_t pnanovdb_uint32_as_int32(pnanovdb_uint32_t v) { return int(v); }
 pnanovdb_int64_t pnanovdb_uint64_as_int64(pnanovdb_uint64_t v) { return ivec2(v); }
 pnanovdb_uint64_t pnanovdb_int64_as_uint64(pnanovdb_int64_t v) { return uvec2(v); }
@@ -525,6 +606,35 @@ PNANOVDB_FORCE_INLINE pnanovdb_coord_t pnanovdb_coord_add(pnanovdb_coord_t a, pn
    v.z = a.z + b.z;
    return v;
 }
+PNANOVDB_FORCE_INLINE pnanovdb_uvec4_t pnanovdb_uvec4_add(pnanovdb_uvec4_t a, pnanovdb_uvec4_t b)
+{
+    pnanovdb_uvec4_t ret;
+    ret.x = a.x + b.x;
+    ret.y = a.y + b.y;
+    ret.z = a.z + b.z;
+    ret.w = a.w + b.w;
+    return ret;
+}
+
+PNANOVDB_FORCE_INLINE pnanovdb_uvec4_t pnanovdb_uvec4_sub(pnanovdb_uvec4_t a, pnanovdb_uvec4_t b)
+{
+    pnanovdb_uvec4_t ret;
+    ret.x = a.x - b.x;
+    ret.y = a.y - b.y;
+    ret.z = a.z - b.z;
+    ret.w = a.w - b.w;
+    return ret;
+}
+
+PNANOVDB_FORCE_INLINE pnanovdb_vec4_t pnanovdb_vec4_add(pnanovdb_vec4_t a, pnanovdb_vec4_t b)
+{
+    pnanovdb_vec4_t ret;
+    ret.x = a.x + b.x;
+    ret.y = a.y + b.y;
+    ret.z = a.z + b.z;
+    ret.w = a.w + b.w;
+    return ret;
+}
 #elif defined(PNANOVDB_HLSL)
 pnanovdb_vec3_t pnanovdb_vec3_uniform(float a) { return float3(a, a, a); }
 pnanovdb_vec3_t pnanovdb_vec3_add(pnanovdb_vec3_t a, pnanovdb_vec3_t b) { return a + b; }
@@ -536,6 +646,9 @@ pnanovdb_vec3_t pnanovdb_vec3_max(pnanovdb_vec3_t a, pnanovdb_vec3_t b) { return
 pnanovdb_vec3_t pnanovdb_coord_to_vec3(pnanovdb_coord_t coord) { return float3(coord); }
 pnanovdb_coord_t pnanovdb_coord_uniform(pnanovdb_int32_t a) { return int3(a, a, a); }
 pnanovdb_coord_t pnanovdb_coord_add(pnanovdb_coord_t a, pnanovdb_coord_t b) { return a + b; }
+pnanovdb_uvec4_t pnanovdb_uvec4_add(pnanovdb_uvec4_t a, pnanovdb_uvec4_t b) { return a + b; }
+pnanovdb_uvec4_t pnanovdb_uvec4_sub(pnanovdb_uvec4_t a, pnanovdb_uvec4_t b) { return a - b; }
+pnanovdb_vec4_t pnanovdb_vec4_add(pnanovdb_vec4_t a, pnanovdb_vec4_t b) { return a + b; }
 #elif defined(PNANOVDB_GLSL)
 pnanovdb_vec3_t pnanovdb_vec3_uniform(float a) { return vec3(a, a, a); }
 pnanovdb_vec3_t pnanovdb_vec3_add(pnanovdb_vec3_t a, pnanovdb_vec3_t b) { return a + b; }
@@ -547,6 +660,9 @@ pnanovdb_vec3_t pnanovdb_vec3_max(pnanovdb_vec3_t a, pnanovdb_vec3_t b) { return
 pnanovdb_vec3_t pnanovdb_coord_to_vec3(const pnanovdb_coord_t coord) { return vec3(coord); }
 pnanovdb_coord_t pnanovdb_coord_uniform(pnanovdb_int32_t a) { return ivec3(a, a, a); }
 pnanovdb_coord_t pnanovdb_coord_add(pnanovdb_coord_t a, pnanovdb_coord_t b) { return a + b; }
+pnanovdb_uvec4_t pnanovdb_uvec4_add(pnanovdb_uvec4_t a, pnanovdb_uvec4_t b) { return a + b; }
+pnanovdb_uvec4_t pnanovdb_uvec4_sub(pnanovdb_uvec4_t a, pnanovdb_uvec4_t b) { return a - b; }
+pnanovdb_vec4_t pnanovdb_vec4_add(pnanovdb_vec4_t a, pnanovdb_vec4_t b) { return a + b; }
 #endif

 // ------------------------------------------------ Uint64 Utils -----------------------------------------------------------
@@ -589,6 +705,19 @@ PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_uint64_offset(pnanovdb_uint64_t
    return pnanovdb_uint32_as_uint64(low, high);
 }

+PNANOVDB_FORCE_INLINE pnanovdb_int64_t pnanovdb_uint64_diff(pnanovdb_uint64_t a, pnanovdb_uint64_t b)
+{
+    pnanovdb_uint32_t low = pnanovdb_uint64_low(a);
+    pnanovdb_uint32_t high = pnanovdb_uint64_high(a);
+    low -= pnanovdb_uint64_low(b);
+    if (low > pnanovdb_uint64_low(a))
+    {
+        high -= 1u;
+    }
+    high -= pnanovdb_uint64_high(b);
+    return pnanovdb_uint64_as_int64(pnanovdb_uint32_as_uint64(low, high));
+}
+
 PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_uint64_dec(pnanovdb_uint64_t a)
 {
    pnanovdb_uint32_t low = pnanovdb_uint64_low(a);
@@ -636,6 +765,11 @@ PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_uint64_offset(pnanovdb_uint64_t
    return a + b;
 }

+PNANOVDB_FORCE_INLINE pnanovdb_int64_t pnanovdb_uint64_diff(pnanovdb_uint64_t a, pnanovdb_uint64_t b)
+{
+    return pnanovdb_uint64_as_int64(a - b);
+}
+
 PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_uint64_dec(pnanovdb_uint64_t a)
 {
    return a - 1u;
@@ -702,6 +836,10 @@ PNANOVDB_FORCE_INLINE pnanovdb_address_t pnanovdb_address_offset64_product(pnano
    ret.byte_offset += pnanovdb_uint64_low(byte_offset) * multiplier;
    return ret;
 }
+PNANOVDB_FORCE_INLINE pnanovdb_int64_t pnanovdb_address_diff(pnanovdb_address_t a, pnanovdb_address_t b)
+{
+    return pnanovdb_uint64_diff(pnanovdb_uint32_as_uint64_low(a.byte_offset),pnanovdb_uint32_as_uint64_low(b.byte_offset));
+}
 PNANOVDB_FORCE_INLINE pnanovdb_uint32_t pnanovdb_address_mask(pnanovdb_address_t address, pnanovdb_uint32_t mask)
 {
    return address.byte_offset & mask;
@@ -725,6 +863,12 @@ PNANOVDB_FORCE_INLINE pnanovdb_bool_t pnanovdb_address_in_interval(pnanovdb_addr
 {
    return address.byte_offset >= min_address.byte_offset && address.byte_offset < max_address.byte_offset;
 }
+PNANOVDB_FORCE_INLINE pnanovdb_address_t pnanovdb_address_add(pnanovdb_address_t a, pnanovdb_address_t b)
+{
+    pnanovdb_address_t sum = a;
+    sum.byte_offset += b.byte_offset;
+    return sum;
+}
 #elif defined(PNANOVDB_ADDRESS_64)
 struct pnanovdb_address_t
 {
@@ -762,6 +906,10 @@ PNANOVDB_FORCE_INLINE pnanovdb_address_t pnanovdb_address_offset64_product(pnano
    ret.byte_offset += byte_offset * pnanovdb_uint32_as_uint64_low(multiplier);
    return ret;
 }
+PNANOVDB_FORCE_INLINE pnanovdb_int64_t pnanovdb_address_diff(pnanovdb_address_t a, pnanovdb_address_t b)
+{
+    return pnanovdb_uint64_diff(a.byte_offset, b.byte_offset);
+}
 PNANOVDB_FORCE_INLINE pnanovdb_uint32_t pnanovdb_address_mask(pnanovdb_address_t address, pnanovdb_uint32_t mask)
 {
    return pnanovdb_uint64_low(address.byte_offset) & mask;
@@ -785,6 +933,12 @@ PNANOVDB_FORCE_INLINE pnanovdb_bool_t pnanovdb_address_in_interval(pnanovdb_addr
 {
    return address.byte_offset >= min_address.byte_offset && address.byte_offset < max_address.byte_offset;
 }
+PNANOVDB_FORCE_INLINE pnanovdb_address_t pnanovdb_address_add(pnanovdb_address_t a, pnanovdb_address_t b)
+{
+    pnanovdb_address_t sum = a;
+    sum.byte_offset += b.byte_offset;
+    return sum;
+}
 #endif

 // ------------------------------------------------ High Level Buffer Read -----------------------------------------------------------
@@ -933,7 +1087,7 @@ PNANOVDB_FORCE_INLINE void pnanovdb_write_vec3(pnanovdb_buf_t buf, pnanovdb_addr
 #define PNANOVDB_MAGIC_FILE   0x324244566f6e614eUL// "NanoVDB2" in hex - little endian (uint64_t)

 #define PNANOVDB_MAJOR_VERSION_NUMBER 32// reflects changes to the ABI
-#define PNANOVDB_MINOR_VERSION_NUMBER  7// reflects changes to the API but not ABI
+#define PNANOVDB_MINOR_VERSION_NUMBER  9// reflects changes to the API but not ABI
 #define PNANOVDB_PATCH_VERSION_NUMBER  0// reflects bug-fixes with no ABI or API changes

 #define PNANOVDB_GRID_TYPE_UNKNOWN 0
@@ -957,14 +1111,16 @@ PNANOVDB_FORCE_INLINE void pnanovdb_write_vec3(pnanovdb_buf_t buf, pnanovdb_addr
 #define PNANOVDB_GRID_TYPE_VEC4D 18
 #define PNANOVDB_GRID_TYPE_INDEX 19
 #define PNANOVDB_GRID_TYPE_ONINDEX 20
-#define PNANOVDB_GRID_TYPE_INDEXMASK 21
-#define PNANOVDB_GRID_TYPE_ONINDEXMASK 22
+//#define PNANOVDB_GRID_TYPE_INDEXMASK 21   // retired - available for future use
+//#define PNANOVDB_GRID_TYPE_ONINDEXMASK 22 // retired - available for future use
 #define PNANOVDB_GRID_TYPE_POINTINDEX 23
 #define PNANOVDB_GRID_TYPE_VEC3U8 24
 #define PNANOVDB_GRID_TYPE_VEC3U16 25
 #define PNANOVDB_GRID_TYPE_UINT8 26
 #define PNANOVDB_GRID_TYPE_END 27

+#define PNANOVDB_GRID_TYPE_CAP 32
+
 #define PNANOVDB_GRID_CLASS_UNKNOWN 0
 #define PNANOVDB_GRID_CLASS_LEVEL_SET 1     // narrow band level set, e.g. SDF
 #define PNANOVDB_GRID_CLASS_FOG_VOLUME 2    // fog volume, e.g. density
@@ -989,22 +1145,22 @@ PNANOVDB_FORCE_INLINE void pnanovdb_write_vec3(pnanovdb_buf_t buf, pnanovdb_addr
 #define PNANOVDB_LEAF_TYPE_LITE 1
 #define PNANOVDB_LEAF_TYPE_FP 2
 #define PNANOVDB_LEAF_TYPE_INDEX 3
-#define PNANOVDB_LEAF_TYPE_INDEXMASK 4
+//#define PNANOVDB_LEAF_TYPE_INDEXMASK 4 // retired - available for future use
 #define PNANOVDB_LEAF_TYPE_POINTINDEX 5

 // BuildType = Unknown, float, double, int16_t, int32_t, int64_t, Vec3f, Vec3d, Mask, ...
 // bit count of values in leaf nodes, i.e. 8*sizeof(*nanovdb::LeafNode<BuildType>::mValues) or zero if no values are stored
-PNANOVDB_STATIC_CONST pnanovdb_uint32_t pnanovdb_grid_type_value_strides_bits[PNANOVDB_GRID_TYPE_END]  = {  0, 32, 64, 16, 32, 64,  96, 192,  0, 16, 32,  1, 32,  4,  8, 16,  0, 128, 256,  0,  0,  0,  0, 16, 24, 48,  8 };
+PNANOVDB_STATIC_CONST pnanovdb_uint32_t pnanovdb_grid_type_value_strides_bits[PNANOVDB_GRID_TYPE_CAP]  = {  0, 32, 64, 16, 32, 64,  96, 192,  0, 16, 32,  1, 32,  4,  8, 16,  0, 128, 256,  0,  0,  0,  0, 16, 24, 48,  8,  0 };
 // bit count of the Tile union in InternalNodes, i.e. 8*sizeof(nanovdb::InternalData::Tile)
-PNANOVDB_STATIC_CONST pnanovdb_uint32_t pnanovdb_grid_type_table_strides_bits[PNANOVDB_GRID_TYPE_END]  = { 64, 64, 64, 64, 64, 64, 128, 192, 64, 64, 64, 64, 64, 64, 64, 64, 64, 128, 256, 64, 64, 64, 64, 64, 64, 64, 64 };
+PNANOVDB_STATIC_CONST pnanovdb_uint32_t pnanovdb_grid_type_table_strides_bits[PNANOVDB_GRID_TYPE_CAP]  = { 64, 64, 64, 64, 64, 64, 128, 192, 64, 64, 64, 64, 64, 64, 64, 64, 64, 128, 256, 64, 64, 64, 64, 64, 64, 64, 64, 64 };
 // bit count of min/max values, i.e. 8*sizeof(nanovdb::LeafData::mMinimum) or zero if no min/max exists
-PNANOVDB_STATIC_CONST pnanovdb_uint32_t pnanovdb_grid_type_minmax_strides_bits[PNANOVDB_GRID_TYPE_END] = {  0, 32, 64, 16, 32, 64,  96, 192,  8, 16, 32,  8, 32, 32, 32, 32, 32, 128, 256, 64, 64, 64, 64, 64, 24, 48,  8 };
+PNANOVDB_STATIC_CONST pnanovdb_uint32_t pnanovdb_grid_type_minmax_strides_bits[PNANOVDB_GRID_TYPE_CAP] = {  0, 32, 64, 16, 32, 64,  96, 192,  8, 16, 32,  8, 32, 32, 32, 32, 32, 128, 256, 64, 64,  0,  0, 64, 24, 48,  8,  0 };
 // bit alignment of the value type, controlled by the smallest native type, which is why it is always 0, 8, 16, 32, or 64, e.g. for Vec3f it is 32
-PNANOVDB_STATIC_CONST pnanovdb_uint32_t pnanovdb_grid_type_minmax_aligns_bits[PNANOVDB_GRID_TYPE_END]  = {  0, 32, 64, 16, 32, 64,  32,  64,  8, 16, 32,  8, 32, 32, 32, 32, 32,  32,  64, 64, 64, 64, 64, 64,  8, 16,  8 };
+PNANOVDB_STATIC_CONST pnanovdb_uint32_t pnanovdb_grid_type_minmax_aligns_bits[PNANOVDB_GRID_TYPE_CAP]  = {  0, 32, 64, 16, 32, 64,  32,  64,  8, 16, 32,  8, 32, 32, 32, 32, 32,  32,  64, 64, 64,  0,  0, 64,  8, 16,  8,  0 };
 // bit alignment of the stats (avg/std-dev) types, e.g. 8*sizeof(nanovdb::LeafData::mAverage)
-PNANOVDB_STATIC_CONST pnanovdb_uint32_t pnanovdb_grid_type_stat_strides_bits[PNANOVDB_GRID_TYPE_END]   = {  0, 32, 64, 32, 32, 64,  32,  64,  8, 32, 32,  8, 32, 32, 32, 32, 32,  32,  64, 64, 64, 64, 64, 64, 32, 32, 32 };
+PNANOVDB_STATIC_CONST pnanovdb_uint32_t pnanovdb_grid_type_stat_strides_bits[PNANOVDB_GRID_TYPE_CAP]   = {  0, 32, 64, 32, 32, 64,  32,  64,  8, 32, 32,  8, 32, 32, 32, 32, 32,  32,  64, 64, 64,  0,  0, 64, 32, 32, 32,  0 };
 // one of the 4 leaf types defined above, e.g. PNANOVDB_LEAF_TYPE_INDEX = 3
-PNANOVDB_STATIC_CONST pnanovdb_uint32_t pnanovdb_grid_type_leaf_type[PNANOVDB_GRID_TYPE_END]           = {  0,  0,  0,  0,  0,  0,  0,    0,  1,  0,  0,  1,  0,  2,  2,  2,  2,   0,   0,  3,  3,  4,  4,  5,  0,  0,  0 };
+PNANOVDB_STATIC_CONST pnanovdb_uint32_t pnanovdb_grid_type_leaf_type[PNANOVDB_GRID_TYPE_CAP]           = {  0,  0,  0,  0,  0,  0,  0,    0,  1,  0,  0,  1,  0,  2,  2,  2,  2,   0,   0,  3,  3,  0,  0,  5,  0,  0,  0,  0 };

 struct pnanovdb_map_t
 {
@@ -1099,7 +1255,9 @@ struct pnanovdb_grid_t
    pnanovdb_uint32_t grid_type;                // 4 bytes,     636
    pnanovdb_int64_t blind_metadata_offset;     // 8 bytes,     640
    pnanovdb_uint32_t blind_metadata_count;     // 4 bytes,     648
-    pnanovdb_uint32_t pad[5];                   // 20 bytes,    652
+    pnanovdb_uint32_t data0;                    // 4 bytes,     652
+    pnanovdb_uint64_t data1;                    // 8 bytes,     656
+    pnanovdb_uint64_t data2;                    // 8 bytes,     664
 };
 PNANOVDB_STRUCT_TYPEDEF(pnanovdb_grid_t)
 struct pnanovdb_grid_handle_t { pnanovdb_address_t address; };
@@ -1122,6 +1280,9 @@ PNANOVDB_STRUCT_TYPEDEF(pnanovdb_grid_handle_t)
 #define PNANOVDB_GRID_OFF_GRID_TYPE 636
 #define PNANOVDB_GRID_OFF_BLIND_METADATA_OFFSET 640
 #define PNANOVDB_GRID_OFF_BLIND_METADATA_COUNT 648
+#define PNANOVDB_GRID_OFF_DATA0 652
+#define PNANOVDB_GRID_OFF_DATA1 656
+#define PNANOVDB_GRID_OFF_DATA2 664

 PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_grid_get_magic(pnanovdb_buf_t buf, pnanovdb_grid_handle_t p) {
    return pnanovdb_read_uint64(buf, pnanovdb_address_offset(p.address, PNANOVDB_GRID_OFF_MAGIC));
@@ -1170,6 +1331,15 @@ PNANOVDB_FORCE_INLINE pnanovdb_int64_t pnanovdb_grid_get_blind_metadata_offset(p
 PNANOVDB_FORCE_INLINE pnanovdb_uint32_t pnanovdb_grid_get_blind_metadata_count(pnanovdb_buf_t buf, pnanovdb_grid_handle_t p) {
    return pnanovdb_read_uint32(buf, pnanovdb_address_offset(p.address, PNANOVDB_GRID_OFF_BLIND_METADATA_COUNT));
 }
+PNANOVDB_FORCE_INLINE pnanovdb_uint32_t pnanovdb_grid_get_data0(pnanovdb_buf_t buf, pnanovdb_grid_handle_t p) {
+    return pnanovdb_read_uint32(buf, pnanovdb_address_offset(p.address, PNANOVDB_GRID_OFF_DATA0));
+}
+PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_grid_get_data1(pnanovdb_buf_t buf, pnanovdb_grid_handle_t p) {
+    return pnanovdb_read_uint64(buf, pnanovdb_address_offset(p.address, PNANOVDB_GRID_OFF_DATA1));
+}
+PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_grid_get_data2(pnanovdb_buf_t buf, pnanovdb_grid_handle_t p) {
+    return pnanovdb_read_uint64(buf, pnanovdb_address_offset(p.address, PNANOVDB_GRID_OFF_DATA2));
+}

 PNANOVDB_FORCE_INLINE void pnanovdb_grid_set_magic(pnanovdb_buf_t buf, pnanovdb_grid_handle_t p, pnanovdb_uint64_t magic) {
    pnanovdb_write_uint64(buf, pnanovdb_address_offset(p.address, PNANOVDB_GRID_OFF_MAGIC), magic);
@@ -1213,6 +1383,15 @@ PNANOVDB_FORCE_INLINE void pnanovdb_grid_set_blind_metadata_offset(pnanovdb_buf_
 PNANOVDB_FORCE_INLINE void pnanovdb_grid_set_blind_metadata_count(pnanovdb_buf_t buf, pnanovdb_grid_handle_t p, pnanovdb_uint32_t metadata_count) {
    pnanovdb_write_uint32(buf, pnanovdb_address_offset(p.address, PNANOVDB_GRID_OFF_BLIND_METADATA_COUNT), metadata_count);
 }
+PNANOVDB_FORCE_INLINE void pnanovdb_grid_set_data0(pnanovdb_buf_t buf, pnanovdb_grid_handle_t p, pnanovdb_uint32_t data0) {
+    pnanovdb_write_uint32(buf, pnanovdb_address_offset(p.address, PNANOVDB_GRID_OFF_DATA0), data0);
+}
+PNANOVDB_FORCE_INLINE void pnanovdb_grid_set_data1(pnanovdb_buf_t buf, pnanovdb_grid_handle_t p, pnanovdb_uint64_t data1) {
+    pnanovdb_write_uint64(buf, pnanovdb_address_offset(p.address, PNANOVDB_GRID_OFF_DATA1), data1);
+}
+PNANOVDB_FORCE_INLINE void pnanovdb_grid_set_data2(pnanovdb_buf_t buf, pnanovdb_grid_handle_t p, pnanovdb_uint64_t data2) {
+    pnanovdb_write_uint64(buf, pnanovdb_address_offset(p.address, PNANOVDB_GRID_OFF_DATA2), data2);
+}

 PNANOVDB_FORCE_INLINE pnanovdb_uint32_t pnanovdb_make_version(pnanovdb_uint32_t major, pnanovdb_uint32_t minor, pnanovdb_uint32_t patch_num)
 {
@@ -1232,6 +1411,25 @@ PNANOVDB_FORCE_INLINE pnanovdb_uint32_t pnanovdb_version_get_patch(pnanovdb_uint
    return version & ((1u << 10u) - 1u);
 }

+#define PNANOVDB_GRIDBLINDMETADATA_CLASS_UNKNOWN 0
+#define PNANOVDB_GRIDBLINDMETADATA_CLASS_INDEX_ARRAY 1
+#define PNANOVDB_GRIDBLINDMETADATA_CLASS_ATTRIBUTE_ARRAY 2
+#define PNANOVDB_GRIDBLINDMETADATA_CLASS_GRID_NAME 3
+#define PNANOVDB_GRIDBLINDMETADATA_CLASS_CHANNEL_ARRAY 4
+#define PNANOVDB_GRIDBLINDMETADATA_CLASS_END 5
+
+#define PNANOVDB_GRIDBLINDMETADATA_SEMANTIC_UNKNOWN 0
+#define PNANOVDB_GRIDBLINDMETADATA_SEMANTIC_POINT_POSITION 1
+#define PNANOVDB_GRIDBLINDMETADATA_SEMANTIC_POINT_COLOR 2
+#define PNANOVDB_GRIDBLINDMETADATA_SEMANTIC_POINT_NORMAL 3
+#define PNANOVDB_GRIDBLINDMETADATA_SEMANTIC_POINT_RADIUS 4
+#define PNANOVDB_GRIDBLINDMETADATA_SEMANTIC_POINT_VELOCITY 5
+#define PNANOVDB_GRIDBLINDMETADATA_SEMANTIC_POINT_ID 6
+#define PNANOVDB_GRIDBLINDMETADATA_SEMANTIC_WORLD_COORDS 7
+#define PNANOVDB_GRIDBLINDMETADATA_SEMANTIC_GRID_COORDS 8
+#define PNANOVDB_GRIDBLINDMETADATA_SEMANTIC_VOXEL_COORDS 9
+#define PNANOVDB_GRIDBLINDMETADATA_SEMANTIC_END 10
+
 struct pnanovdb_gridblindmetadata_t
 {
    pnanovdb_int64_t data_offset;       // 8 bytes,     0
@@ -1278,6 +1476,28 @@ PNANOVDB_FORCE_INLINE pnanovdb_uint32_t pnanovdb_gridblindmetadata_get_name(pnan
    return pnanovdb_read_uint32(buf, pnanovdb_address_offset(p.address, PNANOVDB_GRIDBLINDMETADATA_OFF_NAME + 4u * index));
 }

+PNANOVDB_FORCE_INLINE void pnanovdb_gridblindmetadata_set_data_offset(pnanovdb_buf_t buf, pnanovdb_gridblindmetadata_handle_t p, pnanovdb_int64_t data_offset) {
+    pnanovdb_write_int64(buf, pnanovdb_address_offset(p.address, PNANOVDB_GRIDBLINDMETADATA_OFF_DATA_OFFSET), data_offset);
+}
+PNANOVDB_FORCE_INLINE void pnanovdb_gridblindmetadata_set_value_count(pnanovdb_buf_t buf, pnanovdb_gridblindmetadata_handle_t p, pnanovdb_uint64_t value_count) {
+    pnanovdb_write_uint64(buf, pnanovdb_address_offset(p.address, PNANOVDB_GRIDBLINDMETADATA_OFF_VALUE_COUNT), value_count);
+}
+PNANOVDB_FORCE_INLINE void pnanovdb_gridblindmetadata_set_value_size(pnanovdb_buf_t buf, pnanovdb_gridblindmetadata_handle_t p, pnanovdb_uint32_t value_size) {
+    pnanovdb_write_uint32(buf, pnanovdb_address_offset(p.address, PNANOVDB_GRIDBLINDMETADATA_OFF_VALUE_SIZE), value_size);
+}
+PNANOVDB_FORCE_INLINE void pnanovdb_gridblindmetadata_set_semantic(pnanovdb_buf_t buf, pnanovdb_gridblindmetadata_handle_t p, pnanovdb_uint32_t semantic) {
+    pnanovdb_write_uint32(buf, pnanovdb_address_offset(p.address, PNANOVDB_GRIDBLINDMETADATA_OFF_SEMANTIC), semantic);
+}
+PNANOVDB_FORCE_INLINE void pnanovdb_gridblindmetadata_set_data_class(pnanovdb_buf_t buf, pnanovdb_gridblindmetadata_handle_t p, pnanovdb_uint32_t data_class) {
+    pnanovdb_write_uint32(buf, pnanovdb_address_offset(p.address, PNANOVDB_GRIDBLINDMETADATA_OFF_DATA_CLASS), data_class);
+}
+PNANOVDB_FORCE_INLINE void pnanovdb_gridblindmetadata_set_data_type(pnanovdb_buf_t buf, pnanovdb_gridblindmetadata_handle_t p, pnanovdb_uint32_t data_type) {
+    pnanovdb_write_uint32(buf, pnanovdb_address_offset(p.address, PNANOVDB_GRIDBLINDMETADATA_OFF_DATA_TYPE), data_type);
+}
+PNANOVDB_FORCE_INLINE void pnanovdb_gridblindmetadata_set_name(pnanovdb_buf_t buf, pnanovdb_gridblindmetadata_handle_t p, pnanovdb_uint32_t index, pnanovdb_uint32_t name) {
+    pnanovdb_write_uint32(buf, pnanovdb_address_offset(p.address, PNANOVDB_GRIDBLINDMETADATA_OFF_NAME + 4u * index), name);
+}
+
 struct pnanovdb_tree_t
 {
    pnanovdb_uint64_t node_offset_leaf;
@@ -1287,9 +1507,9 @@ struct pnanovdb_tree_t
    pnanovdb_uint32_t node_count_leaf;
    pnanovdb_uint32_t node_count_lower;
    pnanovdb_uint32_t node_count_upper;
-    pnanovdb_uint32_t tile_count_leaf;
    pnanovdb_uint32_t tile_count_lower;
    pnanovdb_uint32_t tile_count_upper;
+    pnanovdb_uint32_t tile_count_root;
    pnanovdb_uint64_t voxel_count;
 };
 PNANOVDB_STRUCT_TYPEDEF(pnanovdb_tree_t)
@@ -1305,9 +1525,9 @@ PNANOVDB_STRUCT_TYPEDEF(pnanovdb_tree_handle_t)
 #define PNANOVDB_TREE_OFF_NODE_COUNT_LEAF 32
 #define PNANOVDB_TREE_OFF_NODE_COUNT_LOWER 36
 #define PNANOVDB_TREE_OFF_NODE_COUNT_UPPER 40
-#define PNANOVDB_TREE_OFF_TILE_COUNT_LEAF 44
-#define PNANOVDB_TREE_OFF_TILE_COUNT_LOWER 48
-#define PNANOVDB_TREE_OFF_TILE_COUNT_UPPER 52
+#define PNANOVDB_TREE_OFF_TILE_COUNT_LOWER 44
+#define PNANOVDB_TREE_OFF_TILE_COUNT_UPPER 48
+#define PNANOVDB_TREE_OFF_TILE_COUNT_ROOT 52
 #define PNANOVDB_TREE_OFF_VOXEL_COUNT 56

 PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_tree_get_node_offset_leaf(pnanovdb_buf_t buf, pnanovdb_tree_handle_t p) {
@@ -1331,15 +1551,15 @@ PNANOVDB_FORCE_INLINE pnanovdb_uint32_t pnanovdb_tree_get_node_count_lower(pnano
 PNANOVDB_FORCE_INLINE pnanovdb_uint32_t pnanovdb_tree_get_node_count_upper(pnanovdb_buf_t buf, pnanovdb_tree_handle_t p) {
    return pnanovdb_read_uint32(buf, pnanovdb_address_offset(p.address, PNANOVDB_TREE_OFF_NODE_COUNT_UPPER));
 }
-PNANOVDB_FORCE_INLINE pnanovdb_uint32_t pnanovdb_tree_get_tile_count_leaf(pnanovdb_buf_t buf, pnanovdb_tree_handle_t p) {
-    return pnanovdb_read_uint32(buf, pnanovdb_address_offset(p.address, PNANOVDB_TREE_OFF_TILE_COUNT_LEAF));
-}
 PNANOVDB_FORCE_INLINE pnanovdb_uint32_t pnanovdb_tree_get_tile_count_lower(pnanovdb_buf_t buf, pnanovdb_tree_handle_t p) {
    return pnanovdb_read_uint32(buf, pnanovdb_address_offset(p.address, PNANOVDB_TREE_OFF_TILE_COUNT_LOWER));
 }
 PNANOVDB_FORCE_INLINE pnanovdb_uint32_t pnanovdb_tree_get_tile_count_upper(pnanovdb_buf_t buf, pnanovdb_tree_handle_t p) {
    return pnanovdb_read_uint32(buf, pnanovdb_address_offset(p.address, PNANOVDB_TREE_OFF_TILE_COUNT_UPPER));
 }
+PNANOVDB_FORCE_INLINE pnanovdb_uint32_t pnanovdb_tree_get_tile_count_root(pnanovdb_buf_t buf, pnanovdb_tree_handle_t p) {
+    return pnanovdb_read_uint32(buf, pnanovdb_address_offset(p.address, PNANOVDB_TREE_OFF_TILE_COUNT_ROOT));
+}
 PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_tree_get_voxel_count(pnanovdb_buf_t buf, pnanovdb_tree_handle_t p) {
    return pnanovdb_read_uint64(buf, pnanovdb_address_offset(p.address, PNANOVDB_TREE_OFF_VOXEL_COUNT));
 }
@@ -1365,15 +1585,15 @@ PNANOVDB_FORCE_INLINE void pnanovdb_tree_set_node_count_lower(pnanovdb_buf_t buf
 PNANOVDB_FORCE_INLINE void pnanovdb_tree_set_node_count_upper(pnanovdb_buf_t buf, pnanovdb_tree_handle_t p, pnanovdb_uint32_t node_count_upper) {
    pnanovdb_write_uint32(buf, pnanovdb_address_offset(p.address, PNANOVDB_TREE_OFF_NODE_COUNT_UPPER), node_count_upper);
 }
-PNANOVDB_FORCE_INLINE void pnanovdb_tree_set_tile_count_leaf(pnanovdb_buf_t buf, pnanovdb_tree_handle_t p, pnanovdb_uint32_t tile_count_leaf) {
-    pnanovdb_write_uint32(buf, pnanovdb_address_offset(p.address, PNANOVDB_TREE_OFF_TILE_COUNT_LEAF), tile_count_leaf);
-}
 PNANOVDB_FORCE_INLINE void pnanovdb_tree_set_tile_count_lower(pnanovdb_buf_t buf, pnanovdb_tree_handle_t p, pnanovdb_uint32_t tile_count_lower) {
    pnanovdb_write_uint32(buf, pnanovdb_address_offset(p.address, PNANOVDB_TREE_OFF_TILE_COUNT_LOWER), tile_count_lower);
 }
 PNANOVDB_FORCE_INLINE void pnanovdb_tree_set_tile_count_upper(pnanovdb_buf_t buf, pnanovdb_tree_handle_t p, pnanovdb_uint32_t tile_count_upper) {
    pnanovdb_write_uint32(buf, pnanovdb_address_offset(p.address, PNANOVDB_TREE_OFF_TILE_COUNT_UPPER), tile_count_upper);
 }
+PNANOVDB_FORCE_INLINE void pnanovdb_tree_set_tile_count_root(pnanovdb_buf_t buf, pnanovdb_tree_handle_t p, pnanovdb_uint32_t tile_count_root) {
+    pnanovdb_write_uint32(buf, pnanovdb_address_offset(p.address, PNANOVDB_TREE_OFF_TILE_COUNT_ROOT), tile_count_root);
+}
 PNANOVDB_FORCE_INLINE void pnanovdb_tree_set_voxel_count(pnanovdb_buf_t buf, pnanovdb_tree_handle_t p, pnanovdb_uint64_t voxel_count) {
    pnanovdb_write_uint64(buf, pnanovdb_address_offset(p.address, PNANOVDB_TREE_OFF_VOXEL_COUNT), voxel_count);
 }
@@ -1501,6 +1721,13 @@ PNANOVDB_FORCE_INLINE void pnanovdb_upper_set_bbox_min(pnanovdb_buf_t buf, pnano
 PNANOVDB_FORCE_INLINE void pnanovdb_upper_set_bbox_max(pnanovdb_buf_t buf, pnanovdb_upper_handle_t p, PNANOVDB_IN(pnanovdb_coord_t) bbox_max) {
    pnanovdb_write_coord(buf, pnanovdb_address_offset(p.address, PNANOVDB_UPPER_OFF_BBOX_MAX), bbox_max);
 }
+PNANOVDB_FORCE_INLINE void pnanovdb_upper_set_value_mask(pnanovdb_buf_t buf, pnanovdb_upper_handle_t p, pnanovdb_uint32_t bit_index, pnanovdb_bool_t value) {
+    pnanovdb_address_t addr = pnanovdb_address_offset(p.address, PNANOVDB_UPPER_OFF_VALUE_MASK + 4u * (bit_index >> 5u));
+    pnanovdb_uint32_t valueMask = pnanovdb_read_uint32(buf, addr);
+    if (!value) { valueMask &= ~(1u << (bit_index & 31u)); }
+    if (value) valueMask |= (1u << (bit_index & 31u));
+    pnanovdb_write_uint32(buf, addr, valueMask);
+}
 PNANOVDB_FORCE_INLINE void pnanovdb_upper_set_child_mask(pnanovdb_buf_t buf, pnanovdb_upper_handle_t p, pnanovdb_uint32_t bit_index, pnanovdb_bool_t value) {
    pnanovdb_address_t addr = pnanovdb_address_offset(p.address, PNANOVDB_UPPER_OFF_CHILD_MASK + 4u * (bit_index >> 5u));
    pnanovdb_uint32_t valueMask = pnanovdb_read_uint32(buf, addr);
@@ -1556,6 +1783,13 @@ PNANOVDB_FORCE_INLINE void pnanovdb_lower_set_bbox_min(pnanovdb_buf_t buf, pnano
 PNANOVDB_FORCE_INLINE void pnanovdb_lower_set_bbox_max(pnanovdb_buf_t buf, pnanovdb_lower_handle_t p, PNANOVDB_IN(pnanovdb_coord_t) bbox_max) {
    pnanovdb_write_coord(buf, pnanovdb_address_offset(p.address, PNANOVDB_LOWER_OFF_BBOX_MAX), bbox_max);
 }
+PNANOVDB_FORCE_INLINE void pnanovdb_lower_set_value_mask(pnanovdb_buf_t buf, pnanovdb_lower_handle_t p, pnanovdb_uint32_t bit_index, pnanovdb_bool_t value) {
+    pnanovdb_address_t addr = pnanovdb_address_offset(p.address, PNANOVDB_LOWER_OFF_VALUE_MASK + 4u * (bit_index >> 5u));
+    pnanovdb_uint32_t valueMask = pnanovdb_read_uint32(buf, addr);
+    if (!value) { valueMask &= ~(1u << (bit_index & 31u)); }
+    if (value) valueMask |= (1u << (bit_index & 31u));
+    pnanovdb_write_uint32(buf, addr, valueMask);
+}
 PNANOVDB_FORCE_INLINE void pnanovdb_lower_set_child_mask(pnanovdb_buf_t buf, pnanovdb_lower_handle_t p, pnanovdb_uint32_t bit_index, pnanovdb_bool_t value) {
    pnanovdb_address_t addr = pnanovdb_address_offset(p.address, PNANOVDB_LOWER_OFF_CHILD_MASK + 4u * (bit_index >> 5u));
    pnanovdb_uint32_t valueMask = pnanovdb_read_uint32(buf, addr);
@@ -1604,6 +1838,13 @@ PNANOVDB_FORCE_INLINE void pnanovdb_leaf_set_bbox_min(pnanovdb_buf_t buf, pnanov
 PNANOVDB_FORCE_INLINE void pnanovdb_leaf_set_bbox_dif_and_flags(pnanovdb_buf_t buf, pnanovdb_leaf_handle_t p, pnanovdb_uint32_t bbox_dif_and_flags) {
    pnanovdb_write_uint32(buf, pnanovdb_address_offset(p.address, PNANOVDB_LEAF_OFF_BBOX_DIF_AND_FLAGS), bbox_dif_and_flags);
 }
+PNANOVDB_FORCE_INLINE void pnanovdb_leaf_set_value_mask(pnanovdb_buf_t buf, pnanovdb_leaf_handle_t p, pnanovdb_uint32_t bit_index, pnanovdb_bool_t value) {
+    pnanovdb_address_t addr = pnanovdb_address_offset(p.address, PNANOVDB_LEAF_OFF_VALUE_MASK + 4u * (bit_index >> 5u));
+    pnanovdb_uint32_t valueMask = pnanovdb_read_uint32(buf, addr);
+    if (!value) { valueMask &= ~(1u << (bit_index & 31u)); }
+    if (value) valueMask |= (1u << (bit_index & 31u));
+    pnanovdb_write_uint32(buf, addr, valueMask);
+}

 struct pnanovdb_grid_type_constants_t
 {
@@ -1639,7 +1880,7 @@ struct pnanovdb_grid_type_constants_t
 PNANOVDB_STRUCT_TYPEDEF(pnanovdb_grid_type_constants_t)

 // The following table with offsets will nedd to be updates as new GridTypes are added in NanoVDB.h
-PNANOVDB_STATIC_CONST pnanovdb_grid_type_constants_t pnanovdb_grid_type_constants[PNANOVDB_GRID_TYPE_END] =
+PNANOVDB_STATIC_CONST pnanovdb_grid_type_constants_t pnanovdb_grid_type_constants[PNANOVDB_GRID_TYPE_CAP] =
 {
 {28, 28, 28, 28, 28, 32,  0, 8, 20, 32,  8224, 8224, 8224, 8224, 8224, 270368,  1056, 1056, 1056, 1056, 1056, 33824,  80, 80, 80, 80, 96, 96},
 {28, 32, 36, 40, 44, 64,  32, 8, 20, 32,  8224, 8228, 8232, 8236, 8256, 270400,  1056, 1060, 1064, 1068, 1088, 33856,  80, 84, 88, 92, 96, 2144},
@@ -1662,12 +1903,13 @@ PNANOVDB_STATIC_CONST pnanovdb_grid_type_constants_t pnanovdb_grid_type_constant
 {32, 64, 96, 128, 136, 160,  256, 32, 24, 64,  8224, 8256, 8288, 8296, 8320, 1056896,  1056, 1088, 1120, 1128, 1152, 132224,  80, 112, 144, 152, 160, 16544},
 {32, 40, 48, 56, 64, 96,  0, 8, 24, 32,  8224, 8232, 8240, 8248, 8256, 270400,  1056, 1064, 1072, 1080, 1088, 33856,  80, 80, 80, 80, 80, 96},
 {32, 40, 48, 56, 64, 96,  0, 8, 24, 32,  8224, 8232, 8240, 8248, 8256, 270400,  1056, 1064, 1072, 1080, 1088, 33856,  80, 80, 80, 80, 80, 96},
-{32, 40, 48, 56, 64, 96,  0, 8, 24, 32,  8224, 8232, 8240, 8248, 8256, 270400,  1056, 1064, 1072, 1080, 1088, 33856,  80, 80, 80, 80, 80, 160},
-{32, 40, 48, 56, 64, 96,  0, 8, 24, 32,  8224, 8232, 8240, 8248, 8256, 270400,  1056, 1064, 1072, 1080, 1088, 33856,  80, 80, 80, 80, 80, 160},
+{28, 28, 28, 28, 28, 32,  0, 8, 20, 32,  8224, 8224, 8224, 8224, 8224, 270368,  1056, 1056, 1056, 1056, 1056, 33824,  80, 80, 80, 80, 96, 96},
+{28, 28, 28, 28, 28, 32,  0, 8, 20, 32,  8224, 8224, 8224, 8224, 8224, 270368,  1056, 1056, 1056, 1056, 1056, 33824,  80, 80, 80, 80, 96, 96},
 {32, 40, 48, 56, 64, 96,  16, 8, 24, 32,  8224, 8232, 8240, 8248, 8256, 270400,  1056, 1064, 1072, 1080, 1088, 33856,  80, 88, 96, 96, 96, 1120},
 {28, 31, 34, 40, 44, 64,  24, 8, 20, 32,  8224, 8227, 8232, 8236, 8256, 270400,  1056, 1059, 1064, 1068, 1088, 33856,  80, 83, 88, 92, 96, 1632},
 {28, 34, 40, 48, 52, 64,  48, 8, 20, 32,  8224, 8230, 8236, 8240, 8256, 270400,  1056, 1062, 1068, 1072, 1088, 33856,  80, 86, 92, 96, 128, 3200},
 {28, 29, 30, 32, 36, 64,  8, 8, 20, 32,  8224, 8225, 8228, 8232, 8256, 270400,  1056, 1057, 1060, 1064, 1088, 33856,  80, 81, 84, 88, 96, 608},
+{28, 28, 28, 28, 28, 32,  0, 8, 20, 32,  8224, 8224, 8224, 8224, 8224, 270368,  1056, 1056, 1056, 1056, 1056, 33824,  80, 80, 80, 80, 96, 96},
 };

 // ------------------------------------------------ Basic Lookup -----------------------------------------------------------
@@ -1719,6 +1961,11 @@ PNANOVDB_FORCE_INLINE pnanovdb_root_tile_handle_t pnanovdb_root_get_tile_zero(pn
    return tile;
 }

+PNANOVDB_FORCE_INLINE pnanovdb_bool_t pnanovdb_root_tile_get_child_mask(pnanovdb_buf_t buf, pnanovdb_root_tile_handle_t tile)
+{
+    return !pnanovdb_int64_is_zero(pnanovdb_root_tile_get_child(buf, tile));
+}
+
 PNANOVDB_FORCE_INLINE pnanovdb_upper_handle_t pnanovdb_root_get_child(pnanovdb_grid_type_t grid_type, pnanovdb_buf_t buf, pnanovdb_root_handle_t root, pnanovdb_root_tile_handle_t tile)
 {
    pnanovdb_upper_handle_t upper = { root.address };
@@ -1726,6 +1973,11 @@ PNANOVDB_FORCE_INLINE pnanovdb_upper_handle_t pnanovdb_root_get_child(pnanovdb_g
    return upper;
 }

+PNANOVDB_FORCE_INLINE void pnanovdb_root_set_child(pnanovdb_grid_type_t grid_type, pnanovdb_buf_t buf, pnanovdb_root_handle_t root, pnanovdb_root_tile_handle_t tile, pnanovdb_upper_handle_t upper)
+{
+    pnanovdb_root_tile_set_child(buf, tile, pnanovdb_address_diff(upper.address, root.address));
+}
+
 PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_coord_to_key(PNANOVDB_IN(pnanovdb_coord_t) ijk)
 {
 #if defined(PNANOVDB_NATIVE_64)
@@ -1760,6 +2012,27 @@ PNANOVDB_FORCE_INLINE pnanovdb_root_tile_handle_t pnanovdb_root_find_tile(pnanov
    return null_handle;
 }

+// ----------------------------- Grid Type Safe Set ---------------------------------------
+
+PNANOVDB_FORCE_INLINE void pnanovdb_grid_set_first_gridblindmetadata(pnanovdb_buf_t buf, pnanovdb_grid_handle_t p, pnanovdb_gridblindmetadata_handle_t gridblindmetadata) {
+    pnanovdb_grid_set_blind_metadata_offset(buf, p, pnanovdb_address_diff(gridblindmetadata.address, p.address));
+}
+
+// ----------------------------- Tree Type Safe Set ---------------------------------------
+
+PNANOVDB_FORCE_INLINE void pnanovdb_tree_set_first_leaf(pnanovdb_buf_t buf, pnanovdb_tree_handle_t p, pnanovdb_leaf_handle_t leaf) {
+    pnanovdb_tree_set_node_offset_leaf(buf, p, pnanovdb_address_diff(leaf.address, p.address));
+}
+PNANOVDB_FORCE_INLINE void pnanovdb_tree_set_first_lower(pnanovdb_buf_t buf, pnanovdb_tree_handle_t p, pnanovdb_lower_handle_t lower) {
+    pnanovdb_tree_set_node_offset_lower(buf, p, pnanovdb_address_diff(lower.address, p.address));
+}
+PNANOVDB_FORCE_INLINE void pnanovdb_tree_set_first_upper(pnanovdb_buf_t buf, pnanovdb_tree_handle_t p, pnanovdb_upper_handle_t upper) {
+    pnanovdb_tree_set_node_offset_upper(buf, p, pnanovdb_address_diff(upper.address, p.address));
+}
+PNANOVDB_FORCE_INLINE void pnanovdb_tree_set_first_root(pnanovdb_buf_t buf, pnanovdb_tree_handle_t p, pnanovdb_root_handle_t root) {
+    pnanovdb_tree_set_node_offset_root(buf, p, pnanovdb_address_diff(root.address, p.address));
+}
+
 // ----------------------------- Leaf Node ---------------------------------------

 PNANOVDB_FORCE_INLINE pnanovdb_uint32_t pnanovdb_leaf_coord_to_offset(PNANOVDB_IN(pnanovdb_coord_t) ijk)
@@ -1880,57 +2153,6 @@ PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_leaf_index_get_value_index(pnan
    return pnanovdb_uint64_offset(offset, n);
 }

-// ----------------------------- Leaf IndexMask Specialization ---------------------------------------
-
-PNANOVDB_FORCE_INLINE pnanovdb_bool_t pnanovdb_leaf_indexmask_has_stats(pnanovdb_buf_t buf, pnanovdb_leaf_handle_t leaf)
-{
-    return pnanovdb_leaf_index_has_stats(buf, leaf);
-}
-PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_leaf_indexmask_get_min_index(pnanovdb_buf_t buf, pnanovdb_address_t min_address)
-{
-    return pnanovdb_leaf_index_get_min_index(buf, min_address);
-}
-PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_leaf_indexmask_get_max_index(pnanovdb_buf_t buf, pnanovdb_address_t max_address)
-{
-    return pnanovdb_leaf_index_get_max_index(buf, max_address);
-}
-PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_leaf_indexmask_get_ave_index(pnanovdb_buf_t buf, pnanovdb_address_t ave_address)
-{
-    return pnanovdb_leaf_index_get_ave_index(buf, ave_address);
-}
-PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_leaf_indexmask_get_dev_index(pnanovdb_buf_t buf, pnanovdb_address_t dev_address)
-{
-    return pnanovdb_leaf_index_get_dev_index(buf, dev_address);
-}
-PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_leaf_indexmask_get_value_index(pnanovdb_buf_t buf, pnanovdb_address_t value_address, PNANOVDB_IN(pnanovdb_coord_t) ijk)
-{
-    return pnanovdb_leaf_index_get_value_index(buf, value_address, ijk);
-}
-PNANOVDB_FORCE_INLINE pnanovdb_bool_t pnanovdb_leaf_indexmask_get_mask_bit(pnanovdb_buf_t buf, pnanovdb_leaf_handle_t leaf, pnanovdb_uint32_t n)
-{
-    pnanovdb_uint32_t word_idx = n >> 5;
-    pnanovdb_uint32_t bit_idx = n & 31;
-    pnanovdb_uint32_t val_mask =
-        pnanovdb_read_uint32(buf, pnanovdb_address_offset(leaf.address, 96u + 4u * word_idx));
-    return (val_mask & (1u << bit_idx)) != 0u;
-}
-PNANOVDB_FORCE_INLINE void pnanovdb_leaf_indexmask_set_mask_bit(pnanovdb_buf_t buf, pnanovdb_leaf_handle_t leaf, pnanovdb_uint32_t n, pnanovdb_bool_t v)
-{
-    pnanovdb_uint32_t word_idx = n >> 5;
-    pnanovdb_uint32_t bit_idx = n & 31;
-    pnanovdb_uint32_t val_mask =
-        pnanovdb_read_uint32(buf, pnanovdb_address_offset(leaf.address, 96u + 4u * word_idx));
-    if (v)
-    {
-        val_mask = val_mask | (1u << bit_idx);
-    }
-    else
-    {
-        val_mask = val_mask & ~(1u << bit_idx);
-    }
-    pnanovdb_write_uint32(buf, pnanovdb_address_offset(leaf.address, 96u + 4u * word_idx), val_mask);
-}
-
 // ----------------------------- Leaf OnIndex Specialization ---------------------------------------

 PNANOVDB_FORCE_INLINE pnanovdb_uint32_t pnanovdb_leaf_onindex_get_value_count(pnanovdb_buf_t buf, pnanovdb_leaf_handle_t leaf)
@@ -2022,65 +2244,6 @@ PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_leaf_onindex_get_value_index(pn
    return value_index;
 }

-// ----------------------------- Leaf OnIndexMask Specialization ---------------------------------------
-
-PNANOVDB_FORCE_INLINE pnanovdb_uint32_t pnanovdb_leaf_onindexmask_get_value_count(pnanovdb_buf_t buf, pnanovdb_leaf_handle_t leaf)
-{
-    return pnanovdb_leaf_onindex_get_value_count(buf, leaf);
-}
-PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_leaf_onindexmask_get_last_offset(pnanovdb_buf_t buf, pnanovdb_leaf_handle_t leaf)
-{
-    return pnanovdb_leaf_onindex_get_last_offset(buf, leaf);
-}
-PNANOVDB_FORCE_INLINE pnanovdb_bool_t pnanovdb_leaf_onindexmask_has_stats(pnanovdb_buf_t buf, pnanovdb_leaf_handle_t leaf)
-{
-    return pnanovdb_leaf_onindex_has_stats(buf, leaf);
-}
-PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_leaf_onindexmask_get_min_index(pnanovdb_buf_t buf, pnanovdb_address_t min_address)
-{
-    return pnanovdb_leaf_onindex_get_min_index(buf, min_address);
-}
-PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_leaf_onindexmask_get_max_index(pnanovdb_buf_t buf, pnanovdb_address_t max_address)
-{
-    return pnanovdb_leaf_onindex_get_max_index(buf, max_address);
-}
-PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_leaf_onindexmask_get_ave_index(pnanovdb_buf_t buf, pnanovdb_address_t ave_address)
-{
-    return pnanovdb_leaf_onindex_get_ave_index(buf, ave_address);
-}
-PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_leaf_onindexmask_get_dev_index(pnanovdb_buf_t buf, pnanovdb_address_t dev_address)
-{
-    return pnanovdb_leaf_onindex_get_dev_index(buf, dev_address);
-}
-PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_leaf_onindexmask_get_value_index(pnanovdb_buf_t buf, pnanovdb_address_t value_address, PNANOVDB_IN(pnanovdb_coord_t) ijk)
-{
-    return pnanovdb_leaf_onindex_get_value_index(buf, value_address, ijk);
-}
-PNANOVDB_FORCE_INLINE pnanovdb_bool_t pnanovdb_leaf_onindexmask_get_mask_bit(pnanovdb_buf_t buf, pnanovdb_leaf_handle_t leaf, pnanovdb_uint32_t n)
-{
-    pnanovdb_uint32_t word_idx = n >> 5;
-    pnanovdb_uint32_t bit_idx = n & 31;
-    pnanovdb_uint32_t val_mask =
-        pnanovdb_read_uint32(buf, pnanovdb_address_offset(leaf.address, 96u + 4u * word_idx));
-    return (val_mask & (1u << bit_idx)) != 0u;
-}
-PNANOVDB_FORCE_INLINE void pnanovdb_leaf_onindexmask_set_mask_bit(pnanovdb_buf_t buf, pnanovdb_leaf_handle_t leaf, pnanovdb_uint32_t n, pnanovdb_bool_t v)
-{
-    pnanovdb_uint32_t word_idx = n >> 5;
-    pnanovdb_uint32_t bit_idx = n & 31;
-    pnanovdb_uint32_t val_mask =
-        pnanovdb_read_uint32(buf, pnanovdb_address_offset(leaf.address, 96u + 4u * word_idx));
-    if (v)
-    {
-        val_mask = val_mask | (1u << bit_idx);
-    }
-    else
-    {
-        val_mask = val_mask & ~(1u << bit_idx);
-    }
-    pnanovdb_write_uint32(buf, pnanovdb_address_offset(leaf.address, 96u + 4u * word_idx), val_mask);
-}
-
 // ----------------------------- Leaf PointIndex Specialization ---------------------------------------

 PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_leaf_pointindex_get_offset(pnanovdb_buf_t buf, pnanovdb_leaf_handle_t leaf)
@@ -2210,6 +2373,17 @@ PNANOVDB_FORCE_INLINE pnanovdb_address_t pnanovdb_lower_get_value_address(pnanov
    return pnanovdb_lower_get_value_address_and_level(grid_type, buf, lower, ijk, PNANOVDB_REF(level));
 }

+PNANOVDB_FORCE_INLINE void pnanovdb_lower_set_table_child(pnanovdb_grid_type_t grid_type, pnanovdb_buf_t buf, pnanovdb_lower_handle_t node, pnanovdb_uint32_t n, pnanovdb_int64_t child)
+{
+    pnanovdb_address_t table_address = pnanovdb_lower_get_table_address(grid_type, buf, node, n);
+    pnanovdb_write_int64(buf, table_address, child);
+}
+
+PNANOVDB_FORCE_INLINE void pnanovdb_lower_set_child(pnanovdb_grid_type_t grid_type, pnanovdb_buf_t buf, pnanovdb_lower_handle_t lower, pnanovdb_uint32_t n, pnanovdb_leaf_handle_t leaf)
+{
+    pnanovdb_lower_set_table_child(grid_type, buf, lower, n, pnanovdb_address_diff(leaf.address, lower.address));
+}
+
 // ------------------------------------------------ Upper Node -----------------------------------------------------------

 PNANOVDB_FORCE_INLINE pnanovdb_uint32_t pnanovdb_upper_coord_to_offset(PNANOVDB_IN(pnanovdb_coord_t) ijk)
@@ -2291,8 +2465,19 @@ PNANOVDB_FORCE_INLINE void pnanovdb_upper_set_table_child(pnanovdb_grid_type_t g
    pnanovdb_write_int64(buf, bufAddress, child);
 }

+PNANOVDB_FORCE_INLINE void pnanovdb_upper_set_child(pnanovdb_grid_type_t grid_type, pnanovdb_buf_t buf, pnanovdb_upper_handle_t upper, pnanovdb_uint32_t n, pnanovdb_lower_handle_t lower)
+{
+    pnanovdb_upper_set_table_child(grid_type, buf, upper, n, pnanovdb_address_diff(lower.address, upper.address));
+}
+
 // ------------------------------------------------ Root -----------------------------------------------------------

+PNANOVDB_FORCE_INLINE pnanovdb_address_t pnanovdb_root_get_background_address(pnanovdb_grid_type_t grid_type, pnanovdb_buf_t buf, pnanovdb_root_handle_t root)
+{
+    pnanovdb_uint32_t byte_offset = PNANOVDB_GRID_TYPE_GET(grid_type, root_off_background);
+    return pnanovdb_address_offset(root.address, byte_offset);
+}
+
 PNANOVDB_FORCE_INLINE pnanovdb_address_t pnanovdb_root_get_min_address(pnanovdb_grid_type_t grid_type, pnanovdb_buf_t buf, pnanovdb_root_handle_t root)
 {
    pnanovdb_uint32_t byte_offset = PNANOVDB_GRID_TYPE_GET(grid_type, root_off_min);
@@ -2591,12 +2776,6 @@ PNANOVDB_FORCE_INLINE pnanovdb_address_t pnanovdb_lower_get_value_address_and_ca
    return pnanovdb_lower_get_value_address_and_level_and_cache(grid_type, buf, lower, ijk, acc, PNANOVDB_REF(level));
 }

-PNANOVDB_FORCE_INLINE void pnanovdb_lower_set_table_child(pnanovdb_grid_type_t grid_type, pnanovdb_buf_t buf, pnanovdb_lower_handle_t node, pnanovdb_uint32_t n, pnanovdb_int64_t child)
-{
-    pnanovdb_address_t table_address = pnanovdb_lower_get_table_address(grid_type, buf, node, n);
-    pnanovdb_write_int64(buf, table_address, child);
-}
-
 PNANOVDB_FORCE_INLINE pnanovdb_address_t pnanovdb_upper_get_value_address_and_level_and_cache(pnanovdb_grid_type_t grid_type, pnanovdb_buf_t buf, pnanovdb_upper_handle_t upper, PNANOVDB_IN(pnanovdb_coord_t) ijk, PNANOVDB_INOUT(pnanovdb_readaccessor_t) acc, PNANOVDB_INOUT(pnanovdb_uint32_t) level)
 {
    pnanovdb_uint32_t n = pnanovdb_upper_coord_to_offset(ijk);
@@ -3227,6 +3406,19 @@ PNANOVDB_FORCE_INLINE pnanovdb_bool_t pnanovdb_hdda_update(PNANOVDB_INOUT(pnanov
    {
        return PNANOVDB_FALSE;
    }
+
+    // compute valid voxel range
+    pnanovdb_coord_t voxel_max = {
+        (PNANOVDB_DEREF(hdda).voxel.x + PNANOVDB_DEREF(hdda).dim - 1) & (~(dim - 1)),
+        (PNANOVDB_DEREF(hdda).voxel.y + PNANOVDB_DEREF(hdda).dim - 1) & (~(dim - 1)),
+        (PNANOVDB_DEREF(hdda).voxel.z + PNANOVDB_DEREF(hdda).dim - 1) & (~(dim - 1))
+    };
+    pnanovdb_coord_t voxel_min = {
+        PNANOVDB_DEREF(hdda).voxel.x & (~(dim - 1)),
+        PNANOVDB_DEREF(hdda).voxel.y & (~(dim - 1)),
+        PNANOVDB_DEREF(hdda).voxel.z & (~(dim - 1))
+    };
+
    PNANOVDB_DEREF(hdda).dim = dim;

    pnanovdb_vec3_t pos = pnanovdb_vec3_add(
@@ -3237,6 +3429,14 @@ PNANOVDB_FORCE_INLINE pnanovdb_bool_t pnanovdb_hdda_update(PNANOVDB_INOUT(pnanov

    PNANOVDB_DEREF(hdda).voxel = pnanovdb_hdda_pos_to_voxel(PNANOVDB_REF(pos), dim);

+    // clamp voxel to valid range
+    PNANOVDB_DEREF(hdda).voxel.x = PNANOVDB_DEREF(hdda).voxel.x < voxel_max.x ? PNANOVDB_DEREF(hdda).voxel.x : voxel_max.x;
+    PNANOVDB_DEREF(hdda).voxel.y = PNANOVDB_DEREF(hdda).voxel.y < voxel_max.y ? PNANOVDB_DEREF(hdda).voxel.y : voxel_max.y;
+    PNANOVDB_DEREF(hdda).voxel.z = PNANOVDB_DEREF(hdda).voxel.z < voxel_max.z ? PNANOVDB_DEREF(hdda).voxel.z : voxel_max.z;
+    PNANOVDB_DEREF(hdda).voxel.x = PNANOVDB_DEREF(hdda).voxel.x > voxel_min.x ? PNANOVDB_DEREF(hdda).voxel.x : voxel_min.x;
+    PNANOVDB_DEREF(hdda).voxel.y = PNANOVDB_DEREF(hdda).voxel.y > voxel_min.y ? PNANOVDB_DEREF(hdda).voxel.y : voxel_min.y;
+    PNANOVDB_DEREF(hdda).voxel.z = PNANOVDB_DEREF(hdda).voxel.z > voxel_min.z ? PNANOVDB_DEREF(hdda).voxel.z : voxel_min.z;
+
    if (PNANOVDB_DEREF(hdda).step.x != 0)
    {
        PNANOVDB_DEREF(hdda).next.x = PNANOVDB_DEREF(hdda).tmin + (PNANOVDB_DEREF(hdda).voxel.x - pos.x) * dir_inv.x;
@@ -7,7 +7,7 @@

 #]=======================================================================]

-cmake_minimum_required(VERSION 3.18)
+cmake_minimum_required(VERSION 3.24)
 project(NanoVDBExamples LANGUAGES CXX)

 include(GNUInstallDirs)
@@ -35,6 +35,8 @@ void usage [[noreturn]] (const std::string& progName, int exitStatus = EXIT_FAIL
              << "--fp8\tQuantize float grids to 8 bits\n"
              << "--fp16\tQuantize float grids to 16 bits\n"
              << "--fpN\tQuantize float grids to variable bit depth (use -a or -r to specify a tolerance)\n"
+              << "--index\tProduce an IndexGrid where all float values are encoded as sidecar data\n"
+              << "--onIndex\tProduce an IndexGrid where active float values are encoded as sidecar data\n"
              << "-g,--grid name\tConvert all grids matching the specified string name\n"
              << "-h,--help\tPrints this message\n"
              << "-r,--rel-error float\t Relative error tolerance used for variable bit depth quantization\n"
@@ -57,14 +59,14 @@ int main(int argc, char* argv[])
 {
    int exitStatus = EXIT_SUCCESS;

-    nanovdb::io::Codec       codec = nanovdb::io::Codec::NONE;// compression codec for the file
-    nanovdb::tools::StatsMode       sMode = nanovdb::tools::StatsMode::Default;
-    nanovdb::CheckMode    cMode = nanovdb::CheckMode::Default;
-    nanovdb::GridType        qMode = nanovdb::GridType::Unknown;//specify the quantization mode
-    bool                     verbose = false, overwrite = false, dither = false, absolute = true;
-    float                    tolerance = -1.0f;
-    std::string              gridName;
-    std::vector<std::string> fileNames;
+    nanovdb::io::Codec        codec = nanovdb::io::Codec::NONE;// compression codec for the file
+    nanovdb::tools::StatsMode sMode = nanovdb::tools::StatsMode::Default;
+    nanovdb::CheckMode        cMode = nanovdb::CheckMode::Default;
+    nanovdb::GridType         qMode = nanovdb::GridType::Unknown;//specify the quantization mode
+    bool                      verbose = false, overwrite = false, dither = false, absolute = true;
+    float                     tolerance = -1.0f;
+    std::string               gridName;
+    std::vector<std::string>  fileNames;
    auto toLowerCase = [](std::string &str) {
        std::transform(str.begin(), str.end(), str.begin(),[](unsigned char c){return std::tolower(c);});
    };
@@ -85,6 +87,10 @@ int main(int argc, char* argv[])
                qMode = nanovdb::GridType::Fp16;
            } else if (arg == "--fpN") {
                qMode = nanovdb::GridType::FpN;
+            } else if (arg == "--index") {
+                qMode = nanovdb::GridType::Index;
+            } else if (arg == "--onIndex") {
+                qMode = nanovdb::GridType::OnIndex;
            } else if (arg == "-h" || arg == "--help") {
                usage(argv[0], EXIT_SUCCESS);
            } else if (arg == "-b" || arg == "--blosc") {
@@ -169,17 +175,22 @@ int main(int argc, char* argv[])
        std::cerr << "Expected at least one input file followed by exactly one output file\n" << std::endl;
        usage(argv[0]);
    }
-    const std::string outputFile = fileNames.back();
+    const std::string outputFile = fileNames.back();// last file is always the output file
    const std::string ext = outputFile.substr(outputFile.find_last_of(".") + 1);
    bool              toNanoVDB = false;
    if (ext == "nvdb") {
        toNanoVDB = true;
-    } else if (ext != "vdb") {
+    } else if (ext == "vdb") {
+        if (qMode != nanovdb::GridType::Unknown) {
+            std::cerr << "The options are incompatible an output of type OpenVDB" << std::endl;
+            usage(argv[0]);
+        }
+    } else {
        std::cerr << "Unrecognized file extension: \"" << ext << "\"\n" << std::endl;
        usage(argv[0]);
    }

-    fileNames.pop_back();
+    fileNames.pop_back();// remove the output file name

    if (!overwrite) {
        std::ifstream is(outputFile, std::ios::in | std::ios::binary);
@@ -203,9 +214,16 @@ int main(int argc, char* argv[])

    auto openToNano = [&](const openvdb::GridBase::Ptr& base)
    {
-        using SrcGridT = openvdb::FloatGrid;
-        if (auto floatGrid = openvdb::GridBase::grid<SrcGridT>(base)) {
-            nanovdb::tools::CreateNanoGrid<SrcGridT> s(*floatGrid);
+        const bool includeStats = false, includeTiles = false;
+        const int verb = verbose ? 1 : 0;
+        if (qMode == nanovdb::GridType::OnIndex) {
+            return nanovdb::tools::openToIndexVDB<nanovdb::ValueOnIndex>(base, 1u, includeStats, includeTiles, verb);
+        } else if (qMode == nanovdb::GridType::Index) {
+            return nanovdb::tools::openToIndexVDB<nanovdb::ValueIndex>(base, 1u, includeStats, includeTiles, verb);
+        }
+
+        if (auto floatGrid = openvdb::GridBase::grid<openvdb::FloatGrid>(base)) {
+            nanovdb::tools::CreateNanoGrid<openvdb::FloatGrid> s(*floatGrid);
            s.setStats(sMode);
            s.setChecksum(cMode);
            s.enableDithering(dither);
@@ -227,7 +245,7 @@ int main(int argc, char* argv[])
                break;
            }// end of switch
        }
-        return nanovdb::tools::openToNanoVDB(base, sMode, cMode, verbose ? 1 : 0);
+        return nanovdb::tools::openToNanoVDB(base, sMode, cMode, verb);
    };
    try {
        if (toNanoVDB) { // OpenVDB -> NanoVDB
@@ -237,26 +255,21 @@ int main(int argc, char* argv[])
                    std::cerr << "Since the last file has extension .nvdb the remaining input files were expected to have extensions .vdb\n" << std::endl;
                    usage(argv[0]);
                }
-                if (verbose)
-                    std::cout << "Opening OpenVDB file named \"" << inputFile << "\"" << std::endl;
+                if (verbose) std::cout << "Opening OpenVDB file named \"" << inputFile << "\"" << std::endl;
                openvdb::io::File file(inputFile);
                file.open(false); //disable delayed loading
                if (gridName.empty()) {// convert all grid in the file
                    auto grids = file.getGrids();
                    std::vector<nanovdb::GridHandle<nanovdb::HostBuffer> > handles;
                    for (auto& grid : *grids) {
-                        if (verbose) {
-                            std::cout << "Converting OpenVDB grid named \"" << grid->getName() << "\" to NanoVDB" << std::endl;
-                        }
+                        if (verbose) std::cout << "Converting OpenVDB grid named \"" << grid->getName() << "\" to NanoVDB" << std::endl;
                        handles.push_back(openToNano(grid));
                    } // loop over OpenVDB grids in file
                    auto handle = nanovdb::mergeGrids<nanovdb::HostBuffer, std::vector>(handles);
                    nanovdb::io::writeGrid(os, handle, codec);
                } else {// convert only grid with matching name
                    auto grid = file.readGrid(gridName);
-                    if (verbose) {
-                        std::cout << "Converting OpenVDB grid named \"" << grid->getName() << "\" to NanoVDB" << std::endl;
-                    }
+                    if (verbose) std::cout << "Converting OpenVDB grid named \"" << grid->getName() << "\" to NanoVDB" << std::endl;
                    auto handle = openToNano(grid);
                    nanovdb::io::writeGrid(os, handle, codec);
                }
@@ -269,15 +282,13 @@ int main(int argc, char* argv[])
                    std::cerr << "Since the last file has extension .vdb the remaining input files were expected to have extensions .nvdb\n" << std::endl;
                    usage(argv[0]);
                }
-                if (verbose)
-                    std::cout << "Opening NanoVDB file named \"" << inputFile << "\"" << std::endl;
+                if (verbose) std::cout << "Opening NanoVDB file named \"" << inputFile << "\"" << std::endl;
                if (gridName.empty()) {
                    auto handles = nanovdb::io::readGrids(inputFile, verbose);
                    for (auto &h : handles) {
                        for (uint32_t i = 0; i < h.gridCount(); ++i) {
-                            if (verbose)
-                                std::cout << "Converting NanoVDB grid named \"" << h.gridMetaData(i)->shortGridName() << "\" to OpenVDB" << std::endl;
-                            grids->push_back(nanovdb::tools::nanoToOpenVDB(h, 0, i));
+                            if (verbose) std::cout << "Converting NanoVDB grid named \"" << h.gridMetaData(i)->shortGridName() << "\" to OpenVDB" << std::endl;
+                            grids->push_back(nanovdb::tools::nanoToOpenVDB(h, i));
                        }
                    }
                } else {
@@ -286,8 +297,7 @@ int main(int argc, char* argv[])
                        std::cerr << "File did not contain a NanoVDB grid named \"" << gridName << "\"\n" << std::endl;
                        usage(argv[0]);
                    }
-                    if (verbose)
-                        std::cout << "Converting NanoVDB grid named \"" << handle.gridMetaData()->shortGridName() << "\" to OpenVDB" << std::endl;
+                    if (verbose) std::cout << "Converting NanoVDB grid named \"" << handle.gridMetaData()->shortGridName() << "\" to OpenVDB" << std::endl;
                    grids->push_back(nanovdb::tools::nanoToOpenVDB(handle));
                }
            } // loop over input files
@@ -166,7 +166,7 @@ int main(int argc, char* argv[])
            auto       classWidth = std::string("Class").length() + padding;
            auto       codecWidth = std::string("Codec").length() + padding;
            auto       ibboxWidth = std::string("Index Bounding Box").length() + padding;
-            auto       wbboxWidth = std::string("World BBox").length() + padding;
+            auto       wbboxWidth = std::string("World Bounding Box").length() + padding;
            auto       sizeWidth = std::string("Size").length() + padding;
            auto       fileWidth = std::string("File").length() + padding;
            auto       voxelsWidth = std::string("# Voxels").length() + padding;
@@ -175,6 +175,7 @@ int main(int argc, char* argv[])
            auto       configWidth = std::string("32^3->16^3->8^3").length() + padding;
            auto       tileWidth = std::string("# Active tiles").length() + padding;
            auto       resWidth = std::string("Resolution").length() + padding;
+            auto       blindWidth = std::string("Blind data count").length() + padding;
            for (auto& m : list) {
                width(nameWidth, m.gridName);
                width(typeWidth, nanovdb::toStr(str, m.gridType));
@@ -190,6 +191,7 @@ int main(int argc, char* argv[])
                width(tileWidth, nodesToStr(m.tileCount));
                width(voxelsWidth, std::to_string(m.voxelCount));
                width(voxelSizeWidth, Vec3dToStr(m.voxelSize));
+                width(blindWidth, std::to_string(m.blindDataCount));
            }
            std::cout << "\nThe file \"" << file << "\" contains the following ";
            if (list.size()>1) {
@@ -214,7 +216,8 @@ int main(int argc, char* argv[])
                std::cout << std::left << std::setw(configWidth) << "32^3->16^3->8^3"
                          << std::left << std::setw(tileWidth)   << "# Active tiles"
                          << std::left << std::setw(ibboxWidth)  << "Index Bounding Box"
-                          << std::left << std::setw(wbboxWidth)  << "World Bounding Box";
+                          << std::left << std::setw(wbboxWidth)  << "World Bounding Box"
+                          << std::left << std::setw(blindWidth)  << "Blind data count";
            }
            std::cout << std::endl;
            int n = 0;
@@ -238,11 +241,12 @@ int main(int argc, char* argv[])
                    std::cout << std::left << std::setw(configWidth) << nodesToStr(m.nodeCount)
                              << std::left << std::setw(tileWidth)   << nodesToStr(m.tileCount)
                              << std::left << std::setw(ibboxWidth)  << ibboxToStr(m.indexBBox)
-                              << std::left << std::setw(wbboxWidth)  << wbboxToStr(m.worldBBox);
+                              << std::left << std::setw(wbboxWidth)  << wbboxToStr(m.worldBBox)
+                              << std::left << std::setw(blindWidth)  << m.blindDataCount;
                }
                std::cout << std::endl;
-            }
-        }
+            }// loop over FileGridMetaData in list from a single file
+        }// loop over files
        if (verbose) {
            size_t w = 0;
            switch (mode) {
@@ -1,3 +1,5 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
 import argparse
 import os
 from pathlib import Path
@@ -1,4 +1,6 @@
 #!/bin/bash
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
 #Usage process all files in this directory or optionally specify a target directory

 # Define directory in which to find files
@@ -8,7 +8,7 @@

    \date January 8, 2020

-    \brief Implements a simple dual (host/device) CUDA buffer.
+    \brief DeviceBuffer has one pinned host buffer and multiple device CUDA buffers

    \note This file has no device-only kernel functions,
          which explains why it's a .h and not .cuh file.
@@ -17,6 +17,8 @@
 #ifndef NANOVDB_CUDA_DEVICEBUFFER_H_HAS_BEEN_INCLUDED
 #define NANOVDB_CUDA_DEVICEBUFFER_H_HAS_BEEN_INCLUDED

+#include <cuda.h>
+#include <memory>// for std::shared_ptr
 #include <nanovdb/HostBuffer.h>// for BufferTraits
 #include <nanovdb/util/cuda/Util.h>// for cudaMalloc/cudaMallocManaged/cudaFree

@@ -34,44 +36,80 @@ namespace cuda {// =============================================================
 class DeviceBuffer
 {
    uint64_t mSize; // total number of bytes managed by this buffer (assumed to be identical for host and device)
-    void *mCpuData, *mGpuData; // raw pointers to the host and device buffers
-    bool mManaged;
+    void *mCpuData, **mGpuData; // raw pointers to the host and device buffers
+    int   mDeviceCount, mManaged;// if mManaged is non-zero this class is responsible for allocating and freeing memory buffers. Otherwise this is assumed to be handled externally
+
+    /// @brief Initialize buffer
+    /// @param size byte size of buffer to be initialized
+    /// @param device id of the device on which to initialize the buffer
+    /// @note All existing buffers are first cleared
+    /// @warning size is expected to be non-zero. Use clear() clear buffer!
+    void init(uint64_t size, int device, cudaStream_t stream);

 public:
-    /// @brief Static factory method that return an instance of this buffer
-    /// @param size byte size of buffer to be initialized
-    /// @param dummy this argument is currently ignored but required to match the API of the HostBuffer
-    /// @param host If true buffer is initialized only on the host/CPU, else on the device/GPU
-    /// @param stream optional stream argument (defaults to stream NULL)
-    /// @return An instance of this class using move semantics
-    static DeviceBuffer create(uint64_t size, const DeviceBuffer* dummy = nullptr, bool host = true, void* stream = nullptr);

-    /// @brief Static factory method that return an instance of this buffer that wraps externally managed memory
-    /// @param size byte size of buffer specified by external memory
-    /// @param cpuData pointer to externally managed host memory
-    /// @param gpuData pointer to externally managed device memory
-    /// @return An instance of this class using move semantics
-    static DeviceBuffer create(uint64_t size, void* cpuData, void* gpuData);
+    using PtrT = std::shared_ptr<DeviceBuffer>;
+
+    /// @brief Default constructor of an empty buffer
+    DeviceBuffer() : mSize(0), mCpuData(nullptr), mGpuData(nullptr), mDeviceCount(0), mManaged(0){}
+
+    /// @brief Constructor with a specified device and size
+    /// @param size byte size of buffer to be initialized
+    /// @param device id of the device on which to initialize the buffer
+    /// @param stream cuda stream
+    DeviceBuffer(uint64_t size, int device = cudaCpuDeviceId, cudaStream_t stream = 0) : DeviceBuffer()
+    {
+        this->init(size, device, stream);
+    }

    /// @brief Constructor
    /// @param size byte size of buffer to be initialized
-    /// @param host If true buffer is initialized only on the host/CPU, else on the device/GPU
+    /// @param host If true buffer is initialized only on the host/CPU, else on the current device/GPU
    /// @param stream optional stream argument (defaults to stream NULL)
-    DeviceBuffer(uint64_t size = 0, bool host = true, void* stream = nullptr)
-        : mSize(0)
-        , mCpuData(nullptr)
-        , mGpuData(nullptr)
-        , mManaged(false)
+    DeviceBuffer(uint64_t size, bool host, void* stream) : DeviceBuffer()
    {
-        if (size > 0) this->init(size, host, stream);
+        int device = cudaCpuDeviceId;
+        if (!host) cudaCheck(cudaGetDevice(&device));
+        this->init(size, device, reinterpret_cast<cudaStream_t>(stream));
    }

+    /// @brief Constructor for externally managed host and device buffers
+    /// @param size byte size of the two external buffers
+    /// @param cpuData host buffer, assumed to NOT be NULL
+    /// @param gpuData device buffer, assumed to NOT be NULL;
+    /// @note The device buffer, @c gpuData, will be associated
+    ///       with the current device ID given by cudaGetDevice
    DeviceBuffer(uint64_t size, void* cpuData, void* gpuData)
        : mSize(size)
        , mCpuData(cpuData)
-        , mGpuData(gpuData)
-        , mManaged(false)
+        , mManaged(0)
    {
+        cudaCheck(cudaGetDeviceCount(&mDeviceCount));
+        mGpuData = new void*[mDeviceCount]();// NULL initialization
+        NANOVDB_ASSERT(cpuData);
+        NANOVDB_ASSERT(gpuData);
+        int device = 0;
+        cudaCheck(cudaGetDevice(&device));
+        mGpuData[device] = gpuData;
+    }
+
+    /// @brief Constructor for externally managed host and multiple device buffers
+    /// @param size byte size of the two external buffers
+    /// @param cpuData host buffer, assumed to NOT be NULL
+    /// @param list list of device IDs and external device buffers, all assumed to not be NULL
+    DeviceBuffer(uint64_t size, void* cpuData, std::initializer_list<std::pair<int,void*>> list)
+        : mSize(size)
+        , mCpuData(cpuData)
+        , mManaged(0)
+    {
+        NANOVDB_ASSERT(cpuData);
+        cudaCheck(cudaGetDeviceCount(&mDeviceCount));
+        mGpuData = new void*[mDeviceCount]();// NULL initialization
+        for (auto &p : list) {
+            NANOVDB_ASSERT(p.first>=0 && p.first<mDeviceCount);
+            NANOVDB_ASSERT(p.second);
+            mGpuData[p.first] = p.second;
+        }
    }

    /// @brief Disallow copy-construction
@@ -82,138 +120,283 @@ public:
        : mSize(other.mSize)
        , mCpuData(other.mCpuData)
        , mGpuData(other.mGpuData)
+        , mDeviceCount(other.mDeviceCount)
        , mManaged(other.mManaged)
    {
-        other.mSize = 0;
-        other.mCpuData = nullptr;
-        other.mGpuData = nullptr;
-        other.mManaged = false;
+        other.mCpuData = other.mGpuData = nullptr;
+        other.mSize = other.mDeviceCount = other.mManaged = 0;
    }

+    /// @brief Copy-constructor from a HostBuffer
+    /// @param buffer host buffer from which to copy data
+    /// @param device id of the device on which to initialize the buffer
+    /// @param stream cuda stream
+    DeviceBuffer(const HostBuffer& buffer, int device = cudaCpuDeviceId, cudaStream_t stream = 0)
+        : DeviceBuffer(buffer.size(), device, stream)
+    {
+        if (mCpuData) {
+            cudaCheck(cudaMemcpy(mCpuData, buffer.data(), mSize, cudaMemcpyHostToHost));
+        } else if (mGpuData[device]) {
+            cudaCheck(cudaMemcpyAsync(mGpuData[device], buffer.data(), mSize, cudaMemcpyHostToDevice, stream));
+        }
+    }
+
+     /// @brief Destructor frees memory on both the host and device
+    ~DeviceBuffer() { this->clear(); };
+
+    /// @brief Static factory method that return an instance of this buffer
+    /// @param size byte size of buffer to be initialized
+    /// @param dummy this argument is currently ignored but required to match the API of the HostBuffer
+    /// @param host If true buffer is initialized only on the host/CPU, else only on the device/GPU
+    /// @param stream optional stream argument (defaults to stream NULL)
+    /// @return An instance of this class using move semantics
+    static DeviceBuffer create(uint64_t size, const DeviceBuffer* dummy, bool host, void* stream){return DeviceBuffer(size, host, stream);}
+
+    /// @brief Static factory method that returns an instance of this buffer
+    /// @param size byte size of buffer to be initialized
+    /// @param dummy this argument is currently ignored but required to match the API of the HostBuffer
+    /// @param device id of the device on which to initialize the buffer
+    /// @param stream cuda stream
+    static DeviceBuffer create(uint64_t size, const DeviceBuffer* dummy = nullptr, int device = cudaCpuDeviceId, cudaStream_t stream = 0){return DeviceBuffer(size, device, stream);}
+
+    /// @brief Static factory method that returns an instance of this buffer that wraps externally managed memory
+    /// @param size byte size of buffer specified by external memory
+    /// @param cpuData pointer to externally managed host memory
+    /// @param gpuData pointer to externally managed device memory
+    /// @return An instance of this class using move semantics
+    static DeviceBuffer create(uint64_t size, void* cpuData, void* gpuData) {return DeviceBuffer(size, cpuData, gpuData);}
+
+    /// @brief  Static factory method that returns an instance of this buffer that wraps externally managed host and device memory
+    /// @param size byte size of buffer to be initialized
+    /// @param cpuData  pointer to externally managed host memory
+    /// @param list list of device IDs and device memory pointers
+    static DeviceBuffer create(uint64_t size, void* cpuData, std::initializer_list<std::pair<int,void*>> list) {return DeviceBuffer(size, cpuData, list);}
+
+    /// @brief Static factory method that returns an instance of this buffer constructed from a HostBuffer
+    /// @param buffer host buffer from which to copy data
+    /// @param device id of the device on which to initialize the buffer
+    /// @param stream cuda stream
+    static DeviceBuffer create(const HostBuffer& buffer, int device = cudaCpuDeviceId, cudaStream_t stream = 0) {return DeviceBuffer(buffer, device, stream);}
+
+    ///////////////////////////////////////////////////////////////////////
+
+    /// @{
+    /// @brief Factory methods that create a shared pointer to an DeviceBuffer instance
+    static PtrT createPtr(uint64_t size, const DeviceBuffer* = nullptr, int device = cudaCpuDeviceId, cudaStream_t stream = 0) {return std::make_shared<DeviceBuffer>(size, device, stream);}
+    static PtrT createPtr(uint64_t size, void* cpuData, void* gpuData) {return std::make_shared<DeviceBuffer>(size, cpuData, gpuData);}
+    static PtrT createPtr(uint64_t size, void* cpuData, std::initializer_list<std::pair<int,void*>> list) {return std::make_shared<DeviceBuffer>(size, cpuData, list);}
+    static PtrT createPtr(const HostBuffer& buffer, int device = cudaCpuDeviceId, cudaStream_t stream = 0) {return std::make_shared<DeviceBuffer>(buffer, device, stream);}
+    /// @}
+
+    ///////////////////////////////////////////////////////////////////////
+
    /// @brief Disallow copy assignment operation
    DeviceBuffer& operator=(const DeviceBuffer&) = delete;

    /// @brief Move copy assignment operation
-    DeviceBuffer& operator=(DeviceBuffer&& other) noexcept
-    {
-        this->clear();
-        mSize = other.mSize;
-        mCpuData = other.mCpuData;
-        mGpuData = other.mGpuData;
-        mManaged = other.mManaged;
-        other.mSize = 0;
-        other.mCpuData = nullptr;
-        other.mGpuData = nullptr;
-        other.mManaged = false;
-        return *this;
-    }
+    DeviceBuffer& operator=(DeviceBuffer&& other) noexcept;

-    /// @brief Destructor frees memory on both the host and device
-    ~DeviceBuffer() { this->clear(); };
+    ///////////////////////////////////////////////////////////////////////

-    /// @brief Initialize buffer
-    /// @param size byte size of buffer to be initialized
-    /// @param host If true buffer is initialized only on the host/CPU, else on the device/GPU
-    /// @note All existing buffers are first cleared
-    /// @warning size is expected to be non-zero. Use clear() clear buffer!
-    void init(uint64_t size, bool host = true, void* stream = nullptr);
-
-    /// @brief Retuns a raw pointer to the host/CPU buffer managed by this allocator.
+    /// @brief Retuns a raw void pointer to the host/CPU buffer managed by this allocator.
    /// @warning Note that the pointer can be NULL!
    void* data() const { return mCpuData; }

-    /// @brief Retuns a raw pointer to the device/GPU buffer managed by this allocator.
+    /// @brief Returns an offset pointer of a specific type from the allocated host memory
+    /// @tparam T Type of the pointer returned
+    /// @param count Numbers of elements of @c parameter type T to skip
+    /// @warning might return NULL
+    template <typename T>
+    T* data(ptrdiff_t count = 0, int device = cudaCpuDeviceId) const
+    {
+        NANOVDB_ASSERT(device >= cudaCpuDeviceId && device < mDeviceCount);
+        void *ptr = device == cudaCpuDeviceId ? mCpuData : mGpuData[device];
+        return ptr ? reinterpret_cast<T*>(ptr) + count : nullptr;
+    }
+
+    /// @brief Returns a byte offset void pointer from the allocated host memory
+    /// @param byteOffset offset of return pointer in units of bytes
+    /// @warning assumes that this instance is not empty!
+    void* data(ptrdiff_t byteOffset, int device = cudaCpuDeviceId) const
+    {
+        NANOVDB_ASSERT(device >= cudaCpuDeviceId && device < mDeviceCount);
+        void *ptr = device == cudaCpuDeviceId ? mCpuData : mGpuData[device];
+        return ptr ? reinterpret_cast<char*>(ptr) + byteOffset : nullptr;
+    }
+
+    ///////////////////////////////////////////////////////////////////////
+
+    /// @brief Retuns a raw pointer to the specified device/GPU buffer managed by this allocator.
    /// @warning Note that the pointer can be NULL!
-    void* deviceData() const { return mGpuData; }
+    void* deviceData(int device) const {
+        NANOVDB_ASSERT(device >= 0 && device < mDeviceCount);
+        return mGpuData[device];
+    }

-    /// @brief  Upload this buffer from the host to the device, i.e. CPU -> GPU.
-    /// @param stream optional CUDA stream (defaults to CUDA stream 0)
-    /// @param sync if false the memory copy is asynchronous
-    /// @note If the device/GPU buffer does not exist it is first allocated
-    /// @warning Assumes that the host/CPU buffer already exists
-    void deviceUpload(void* stream = nullptr, bool sync = true) const;
+    /// @brief Retuns a raw pointer to the current device/GPU buffer managed by this allocator.
+    /// @warning Note that the pointer can be NULL!
+    void* deviceData() const {
+        int device = cudaCpuDeviceId;
+        cudaCheck(cudaGetDevice(&device));
+        return this->deviceData(device);
+    }

-    /// @brief Upload this buffer from the device to the host, i.e. GPU -> CPU.
-    /// @param stream optional CUDA stream (defaults to CUDA stream 0)
+    ///////////////////////////////////////////////////////////////////////
+
+    /// @brief Uploads buffer on the host to a specific device. If it doesn't exist it's created first.
+    /// @param device Device ID that the data is copied to
+    /// @param stream cuda stream
+    /// @param sync if false the memory copy is asynchronous.
+    /// @warning Assumes that the host buffer already exists!
+    /// @note determine the current device with cudaGetDevice
+    void deviceUpload(int device = 0, cudaStream_t stream = 0, bool sync = true);
+    void deviceUpload(int device, void* stream, bool sync){this->deviceUpload(device, cudaStream_t(stream), sync);}
+
+    /// @brief Upload buffer from the host to ALL the existing devices, i.e. CPU -> GPU.
+    ///        If no device buffers exist one is created for the current device (typically 0)
+    ///        and subsequently populated with the host data.
+    /// @param stream CUDA stream.
+    /// @param sync if false the memory copy is asynchronous.
+    /// @warning Assumes that the host buffer already exists!
+    void deviceUpload(cudaStream_t stream, bool sync);
+    void deviceUpload(void* stream, bool sync) {this->deviceUpload(cudaStream_t(stream), sync);}
+
+    ///////////////////////////////////////////////////////////////////////
+
+    /// @brief Download data from a specified device to the host. If the host buffer des not exist it will first be allocated
+    /// @param device device ID to download source data from
+    /// @param stream cuda stream
+    /// @param sync if false the memory copy is asynchronous.
+    /// @warning Assumes that the specifed device buffer already exists!
+    void deviceDownload(int device = 0, cudaStream_t stream = 0, bool sync = true);
+    void deviceDownload(int device, void* stream , bool sync) {this->deviceDownload(device, cudaStream_t(stream), sync);}
+
+    /// @brief Download the buffer from the current device to the host, i.e. GPU -> CPU.
+    ///        If the host buffer des not exist it will first be allocated
+    /// @param stream CUDA stream
    /// @param sync if false the memory copy is asynchronous
    /// @note If the host/CPU buffer does not exist it is first allocated
    /// @warning Assumes that the device/GPU buffer already exists
-    void deviceDownload(void* stream = nullptr, bool sync = true) const;
+    void deviceDownload(void* stream, bool sync);
+
+    ///////////////////////////////////////////////////////////////////////

    /// @brief Returns the size in bytes of the raw memory buffer managed by this allocator.
    uint64_t size() const { return mSize; }
+    uint64_t capacity() const {return this->size();}

-    //@{
+    /// @brief Returns the number of buffers that are not NULL
+    int bufferCount() const {
+        int count = mCpuData ? 1 : 0;
+        for (int i=0; i<mDeviceCount; ++i) if (mGpuData[i]) ++count;
+        return count;
+    }
+
+    int deviceCount() const {return mDeviceCount;}
+
+    /// @{
    /// @brief Returns true if this allocator is empty, i.e. has no allocated memory
    bool empty() const { return mSize == 0; }
-    bool isEmpty() const { return mSize == 0; }
-    //@}
+    bool isEmpty() const { return this->empty(); }
+    /// @}

    /// @brief De-allocate all memory managed by this allocator and set all pointers to NULL
-    void clear(void* stream = nullptr);
+    void clear(cudaStream_t stream = 0);
+    void clear(void* stream){this->clear(cudaStream_t(stream));}

 }; // DeviceBuffer class

 // --------------------------> Implementations below <------------------------------------

-inline DeviceBuffer DeviceBuffer::create(uint64_t size, const DeviceBuffer*, bool host, void* stream)
+inline DeviceBuffer& DeviceBuffer::operator=(DeviceBuffer&& other) noexcept
 {
-    return DeviceBuffer(size, host, stream);
+    if (mManaged) {// first free all the managed data buffers
+        cudaCheck(cudaFreeHost(mCpuData));
+        for (int i=0; i<mDeviceCount; ++i) cudaCheck(util::cuda::freeAsync(mGpuData[i], 0));
+    }
+    delete [] mGpuData;
+    mSize    = other.mSize;
+    mCpuData = other.mCpuData;
+    mGpuData = other.mGpuData;
+    mDeviceCount = other.mDeviceCount;
+    mManaged = other.mManaged;
+    other.mCpuData = nullptr;
+    other.mGpuData = nullptr;
+    other.mSize = 0;
+    other.mDeviceCount = 0;
+    other.mManaged = 0;
+    return *this;
 }

-inline DeviceBuffer DeviceBuffer::create(uint64_t size, void* cpuData, void* gpuData)
+inline void DeviceBuffer::init(uint64_t size, int device, cudaStream_t stream)
 {
-    return DeviceBuffer(size, cpuData, gpuData);
-}
-
-inline void DeviceBuffer::init(uint64_t size, bool host, void* stream)
-{
-    if (mSize>0) this->clear(stream);
-    NANOVDB_ASSERT(size > 0);
-    if (host) {
+    if (size==0) return;
+    cudaCheck(cudaGetDeviceCount(&mDeviceCount));
+    mGpuData = new void*[mDeviceCount]();// NULL initialization
+    NANOVDB_ASSERT(device >= cudaCpuDeviceId && device < mDeviceCount);
+    if (device == cudaCpuDeviceId) {
        cudaCheck(cudaMallocHost((void**)&mCpuData, size)); // un-managed pinned memory on the host (can be slow to access!). Always 32B aligned
        checkPtr(mCpuData, "cuda::DeviceBuffer::init: failed to allocate host buffer");
    } else {
-        cudaCheck(util::cuda::mallocAsync((void**)&mGpuData, size, reinterpret_cast<cudaStream_t>(stream))); // un-managed memory on the device, always 32B aligned!
-        checkPtr(mGpuData, "cuda::DeviceBuffer::init: failed to allocate device buffer");
+        cudaCheck(util::cuda::mallocAsync(mGpuData+device, size, stream)); // un-managed memory on the device, always 32B aligned!
+        checkPtr(mGpuData[device], "cuda::DeviceBuffer::init: failed to allocate device buffer");
    }
    mSize = size;
-    mManaged = true;
+    mManaged = 1;// i.e. this instance is responsible for allocating and delete memory
 } // DeviceBuffer::init

-inline void DeviceBuffer::deviceUpload(void* stream, bool sync) const
+inline void DeviceBuffer::deviceUpload(int device, cudaStream_t stream, bool sync)
 {
-    if (!mManaged) throw std::runtime_error("DeviceBuffer::deviceUpload called on externally managed memory. Replace deviceUpload call with the appropriate external copy operation.");
-
-    checkPtr(mCpuData, "uninitialized cpu data");
-    if (mGpuData == nullptr) {
-        cudaCheck(util::cuda::mallocAsync((void**)&mGpuData, mSize, reinterpret_cast<cudaStream_t>(stream))); // un-managed memory on the device, always 32B aligned!
+    NANOVDB_ASSERT(device >= 0 && device < mDeviceCount);// should be device and not the host
+    checkPtr(mCpuData, "uninitialized cpu source data");
+    if (mGpuData[device] == nullptr) {
+        if (mManaged==0) throw std::runtime_error("DeviceBuffer::deviceUpload called on externally managed memory that wasn\'t allocated.");
+        cudaCheck(util::cuda::mallocAsync(mGpuData+device, mSize, stream)); // un-managed memory on the device, always 32B aligned!
    }
-    checkPtr(mGpuData, "uninitialized gpu data");
-    cudaCheck(cudaMemcpyAsync(mGpuData, mCpuData, mSize, cudaMemcpyHostToDevice, reinterpret_cast<cudaStream_t>(stream)));
-    if (sync) cudaCheck(cudaStreamSynchronize(reinterpret_cast<cudaStream_t>(stream)));
-} // DeviceBuffer::gpuUpload
+    checkPtr(mGpuData[device], "uninitialized gpu destination data");
+    cudaCheck(cudaMemcpyAsync(mGpuData[device], mCpuData, mSize, cudaMemcpyHostToDevice, stream));
+    if (sync) cudaCheck(cudaStreamSynchronize(stream));
+} // DeviceBuffer::deviceUpload

-inline void DeviceBuffer::deviceDownload(void* stream, bool sync) const
+inline void DeviceBuffer::deviceUpload(cudaStream_t stream, bool sync)
 {
-    if (!mManaged) throw std::runtime_error("DeviceBuffer::deviceDownload called on externally managed memory. Replace deviceDownload call with the appropriate external copy operation.");
+    int device = 0;
+    cudaGetDevice(&device);
+    this->deviceUpload(device, stream, sync);
+} // DeviceBuffer::deviceUpload

-    checkPtr(mGpuData, "uninitialized gpu data");
+inline void DeviceBuffer::deviceDownload(int device, cudaStream_t stream, bool sync)
+{
+    NANOVDB_ASSERT(device >= 0 && device < mDeviceCount);
+    checkPtr(mGpuData[device], "uninitialized gpu source data");// no source data on the specified device
    if (mCpuData == nullptr) {
+        if (mManaged==0) throw std::runtime_error("DeviceBuffer::deviceDownload called on uninitialized cpu destination memory that is externally managed.");
        cudaCheck(cudaMallocHost((void**)&mCpuData, mSize)); // un-managed pinned memory on the host (can be slow to access!). Always 32B aligned
    }
-    checkPtr(mCpuData, "uninitialized cpu data");
-    cudaCheck(cudaMemcpyAsync(mCpuData, mGpuData, mSize, cudaMemcpyDeviceToHost, reinterpret_cast<cudaStream_t>(stream)));
-    if (sync) cudaCheck(cudaStreamSynchronize(reinterpret_cast<cudaStream_t>(stream)));
-} // DeviceBuffer::gpuDownload
+    checkPtr(mCpuData, "uninitialized cpu destination data");
+    cudaCheck(cudaMemcpyAsync(mCpuData, mGpuData[device], mSize, cudaMemcpyDeviceToHost, stream));
+    if (sync) cudaCheck(cudaStreamSynchronize(stream));
+} // DeviceBuffer::deviceDownload

-inline void DeviceBuffer::clear(void *stream)
+inline void DeviceBuffer::deviceDownload(void* stream, bool sync)
 {
-    if (mManaged && mGpuData) cudaCheck(util::cuda::freeAsync(mGpuData, reinterpret_cast<cudaStream_t>(stream)));
-    if (mManaged && mCpuData) cudaCheck(cudaFreeHost(mCpuData));
-    mCpuData = mGpuData = nullptr;
+    int device = 0;
+    cudaCheck(cudaGetDevice(&device));
+    this->deviceDownload(device, cudaStream_t(stream), sync);
+} // DeviceBuffer::deviceDownload
+
+inline void DeviceBuffer::clear(cudaStream_t stream)
+{
+    if (mManaged) {// free all the managed data buffers
+        cudaCheck(cudaFreeHost(mCpuData));
+        for (int i=0; i<mDeviceCount; ++i) cudaCheck(util::cuda::freeAsync(mGpuData[i], stream));
+    }
+    delete [] mGpuData;
+    mCpuData = nullptr;
+    mGpuData = nullptr;
    mSize = 0;
-    mManaged = false;
+    mDeviceCount = 0;
+    mManaged = 0;
 } // DeviceBuffer::clear

 }// namespace cuda
@@ -0,0 +1,229 @@
+// Copyright Contributors to the OpenVDB Project
+// SPDX-License-Identifier: Apache-2.0
+
+/*!
+    \file DeviceMesh.h
+
+    \brief nanovdb::cuda::DeviceMesh encapsulates device IDs, CUDA streams,
+           and NCCL communicators in order to facilitate multi-GPU applications.
+*/
+
+#ifndef NANOVDB_CUDA_DEVICEMESH_H_HAS_BEEN_INCLUDED
+#define NANOVDB_CUDA_DEVICEMESH_H_HAS_BEEN_INCLUDED
+
+#include <vector>
+#include <algorithm>
+#include <thread>
+
+#include <nanovdb/util/cuda/Util.h>
+#ifdef NANOVDB_USE_NCCL
+#include <nccl.h>
+#endif
+
+namespace nanovdb {
+
+namespace cuda {
+
+namespace detail {
+
+/// @brief RAII class that caches/restores the current device at construction/destruction
+class DeviceGuard {
+    public:
+        DeviceGuard() { cudaGetDevice(&deviceId); }
+        ~DeviceGuard() { cudaSetDevice(deviceId); }
+
+        /// @{
+        /// @brief DeviceGuard is not copyable nor movable
+        DeviceGuard(const DeviceGuard&) = delete;
+        DeviceGuard& operator=(const DeviceGuard&) = delete;
+        DeviceGuard(DeviceGuard&& other) = delete;
+        DeviceGuard& operator=(DeviceGuard&& other) = delete;
+        /// @}
+    private:
+        int deviceId = -1;
+};
+
+}
+
+/// @brief POD struct representing a device id and a stream on that device
+struct DeviceNode
+{
+    int id = -1;
+    cudaStream_t stream = cudaStream_t(0);
+};
+
+/// @brief This class wraps a vector of per-device IDs and CUDA streams and holds inter-device connectivity information and NCCL comms.
+class DeviceMesh
+{
+    using DeviceVecT = std::vector<DeviceNode>;
+    using const_iterator = DeviceVecT::const_iterator;
+    using size_type = DeviceVecT::size_type;
+public:
+    /// @brief Constructs a device mesh for all devices on the host and initializes a CUDA stream (and NCCL communicator) for each device
+    DeviceMesh();
+    /// @brief Destroys the per-device CUDA stream and finalizes and destroys the per-device NCCL communicators
+    ~DeviceMesh();
+    /// @brief Disallow copy-construction
+    DeviceMesh(const DeviceMesh&) = delete;
+    /// @brief Move constructor.  Underlying CUDA streams and NCCL communicators are not reinitialized.
+    /// @param DeviceMesh instance that will be moved into this DeviceMesh.
+    DeviceMesh(DeviceMesh&&) noexcept;
+    /// @brief Disallow copy-assignment
+    DeviceMesh& operator=(const DeviceMesh&) = delete;
+    /// @brief Move assignment. Underlying CUDA streams and NCCL communicators are not reinitialized.
+    /// @param DeviceMesh instance that will be moved into this DeviceMesh.
+    DeviceMesh& operator=(DeviceMesh&&) noexcept;
+
+    /// @brief Returns the number of devices
+    size_type deviceCount() const { return mDeviceNodes.size(); };
+
+    /// @brief Returns a const reference to the DeviceNode for a particular device
+    const DeviceNode& operator [](int deviceId) const { return mDeviceNodes[deviceId]; }
+
+    /// @brief Returns an iterator to the first device node
+    const_iterator begin() const { return mDeviceNodes.begin(); };
+    /// @brief Returns an iterator past the last device node
+    const_iterator end() const { return mDeviceNodes.end(); };
+
+#ifdef NANOVDB_USE_NCCL
+    /// @brief Returns the NCCL communicator for a particular device
+    ncclComm_t comm(int deviceId) const { return mComms[deviceId]; }
+#endif
+
+    /// @brief Returns whether or not peer to peer access is supported between two devices.
+    bool canAccessPeer(int deviceId, int peerId) const { return mConnectivity[deviceId][peerId] & (1 << cudaDevP2PAttrAccessSupported); }
+
+private:
+    /// @brief Returns whether or not a device supports managed memory
+    bool hasManagedMemory(int deviceId) const {
+        int managedMemoryValue = 0;
+        cudaDeviceGetAttribute(&managedMemoryValue, cudaDevAttrManagedMemory, deviceId);
+        return static_cast<bool>(managedMemoryValue);
+    }
+
+    DeviceVecT mDeviceNodes;
+
+#ifdef NANOVDB_USE_NCCL
+    std::vector<ncclComm_t> mComms;
+#endif
+    std::vector<std::vector<uint32_t>> mConnectivity;
+};
+
+inline DeviceMesh::DeviceMesh()
+{
+    detail::DeviceGuard deviceGuard;
+
+    int deviceCount = -1;
+    cudaGetDeviceCount(&deviceCount);
+
+    for (int deviceId = 0; deviceId < deviceCount; ++deviceId) {
+        NANOVDB_ASSERT(hasManagedMemory(deviceId));
+        cudaSetDevice(deviceId);
+        cudaStream_t stream;
+        cudaStreamCreate(&stream);
+        mDeviceNodes.push_back({deviceId, stream});
+    }
+
+#ifdef NANOVDB_USE_NCCL
+    mComms.resize(deviceCount);
+    ncclCommInitAll(mComms.data(), deviceCount, nullptr);
+#endif
+
+    mConnectivity.resize(deviceCount);
+    for (const auto& deviceNode : mDeviceNodes) {
+        mConnectivity[deviceNode.id].resize(deviceCount);
+        for (const auto& peerNode : mDeviceNodes) {
+            mConnectivity[deviceNode.id][peerNode.id] = 0;
+            if (deviceNode.id != peerNode.id) {
+                int peerAccessSupportedValue = 0;
+                cudaDeviceGetP2PAttribute(&peerAccessSupportedValue, cudaDevP2PAttrAccessSupported, deviceNode.id, peerNode.id);
+                if (peerAccessSupportedValue) {
+                    mConnectivity[deviceNode.id][peerNode.id] |= (1 << cudaDevP2PAttrAccessSupported);
+                }
+            }
+        }
+    }
+}
+
+inline DeviceMesh::DeviceMesh(DeviceMesh&& other) noexcept
+    : mDeviceNodes(std::move(other.mDeviceNodes)),
+#ifdef NANOVDB_USE_NCCL
+      mComms(std::move(other.mComms)),
+#endif
+      mConnectivity(std::move(other.mConnectivity))
+{
+}
+
+inline DeviceMesh::~DeviceMesh()
+{
+    detail::DeviceGuard deviceGuard;
+
+#ifdef NANOVDB_USE_NCCL
+    std::for_each(mComms.begin(), mComms.end(), [](ncclComm_t comm) {
+        ncclCommFinalize(comm);
+        ncclCommDestroy(comm);
+    });
+#endif
+
+    std::for_each(mDeviceNodes.begin(), mDeviceNodes.end(), [](DeviceNode& deviceNode) {
+        cudaSetDevice(deviceNode.id);
+        cudaStreamDestroy(deviceNode.stream);
+    });
+}
+
+inline DeviceMesh& DeviceMesh::operator=(DeviceMesh&& other) noexcept
+{
+    mDeviceNodes = std::move(other.mDeviceNodes);
+#ifdef NANOVDB_USE_NCCL
+    mComms = std::move(other.mComms);
+#endif
+    mConnectivity = std::move(other.mConnectivity);
+    return *this;
+}
+
+namespace detail {
+
+inline void* queryAllocationGranularityEntryPoint()
+{
+    void* entryPoint = nullptr;
+#if CUDART_VERSION >= 12500 // cudaGetDriverEntryPointByVersion was added in CUDA 12.5 with cudaGetDriverEntryPoint being potentially deprecated
+    cudaDriverEntryPointQueryResult queryResult = cudaDriverEntryPointSymbolNotFound;
+    cudaCheck(cudaGetDriverEntryPointByVersion("cuMemGetAllocationGranularity", &entryPoint, 12000, cudaEnableDefault, &queryResult));
+    NANOVDB_ASSERT(queryResult == cudaDriverEntryPointSuccess);
+#elif CUDART_VERSION >= 12000 // queryResult argument was added in CUDA 12
+    cudaDriverEntryPointQueryResult queryResult = cudaDriverEntryPointSymbolNotFound;
+    cudaCheck(cudaGetDriverEntryPoint("cuMemGetAllocationGranularity", &entryPoint, cudaEnableDefault, &queryResult));
+    NANOVDB_ASSERT(queryResult == cudaDriverEntryPointSuccess);
+#else
+    cudaCheck(cudaGetDriverEntryPoint("cuMemGetAllocationGranularity", &entryPoint, cudaEnableDefault));
+#endif
+    return entryPoint;
+}// queryAllocationGranularityEntryPoint
+
+}
+
+/// @brief Returns the minimum page size (in bytes) across all devices on the system
+inline size_t minDevicePageSize(const DeviceMesh& mesh)
+{
+    using FuncT = CUresult(size_t*, const CUmemAllocationProp*, CUmemAllocationGranularity_flags);
+    static FuncT* functPtr = reinterpret_cast<FuncT*>(detail::queryAllocationGranularityEntryPoint());
+
+    NANOVDB_ASSERT(functPtr);
+    CUmemAllocationProp prop = {};
+    prop.type = CU_MEM_ALLOCATION_TYPE_PINNED;
+    prop.location.type = CU_MEM_LOCATION_TYPE_DEVICE;
+    size_t minGranularity = 0;
+    for (const auto& node : mesh) {
+        prop.location.id = node.id;
+        size_t granularity = 0;
+        (*functPtr)(&granularity, &prop, CU_MEM_ALLOC_GRANULARITY_MINIMUM);
+        if (minGranularity < granularity) minGranularity = granularity;
+    }
+    return minGranularity;
+}// minDevicePageSize
+
+} // namespace cuda
+
+} // namespace nanovdb
+
+#endif // end of NANOVDB_CUDA_DEVICEMESH_H_HAS_BEEN_INCLUDED
@@ -0,0 +1,35 @@
+// Copyright Contributors to the OpenVDB Project
+// SPDX-License-Identifier: Apache-2.0
+//
+#ifndef NANOVDB_CUDA_DEVICERESOURCE_H_HAS_BEEN_INCLUDED
+#define NANOVDB_CUDA_DEVICERESOURCE_H_HAS_BEEN_INCLUDED
+
+#include <cuda_runtime_api.h>
+#include <nanovdb/util/cuda/Util.h>
+
+namespace nanovdb {
+
+namespace cuda {
+
+class DeviceResource
+{
+public:
+    // cudaMalloc aligns memory to 256 bytes by default
+    static constexpr size_t DEFAULT_ALIGNMENT = 256;
+
+    static void* allocateAsync(size_t bytes, size_t, cudaStream_t stream) {
+        void* p = nullptr;
+        cudaCheck(util::cuda::mallocAsync(&p, bytes, stream));
+        return p;
+    }
+
+    static void deallocateAsync(void *p, size_t, size_t, cudaStream_t stream) {
+        cudaCheck(util::cuda::freeAsync(p, stream));
+    }
+};
+
+}
+
+} // namespace nanovdb::cuda
+
+#endif // end of NANOVDB_CUDA_DEVICERESOURCE_H_HAS_BEEN_INCLUDED
@@ -0,0 +1,124 @@
+// Copyright Contributors to the OpenVDB Project
+// SPDX-License-Identifier: Apache-2.0
+
+/*!
+    \file DeviceStreamMap.h
+
+    \author Ken Museth
+
+    \date October 15, 2024
+
+    \brief nanovdb::cuda::DeviceStreamMap maps device IDs to CUDA streams,
+           which is useful for multi-GPU applications.
+*/
+
+#ifndef NANOVDB_CUDA_DEVICESTREAMMAP_H_HAS_BEEN_INCLUDED
+#define NANOVDB_CUDA_DEVICESTREAMMAP_H_HAS_BEEN_INCLUDED
+
+#include <cuda.h>
+#include <map>
+#include <vector>
+#include <nanovdb/util/cuda/Util.h>// for cudaCheck, deviceCount etc
+
+namespace nanovdb {// ================================================================
+
+namespace cuda {// ===================================================================
+
+/// @brief map from a device ID to an associated cuda stream. Useful for multi-GPU applications.
+class DeviceStreamMap : public std::map<int, cudaStream_t>
+{
+     using FuncT = CUresult(size_t*, const CUmemAllocationProp*, CUmemAllocationGranularity_flags);
+     FuncT *mFunctPtr;
+
+public:
+
+    enum DeviceType { Any = 0, PeerToPeer = 1, Unified = 3 };
+
+    /// @brief Initiates a map between CUDA device IDs and corresponding streams that satisfy certain constraints.
+    ///        All devices should be able to access memory on all the other devices!
+    /// @param t Type of device to include in map. Any means all available devices, PeerToPeer means only devices that
+    ///          access all another devices are included, and Unified means all devices support unified memory, concurrent access,
+    ///          and can be access by all other devices.
+    /// @param exclude optional list of device IDs to exclude from the map
+    /// @param verbose  0 means quiet, 1 means print if a device is ignores and 2 means print is a device is included
+    DeviceStreamMap(DeviceType t = DeviceType::Unified, std::vector<int> exclude = {}, int verbose = 0);
+
+    /// @brief Destructor
+    ~DeviceStreamMap();
+
+    /// @brief returns the minimum page size of all the devices in this map
+    size_t getMinPageSize() const;
+
+    /// @brief Print information about all the devices included in this map
+    void printDevInfo(std::FILE* file = stdout) const {for (auto &p : *this) util::cuda::printDevInfo(p.first, nullptr, file);}
+
+    /// @brief Returns the number of device associated with this map
+    int deviceCount() const {return this->size();}
+
+};// DeviceStreamMap
+
+DeviceStreamMap::DeviceStreamMap(DeviceType t, std::vector<int> exclude, int verbose)
+{
+    std::initializer_list<cudaDeviceAttr> filter = {cudaDevAttrUnifiedAddressing, cudaDevAttrConcurrentManagedAccess};
+    const int devCount = util::cuda::deviceCount(), current = util::cuda::currentDevice();
+    for (int dev = 0; dev < devCount; ++dev) {
+        int check = 1;
+        for (auto it=exclude.begin();         check && it!=exclude.end(); ++it) if (dev == *it) check = 0;
+        for (auto it=filter.begin(); (t&2) && check && it!=filter.end(); ++it) cudaCheck(cudaDeviceGetAttribute( &check, *it, dev));
+        for (auto it= this->begin(); (t&1) && check && it!= this->end(); ++it) cudaCheck(cudaDeviceCanAccessPeer(&check, dev, it->first));
+        if (check) {
+            cudaCheck(cudaSetDevice(dev));
+            cudaStream_t stream;
+            cudaCheck(cudaStreamCreate(&stream));
+            if (verbose>1) util::cuda::printDevInfo(dev, "Using");
+            (*this)[dev] = stream;
+        } else if (verbose) util::cuda::printDevInfo(dev, "Ignoring");
+    }
+    cudaCheck(cudaSetDevice(current));// reset to the previous device
+
+    void* entryPoint = nullptr;
+#if CUDART_VERSION >= 13000
+    cudaDriverEntryPointQueryResult queryResult;
+    cudaCheck(cudaGetDriverEntryPointByVersion("cuMemGetAllocationGranularity", &entryPoint, 13000, cudaEnableDefault, &queryResult));
+    NANOVDB_ASSERT(queryResult == cudaDriverEntryPointSuccess);
+#elif CUDART_VERSION >= 12000// queryResult argument was added in CUDA 12
+    cudaDriverEntryPointQueryResult queryResult;
+    cudaCheck(cudaGetDriverEntryPoint("cuMemGetAllocationGranularity", &entryPoint, cudaEnableDefault, &queryResult));
+    NANOVDB_ASSERT(queryResult == cudaDriverEntryPointSuccess);
+#else
+    cudaCheck(cudaGetDriverEntryPoint("cuMemGetAllocationGranularity", &entryPoint, cudaEnableDefault));
+#endif
+    mFunctPtr = reinterpret_cast<FuncT*>(entryPoint);
+}// DeviceStreamMap::DeviceStreamMap
+
+DeviceStreamMap::~DeviceStreamMap()
+{
+    const int current = util::cuda::currentDevice();
+    for (auto& [device, stream] : *this) {
+        cudaCheck(cudaSetDevice(device));
+        cudaCheck(cudaStreamDestroy(stream));
+    }
+    cudaCheck(cudaSetDevice(current));// reset to the previous device
+}
+
+inline size_t DeviceStreamMap::getMinPageSize() const
+{
+    NANOVDB_ASSERT(mFunctPtr);
+    CUmemAllocationProp prop = {};
+    prop.type = CU_MEM_ALLOCATION_TYPE_PINNED;
+    prop.location.type = CU_MEM_LOCATION_TYPE_DEVICE;
+    size_t minGranularity = 0;
+    for (auto it = this->begin(); it!=this->end(); ++it) {
+        prop.location.id = it->first;
+        size_t granularity = 0;
+        (*mFunctPtr)(&granularity, &prop, CU_MEM_ALLOC_GRANULARITY_MINIMUM);
+        if (minGranularity < granularity) minGranularity = granularity;
+    }
+    return minGranularity;
+}// DeviceStreamMap::getMinPageSize
+
+}// namespace cuda
+
+}// namespace nanovdb
+
+#endif // end of NANOVDB_CUDA_DEVICESTREAMMAP_H_HAS_BEEN_INCLUDED
@@ -25,22 +25,25 @@ namespace nanovdb {

 namespace cuda {

-namespace {// anonymous namespace
-__global__ void cpyGridHandleMeta(const GridData *d_data, GridHandleMetaData *d_meta)
+namespace detail {
+
+static __global__ void cpyGridHandleMeta(const GridData *d_data, GridHandleMetaData *d_meta)
 {
    nanovdb::cpyGridHandleMeta(d_data, d_meta);
 }

-__global__ void updateGridCount(GridData *d_data, uint32_t gridIndex, uint32_t gridCount, bool *d_dirty)
+static __global__ void updateGridCount(GridData *d_data, uint32_t gridIndex, uint32_t gridCount, bool *d_dirty)
 {
    NANOVDB_ASSERT(gridIndex < gridCount);
-    if (*d_dirty = d_data->mGridIndex != gridIndex || d_data->mGridCount != gridCount) {
+    *d_dirty = (d_data->mGridIndex != gridIndex) || (d_data->mGridCount != gridCount);
+    if (*d_dirty) {
        d_data->mGridIndex = gridIndex;
        d_data->mGridCount = gridCount;
        if (d_data->mChecksum.isEmpty()) *d_dirty = false;// no need to update checksum if it didn't already exist
    }
 }
-}// anonymous namespace
+
+}// namespace detail

 template<typename BufferT, template <class, class...> class VectorT = std::vector>
 inline typename util::enable_if<BufferTraits<BufferT>::hasDeviceDual, VectorT<GridHandle<BufferT>>>::type
@@ -51,20 +54,22 @@ splitGridHandles(const GridHandle<BufferT> &handle, const BufferT* other = nullp
    VectorT<GridHandle<BufferT>> handles(handle.gridCount());
    bool dirty, *d_dirty;// use this to check if the checksum needs to be recomputed
    cudaCheck(util::cuda::mallocAsync((void**)&d_dirty, sizeof(bool), stream));
+    int device = util::cuda::currentDevice();
    for (uint32_t n=0; n<handle.gridCount(); ++n) {
-        auto buffer = BufferT::create(handle.gridSize(n), other, false, stream);
+        auto buffer = BufferT::create(handle.gridSize(n), other, device, stream);
        GridData *dst = reinterpret_cast<GridData*>(buffer.deviceData());
        const GridData *src = reinterpret_cast<const GridData*>(ptr);
        cudaCheck(cudaMemcpyAsync(dst, src, handle.gridSize(n), cudaMemcpyDeviceToDevice, stream));
-        updateGridCount<<<1, 1, 0, stream>>>(dst, 0u, 1u, d_dirty);
+        detail::updateGridCount<<<1, 1, 0, stream>>>(dst, 0u, 1u, d_dirty);
        cudaCheckError();
        cudaCheck(cudaMemcpyAsync(&dirty, d_dirty, sizeof(bool), cudaMemcpyDeviceToHost, stream));
+        cudaCheck(cudaStreamSynchronize(stream));
        if (dirty) tools::cuda::updateChecksum(dst, CheckMode::Partial, stream);
        handles[n] = nanovdb::GridHandle<BufferT>(std::move(buffer));
        ptr = util::PtrAdd(ptr, handle.gridSize(n));
    }
    cudaCheck(util::cuda::freeAsync(d_dirty, stream));
-    return std::move(handles);
+    return handles;
 }// cuda::splitGridHandles

 template<typename BufferT, template <class, class...> class VectorT>
@@ -77,7 +82,8 @@ mergeGridHandles(const VectorT<GridHandle<BufferT>> &handles, const BufferT* oth
        gridCount += h.gridCount();
        for (uint32_t n=0; n<h.gridCount(); ++n) size += h.gridSize(n);
    }
-    auto buffer = BufferT::create(size, other, false, stream);
+    int device = util::cuda::currentDevice();
+    auto buffer = BufferT::create(size, other, device, stream);
    void *dst = buffer.deviceData();
    bool dirty, *d_dirty;// use this to check if the checksum needs to be recomputed
    cudaCheck(util::cuda::mallocAsync((void**)&d_dirty, sizeof(bool), stream));
@@ -86,9 +92,10 @@ mergeGridHandles(const VectorT<GridHandle<BufferT>> &handles, const BufferT* oth
        for (uint32_t n=0; n<h.gridCount(); ++n) {
            cudaCheck(cudaMemcpyAsync(dst, src, h.gridSize(n), cudaMemcpyDeviceToDevice, stream));
            GridData *data = reinterpret_cast<GridData*>(dst);
-            updateGridCount<<<1, 1, 0, stream>>>(data, counter++, gridCount, d_dirty);
+            detail::updateGridCount<<<1, 1, 0, stream>>>(data, counter++, gridCount, d_dirty);
            cudaCheckError();
            cudaCheck(cudaMemcpyAsync(&dirty, d_dirty, sizeof(bool), cudaMemcpyDeviceToHost, stream));
+            cudaCheck(cudaStreamSynchronize(stream));
            if (dirty) tools::cuda::updateChecksum(data, CheckMode::Partial, stream);
            dst = util::PtrAdd(dst, h.gridSize(n));
            src = util::PtrAdd(src, h.gridSize(n));
@@ -129,7 +136,7 @@ GridHandle<BufferT>::GridHandle(T&& buffer)
            if (!tmp.isValid()) throw std::runtime_error("GridHandle was constructed with an invalid device buffer");
            GridHandleMetaData *d_metaData;
            cudaMalloc((void**)&d_metaData, tmp.mGridCount*sizeof(GridHandleMetaData));
-            cuda::cpyGridHandleMeta<<<1,1>>>(d_data, d_metaData);
+            cuda::detail::cpyGridHandleMeta<<<1,1>>>(d_data, d_metaData);
            mMetaData.resize(tmp.mGridCount);
            cudaCheck(cudaMemcpy(mMetaData.data(), d_metaData,tmp.mGridCount*sizeof(GridHandleMetaData), cudaMemcpyDeviceToHost));
            cudaCheck(cudaFree(d_metaData));
@@ -37,16 +37,14 @@ createNodeManager(const NanoGrid<BuildT> *d_grid,
                  const BufferT& pool = BufferT(),
                  cudaStream_t stream = 0)
 {
-    auto buffer = BufferT::create(sizeof(NodeManagerData), &pool, false, stream);
+    int device = 0;
+    cudaCheck(cudaGetDevice(&device));
+    auto buffer = BufferT::create(sizeof(NodeManagerData), &pool, device, stream);
    auto *d_data = (NodeManagerData*)buffer.deviceData();
    size_t size = 0u, *d_size;
    cudaCheck(util::cuda::mallocAsync((void**)&d_size, sizeof(size_t), stream));
    util::cuda::lambdaKernel<<<1, 1, 0, stream>>>(1, [=] __device__(size_t) {
-#ifdef NANOVDB_USE_NEW_MAGIC_NUMBERS
-        *d_data = NodeManagerData{NANOVDB_MAGIC_NODE,   0u, (void*)d_grid, {0u,0u,0u}};
-#else
-        *d_data = NodeManagerData{NANOVDB_MAGIC_NUMB, 0u, (void*)d_grid, {0u,0u,0u}};
-#endif
+        *d_data = NodeManagerData((void*)d_grid);
        *d_size = sizeof(NodeManagerData);
        auto &tree = d_grid->tree();
        if (NodeManager<BuildT>::FIXED_SIZE && d_grid->isBreadthFirst()) {
@@ -62,7 +60,7 @@ createNodeManager(const NanoGrid<BuildT> *d_grid,
    cudaCheck(cudaMemcpyAsync(&size, d_size, sizeof(size_t), cudaMemcpyDeviceToHost, stream));
    cudaCheck(util::cuda::freeAsync(d_size, stream));
    if (size > sizeof(NodeManagerData)) {
-        auto tmp = BufferT::create(size, &pool, false, stream);// only allocate buffer on the device
+        auto tmp = BufferT::create(size, &pool, device, stream);// only allocate buffer on the device
        cudaCheck(cudaMemcpyAsync(tmp.deviceData(), buffer.deviceData(), sizeof(NodeManagerData), cudaMemcpyDeviceToDevice, stream));
        buffer = std::move(tmp);
        d_data = reinterpret_cast<NodeManagerData*>(buffer.deviceData());
@@ -0,0 +1,66 @@
+// Copyright Contributors to the OpenVDB Project
+// SPDX-License-Identifier: Apache-2.0
+//
+/// @brief Defines a simple memory pool used to call cub functions that use dynamic temporary storage
+///
+/// @details See nanovdb/tools/cuda/PointToGrid.cuh and nanovdb/tools/cuda/DistributedPointToGrid.cuh
+///          for examples. Also note that this explains the somewhat unusual API with direct access to
+///          protected member data.
+//
+#ifndef NANOVDB_CUDA_TEMPPOOL_H_HAS_BEEN_INCLUDED
+#define NANOVDB_CUDA_TEMPPOOL_H_HAS_BEEN_INCLUDED
+
+#include <nanovdb/cuda/DeviceResource.h>
+
+namespace nanovdb {
+
+namespace cuda {
+
+template <class Resource>
+class TempPool {
+public:
+
+    /// @brief Default c-tor of an empty memory pool.
+    TempPool() : mData(nullptr), mSize(0), mRequestedSize(0) {}
+
+    /// @brief Destructor
+    ~TempPool() {
+        mRequestedSize = 0;
+        Resource::deallocateAsync(mData, mSize, Resource::DEFAULT_ALIGNMENT, nullptr);
+        mData = nullptr;
+        mSize = 0;
+    }
+
+    /// @brief Returns a non-const void pointer to the data managed by this instance.
+    void* data() {return mData;}
+
+    /// @brief Returns a non-const reference to the actual size of the data managed by this instance.
+    size_t& size() {return mSize;}
+
+    /// @brief Returns a non-const reference to the requested size of the data managed by this instance.
+    /// @note This requested size should always be less than or smaller than the actual size().
+    size_t& requestedSize() {return mRequestedSize;}
+
+    /// @brief Re-allocation of the data managed by this instance. Only has affect if the pool in empty or
+    ///        the requested memory is larger than the existing size.
+    /// @param stream cuda stream used for asynchronous de-allocation and allocation.
+    void reallocate(cudaStream_t stream) {
+        if (!mData || mRequestedSize > mSize) {
+            Resource::deallocateAsync(mData, mSize, Resource::DEFAULT_ALIGNMENT, stream);
+            mData = Resource::allocateAsync(mRequestedSize, Resource::DEFAULT_ALIGNMENT, stream);
+            mSize = mRequestedSize;
+        }
+    }
+private:
+    void  *mData;
+    size_t mSize;
+    size_t mRequestedSize;
+};// TempPool<Resource> class
+
+using TempDevicePool = TempPool<DeviceResource>;
+
+} // namespace cuda
+
+} // namespace nanovdb
+
+#endif // end of NANOVDB_CUDA_TEMPPOOL_H_HAS_BEEN_INCLUDED
@@ -0,0 +1,326 @@
+// Copyright Contributors to the OpenVDB Project
+// SPDX-License-Identifier: Apache-2.0
+
+/*!
+    \file UnifiedBuffer.h
+
+    \author Ken Museth
+
+    \date October 15, 2024
+
+    \brief nanovdb::cuda::UnifiedBuffer that uses unified memory management
+
+    \note This file has no device-only kernel functions,
+          which explains why it's a .h and not .cuh file.
+*/
+
+#ifndef NANOVDB_CUDA_UNIFIEDBUFFER_H_HAS_BEEN_INCLUDED
+#define NANOVDB_CUDA_UNIFIEDBUFFER_H_HAS_BEEN_INCLUDED
+
+#include <cuda.h>
+#include <memory>// for std::shared_ptr
+#include <nanovdb/HostBuffer.h>// for BufferTraits
+#include <nanovdb/util/cuda/Util.h>// for cudaCheck
+
+namespace nanovdb {// ================================================================
+
+namespace cuda {// ===================================================================
+
+/// @brief  buffer, used for instance by the GridHandle, to allocate unified memory that
+///         can be resized and shared between multiple devices and the host.
+class UnifiedBuffer
+{
+    void *mPtr;
+    size_t mSize, mCapacity;
+public:
+
+    using PtrT = std::shared_ptr<UnifiedBuffer>;
+
+    /// @brief Default constructor of an empty buffer
+    UnifiedBuffer() : mPtr(nullptr), mSize(0), mCapacity(0){}
+
+    /// @brief Constructor that specifies both the size and capacity
+    /// @param size size of the buffer in bytes, indication what is actually used
+    /// @param capacity number of bytes in the virtual page table, i.e max size for growing
+    /// @note Capacity can be over-estimated to allow for future growth. Memory is not allocated
+    ///       with this constructor, only a page table. Allocation happens on usage or when calling prefetch
+    UnifiedBuffer(size_t size, size_t capacity) : mPtr(nullptr), mSize(size), mCapacity(capacity)
+    {
+        assert(mSize <= mCapacity);
+        cudaCheck(cudaMallocManaged(&mPtr, mCapacity, cudaMemAttachGlobal));
+    }
+
+     /// @brief Similar to the constructor above except the size and capacity are equal, so no future growth is supported
+    UnifiedBuffer(size_t size) : UnifiedBuffer(size, size){}
+
+    /// @brief Constructor that specifies the size, capacity, and device (for prefetching)
+    /// @param size
+    /// @param capacity
+    /// @param device
+    /// @param stream
+    UnifiedBuffer(uint64_t size, uint64_t capacity, int device, cudaStream_t stream = 0) : mPtr(nullptr), mSize(size), mCapacity(size)
+    {
+        assert(mSize <= mCapacity);
+        cudaCheck(cudaMallocManaged(&mPtr, mCapacity, cudaMemAttachGlobal));
+        cudaCheck(util::cuda::memAdvise(mPtr, size, cudaMemAdviseSetPreferredLocation, device));
+        cudaCheck(util::cuda::memPrefetchAsync(mPtr, size, device, stream));
+    }
+
+    /// @brief Constructor with a specified device
+    /// @param size
+    /// @param device
+    /// @param stream
+    UnifiedBuffer(uint64_t size, int device, cudaStream_t stream = 0) : UnifiedBuffer(size, size, device, stream){}
+
+     /// @brief Disallow copy-construction
+    UnifiedBuffer(const UnifiedBuffer&) = delete;
+
+    /// @brief Move copy-constructor
+    UnifiedBuffer(UnifiedBuffer&& other) noexcept
+        : mPtr(other.mPtr)
+        , mSize(other.mSize)
+        , mCapacity(other.mCapacity)
+    {
+        other.mPtr = nullptr;
+        other.mSize = other.mCapacity = 0;
+    }
+
+    /// @brief Destructor
+    ~UnifiedBuffer(){cudaCheck(cudaFree(mPtr));}
+
+    ///////////////////////////////////////////////////////////////////////
+
+    //@{
+    /// @brief Factory methods that create an UnifiedBuffer instance and returns it with move semantics
+    static UnifiedBuffer create(size_t size, size_t capacity) {return UnifiedBuffer(size, capacity);}
+    static UnifiedBuffer create(size_t size) {return UnifiedBuffer(size);}
+    ///@}
+
+    //@{
+    /// @brief Factory methods that create a shared pointer to an UnifiedBuffer instance
+    static PtrT createPtr(size_t size, size_t capacity) {return std::make_shared<UnifiedBuffer>(size, capacity);}
+    static PtrT createPtr(size_t size) {return std::make_shared<UnifiedBuffer>(size);}
+    ///@}
+
+    /// @brief Legacy factory method that mirrors DeviceBuffer. It creates a UnifiedBuffer from a size and a reference buffer.
+    ///        If a reference buffer is provided and its non-empty, it is used to defined the capacity of the new buffer
+    /// @param size Size on bytes of the new buffer
+    /// @param reference reference buffer optionally used to define the capacity
+    /// @param host Ignored for now
+    /// @param stream cuda stream
+    /// @return An instance of a new UnifiedBuffer using move semantics
+    static UnifiedBuffer create(size_t size, const UnifiedBuffer* reference, int device, cudaStream_t stream)
+    {
+        const size_t capacity = (reference && reference->capacity()) ? reference->capacity() : size;
+        UnifiedBuffer buffer(size, capacity);
+        cudaCheck(util::cuda::memAdvise(buffer.mPtr, size, cudaMemAdviseSetPreferredLocation, device));
+        cudaCheck(util::cuda::memPrefetchAsync(buffer.mPtr, size, device, stream));
+        return buffer;
+    }
+
+    /// @brief Factory method that created a buffer on the host of the specified size. If the
+    ///        reference buffer has a capacity it is used. Also the buffer is prefetched to the host
+    /// @param size byte size of buffer initiated on the host
+    /// @param reference optional reference buffer from which the capacity is derived
+    static UnifiedBuffer create(size_t size, const UnifiedBuffer* reference){return create(size, reference, cudaCpuDeviceId, (cudaStream_t)0);}
+
+    /// @brief Factory method that created a buffer on the host or device of the specified size. If the
+    ///        reference buffer has a capacity it is used. Also the buffer is prefetched to the host or (current) device
+    /// @param size byte size of buffer initiated on the device or host
+    /// @param reference optional reference buffer from which the capacity is derived
+    /// @param host If true the buffer will be prefetched to the host, else to the current device
+    /// @param stream optional cuda stream
+    static UnifiedBuffer create(size_t size, const UnifiedBuffer* reference, bool host, void* stream = nullptr)
+    {
+        int device = cudaCpuDeviceId;
+        if (!host) cudaGetDevice(&device);
+        return create(size, reference, device, (cudaStream_t)stream);
+    }
+
+    /// @brief Free all memory and reset this instance to empty
+    void clear()
+    {
+        cudaCheck(cudaFree(mPtr));
+        mPtr = nullptr;
+        mSize = mCapacity = 0;
+    }
+
+    /// @brief Disallow copy assignment operation
+    UnifiedBuffer& operator=(const UnifiedBuffer&) = delete;
+
+    /// @brief Allow move assignment operation
+    UnifiedBuffer& operator=(UnifiedBuffer&& other)
+    {
+        cudaCheck(cudaFree(mPtr));
+        mPtr = other.mPtr;
+        mSize = other.mSize;
+        mCapacity = other.mCapacity;
+        other.mPtr = nullptr;
+        other.mSize = other.mCapacity = 0;
+        return *this;
+    }
+
+    /// @brief initialize buffer as a new with the specified size and capacity
+    /// @param size size of memory block to be used in bytes
+    /// @param capacity size of page table in bytes
+    void init(size_t size, size_t capacity)
+    {
+        NANOVDB_ASSERT(size <= capacity);
+        cudaCheck(cudaFree(mPtr));
+        mSize = size;
+        mCapacity = capacity;
+        cudaCheck(cudaMallocManaged(&mPtr, capacity, cudaMemAttachGlobal));
+    }
+
+    /// @brief Resize the memory block managed by this buffer. If the current capacity is larger than the new size this method
+    ///        simply redefines size. Otherwise a new page-table is defined, with the specified advice, and the old block is copied to the new block.
+    /// @param size size of the new memory block
+    /// @param dev the device ID on which to apply each advice provided in list, cudaCpuDeviceId = -1, 0, 1, ...
+    /// @param list advices to be applied to the resized range
+    void resize(size_t size, int dev = cudaCpuDeviceId, std::initializer_list<cudaMemoryAdvise> list = {cudaMemAdviseSetPreferredLocation})
+    {
+        if (size <= mCapacity) {
+            mSize = size;
+        } else {
+            void *ptr = 0;
+            cudaCheck(cudaMallocManaged(&ptr, size, cudaMemAttachGlobal));
+            if (dev > -2) for (auto a : list) cudaCheck(util::cuda::memAdvise(ptr, size, a, dev));
+            if (mSize > 0) {// copy over data from the old memory block
+                cudaCheck(cudaMemcpy(ptr, mPtr, std::min(mSize, size), cudaMemcpyDefault));
+                cudaCheck(cudaFree(mPtr));
+            }
+            mPtr = ptr;
+            mSize = mCapacity = size;
+        }
+    }
+
+    /// @brief Apply a single advise to a memory block
+    /// @param byteOffset offset in bytes marking the beginning of the memory block to be advised
+    /// @param size size in bytes of the memory block to be advised.
+    /// @param dev the device ID on which to apply the advice provided in adv, cudaCpuDeviceId = -1, 0, 1, ...
+    /// @param adv advice to be applied to the resized range
+    void advise(ptrdiff_t byteOffset, size_t size, int dev, cudaMemoryAdvise adv) const
+    {
+        cudaCheck(util::cuda::memAdvise(util::PtrAdd(mPtr, byteOffset), size, adv, dev));
+    }
+
+    /// @brief Apply a list of advices to a memory block
+    /// @param byteOffset offset in bytes marking the beginning of the memory block to be advised
+    /// @param size size in bytes of the memory block to be advised.
+    /// @param dev the device ID to prefetch to, cudaCpuDeviceId = -1, 0, 1, ...
+    /// @param list list of cuda advises
+    void advise(ptrdiff_t byteOffset, size_t size, int dev, std::initializer_list<cudaMemoryAdvise> list) const
+    {
+        void *ptr = util::PtrAdd(mPtr, byteOffset);
+        for (auto a : list)  cudaCheck(util::cuda::memAdvise(ptr, size, a, dev));
+    }
+
+    /// @brief Prefetches data to the specified device, i.e. ensure the device has an up-to-date copy of the memory specified
+    /// @param byteOffset offset in bytes marking the beginning of the memory block to be prefetched
+    /// @param size size in bytes of the memory block to be prefetched. The default value of zero means copy all @c this->size() bytes.
+    /// @param dev the device ID to prefetch to, cudaCpuDeviceId = -1, 0, 1, ...
+    /// @param stream  cuda stream
+    void prefetch(ptrdiff_t byteOffset = 0, size_t size = 0, int dev = cudaCpuDeviceId, cudaStream_t stream = cudaStreamPerThread) const
+    {
+        cudaCheck(util::cuda::memPrefetchAsync(util::PtrAdd(mPtr, byteOffset), size ? size : mSize, dev, stream));
+    }
+
+    ///////////////////////////////////////////////////////////////////////
+
+    /// @brief Prefetches all data to the specified device
+    /// @param device device ID, cudaCpuDeviceId = -1, 0, 1, ...
+    /// @param stream cuda stream
+    /// @param sync if false the memory copy is asynchronous
+    /// @note Legacy method included for compatibility with DeviceBuffer
+    void deviceUpload(int device = 0, cudaStream_t stream = cudaStreamPerThread, bool sync = false) const
+    {
+        cudaCheck(util::cuda::memPrefetchAsync(mPtr, mSize, device, stream));
+        if (sync) cudaCheck(cudaStreamSynchronize(stream));
+    }
+    void deviceUpload(int device, void* stream, bool sync) const{this->deviceUpload(device, cudaStream_t(stream));}
+
+    /// @brief Prefetches all data to the current device, as given by cudaGetDevice
+    /// @param stream cuda stream
+    /// @param sync if false the memory copy is asynchronous
+    /// @note Legacy method included for compatibility with DeviceBuffer
+    void deviceUpload(void* stream, bool sync) const{
+        int device = 0;
+        cudaCheck(cudaGetDevice(&device));
+        this->deviceUpload(device, cudaStream_t(stream), sync);
+    }
+
+    ///////////////////////////////////////////////////////////////////////
+
+    /// @brief Prefetches all data to the host
+    /// @param stream cuda stream
+    /// @param sync if false the memory copy is asynchronous
+    void deviceDownload(cudaStream_t stream = 0, bool sync = false) const
+    {
+        cudaCheck(util::cuda::memPrefetchAsync(mPtr, mSize, cudaCpuDeviceId, stream));
+        if (sync) cudaCheck(cudaStreamSynchronize(stream));
+    }
+
+    /// @brief Legacy
+    /// @param stream
+    /// @param sync
+    void deviceDownload(void* stream, bool sync) const{this->deviceDownload(cudaStream_t(stream), sync);}
+
+    // used by GridHandle
+    void deviceDownload(int dummmy, void* stream, bool sync) const{this->deviceDownload(cudaStream_t(stream), sync);}
+
+    ///////////////////////////////////////////////////////////////////////
+
+    /// @brief Returns a raw pointer to the unified memory managed by this instance.
+    /// @warning Note that the pointer can be NULL!
+    void* data() const {return mPtr;}
+
+    /// @brief Returns an offset pointer of a specific type from the allocated unified memory
+    /// @tparam T Type of the pointer returned
+    /// @param count Numbers of elements of @c parameter type T to skip (or offset) the return pointer
+    /// @warning assumes that this instance is not empty!
+    template <typename T>
+    T* data(ptrdiff_t count = 0) const {
+        NANOVDB_ASSERT(mPtr != nullptr || count == 0);
+        return reinterpret_cast<T*>(mPtr) + count;
+    }
+
+    /// @brief Returns a byte offset void pointer from the unified memory
+    /// @param byteOffset Number of bytes to skip (or offset) the return pointer
+    /// @warning assumes that this instance is not empty!
+    void* data(ptrdiff_t byteOffset) const {
+        NANOVDB_ASSERT(mPtr != nullptr || byteOffset == 0);
+        return util::PtrAdd(mPtr, byteOffset);
+    }
+
+    /// @brief Legacy
+    /// @return
+    void* deviceData()    const {return mPtr;}
+    void* deviceData(int) const {return mPtr;}
+
+    /// @brief Size of the allocated pages in this instance
+    /// @return number bytes allocated by this instance
+    size_t size() const {return mSize;}
+
+    /// @brief Capacity of this instance, i.e. room in page table
+    /// @return number of bytes reserved, but not necessarily allocated, by this instance
+    size_t capacity() const {return mCapacity;}
+
+    //@{
+    /// @brief Returns true if this allocator is empty, i.e. has no allocated memory
+    inline bool empty() const { return mPtr == nullptr; }
+    inline bool isEmpty() const { return this->empty(); }
+    //@}
+
+};// UnifiedBuffer
+
+}// namespace cuda
+
+template<>
+struct BufferTraits<cuda::UnifiedBuffer>
+{
+    static constexpr bool hasDeviceDual = true;
+};
+
+}// namespace nanovdb
+
+#endif // end of NANOVDB_CUDA_UNIFIEDBUFFER_H_HAS_BEEN_INCLUDED
@@ -1,3 +1,5 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
 # find_package(doxygen REQUIRED dot  )
 if(WIN32)
  set(DOXYGEN_EXECUTABLE "C:/Program Files/doxygen/bin/doxygen.exe")
@@ -7,7 +7,7 @@

 #]=======================================================================]

-cmake_minimum_required(VERSION 3.18)
+cmake_minimum_required(VERSION 3.24)
 project(NanoVDBExamples LANGUAGES CXX)

 include(GNUInstallDirs)
@@ -100,14 +100,24 @@ nanovdb_example(NAME "ex_read_nanovdb_sphere")
 nanovdb_example(NAME "ex_read_nanovdb_sphere_accessor")
 nanovdb_example(NAME "ex_read_nanovdb_sphere_accessor_cuda")
 nanovdb_example(NAME "ex_index_grid_cuda")
-nanovdb_example(NAME "ex_nodemanager_cuda")
+nanovdb_example(NAME "ex_nodemanager_cuda" OPENVDB)
 nanovdb_example(NAME "ex_voxels_to_grid_cuda")
+nanovdb_example(NAME "ex_voxels_to_grid_mgpu")
 nanovdb_example(NAME "ex_modify_nanovdb_thrust")
 nanovdb_example(NAME "ex_map_pool_buffer")
 nanovdb_example(NAME "ex_bump_pool_buffer")
 nanovdb_example(NAME "ex_collide_level_set")
 nanovdb_example(NAME "ex_raytrace_fog_volume")
 nanovdb_example(NAME "ex_raytrace_level_set")
+nanovdb_example(NAME "ex_dilate_nanovdb_cuda" OPENVDB)
+nanovdb_example(NAME "ex_merge_nanovdb_cuda" OPENVDB)
+nanovdb_example(NAME "ex_refine_nanovdb_cuda" OPENVDB)
+nanovdb_example(NAME "ex_coarsen_nanovdb_cuda" OPENVDB)
+
+if(CUDAToolkit_FOUND)
+  nanovdb_example(NAME "ex_make_mgpu_nanovdb") # requires cuRAND
+  target_link_libraries(ex_make_mgpu_nanovdb PRIVATE CUDA::curand)
+endif()

 if(NANOVDB_USE_MAGICAVOXEL)
  nanovdb_example(NAME "ex_vox_to_nanovdb")
@@ -0,0 +1,150 @@
+// Copyright Contributors to the OpenVDB Project
+// SPDX-License-Identifier: Apache-2.0
+
+// the following files are from OpenVDB
+#include <openvdb/tools/Composite.h>
+#include <openvdb/util/CpuTimer.h>
+
+// the following files are from NanoVDB
+#include <nanovdb/NanoVDB.h>
+#include <nanovdb/cuda/DeviceBuffer.h>
+#include <nanovdb/tools/CreateNanoGrid.h>
+
+template<class CoordT>
+inline CoordT
+coarsenCoord(const CoordT& coord)
+{
+    auto coarsenComponent = [](const typename CoordT::ValueType n) {return (n>=0) ? (n>>1) : -((-n+1)>>1);};
+    CoordT result;
+    result[0] = coarsenComponent(coord[0]);
+    result[1] = coarsenComponent(coord[1]);
+    result[2] = coarsenComponent(coord[2]);
+    return result;
+}
+
+template<typename BuildT>
+void mainCoarsenGrid(
+    nanovdb::NanoGrid<BuildT> *deviceGridOriginal,
+    nanovdb::NanoGrid<BuildT> *deviceGridCoarsened,
+    nanovdb::NanoGrid<BuildT> *indexGridOriginal,
+    nanovdb::NanoGrid<BuildT> *indexGridCoarsened,
+    uint32_t benchmark_iters
+);
+
+/// @brief This example depends on OpenVDB, NanoVDB, and CUDA
+int main(int argc, char *argv[])
+{
+    using GridT = openvdb::FloatGrid;
+    using BuildT = nanovdb::ValueOnIndex;
+
+    openvdb::util::CpuTimer cpuTimer;
+    const bool printGridDiagnostics = true;
+
+    try {
+        if (argc<2) OPENVDB_THROW(openvdb::ValueError, "usage: "+std::string(argv[0])+" input.vdb [<iterations>]\n");
+        int benchmark_iters = 10;
+        if (argc > 2) sscanf(argv[2], "%d", &benchmark_iters);
+
+        // Read the initial level set from file
+
+        cpuTimer.start("Read input VDB file");
+        openvdb::initialize();
+        openvdb::io::File inFile(argv[1]);
+        inFile.open(false); // disable delayed loading
+        auto baseGrids = inFile.getGrids();
+        inFile.close();
+        auto grid = openvdb::gridPtrCast<GridT>(baseGrids->at(0));
+        if (!grid) OPENVDB_THROW(openvdb::ValueError, "First grid is not a FloatGrid\n");
+        cpuTimer.stop();
+
+        // Convert to indexGrid (original, un-coarsened)
+        cpuTimer.start("Converting openVDB input to indexGrid (original version)");
+        auto handleOriginal = nanovdb::tools::openToIndexVDB<BuildT, nanovdb::cuda::DeviceBuffer>(
+            grid,
+            0u,    // Don't copy data channel
+            false, // No stats
+            false, // No tiles
+            1      // Verbose mode
+        );
+        auto *indexGridOriginal = handleOriginal.grid<BuildT>();
+        cpuTimer.stop();
+
+        if (printGridDiagnostics) {
+            std::cout << "============ Original Grid ===========" << std::endl;
+            std::cout << "Allocated values [valueCount()]       : " << indexGridOriginal->valueCount() << std::endl;
+            std::cout << "Active voxels    [activeVoxelCount()] : " << indexGridOriginal->activeVoxelCount() << std::endl;
+            auto minCorner = indexGridOriginal->indexBBox().min(), maxCorner = indexGridOriginal->indexBBox().max();
+            std::cout << "Index-space bounding box              : [" << minCorner.x() << "," << minCorner.y() << "," << minCorner.z()
+                      << "] -> [" << maxCorner.x() << "," << maxCorner.y() << "," << maxCorner.z() << "]" << std::endl;
+            std::cout << "Leaf nodes                            : " << indexGridOriginal->tree().nodeCount(0) << std::endl;
+            std::cout << "Lower internal nodes                  : " << indexGridOriginal->tree().nodeCount(1) << std::endl;
+            std::cout << "Upper internal nodes                  : " << indexGridOriginal->tree().nodeCount(2) << std::endl;
+            std::cout << "Leaf-level occupancy                  : "
+                      << 100.f * (float)(indexGridOriginal->activeVoxelCount())/(float)(indexGridOriginal->tree().nodeCount(0) * 512)
+                      << "%" << std::endl;
+            std::cout << "Memory usage                          : " << indexGridOriginal->gridSize() << " bytes" << std::endl;
+        }
+        // Coarsening (CPU/OpenVDB version)
+        cpuTimer.start("Coarsening OpenVDB (on CPU)");
+        using TreeT = GridT::TreeType;
+        using LeafManagerT = openvdb::tree::LeafManager<const TreeT>;
+        LeafManagerT leafMgr(grid->tree());
+        auto coarsenedGrid = openvdb::FloatGrid::create(grid->background());
+        coarsenedGrid->setTransform(grid->transform().copy());
+        coarsenedGrid->setName(grid->getName());
+        auto dstAcc = coarsenedGrid->getAccessor();
+        for (std::size_t leafID = 0; leafID < leafMgr.leafCount(); ++leafID) {
+            const auto& srcLeaf = leafMgr.leaf(leafID);
+            for (auto iter = srcLeaf.cbeginValueOn(); iter; ++iter) {
+                const auto dstCoord = coarsenCoord(iter.getCoord());
+                if (!dstAcc.isValueOn(dstCoord))
+                    dstAcc.setValue(dstCoord, iter.getValue());
+            }
+        }
+        cpuTimer.stop();
+
+        // Convert to indexGrid (coarsened)
+        cpuTimer.start("Converting openVDB input to indexGrid (coarsened version)");
+        auto handleCoarsened = nanovdb::tools::openToIndexVDB<BuildT, nanovdb::cuda::DeviceBuffer>(
+            coarsenedGrid,
+            0u,    // Don't copy data channel
+            false, // No stats
+            false, // No tiles
+            1      // Verbose mode
+        );
+        cpuTimer.stop();
+
+        auto *indexGridCoarsened = handleCoarsened.grid<BuildT>();
+
+        if (printGridDiagnostics) {
+            std::cout << "=========== Coarsened Grid ===========" << std::endl;
+            std::cout << "Allocated values [valueCount()]       : " << indexGridCoarsened->valueCount() << std::endl;
+            std::cout << "Active voxels    [activeVoxelCount()] : " << indexGridCoarsened->activeVoxelCount() << std::endl;
+            auto minCorner = indexGridCoarsened->indexBBox().min(), maxCorner = indexGridCoarsened->indexBBox().max();
+            std::cout << "Index-space bounding box              : [" << minCorner.x() << "," << minCorner.y() << "," << minCorner.z()
+                      << "] -> [" << maxCorner.x() << "," << maxCorner.y() << "," << maxCorner.z() << "]" << std::endl;
+            std::cout << "Leaf nodes                            : " << indexGridCoarsened->tree().nodeCount(0) << std::endl;
+            std::cout << "Lower internal nodes                  : " << indexGridCoarsened->tree().nodeCount(1) << std::endl;
+            std::cout << "Upper internal nodes                  : " << indexGridCoarsened->tree().nodeCount(2) << std::endl;
+            std::cout << "Leaf-level occupancy                  : "
+                      << 100.f * (float)(indexGridCoarsened->activeVoxelCount())/(float)(indexGridCoarsened->tree().nodeCount(0) * 512)
+                      << "%" << std::endl;
+            std::cout << "Memory usage                          : " << indexGridCoarsened->gridSize() << " bytes" << std::endl;
+        }
+
+        // Copy both NanoVDB grids to GPU
+        handleOriginal.deviceUpload();
+        handleCoarsened.deviceUpload();
+        auto* deviceGridOriginal = handleOriginal.deviceGrid<BuildT>();
+        auto* deviceGridCoarsened = handleCoarsened.deviceGrid<BuildT>();
+        if (!deviceGridOriginal || !deviceGridCoarsened)
+            OPENVDB_THROW(openvdb::RuntimeError, "Failure while uploading indexGrids to GPU");
+
+        // Launch benchmark
+        mainCoarsenGrid( deviceGridOriginal, deviceGridCoarsened, indexGridOriginal, indexGridCoarsened, benchmark_iters );
+    }
+    catch (const std::exception& e) {
+        std::cerr << "An exception occurred: \"" << e.what() << "\"" << std::endl;
+    }
+    return 0;
+}
@@ -0,0 +1,56 @@
+// Copyright Contributors to the OpenVDB Project
+// SPDX-License-Identifier: Apache-2.0
+
+#include <nanovdb/tools/cuda/CoarsenGrid.cuh>
+
+template<typename T>
+bool bufferCheck(const T* deviceBuffer, const T* hostBuffer, size_t elem_count) {
+    T* tmpBuffer = new T[elem_count];
+    cudaCheck(cudaMemcpy(tmpBuffer, deviceBuffer, elem_count * sizeof(T), cudaMemcpyDeviceToHost));
+    bool same = true;
+    for (int i=0; same && i< elem_count; ++i) { same = (tmpBuffer[i] == hostBuffer[i]); }
+    delete [] tmpBuffer;
+    return same;
+}
+
+template<typename BuildT>
+void mainCoarsenGrid(
+    nanovdb::NanoGrid<BuildT> *deviceGridOriginal,
+    nanovdb::NanoGrid<BuildT> *deviceGridCoarsened,
+    nanovdb::NanoGrid<BuildT> *indexGridOriginal,
+    nanovdb::NanoGrid<BuildT> *indexGridCoarsened,
+    uint32_t benchmark_iters)
+{
+    nanovdb::util::cuda::Timer gpuTimer;
+
+    // Initialize coarsener
+    nanovdb::tools::cuda::CoarsenGrid<BuildT> coarsener( deviceGridOriginal );
+    coarsener.setChecksum(nanovdb::CheckMode::Default);
+    coarsener.setVerbose(1);
+
+    auto handle = coarsener.getHandle();
+    auto dstGrid = handle.template deviceGrid<BuildT>();
+
+    // Check for correctness
+    if (bufferCheck((char*)dstGrid, (char*)indexGridCoarsened->data(), indexGridCoarsened->gridSize()))
+        std::cout << "Result of CoarsenGrid check out CORRECT against reference" << std::endl;
+    else
+        std::cout << "Result of CoarsenGrid compares INCORRECT against reference" << std::endl;
+
+    // Re-run warm-started iterations
+    coarsener.setVerbose(0);
+    for (int i = 0; i < benchmark_iters; i++) {
+        gpuTimer.start("Re-running entire coarsening after warmstart");
+        auto dummyHandle = coarsener.getHandle();
+        gpuTimer.stop();
+    }
+}
+
+template
+void mainCoarsenGrid(
+    nanovdb::NanoGrid<nanovdb::ValueOnIndex> *deviceGridOriginal,
+    nanovdb::NanoGrid<nanovdb::ValueOnIndex> *deviceGridCoarsened,
+    nanovdb::NanoGrid<nanovdb::ValueOnIndex> *indexGridOriginal,
+    nanovdb::NanoGrid<nanovdb::ValueOnIndex> *indexGridCoarsened,
+    uint32_t benchmark_iters
+);
@@ -37,10 +37,8 @@ int main(int ac, char** av)

        const int numPoints = 10000000;

-        BufferT positionBuffer;
-        positionBuffer.init(numPoints * sizeof(float) * 3);
-        BufferT velocityBuffer;
-        velocityBuffer.init(numPoints * sizeof(float) * 3);
+        BufferT positionBuffer(numPoints * sizeof(float) * 3);
+        BufferT velocityBuffer(numPoints * sizeof(float) * 3);

        runNanoVDB(handle, numIterations, numPoints, positionBuffer, velocityBuffer);
 #if defined(NANOVDB_USE_OPENVDB)
@@ -0,0 +1,133 @@
+// Copyright Contributors to the OpenVDB Project
+// SPDX-License-Identifier: Apache-2.0
+
+// the following files are from OpenVDB
+#include <openvdb/tools/Morphology.h>
+#include <openvdb/util/CpuTimer.h>
+
+// the following files are from NanoVDB
+#include <nanovdb/NanoVDB.h>
+#include <nanovdb/cuda/DeviceBuffer.h>
+#include <nanovdb/tools/CreateNanoGrid.h>
+
+template<typename BuildT>
+void mainDilateGrid(
+    nanovdb::NanoGrid<BuildT> *deviceGridOriginal,
+    nanovdb::NanoGrid<BuildT> *deviceGridDilated,
+    nanovdb::NanoGrid<BuildT> *indexGridOriginal,
+    nanovdb::NanoGrid<BuildT> *indexGridDilated,
+    uint32_t nnType,
+    uint32_t benchmark_iters
+);
+
+/// @brief This example depends on OpenVDB, NanoVDB, and CUDA
+int main(int argc, char *argv[])
+{
+    using GridT = openvdb::FloatGrid;
+    using BuildT = nanovdb::ValueOnIndex;
+
+    // Select the type of dilation here. The NN_EDGE case supports leaf dilation too (currently)
+    // openvdb::tools::NearestNeighbors nnType = openvdb::tools::NN_FACE_EDGE_VERTEX;
+    openvdb::tools::NearestNeighbors nnType = openvdb::tools::NN_FACE;
+
+    openvdb::util::CpuTimer cpuTimer;
+    const bool printGridDiagnostics = true;
+
+    try {
+
+        if (argc<2) OPENVDB_THROW(openvdb::ValueError, "usage: "+std::string(argv[0])+" input.vdb [<iterations>]\n");
+        int benchmark_iters = 10;
+        if (argc > 2) sscanf(argv[2], "%d", &benchmark_iters);
+
+        // Read the initial level set from file
+
+        cpuTimer.start("Read input VDB file");
+        openvdb::initialize();
+        openvdb::io::File inFile(argv[1]);
+        inFile.open(false); // disable delayed loading
+        auto baseGrids = inFile.getGrids();
+        inFile.close();
+        auto grid = openvdb::gridPtrCast<GridT>(baseGrids->at(0));
+        openvdb::FloatGrid* ptr = grid.get(); // raw pointer
+        if (!grid) OPENVDB_THROW(openvdb::ValueError, "First grid is not a FloatGrid\n");
+        cpuTimer.stop();
+
+        // Convert to indexGrid (original, un-dilated)
+        cpuTimer.start("Converting openVDB input to indexGrid (original version)");
+        auto handleOriginal = nanovdb::tools::openToIndexVDB<BuildT, nanovdb::cuda::DeviceBuffer>(
+            grid,
+            0u,    // Don't copy data channel
+            false, // No stats
+            false, // No tiles
+            1      // Verbose mode
+        );
+        auto *indexGridOriginal = handleOriginal.grid<BuildT>();
+        cpuTimer.stop();
+
+        if (printGridDiagnostics) {
+            std::cout << "============ Original Grid ===========" << std::endl;
+            std::cout << "Allocated values [valueCount()]       : " << indexGridOriginal->valueCount() << std::endl;
+            std::cout << "Active voxels    [activeVoxelCount()] : " << indexGridOriginal->activeVoxelCount() << std::endl;
+            auto minCorner = indexGridOriginal->indexBBox().min(), maxCorner = indexGridOriginal->indexBBox().max();
+            std::cout << "Index-space bounding box              : [" << minCorner.x() << "," << minCorner.y() << "," << minCorner.z()
+                      << "] -> [" << maxCorner.x() << "," << maxCorner.y() << "," << maxCorner.z() << "]" << std::endl;
+            std::cout << "Leaf nodes                            : " << indexGridOriginal->tree().nodeCount(0) << std::endl;
+            std::cout << "Lower internal nodes                  : " << indexGridOriginal->tree().nodeCount(1) << std::endl;
+            std::cout << "Upper internal nodes                  : " << indexGridOriginal->tree().nodeCount(2) << std::endl;
+            std::cout << "Leaf-level occupancy                  : "
+                      << 100.f * (float)(indexGridOriginal->activeVoxelCount())/(float)(indexGridOriginal->tree().nodeCount(0) * 512)
+                      << "%" << std::endl;
+            std::cout << "Memory usage                          : " << indexGridOriginal->gridSize() << " bytes" << std::endl;
+        }
+
+        // Dilation (CPU/OpenVDB version)
+        cpuTimer.start("Dilating openVDB (on CPU)");
+        openvdb::tools::dilateActiveValues(grid->tree(), 1, nnType);
+        cpuTimer.stop();
+
+        // Convert to indexGrid (dilated)
+        cpuTimer.start("Converting openVDB input to indexGrid (dilated version)");
+        auto handleDilated = nanovdb::tools::openToIndexVDB<BuildT, nanovdb::cuda::DeviceBuffer>(
+            grid,
+            0u,    // Don't copy data channel
+            false, // No stats
+            false, // No tiles
+            1      // Verbose mode
+        );
+        cpuTimer.stop();
+
+        auto *indexGridDilated = handleDilated.grid<BuildT>();
+
+        if (printGridDiagnostics) {
+            std::cout << "============ Dilated Grid ============" << std::endl;
+            std::cout << "Allocated values [valueCount()]       : " << indexGridDilated->valueCount() << std::endl;
+            std::cout << "Active voxels    [activeVoxelCount()] : " << indexGridDilated->activeVoxelCount() << std::endl;
+            auto minCorner = indexGridDilated->indexBBox().min(), maxCorner = indexGridDilated->indexBBox().max();
+            std::cout << "Index-space bounding box              : [" << minCorner.x() << "," << minCorner.y() << "," << minCorner.z()
+                      << "] -> [" << maxCorner.x() << "," << maxCorner.y() << "," << maxCorner.z() << "]" << std::endl;
+            std::cout << "Leaf nodes                            : " << indexGridDilated->tree().nodeCount(0) << std::endl;
+            std::cout << "Lower internal nodes                  : " << indexGridDilated->tree().nodeCount(1) << std::endl;
+            std::cout << "Upper internal nodes                  : " << indexGridDilated->tree().nodeCount(2) << std::endl;
+            std::cout << "Leaf-level occupancy                  : "
+                      << 100.f * (float)(indexGridDilated->activeVoxelCount())/(float)(indexGridDilated->tree().nodeCount(0) * 512)
+                      << "%" << std::endl;
+            std::cout << "Memory usage                          : " << indexGridDilated->gridSize() << " bytes" << std::endl;
+        }
+
+        // Copy both NanoVDB grids to GPU
+        handleOriginal.deviceUpload();
+        handleDilated.deviceUpload();
+        auto* deviceGridOriginal = handleOriginal.deviceGrid<BuildT>();
+        auto* deviceGridDilated = handleDilated.deviceGrid<BuildT>();
+        if (!deviceGridOriginal || !deviceGridDilated)
+            OPENVDB_THROW(openvdb::RuntimeError, "Failure while uploading indexGrids to GPU");
+
+        // Launch benchmark
+        mainDilateGrid( deviceGridOriginal, deviceGridDilated, indexGridOriginal, indexGridDilated, nnType, benchmark_iters );
+
+    }
+    catch (const std::exception& e) {
+        std::cerr << "An exception occurred: \"" << e.what() << "\"" << std::endl;
+    }
+    return 0;
+}
@@ -0,0 +1,102 @@
+// Copyright Contributors to the OpenVDB Project
+// SPDX-License-Identifier: Apache-2.0
+
+#include <nanovdb/tools/cuda/DilateGrid.cuh>
+#include <nanovdb/tools/cuda/PruneGrid.cuh>
+#include <nanovdb/util/cuda/Injection.cuh>
+
+template<typename T>
+bool bufferCheck(const T* deviceBuffer, const T* hostBuffer, size_t elem_count) {
+    T* tmpBuffer = new T[elem_count];
+    cudaCheck(cudaMemcpy(tmpBuffer, deviceBuffer, elem_count * sizeof(T), cudaMemcpyDeviceToHost));
+    bool same = true;
+    for (int i=0; same && i< elem_count; ++i) { same = (tmpBuffer[i] == hostBuffer[i]); }
+    delete [] tmpBuffer;
+    return same;
+}
+
+template<typename BuildT>
+void mainDilateGrid(
+    nanovdb::NanoGrid<BuildT> *deviceGridOriginal,
+    nanovdb::NanoGrid<BuildT> *deviceGridDilated,
+    nanovdb::NanoGrid<BuildT> *indexGridOriginal,
+    nanovdb::NanoGrid<BuildT> *indexGridDilated,
+    uint32_t nnType,
+    uint32_t benchmark_iters)
+{
+    nanovdb::util::cuda::Timer gpuTimer;
+
+    // Initialize dilator
+    nanovdb::tools::cuda::DilateGrid<BuildT> dilator( deviceGridOriginal );
+    dilator.setOperation(nanovdb::tools::morphology::NearestNeighbors(nnType));
+    dilator.setChecksum(nanovdb::CheckMode::Default);
+    dilator.setVerbose(1);
+
+    auto handle = dilator.getHandle();
+    auto dstGrid = handle.template deviceGrid<BuildT>();
+
+    // Check for correctness
+    if (bufferCheck((char*)dstGrid, (char*)indexGridDilated->data(), indexGridDilated->gridSize()))
+        std::cout << "Result of DilateGrid check out CORRECT against reference" << std::endl;
+    else
+        std::cout << "Result of DilateGrid compares INCORRECT against reference" << std::endl;
+
+    // Re-run warm-started iterations
+    dilator.setVerbose(0);
+    for (int i = 0; i < benchmark_iters; i++) {
+        gpuTimer.start("Re-running entire dilation after warmstart");
+        auto dummyHandle = dilator.getHandle();
+        gpuTimer.stop();
+    }
+
+    uint32_t dstLeafCount = nanovdb::util::cuda::DeviceGridTraits<BuildT>::getTreeData(dstGrid).mNodeCount[0];
+    nanovdb::cuda::DeviceBuffer dstLeafMaskBuffer;
+    nanovdb::Mask<3>* dstLeafMasks = nullptr;
+    if (dstLeafCount) {
+        dstLeafMaskBuffer = nanovdb::cuda::DeviceBuffer::create( std::size_t(dstLeafCount) * sizeof(nanovdb::Mask<3>), nullptr, false );
+        dstLeafMasks = static_cast<nanovdb::Mask<3>*>(dstLeafMaskBuffer.deviceData());
+        if (!dstLeafMasks) throw std::runtime_error("No GPU buffer for dstLeafMask");
+    }
+
+    const unsigned int numThreads = 128;
+    auto numBlocks = [numThreads] (unsigned int n) {return (n + numThreads - 1) / numThreads;};
+    gpuTimer.start("Injecting un-dilated topology as a pruning mask");
+    if (dstLeafCount)
+        nanovdb::util::cuda::lambdaKernel<<<numBlocks(dstLeafCount), numThreads>>>(dstLeafCount,
+            nanovdb::util::cuda::InjectGridMaskFunctor<BuildT>(),
+            deviceGridOriginal, dstGrid, dstLeafMasks );
+    gpuTimer.stop();
+
+    // Initialize pruner
+    nanovdb::tools::cuda::PruneGrid<BuildT> pruner( dstGrid, dstLeafMasks );
+    pruner.setChecksum(nanovdb::CheckMode::Default);
+    pruner.setVerbose(1);
+
+    auto prunedHandle = pruner.getHandle();
+    auto prunedGrid = prunedHandle.template deviceGrid<BuildT>();
+
+    // Check for correctness
+    if (bufferCheck((char*)prunedGrid, (char*)indexGridOriginal->data(), indexGridOriginal->gridSize()))
+        std::cout << "Result of PruneGrid check out CORRECT against reference" << std::endl;
+    else
+        std::cout << "Result of PruneGrid compares INCORRECT against reference" << std::endl;
+
+    // Re-run warm-started iterations
+    pruner.setVerbose(0);
+    for (int i = 0; i < benchmark_iters; i++) {
+        gpuTimer.start("Re-running entire pruning after warmstart");
+        auto dummyHandle = pruner.getHandle();
+        gpuTimer.stop();
+    }
+
+}
+
+template
+void mainDilateGrid(
+    nanovdb::NanoGrid<nanovdb::ValueOnIndex> *deviceGridOriginal,
+    nanovdb::NanoGrid<nanovdb::ValueOnIndex> *deviceGridDilated,
+    nanovdb::NanoGrid<nanovdb::ValueOnIndex> *indexGridOriginal,
+    nanovdb::NanoGrid<nanovdb::ValueOnIndex> *indexGridDilated,
+    uint32_t nnType,
+    uint32_t benchmark_iters
+);
@@ -3,16 +3,17 @@

 #include <nanovdb/NanoVDB.h> // this defined the core tree data structure of NanoVDB accessable on both the host and device
 #include <nanovdb/cuda/GridHandle.cuh>// required since GridHandle<DeviceBuffer> has device code
-#include <stdio.h> // for printf
+#include <cstdio> // for printf
+#include <cinttypes>

 // This is called by the host only
 void cpu_kernel(const nanovdb::NanoGrid<nanovdb::ValueOnIndex>* cpuGrid)
 {
    nanovdb::ChannelAccessor<float, nanovdb::ValueOnIndex> acc(*cpuGrid);
    //printf("\nNanoVDB CPU: channels=%u values=%lu\n", acc.grid().blindDataCount(), acc.root().maximum());
-    printf("NanoVDB CPU; %lu\n", acc.idx(  0, 0, 0));
-    printf("NanoVDB CPU; %lu\n", acc.idx( 99, 0, 0));
-    printf("NanoVDB CPU; %lu\n", acc.idx(100, 0, 0));
+    printf("NanoVDB CPU; %" PRIu64 "\n", acc.idx(  0, 0, 0));
+    printf("NanoVDB CPU; %" PRIu64 "\n", acc.idx( 99, 0, 0));
+    printf("NanoVDB CPU; %" PRIu64 "\n", acc.idx(100, 0, 0));
    printf("NanoVDB CPU; %4.2f\n",   acc(  0, 0, 0));
    printf("NanoVDB CPU; %4.2f\n",   acc( 99, 0, 0));
    printf("NanoVDB CPU; %4.2f\n",   acc(100, 0, 0));
@@ -23,9 +24,9 @@ __global__ void gpu_kernel(const nanovdb::NanoGrid<nanovdb::ValueOnIndex>* gpuGr
 {
    nanovdb::ChannelAccessor<float, nanovdb::ValueOnIndex> acc(*gpuGrid);
    //printf("\nNanoVDB GPU: channels=%u values=%lu\n", gpuGrid->blindDataCount(), acc.root().maximum());
-    printf("NanoVDB GPU; %lu\n", acc.idx(  0, 0, 0));
-    printf("NanoVDB GPU; %lu\n", acc.idx( 99, 0, 0));
-    printf("NanoVDB GPU; %lu\n", acc.idx(100, 0, 0));
+    printf("NanoVDB GPU; %" PRIu64 "\n", acc.idx(  0, 0, 0));
+    printf("NanoVDB GPU; %" PRIu64 "\n", acc.idx( 99, 0, 0));
+    printf("NanoVDB GPU; %" PRIu64 "\n", acc.idx(100, 0, 0));
    printf("NanoVDB GPU; %4.2f\n",   acc(  0, 0, 0));
    printf("NanoVDB GPU; %4.2f\n",   acc( 99, 0, 0));
    printf("NanoVDB GPU; %4.2f\n",   acc(100, 0, 0));
@@ -39,4 +40,4 @@ extern "C" void launch_kernels(const nanovdb::NanoGrid<nanovdb::ValueOnIndex>* g
    gpu_kernel<<<1, 1, 0, stream>>>(gpuGrid); // Launch the device kernel asynchronously

    cpu_kernel(cpuGrid); // Launch the host "kernel" (synchronously)
-}
+}
@@ -34,6 +34,7 @@ int main()

        cudaStream_t stream; // Create a CUDA stream to allow for asynchronous copy of pinned CUDA memory.
        cudaStreamCreate(&stream);
+
        handle.deviceUpload(stream, false); // Copy the NanoVDB grid to the GPU asynchronously
        auto* gpuGrid = handle.deviceGrid<float>(); // get a (raw) pointer to a NanoVDB grid of value type float on the GPU

@@ -0,0 +1,302 @@
+// Copyright Contributors to the OpenVDB Project
+// SPDX-License-Identifier: Apache-2.0
+#include <nanovdb/tools/CreatePrimitives.h>
+#include <nanovdb/cuda/UnifiedBuffer.h>
+#include <nanovdb/cuda/DeviceMesh.h>
+#include <nanovdb/util/cuda/Timer.h>
+
+#include <cassert>
+#include <cinttypes>
+#include <cuda.h>
+#include <curand.h>
+#include <mma.h>
+#include <device_launch_parameters.h>
+#include <iostream>
+#include <random>
+#include <thread>
+#include <type_traits>
+#include <map>
+
+namespace {// anonymous namespace
+
+template <typename frag_t>
+inline __device__
+void to_TF32(frag_t& frag) {
+#pragma unroll
+    for (int t = 0; t < frag.num_elements; t++)
+        frag.x[t] =  nvcuda::wmma::__float_to_tf32(frag.x[t]);
+}
+
+__global__
+__launch_bounds__(256)
+void stencilConvolve_v7(nanovdb::NanoGrid<nanovdb::ValueOnIndex> *deviceGrid, int leafOffset,
+    float *inputBuffer, uint32_t *haloIndices, float *haloBuffer,
+    float *stencil, float *outputBuffer, float *denseOutputBuffer)
+{
+    static constexpr int Di = 64;
+    static constexpr int Do = 128;
+    static constexpr int M = 16;
+    static constexpr int N = 16;
+    static constexpr int K = 8;
+
+    using InputBufferType = float (&)[][Di];
+    using HaloBufferType = float (&)[4][4][6][Di];
+    using StencilType = float (&)[3][3][3][Di][Do];
+    using OutputBufferType = float (&)[][Do];
+    using DenseOutputBufferType = float (&)[2][2][4][Do];
+    using SpokeStencilType = float (&)[2][2][4][Di];
+
+    InputBufferType mInputBuffer = reinterpret_cast<InputBufferType>(*inputBuffer);
+    OutputBufferType mOutputBuffer = reinterpret_cast<OutputBufferType>(*outputBuffer);
+    StencilType mStencil = reinterpret_cast<StencilType>(*stencil);
+
+    const int Bk = (blockIdx.x & 1) * 4;
+    const int Bj = ((blockIdx.x >> 1) & 3) * 2;
+    const int Bi = ((blockIdx.x >> 3) & 3) * 2;
+    const int leafId = (blockIdx.x >> 5) + leafOffset;
+    const int tid = threadIdx.x;
+
+    using LeafNodeType = nanovdb::NanoGrid<nanovdb::ValueOnIndex>::TreeType::LeafNodeType;
+    const auto& tree = deviceGrid->tree();
+    auto acc = tree.getAccessor();
+    const LeafNodeType& leaf = tree.template getFirstNode<LeafNodeType>()[leafId];
+    const nanovdb::Coord leafOrigin = leaf.origin();
+    const auto& valueMask = leaf.valueMask();
+
+    uint64_t activeMask = valueMask.words()[Bi] | valueMask.words()[Bi + 1];
+    activeMask &= (0xffffUL << (Bj << 3));
+    activeMask &= (0xf0f0f0f0f0f0f0fUL << Bk);
+    if (!activeMask) return;
+
+    int II = (tid >> 6) & 0x3;
+    int E  =  tid       & 0x3f;
+
+    __shared__ float sBufferRaw[6144]; // 4x4x6 array of elements of size Di=64
+    HaloBufferType sHaloBuffer = reinterpret_cast<HaloBufferType>(sBufferRaw[0]);
+    __shared__ float sSpokeStencil[2][2][4][Di];
+    __shared__ float sOutputBuffer[2][2][4][Do];
+
+    auto origin = leafOrigin.offsetBy(Bi, Bj, Bk);
+
+    // -----------------------------
+    for (int jj = 0; jj < 4; jj++)
+        for (int kk = 0; kk < 6; kk++) {
+            const auto& offset = acc.getValue(origin.offsetBy(II - 1, jj - 1, kk - 1));
+            sHaloBuffer[II][jj][kk][E] = mInputBuffer[offset][E];
+        }
+    __syncthreads();
+
+// -----------------------------
+
+    using a_frag_t = nvcuda::wmma::fragment<nvcuda::wmma::matrix_a, M, N, K, nvcuda::wmma::precision::tf32, nvcuda::wmma::row_major>;
+    nvcuda::wmma::fragment<nvcuda::wmma::matrix_b, M, N, K, nvcuda::wmma::precision::tf32, nvcuda::wmma::row_major> b_frag;
+    nvcuda::wmma::fragment<nvcuda::wmma::accumulator, M, N, K, float> c_frag;
+    __shared__ float sFragBuffer[8][32][4];
+    const int warpID = tid >> 5;
+    const int laneID = tid & 0x1f;
+    nvcuda::wmma::fill_fragment(c_frag, 0.f);
+
+    for (int di = 0; di <= 2; di++)
+    for (int dj = 0; dj <= 2; dj++)
+    for (int dk = 0; dk <= 2; dk++) {
+        // Create a contiguous copy of the 2x2x4 Spoke-Stencil
+        // No sychthreads here, because we take care of that at the end of the loop
+        for (int b = 0; b < 2 * 2 * 4 * Di; b += blockDim.x) {
+            auto eid = tid + b;
+            int ii = ( eid >> 9 ) & 0x1;  // 1-wide in X
+            int jj = ( eid >> 8 ) & 0x1;  // 2-wide in Y
+            int kk = ( eid >> 6 ) & 0x3;  // 4-wide in Z
+            int ee = ( eid >> 0 ) & 0x3f; // 64 entries
+            sSpokeStencil[ii][jj][kk][ee] = sHaloBuffer[ii + di][jj + dj][kk + dk][ee];
+        }
+        __syncthreads();
+
+        {
+            int inBase = warpID << 3;
+
+            a_frag_t& a_frag = *reinterpret_cast<a_frag_t*>(sFragBuffer[warpID][laneID]);
+            nvcuda::wmma::load_matrix_sync(a_frag, &sSpokeStencil[0][0][0][inBase], 64);
+            to_TF32(a_frag);
+
+            __syncthreads();
+        }
+
+#pragma unroll
+        for (int sweep = 0; sweep < 8; ++sweep) {
+            const int inBlock = sweep;
+            const int outBlock = warpID;
+            const int inBase = inBlock << 3;
+            const int outBase = outBlock << 4;
+
+            a_frag_t& a_frag = *reinterpret_cast<a_frag_t*>(sFragBuffer[inBlock][laneID]);
+
+            nvcuda::wmma::load_matrix_sync(b_frag, &mStencil[di][dj][dk][inBase][outBase], 128);
+            to_TF32(b_frag);
+            nvcuda::wmma::mma_sync(c_frag, a_frag, b_frag, c_frag);
+
+        }
+    }
+    nvcuda::wmma::store_matrix_sync(&sOutputBuffer[0][0][0][warpID << 4], c_frag, 128, nvcuda::wmma::mem_row_major);
+
+    // Sparse commit phase
+    __syncthreads();
+    const int warpI     = (tid >> 8) & 0x1;
+    const int warpJ     = (tid >> 7) & 0x1;
+    const int warpK     = (tid >> 5) & 0x3;
+    const int elementID = laneID | ((tid & 0x200) >> 4);
+
+    // NB: this if fixed for 256 threads/block
+    const int elementsPerSM = 32;
+    const int xSpanPerSM = 1;
+    const int ySpanPerSM = 2;
+
+#pragma unroll
+    for (int xOffset = 0; xOffset < 2; xOffset += xSpanPerSM)
+    for (int yOffset = 0; yOffset < 2; yOffset += ySpanPerSM) {
+        const auto coord = origin.offsetBy(warpI + xOffset, warpJ + yOffset, warpK);
+        const auto& offset = acc.getValue(coord);
+#pragma unroll
+        for (int elementOffset = 0; elementOffset < Do; elementOffset += elementsPerSM)
+            mOutputBuffer[offset][elementID + elementOffset] = sOutputBuffer[warpI + xOffset][warpJ + yOffset][warpK][elementID + elementOffset];
+    }
+}// stencilConvolve_v7
+
+}// anonymous namespace
+
+void testConvolution()
+{
+    static constexpr int Di = 64, Do = 128;
+    using InputBufferType  = std::array<float, Di>;
+    using OutputBufferType = std::array<float, Do>;
+    using StencilType = float (&)[3][3][3][Di][Do];
+
+    float* stencilHostPtr = new float[Do * 27 * Di];
+
+    std::mt19937 gen(42u);
+    std::uniform_real_distribution<float> uniform_dist(-1., 1.);
+
+    for (int i = 0; i < Do * 27 * Di; i++) stencilHostPtr[i] = uniform_dist(gen);
+
+    nanovdb::cuda::DeviceMesh deviceMesh;
+    const size_t deviceCount = deviceMesh.deviceCount();
+    std::cout << "Number of devices that supports unified memory: " << deviceCount << std::endl;
+
+    // Calculate minimum page size which corresponds to minimum physical allocation granularity
+    size_t minGranularity = nanovdb::cuda::minDevicePageSize(deviceMesh);
+
+    // Ensure that we don't split an input and/or output feature across page boundaries
+    minGranularity = std::lcm(std::lcm(minGranularity, sizeof(InputBufferType)), sizeof(OutputBufferType));
+    size_t valueCountGranularity = minGranularity / min(sizeof(InputBufferType), sizeof(OutputBufferType));
+
+    // Initialize and replicate an IndexGrid on each device
+    auto floatHandle = nanovdb::tools::createLevelSetSphere<float>(100, nanovdb::Vec3d(0), 1, 3, nanovdb::Vec3d(0), "test");
+    nanovdb::FloatGrid* floatGrid = floatHandle.grid<float>();
+
+    using BufferT = nanovdb::cuda::DeviceBuffer;
+    auto indexHandle = nanovdb::tools::createNanoGrid<nanovdb::FloatGrid, nanovdb::ValueOnIndex, BufferT>(*floatGrid, 0u, false, false, 1);
+    std::for_each(deviceMesh.begin(), deviceMesh.end(), [&](const nanovdb::cuda::DeviceNode& node) {// copy host buffer to all the device buffers
+        cudaCheck(cudaSetDevice(node.id));
+        indexHandle.deviceUpload(node.id, node.stream, true);
+    });
+    auto* indexGrid = indexHandle.grid<nanovdb::ValueOnIndex>();
+
+    auto ceil = [](size_t x, size_t y)->size_t{return ((x + y - 1) / y) * y;};
+    const size_t valueCount = ceil(indexGrid->valueCount(), deviceCount * valueCountGranularity);
+
+    const size_t inputAllocationSize  = valueCount * sizeof(InputBufferType);
+    const size_t outputAllocationSize = valueCount * sizeof(OutputBufferType);
+
+    nanovdb::cuda::UnifiedBuffer inputBuffer(inputAllocationSize, 2*inputAllocationSize);// over-allocate
+    nanovdb::cuda::UnifiedBuffer outputBuffer(outputAllocationSize);
+
+    const size_t deviceValueCount = valueCount / deviceCount;
+
+    // Randomly initialize input features
+    std::vector<std::thread> threads;
+    std::for_each(deviceMesh.begin(), deviceMesh.end(), [&](const nanovdb::cuda::DeviceNode& node) {
+        threads.emplace_back([&](int device, cudaStream_t stream) {
+            cudaCheck(cudaSetDevice(device));
+            float* inputStripePtr  =  inputBuffer.data<float>(deviceValueCount * Di * device);
+            float* outputStripePtr = outputBuffer.data<float>(deviceValueCount * Do * device);
+
+            unsigned long long seed = 42u;
+            curandGenerator_t rng;
+            curandStatus_t curandStatus = curandCreateGenerator(&rng, CURAND_RNG_PSEUDO_DEFAULT);
+            curandStatus = curandSetStream(rng, stream);
+            curandStatus = curandSetPseudoRandomGeneratorSeed(rng, seed);
+            curandStatus = curandGenerateUniform(rng, inputStripePtr,  deviceValueCount * Di);
+            curandStatus = curandGenerateUniform(rng, outputStripePtr, deviceValueCount * Do);
+            curandStatus = curandDestroyGenerator(rng);
+            NANOVDB_ASSERT(curandStatus == CURAND_STATUS_SUCCESS);
+
+            cudaCheck(cudaStreamSynchronize(stream));
+
+            // If we use managed memory, we need to advise about the usage of the memory range in order to obtain an
+            // equivalently optimal paging strategy. cudaMemAdviseSetReadMostly instructs the "paging policy" that data
+            // is far more likely to be read than written, cudaMemAdviseSetPreferredLocation suggests the preferred device to place the data on, and cudaMemAdviseSetAccessedBy is a hint about the which devices are accessing the data.
+            const size_t inPageSize  = deviceValueCount * Di * sizeof(float), inPageOffset  =  inPageSize * device;// in bytes
+            const size_t outPageSize = deviceValueCount * Do * sizeof(float), outPageOffset = outPageSize * device;// in bytes
+            inputBuffer.advise(inPageOffset, inPageSize, device, {cudaMemAdviseSetReadMostly, cudaMemAdviseSetPreferredLocation});
+            inputBuffer.prefetch(inPageOffset, inPageSize, device, stream);
+            outputBuffer.advise(outPageOffset, outPageSize, device, cudaMemAdviseSetPreferredLocation);
+            std::for_each(deviceMesh.begin(), deviceMesh.end(), [&](const nanovdb::cuda::DeviceNode& otherNode) {
+                inputBuffer.advise(  inPageOffset,  inPageSize, otherNode.id, cudaMemAdviseSetAccessedBy);
+                outputBuffer.advise(outPageOffset, outPageSize, otherNode.id, cudaMemAdviseSetAccessedBy);
+            });
+        }, node.id, node.stream);
+    });
+    std::for_each(threads.begin(), threads.end(), [](std::thread& t) { t.join(); });
+    threads.clear();
+
+    // Spawn a convolution kernel on each device that operates on a disjoint subset of the leaves. The leaves operated on by
+    // each device corresponds approximately to the features in the virtual address range that are physically allocated on
+    // the same device.
+    auto **timers = new nanovdb::util::cuda::Timer*[deviceCount];
+    const size_t leafNodeCount = indexGrid->tree().nodeCount(0);
+    std::for_each(deviceMesh.begin(), deviceMesh.end(), [&](const nanovdb::cuda::DeviceNode& node) {
+        threads.emplace_back([&](int device, cudaStream_t stream) {
+            cudaCheck(cudaSetDevice(device));
+            timers[device] = new nanovdb::util::cuda::Timer(stream);
+
+            void* stencilDevicePtr;
+            cudaCheck(nanovdb::util::cuda::mallocAsync(&stencilDevicePtr, sizeof(float) * Do * 27 * Di, stream));
+            cudaCheck(cudaMemcpyAsync(stencilDevicePtr, stencilHostPtr, sizeof(float) * Do * 27 * Di, cudaMemcpyHostToDevice, stream));
+
+            size_t deviceLeafNodeCount = (leafNodeCount + deviceCount - 1) / deviceCount;
+            const size_t deviceLeafNodeOffset = deviceLeafNodeCount * device;
+            deviceLeafNodeCount = std::min(deviceLeafNodeCount, leafNodeCount - deviceLeafNodeOffset);
+            auto deviceIndexGrid =  reinterpret_cast<nanovdb::OnIndexGrid*>(indexHandle.deviceData(device));
+
+            // Run 10 warmup iterations
+            float* stencilPtr = (float*)stencilDevicePtr;
+            dim3 blockDim(256);
+            for (int k = 0; k < 10; ++k) {
+                stencilConvolve_v7<<<deviceLeafNodeCount * 2 * 4 * 4, blockDim, 0, stream>>>(
+                    deviceIndexGrid, deviceLeafNodeOffset, inputBuffer.data<float>(), nullptr, nullptr, stencilPtr, outputBuffer.data<float>(), nullptr);
+            }
+
+            timers[device]->start();
+            stencilConvolve_v7<<<deviceLeafNodeCount * 2 * 4 * 4, blockDim, 0, stream>>>(
+                deviceIndexGrid, deviceLeafNodeOffset, inputBuffer.data<float>(), nullptr, nullptr, stencilPtr, outputBuffer.data<float>(), nullptr);
+            timers[device]->record();
+            cudaCheck(nanovdb::util::cuda::freeAsync(stencilDevicePtr, stream));
+            cudaCheck(cudaStreamSynchronize(stream));
+        }, node.id, node.stream);
+    });
+    std::for_each(threads.begin(), threads.end(), [](std::thread& t) { t.join(); });
+    threads.clear();
+    delete[] stencilHostPtr;
+
+    std::for_each(deviceMesh.begin(), deviceMesh.end(), [&](const nanovdb::cuda::DeviceNode& node) {
+        timers[node.id]->print("Device " + std::to_string(node.id) + " GPU convolution ", std::cout);
+        delete timers[node.id];
+    });
+    delete[] timers;
+    std::cout << "Multi-GPU sparse convolution test complete!" << std::endl;
+}// testConvolution
+
+int main(int argc, char** argv)
+{
+    testConvolution();
+    return 0;
+}
@@ -0,0 +1,166 @@
+// Copyright Contributors to the OpenVDB Project
+// SPDX-License-Identifier: Apache-2.0
+
+// the following files are from OpenVDB
+#include <openvdb/tools/Composite.h>
+#include <openvdb/util/CpuTimer.h>
+
+// the following files are from NanoVDB
+#include <nanovdb/NanoVDB.h>
+#include <nanovdb/cuda/DeviceBuffer.h>
+#include <nanovdb/tools/CreateNanoGrid.h>
+
+template<typename BuildT>
+void mainMergeGrids(
+    nanovdb::NanoGrid<BuildT> *deviceSrcGrid1,
+    nanovdb::NanoGrid<BuildT> *deviceSrcGrid2,
+    nanovdb::NanoGrid<BuildT> *deviceDstReferenceGrid,
+    nanovdb::NanoGrid<BuildT> *hostSrcGrid1,
+    nanovdb::NanoGrid<BuildT> *hostSrcGrid2,
+    nanovdb::NanoGrid<BuildT> *hostDstReferenceGrid,
+    uint32_t benchmark_iters
+);
+
+/// @brief This example depends on OpenVDB, NanoVDB, and CUDA
+int main(int argc, char *argv[])
+{
+    using GridT = openvdb::FloatGrid;
+    using BuildT = nanovdb::ValueOnIndex;
+
+    openvdb::util::CpuTimer cpuTimer;
+    const bool printGridDiagnostics = true;
+
+    try {
+
+        if (argc < 3) OPENVDB_THROW(openvdb::ValueError, "usage: "+std::string(argv[0])+" input1.vdb input2.vdb [<iterations>]\n");
+        int benchmark_iters = 10;
+        if (argc > 3) sscanf(argv[3], "%d", &benchmark_iters);
+
+        // Read the openvdb inputs from files
+
+        cpuTimer.start("Read first input VDB file");
+        openvdb::initialize();
+        openvdb::io::File inFile1(argv[1]);
+        inFile1.open(false); // disable delayed loading
+        auto baseGrids1 = inFile1.getGrids();
+        inFile1.close();
+        auto grid1 = openvdb::gridPtrCast<GridT>(baseGrids1->at(0));
+        if (!grid1) OPENVDB_THROW(openvdb::ValueError, "First grid is not a FloatGrid\n");
+        cpuTimer.stop();
+
+        cpuTimer.start("Read second input VDB file");
+        openvdb::initialize();
+        openvdb::io::File inFile2(argv[2]);
+        inFile2.open(false); // disable delayed loading
+        auto baseGrids2 = inFile2.getGrids();
+        inFile2.close();
+        auto grid2 = openvdb::gridPtrCast<GridT>(baseGrids2->at(0));
+        if (!grid2) OPENVDB_THROW(openvdb::ValueError, "Second grid is not a FloatGrid\n");
+        cpuTimer.stop();
+
+        // Convert to indexGrid
+        cpuTimer.start("Converting openVDB input to indexGrid (first component)");
+        auto srcHandle1 = nanovdb::tools::openToIndexVDB<BuildT, nanovdb::cuda::DeviceBuffer>(
+            grid1,
+            0u,    // Don't copy data channel
+            false, // No stats
+            false, // No tiles
+            1      // Verbose mode
+        );
+        auto *srcIndexGrid1 = srcHandle1.grid<BuildT>();
+        cpuTimer.stop();
+
+        cpuTimer.start("Converting openVDB input to indexGrid (second component)");
+        auto srcHandle2 = nanovdb::tools::openToIndexVDB<BuildT, nanovdb::cuda::DeviceBuffer>(
+            grid2,
+            0u,    // Don't copy data channel
+            false, // No stats
+            false, // No tiles
+            1      // Verbose mode
+        );
+        auto *srcIndexGrid2 = srcHandle2.grid<BuildT>();
+        cpuTimer.stop();
+
+        if (printGridDiagnostics) {
+            std::cout << "============ Component Grid #1 =======  " << std::endl;
+            std::cout << "Allocated values [valueCount()]       : " << srcIndexGrid1->valueCount() << std::endl;
+            std::cout << "Active voxels    [activeVoxelCount()] : " << srcIndexGrid1->activeVoxelCount() << std::endl;
+            auto minCorner = srcIndexGrid1->indexBBox().min(), maxCorner = srcIndexGrid1->indexBBox().max();
+            std::cout << "Index-space bounding box              : [" << minCorner.x() << "," << minCorner.y() << "," << minCorner.z()
+                      << "] -> [" << maxCorner.x() << "," << maxCorner.y() << "," << maxCorner.z() << "]" << std::endl;
+            std::cout << "Leaf nodes                            : " << srcIndexGrid1->tree().nodeCount(0) << std::endl;
+            std::cout << "Lower internal nodes                  : " << srcIndexGrid1->tree().nodeCount(1) << std::endl;
+            std::cout << "Upper internal nodes                  : " << srcIndexGrid1->tree().nodeCount(2) << std::endl;
+            std::cout << "Leaf-level occupancy                  : "
+                      << 100.f * (float)(srcIndexGrid1->activeVoxelCount())/(float)(srcIndexGrid1->tree().nodeCount(0) * 512)
+                      << "%" << std::endl;
+            std::cout << "Memory usage                          : " << srcIndexGrid1->gridSize() << " bytes" << std::endl;
+            std::cout << "============ Component Grid #2 =======  " << std::endl;
+            std::cout << "Allocated values [valueCount()]       : " << srcIndexGrid2->valueCount() << std::endl;
+            std::cout << "Active voxels    [activeVoxelCount()] : " << srcIndexGrid2->activeVoxelCount() << std::endl;
+            minCorner = srcIndexGrid2->indexBBox().min(), maxCorner = srcIndexGrid2->indexBBox().max();
+            std::cout << "Index-space bounding box              : [" << minCorner.x() << "," << minCorner.y() << "," << minCorner.z()
+                      << "] -> [" << maxCorner.x() << "," << maxCorner.y() << "," << maxCorner.z() << "]" << std::endl;
+            std::cout << "Leaf nodes                            : " << srcIndexGrid2->tree().nodeCount(0) << std::endl;
+            std::cout << "Lower internal nodes                  : " << srcIndexGrid2->tree().nodeCount(1) << std::endl;
+            std::cout << "Upper internal nodes                  : " << srcIndexGrid2->tree().nodeCount(2) << std::endl;
+            std::cout << "Leaf-level occupancy                  : "
+                      << 100.f * (float)(srcIndexGrid2->activeVoxelCount())/(float)(srcIndexGrid2->tree().nodeCount(0) * 512)
+                      << "%" << std::endl;
+            std::cout << "Memory usage                          : " << srcIndexGrid2->gridSize() << " bytes" << std::endl;
+        }
+
+        // Merge (CPU/OpenVDB version)
+        cpuTimer.start("Merging openVDBs (on CPU)");
+        auto mergedGrid = grid1->deepCopy();
+        openvdb::tools::compSum(*mergedGrid, *grid2);
+        cpuTimer.stop();
+
+        // Convert to indexGrid
+        cpuTimer.start("Converting merged openVDB output to indexGrid");
+        auto dstReferenceHandle = nanovdb::tools::openToIndexVDB<BuildT, nanovdb::cuda::DeviceBuffer>(
+            mergedGrid,
+            0u,    // Don't copy data channel
+            false, // No stats
+            false, // No tiles
+            1      // Verbose mode
+        );
+        auto *dstReferenceGrid = dstReferenceHandle.grid<BuildT>();
+        cpuTimer.stop();
+
+        if (printGridDiagnostics) {
+            std::cout << "============ Merged Grid (CPU) =======  " << std::endl;
+            std::cout << "Allocated values [valueCount()]       : " << dstReferenceGrid->valueCount() << std::endl;
+            std::cout << "Active voxels    [activeVoxelCount()] : " << dstReferenceGrid->activeVoxelCount() << std::endl;
+            auto minCorner = dstReferenceGrid->indexBBox().min(), maxCorner = dstReferenceGrid->indexBBox().max();
+            std::cout << "Index-space bounding box              : [" << minCorner.x() << "," << minCorner.y() << "," << minCorner.z()
+                      << "] -> [" << maxCorner.x() << "," << maxCorner.y() << "," << maxCorner.z() << "]" << std::endl;
+            std::cout << "Leaf nodes                            : " << dstReferenceGrid->tree().nodeCount(0) << std::endl;
+            std::cout << "Lower internal nodes                  : " << dstReferenceGrid->tree().nodeCount(1) << std::endl;
+            std::cout << "Upper internal nodes                  : " << dstReferenceGrid->tree().nodeCount(2) << std::endl;
+            std::cout << "Leaf-level occupancy                  : "
+                      << 100.f * (float)(dstReferenceGrid->activeVoxelCount())/(float)(dstReferenceGrid->tree().nodeCount(0) * 512)
+                      << "%" << std::endl;
+            std::cout << "Memory usage                          : " << dstReferenceGrid->gridSize() << " bytes" << std::endl;
+        }
+
+        // Copy both NanoVDB grids to GPU
+        srcHandle1.deviceUpload();
+        srcHandle2.deviceUpload();
+        dstReferenceHandle.deviceUpload();
+        auto* deviceSrcGrid1 = srcHandle1.deviceGrid<BuildT>();
+        auto* deviceSrcGrid2 = srcHandle2.deviceGrid<BuildT>();
+        auto* deviceDstReferenceGrid = dstReferenceHandle.deviceGrid<BuildT>();
+        if (!deviceSrcGrid1 || !deviceSrcGrid2 || !deviceDstReferenceGrid)
+            OPENVDB_THROW(openvdb::RuntimeError, "Failure while uploading indexGrids to GPU");
+
+        // Launch benchmark
+        mainMergeGrids(
+            deviceSrcGrid1, deviceSrcGrid2, deviceDstReferenceGrid,
+            srcIndexGrid1, srcIndexGrid2, dstReferenceGrid, benchmark_iters);
+    }
+    catch (const std::exception& e) {
+        std::cerr << "An exception occurred: \"" << e.what() << "\"" << std::endl;
+    }
+    return 0;
+}
@@ -0,0 +1,59 @@
+// Copyright Contributors to the OpenVDB Project
+// SPDX-License-Identifier: Apache-2.0
+
+#include <nanovdb/tools/cuda/MergeGrids.cuh>
+
+template<typename T>
+bool bufferCheck(const T* deviceBuffer, const T* hostBuffer, size_t elem_count) {
+    T* tmpBuffer = new T[elem_count];
+    cudaCheck(cudaMemcpy(tmpBuffer, deviceBuffer, elem_count * sizeof(T), cudaMemcpyDeviceToHost));
+    bool same = true;
+    for (int i=0; same && i< elem_count; ++i) { same = (tmpBuffer[i] == hostBuffer[i]); }
+    delete [] tmpBuffer;
+    return same;
+}
+
+template<typename BuildT>
+void mainMergeGrids(
+    nanovdb::NanoGrid<BuildT> *deviceSrcGrid1,
+    nanovdb::NanoGrid<BuildT> *deviceSrcGrid2,
+    nanovdb::NanoGrid<BuildT> *deviceDstReferenceGrid,
+    nanovdb::NanoGrid<BuildT> *hostSrcGrid1,
+    nanovdb::NanoGrid<BuildT> *hostSrcGrid2,
+    nanovdb::NanoGrid<BuildT> *hostDstReferenceGrid,
+    uint32_t benchmark_iters)
+{
+    nanovdb::util::cuda::Timer gpuTimer;
+
+    // Initialize converter
+    nanovdb::tools::cuda::MergeGrids<BuildT> converter( deviceSrcGrid1, deviceSrcGrid2 );
+    converter.setChecksum(nanovdb::CheckMode::Default);
+    converter.setVerbose(1);
+
+    auto handle = converter.getHandle();
+    auto dstGrid = handle.template deviceGrid<BuildT>();
+
+    // Check for correctness
+    if (bufferCheck((char*)dstGrid, (char*)hostDstReferenceGrid->data(), hostDstReferenceGrid->gridSize()))
+        std::cout << "Result of MergeGrids check out CORRECT against reference" << std::endl;
+    else
+        std::cout << "Result of MergeGrids compares INCORRECT against reference" << std::endl;
+
+    // Re-run warm-started iterations
+    converter.setVerbose(0);
+    for (int i = 0; i < benchmark_iters; i++) {
+        gpuTimer.start("Re-running entire dilation after warmstart");
+        auto handle2 = converter.getHandle();
+        gpuTimer.stop();
+    }
+}
+
+template
+void mainMergeGrids(
+    nanovdb::NanoGrid<nanovdb::ValueOnIndex> *deviceSrcGrid1,
+    nanovdb::NanoGrid<nanovdb::ValueOnIndex> *deviceSrcGrid2,
+    nanovdb::NanoGrid<nanovdb::ValueOnIndex> *deviceDstReferenceGrid,
+    nanovdb::NanoGrid<nanovdb::ValueOnIndex> *hostSrcGrid1,
+    nanovdb::NanoGrid<nanovdb::ValueOnIndex> *hostSrcGrid2,
+    nanovdb::NanoGrid<nanovdb::ValueOnIndex> *hostDstReferenceGrid,
+    uint32_t benchmark_iters);
@@ -16,7 +16,7 @@ int main()
        auto handle = nanovdb::tools::createLevelSetSphere<float, nanovdb::cuda::DeviceBuffer>(100.0f);
        using GridT = nanovdb::FloatGrid;

-        handle.deviceUpload(0, false); // Copy the NanoVDB grid to the GPU asynchronously
+        handle.deviceUpload(nullptr, false); // Copy the NanoVDB grid to the GPU asynchronously

        const GridT* grid = handle.grid<float>(); // get a (raw) const pointer to a NanoVDB grid of value type float on the CPU
        GridT* deviceGrid = handle.deviceGrid<float>(); // get a (raw) pointer to a NanoVDB grid of value type float on the GPU
@@ -32,7 +32,7 @@ int main()

        scaleActiveVoxels(deviceGrid, grid->tree().nodeCount(0), 2.0f);

-        handle.deviceDownload(0, true); // Copy the NanoVDB grid to the CPU synchronously
+        handle.deviceDownload(nullptr, true); // Copy the NanoVDB grid to the CPU synchronously

        std::cout << "Value after scaling  = " << grid->tree().getValue(nanovdb::Coord(101,0,0)) << std::endl;
    }
@@ -40,4 +40,4 @@ int main()
        std::cerr << "An exception occurred: \"" << e.what() << "\"" << std::endl;
    }
    return 0;
-}
+}
@@ -37,8 +37,7 @@ int main(int ac, char** av)

        const int width = 1024;
        const int height = 1024;
-        BufferT   imageBuffer;
-        imageBuffer.init(width * height * sizeof(float));
+        BufferT   imageBuffer(width * height * sizeof(float));

        runNanoVDB(handle, numIterations, width, height, imageBuffer);
 #if defined(NANOVDB_USE_OPENVDB)
@@ -11,12 +11,12 @@
 #include <openvdb/math/Ray.h>
 #include <openvdb/tools/LevelSetSphere.h>

-#include <nanovdb/cuda/DeviceBuffer.h>
 #include <nanovdb/tools/NanoToOpenVDB.h>

 #include "common.h"

 #if defined(NANOVDB_USE_CUDA)
+#include <nanovdb/cuda/DeviceBuffer.h>
 using BufferT = nanovdb::cuda::DeviceBuffer;
 #else
 using BufferT = nanovdb::HostBuffer;
@@ -30,10 +30,9 @@ void runOpenVDB(nanovdb::GridHandle<BufferT>& handle, int numIterations, int wid
    using Vec3T = openvdb::math::Vec3<RealT>;
    using RayT = openvdb::math::Ray<RealT>;

-    auto srcGrid = nanovdb::tools::nanoToOpenVDB(handle);
+    openvdb::GridBase::Ptr srcGrid = nanovdb::tools::nanoToOpenVDB(handle);
    std::cout << "Exporting to OpenVDB grid[" << srcGrid->getName() << "]...\n";
-
-    auto h_grid = (GridT*)srcGrid.get();
+    GridT::Ptr h_grid = openvdb::GridBase::grid<GridT>(srcGrid);

    float* h_outImage = reinterpret_cast<float*>(imageBuffer.data());

@@ -82,7 +81,7 @@ void runOpenVDB(nanovdb::GridHandle<BufferT>& handle, int numIterations, int wid
    {
        float durationAvg = 0;
        for (int i = 0; i < numIterations; ++i) {
-            float duration = renderImage(false, renderOp, width, height, h_outImage, h_grid);
+            float duration = renderImage(false, renderOp, width, height, h_outImage, h_grid.get());
            //std::cout << "Duration(OpenVDB-Host) = " << duration << " ms" << std::endl;
            durationAvg += duration;
        }
@@ -37,8 +37,7 @@ int main(int ac, char** av)

        const int width = 1024;
        const int height = 1024;
-        BufferT   imageBuffer;
-        imageBuffer.init(width * height * sizeof(float));
+        BufferT   imageBuffer(width * height * sizeof(float));

        runNanoVDB(handle, numIterations, width, height, imageBuffer);
 #if defined(NANOVDB_USE_OPENVDB)
@@ -11,12 +11,12 @@
 #include <openvdb/math/Ray.h>
 #include <openvdb/tools/RayIntersector.h>

-#include <nanovdb/cuda/DeviceBuffer.h>
 #include <nanovdb/tools/NanoToOpenVDB.h>

 #include "common.h"

 #if defined(NANOVDB_USE_CUDA)
+#include <nanovdb/cuda/DeviceBuffer.h>
 using BufferT = nanovdb::cuda::DeviceBuffer;
 #else
 using BufferT = nanovdb::HostBuffer;
@@ -31,7 +31,7 @@ void runOpenVDB(nanovdb::GridHandle<nanovdb::cuda::DeviceBuffer>& handle, int nu
    using RayT = openvdb::math::Ray<RealT>;

 #if 1
-    auto srcGrid = nanovdb::tools::nanoToOpenVDB(handle);
+    openvdb::GridBase::Ptr srcGrid = nanovdb::tools::nanoToOpenVDB(handle);
    std::cout << "Exporting to OpenVDB grid[" << srcGrid->getName() << "]...\n";
 #else
    openvdb::initialize();
@@ -42,7 +42,7 @@ void runOpenVDB(nanovdb::GridHandle<nanovdb::cuda::DeviceBuffer>& handle, int nu
    std::cout << "Loading OpenVDB grid[" << srcGrid->getName() << "]...\n";
 #endif

-    auto h_grid = (GridT*)srcGrid.get();
+    GridT::Ptr h_grid = openvdb::GridBase::grid<GridT>(srcGrid);

    float* h_outImage = reinterpret_cast<float*>(imageBuffer.data());

@@ -85,7 +85,7 @@ void runOpenVDB(nanovdb::GridHandle<nanovdb::cuda::DeviceBuffer>& handle, int nu
    {
        float durationAvg = 0;
        for (int i = 0; i < numIterations; ++i) {
-            float duration = renderImage(false, renderOp, width, height, h_outImage, h_grid);
+            float duration = renderImage(false, renderOp, width, height, h_outImage, h_grid.get());
            //std::cout << "Duration(OpenVDB-Host) = " << duration << " ms" << std::endl;
            durationAvg += duration;
        }
@@ -1,3 +1,5 @@
+// Copyright Contributors to the OpenVDB Project
+// SPDX-License-Identifier: Apache-2.0
 #include <nanovdb/io/IO.h> // this is required to read (and write) NanoVDB files on the host

 /// @brief Read a NanoVDB grid from a file and print out multiple values.
@@ -0,0 +1,200 @@
+// Copyright Contributors to the OpenVDB Project
+// SPDX-License-Identifier: Apache-2.0
+
+#ifdef NANOVDB_USE_TBB
+#include <tbb/tbb.h>
+#endif
+
+// the following files are from OpenVDB
+#include <openvdb/tools/Composite.h>
+#include <openvdb/util/CpuTimer.h>
+
+// the following files are from NanoVDB
+#include <nanovdb/NanoVDB.h>
+#include <nanovdb/cuda/DeviceBuffer.h>
+#include <nanovdb/tools/CreateNanoGrid.h>
+
+template<typename BuildT>
+void mainRefineGrid(
+    nanovdb::NanoGrid<BuildT> *deviceGridOriginal,
+    nanovdb::NanoGrid<BuildT> *deviceGridRefined,
+    nanovdb::NanoGrid<BuildT> *indexGridOriginal,
+    nanovdb::NanoGrid<BuildT> *indexGridRefined,
+    uint32_t benchmark_iters
+);
+
+#ifdef NANOVDB_USE_TBB
+template<typename GridT>
+class VoxelRefiner
+{
+    using TreeT = typename GridT::TreeType;
+    using LeafMgrT = openvdb::tree::LeafManager<const TreeT>;
+
+public:
+    VoxelRefiner(typename GridT::Ptr srcGrid, tbb::combinable<typename GridT::Ptr>& destGrids) : mSrcGrid(srcGrid), mDestGrids(destGrids) {}
+    VoxelRefiner(const VoxelRefiner& other, tbb::split) : mSrcGrid(other.mSrcGrid), mDestGrids(other.mDestGrids) {}
+
+    void operator()(const typename LeafMgrT::LeafRange& range) const
+    {
+        // Refine into a thread-local copy, for safety
+        typename GridT::Ptr& dstGrid = mDestGrids.local();
+        if (!dstGrid) { // Create if it doesn't exist
+            dstGrid = GridT::create(mSrcGrid->background());
+            dstGrid->setTransform(mSrcGrid->transform().copy());
+        }
+
+        auto unitBox = openvdb::CoordBBox::createCube(openvdb::Coord(), 2);
+        auto dstAcc = dstGrid->getAccessor();
+        for (auto srcLeafIter = range.begin(); srcLeafIter; ++srcLeafIter) {
+            for (auto srcIter = srcLeafIter->cbeginValueOn(); srcIter; ++srcIter) {
+                const auto& ijk = srcIter.getCoord();
+                const auto& value = srcIter.getValue();
+                for (auto boxIter = unitBox.begin(); boxIter != unitBox.end(); ++boxIter)
+                    dstAcc.setValue(ijk+ijk+*boxIter, value);
+            }
+        }
+    }
+
+private:
+    typename GridT::Ptr mSrcGrid;
+    tbb::combinable<typename GridT::Ptr>& mDestGrids;
+};
+#endif
+
+/// @brief This example depends on OpenVDB, NanoVDB, and CUDA
+int main(int argc, char *argv[])
+{
+    using GridT = openvdb::FloatGrid;
+    using BuildT = nanovdb::ValueOnIndex;
+
+    openvdb::util::CpuTimer cpuTimer;
+    const bool printGridDiagnostics = true;
+
+    try {
+
+        if (argc<2) OPENVDB_THROW(openvdb::ValueError, "usage: "+std::string(argv[0])+" input.vdb [<iterations>]\n");
+        int benchmark_iters = 10;
+        if (argc > 2) sscanf(argv[2], "%d", &benchmark_iters);
+
+        // Read the initial level set from file
+
+        cpuTimer.start("Read input VDB file");
+        openvdb::initialize();
+        openvdb::io::File inFile(argv[1]);
+        inFile.open(false); // disable delayed loading
+        auto baseGrids = inFile.getGrids();
+        inFile.close();
+        auto grid = openvdb::gridPtrCast<GridT>(baseGrids->at(0));
+        if (!grid) OPENVDB_THROW(openvdb::ValueError, "First grid is not a FloatGrid\n");
+        cpuTimer.stop();
+
+        // Convert to indexGrid (original, un-refined)
+        cpuTimer.start("Converting openVDB input to indexGrid (original version)");
+        auto handleOriginal = nanovdb::tools::openToIndexVDB<BuildT, nanovdb::cuda::DeviceBuffer>(
+            grid,
+            0u,    // Don't copy data channel
+            false, // No stats
+            false, // No tiles
+            1      // Verbose mode
+        );
+        auto *indexGridOriginal = handleOriginal.grid<BuildT>();
+        cpuTimer.stop();
+
+        if (printGridDiagnostics) {
+            std::cout << "============ Original Grid ===========" << std::endl;
+            std::cout << "Allocated values [valueCount()]       : " << indexGridOriginal->valueCount() << std::endl;
+            std::cout << "Active voxels    [activeVoxelCount()] : " << indexGridOriginal->activeVoxelCount() << std::endl;
+            auto minCorner = indexGridOriginal->indexBBox().min(), maxCorner = indexGridOriginal->indexBBox().max();
+            std::cout << "Index-space bounding box              : [" << minCorner.x() << "," << minCorner.y() << "," << minCorner.z()
+                      << "] -> [" << maxCorner.x() << "," << maxCorner.y() << "," << maxCorner.z() << "]" << std::endl;
+            std::cout << "Leaf nodes                            : " << indexGridOriginal->tree().nodeCount(0) << std::endl;
+            std::cout << "Lower internal nodes                  : " << indexGridOriginal->tree().nodeCount(1) << std::endl;
+            std::cout << "Upper internal nodes                  : " << indexGridOriginal->tree().nodeCount(2) << std::endl;
+            std::cout << "Leaf-level occupancy                  : "
+                      << 100.f * (float)(indexGridOriginal->activeVoxelCount())/(float)(indexGridOriginal->tree().nodeCount(0) * 512)
+                      << "%" << std::endl;
+            std::cout << "Memory usage                          : " << indexGridOriginal->gridSize() << " bytes" << std::endl;
+        }
+
+        // Refinement (CPU/OpenVDB version)
+        cpuTimer.start("Refining OpenVDB (on CPU)");
+        using TreeT = GridT::TreeType;
+        using LeafManagerT = openvdb::tree::LeafManager<const TreeT>;
+#ifdef NANOVDB_USE_TBB
+        // Phase 1: Multithreaded refinement
+        tbb::combinable<GridT::Ptr> localGrids;
+        LeafManagerT leafMgr(grid->tree());
+        VoxelRefiner<GridT> op(grid, localGrids);
+        tbb::parallel_for( leafMgr.leafRange(), op);
+        // Phase 2: Recombine partial grids into final refined result
+        auto refinedGrid = openvdb::FloatGrid::create(grid->background());
+        refinedGrid->setTransform(grid->transform().copy());
+        refinedGrid->setName(grid->getName());
+        localGrids.combine_each(
+            [&refinedGrid](const openvdb::FloatGrid::Ptr& localGrid) { if (localGrid) openvdb::tools::compSum(*refinedGrid, *localGrid); } );
+#else
+        // Single-threaded refinement
+        LeafManagerT leafMgr(grid->tree());
+        auto refinedGrid = openvdb::FloatGrid::create(grid->background());
+        refinedGrid->setTransform(grid->transform().copy());
+        refinedGrid->setName(grid->getName());
+        auto dstAcc = refinedGrid->getAccessor();
+        auto unitBox = openvdb::CoordBBox::createCube(openvdb::Coord(), 2);
+        for (std::size_t leafID = 0; leafID < leafMgr.leafCount(); ++leafID) {
+            const auto& srcLeaf = leafMgr.leaf(leafID);
+            for (auto iter = srcLeaf.cbeginValueOn(); iter; ++iter) {
+                const auto& ijk = iter.getCoord();
+                const auto& value = iter.getValue();
+                for (auto boxIter = unitBox.begin(); boxIter != unitBox.end(); ++boxIter)
+                    dstAcc.setValue(ijk+ijk+*boxIter, value);
+            }
+        }
+#endif
+        cpuTimer.stop();
+
+        // Convert to indexGrid (refined)
+        cpuTimer.start("Converting openVDB input to indexGrid (refineed version)");
+        auto handleRefined = nanovdb::tools::openToIndexVDB<BuildT, nanovdb::cuda::DeviceBuffer>(
+            refinedGrid,
+            0u,    // Don't copy data channel
+            false, // No stats
+            false, // No tiles
+            1      // Verbose mode
+        );
+        cpuTimer.stop();
+
+        auto *indexGridRefined = handleRefined.grid<BuildT>();
+
+        if (printGridDiagnostics) {
+            std::cout << "============ Refined Grid ============" << std::endl;
+            std::cout << "Allocated values [valueCount()]       : " << indexGridRefined->valueCount() << std::endl;
+            std::cout << "Active voxels    [activeVoxelCount()] : " << indexGridRefined->activeVoxelCount() << std::endl;
+            auto minCorner = indexGridRefined->indexBBox().min(), maxCorner = indexGridRefined->indexBBox().max();
+            std::cout << "Index-space bounding box              : [" << minCorner.x() << "," << minCorner.y() << "," << minCorner.z()
+                      << "] -> [" << maxCorner.x() << "," << maxCorner.y() << "," << maxCorner.z() << "]" << std::endl;
+            std::cout << "Leaf nodes                            : " << indexGridRefined->tree().nodeCount(0) << std::endl;
+            std::cout << "Lower internal nodes                  : " << indexGridRefined->tree().nodeCount(1) << std::endl;
+            std::cout << "Upper internal nodes                  : " << indexGridRefined->tree().nodeCount(2) << std::endl;
+            std::cout << "Leaf-level occupancy                  : "
+                      << 100.f * (float)(indexGridRefined->activeVoxelCount())/(float)(indexGridRefined->tree().nodeCount(0) * 512)
+                      << "%" << std::endl;
+            std::cout << "Memory usage                          : " << indexGridRefined->gridSize() << " bytes" << std::endl;
+        }
+
+        // Copy both NanoVDB grids to GPU
+        handleOriginal.deviceUpload();
+        handleRefined.deviceUpload();
+        auto* deviceGridOriginal = handleOriginal.deviceGrid<BuildT>();
+        auto* deviceGridRefined = handleRefined.deviceGrid<BuildT>();
+        if (!deviceGridOriginal || !deviceGridRefined)
+            OPENVDB_THROW(openvdb::RuntimeError, "Failure while uploading indexGrids to GPU");
+
+        // Launch benchmark
+        mainRefineGrid( deviceGridOriginal, deviceGridRefined, indexGridOriginal, indexGridRefined, benchmark_iters );
+
+    }
+    catch (const std::exception& e) {
+        std::cerr << "An exception occurred: \"" << e.what() << "\"" << std::endl;
+    }
+    return 0;
+}
@@ -0,0 +1,56 @@
+// Copyright Contributors to the OpenVDB Project
+// SPDX-License-Identifier: Apache-2.0
+
+#include <nanovdb/tools/cuda/RefineGrid.cuh>
+
+template<typename T>
+bool bufferCheck(const T* deviceBuffer, const T* hostBuffer, size_t elem_count) {
+    T* tmpBuffer = new T[elem_count];
+    cudaCheck(cudaMemcpy(tmpBuffer, deviceBuffer, elem_count * sizeof(T), cudaMemcpyDeviceToHost));
+    bool same = true;
+    for (int i=0; same && i< elem_count; ++i) { same = (tmpBuffer[i] == hostBuffer[i]); }
+    delete [] tmpBuffer;
+    return same;
+}
+
+template<typename BuildT>
+void mainRefineGrid(
+    nanovdb::NanoGrid<BuildT> *deviceGridOriginal,
+    nanovdb::NanoGrid<BuildT> *deviceGridRefined,
+    nanovdb::NanoGrid<BuildT> *indexGridOriginal,
+    nanovdb::NanoGrid<BuildT> *indexGridRefined,
+    uint32_t benchmark_iters)
+{
+    nanovdb::util::cuda::Timer gpuTimer;
+
+    // Initialize refiner
+    nanovdb::tools::cuda::RefineGrid<BuildT> refiner( deviceGridOriginal );
+    refiner.setChecksum(nanovdb::CheckMode::Default);
+    refiner.setVerbose(1);
+
+    auto handle = refiner.getHandle();
+    auto dstGrid = handle.template deviceGrid<BuildT>();
+
+    // Check for correctness
+    if (bufferCheck((char*)dstGrid, (char*)indexGridRefined->data(), indexGridRefined->gridSize()))
+        std::cout << "Result of RefineGrid check out CORRECT against reference" << std::endl;
+    else
+        std::cout << "Result of RefineGrid compares INCORRECT against reference" << std::endl;
+
+    // Re-run warm-started iterations
+    refiner.setVerbose(0);
+    for (int i = 0; i < benchmark_iters; i++) {
+        gpuTimer.start("Re-running entire refinment after warmstart");
+        auto dummyHandle = refiner.getHandle();
+        gpuTimer.stop();
+    }
+}
+
+template
+void mainRefineGrid(
+    nanovdb::NanoGrid<nanovdb::ValueOnIndex> *deviceGridOriginal,
+    nanovdb::NanoGrid<nanovdb::ValueOnIndex> *deviceGridRefined,
+    nanovdb::NanoGrid<nanovdb::ValueOnIndex> *indexGridOriginal,
+    nanovdb::NanoGrid<nanovdb::ValueOnIndex> *indexGridRefined,
+    uint32_t benchmark_iters
+);
@@ -29,7 +29,7 @@ int main()
            using OpT = nanovdb::SetVoxel<float>;// defines type of random-access operation (set value)
            const nanovdb::Coord &ijk = d_coords[tid];
            d_grid->tree().set<OpT>(ijk, d_values[tid]);// normally one should use a ValueAccessor
-            printf("GPU: voxel # %lu, grid(%4i,%4i,%4i) = %5.1f\n", tid, ijk[0], ijk[1], ijk[2], d_grid->tree().getValue(ijk));
+            printf("GPU: voxel # %zu, grid(%4i,%4i,%4i) = %5.1f\n", tid, ijk[0], ijk[1], ijk[2], d_grid->tree().getValue(ijk));
        }); cudaCheckError();

        // Copy grid from GPU to CPU and print the voxel values for validation
@@ -37,7 +37,7 @@ int main()
        auto *grid = handle.grid<float>();
        for (size_t i=0; i<numVoxels; ++i) {
            const nanovdb::Coord &ijk = coords[i];
-            printf("CPU: voxel # %lu, grid(%4i,%4i,%4i) = %5.1f\n", i, ijk[0], ijk[1], ijk[2], grid->tree().getValue(ijk));
+            printf("CPU: voxel # %zu, grid(%4i,%4i,%4i) = %5.1f\n", i, ijk[0], ijk[1], ijk[2], grid->tree().getValue(ijk));
        }

        // free arrays allocated on the device
@@ -49,4 +49,4 @@ int main()
    }

    return 0;
-}
+}
@@ -0,0 +1,60 @@
+// Copyright Contributors to the OpenVDB Project
+// SPDX-License-Identifier: Apache-2.0
+
+#include <nanovdb/tools/cuda/DistributedPointsToGrid.cuh>
+
+namespace
+{
+
+template<typename BufferT = nanovdb::HostBuffer>
+std::vector<nanovdb::Coord>
+initializeCoordListFromCSV(const std::string& filename)
+{
+    std::ifstream file;
+    file.open(filename);
+    std::string line;
+    std::vector<nanovdb::Coord> coords;
+    while (getline(file, line)) {
+        std::istringstream stream(line);
+        nanovdb::Coord coord;
+        stream >> coord.x(); stream.ignore(1, ',' );
+        stream >> coord.y(); stream.ignore(1, ',' );
+        stream >> coord.z();
+        coords.emplace_back(coord);
+    }
+    file.close();
+    return coords;
+}
+
+}
+
+/// @brief Demonstrates how to create a NanoVDB grid from voxel coordinates on the GPU
+void testVoxelsToGrid(const std::string& filename)
+{
+    nanovdb::cuda::DeviceMesh deviceMesh;
+
+    std::cout << "CSV index source : " << filename << std::endl;
+    std::cout << "==================================================" << std::endl;
+    std::vector<nanovdb::Coord> hostCoords = initializeCoordListFromCSV(filename);
+    std::cout << "Number of coords : " << hostCoords.size() << std::endl;
+
+    size_t coordCount = hostCoords.size();
+    nanovdb::Coord* coords = nullptr;
+    cudaMallocManaged(&coords, coordCount * sizeof(nanovdb::Coord));
+    cudaMemcpy(coords, hostCoords.data(), coordCount * sizeof(nanovdb::Coord), cudaMemcpyHostToDevice);
+
+    nanovdb::tools::cuda::DistributedPointsToGrid<nanovdb::ValueOnIndex> distributedConverter(deviceMesh);
+    auto distributedHandle = distributedConverter.getHandle(coords, coordCount);
+
+    cudaFree(coords);
+}
+
+int main(int argc, char** argv)
+{
+    if (argc != 2)
+        throw std::runtime_error("Expected CSV filename as command-line argument");
+    std::string filename(argv[1]);
+    testVoxelsToGrid(filename);
+
+    return 0;
+}
@@ -14,13 +14,20 @@

    \note  This file does NOT depend on OpenVDB, but optionally on ZIP and BLOSC

-    \details NanoVDB files take on of two formats:
+    \details NanoVDB files take on one of the two following formats:
             1) multiple segments each with multiple grids (segments have easy to access metadata about its grids)
             2) starting with verion 32.6.0 nanovdb files also support a raw buffer with one or more grids (just a
             dump of a raw grid buffer, so no new metadata in headers as when using segments mentioned above).

-    // 1: Segment:  FileHeader, MetaData0, gridName0...MetaDataN, gridNameN, compressed Grid0, ... compressed GridN
-    // 2: Raw: Grid0, ... GridN
+    Example of case 1:
+        | <------------------------------------ segment 1 with N grids --------------------------------------> | <--- segment 2 ...
+        FileHeader, FileMetaData0, gridName0...FileMetaDataN, gridNameN, compressed Grid0, ... compressed GridN FileHeader ...
+    Example of case 2:
+        | <-- grid buffer ---> |
+        Grid0, Grid1, ... GridN
+
+    Note that FileHeader and FileMetaData (both defined in NanoVDB.h) have fixed sizes of respectively 16B and 176B.
+    However, GridNameX and GridX have variable sizes!
 */

 #ifndef NANOVDB_IO_H_HAS_BEEN_INCLUDED
@@ -216,7 +223,7 @@ fileSize_t Internal::write(std::ostream& os, const GridHandle<BufferT>& handle,
        uLongf                   size = compressBound(static_cast<uLongf>(residual)); // Get an upper bound on the size of the compressed data.
        std::unique_ptr<Bytef[]> tmp(new Bytef[size]);
        const int                status = compress(tmp.get(), &size, reinterpret_cast<const Bytef*>(data), static_cast<uLongf>(residual));
-        if (status != Z_OK) std::runtime_error("Internal write error in ZIP");
+        if (status != Z_OK) throw std::runtime_error("Internal write error in ZIP");
        if (size > residual) std::cerr << "\nWarning: Unexpected ZIP compression from " << residual << " to " << size << " bytes\n";
        const fileSize_t outBytes = size;
        os.write(reinterpret_cast<const char*>(&outBytes), sizeof(fileSize_t));
@@ -233,7 +240,7 @@ fileSize_t Internal::write(std::ostream& os, const GridHandle<BufferT>& handle,
            fileSize_t              chunk = residual < MAX_SIZE ? residual : MAX_SIZE, size = chunk + BLOSC_MAX_OVERHEAD;
            std::unique_ptr<char[]> tmp(new char[size]);
            const int               count = blosc_compress_ctx(9, 1, sizeof(float), chunk, data, tmp.get(), size, BLOSC_LZ4_COMPNAME, 1 << 18, 1);
-            if (count <= 0) std::runtime_error("Internal write error in BLOSC");
+            if (count <= 0) throw std::runtime_error("Internal write error in BLOSC");
            const fileSize_t outBytes = count;
            os.write(reinterpret_cast<const char*>(&outBytes), sizeof(fileSize_t));
            os.write(reinterpret_cast<const char*>(tmp.get()), outBytes);
@@ -277,7 +284,7 @@ void Internal::read(std::istream& is, char* data, fileSize_t residual, Codec cod
        is.read(reinterpret_cast<char*>(tmp.get()), size);
        uLongf numBytes = static_cast<uLongf>(residual);
        int status = uncompress(reinterpret_cast<Bytef*>(data), &numBytes, tmp.get(), static_cast<uLongf>(size));
-        if (status != Z_OK) std::runtime_error("Internal read error in ZIP");
+        if (status != Z_OK) throw std::runtime_error("Internal read error in ZIP");
        if (fileSize_t(numBytes) != residual) throw std::runtime_error("UNZIP failed on byte size");
 #else
        throw std::runtime_error("ZIP compression codec was disabled during build");
@@ -293,7 +300,7 @@ void Internal::read(std::istream& is, char* data, fileSize_t residual, Codec cod
            is.read(reinterpret_cast<char*>(tmp.get()), size);
            const fileSize_t chunk = residual < MAX_SIZE ? residual : MAX_SIZE;
            const int        count = blosc_decompress_ctx(tmp.get(), data, size_t(chunk), 1); //fails with more threads :(
-            if (count < 1) std::runtime_error("Internal read error in BLOSC");
+            if (count < 1) throw std::runtime_error("Internal read error in BLOSC");
            if (count != int(chunk)) throw std::runtime_error("BLOSC failed on byte size");
            data += size_t(chunk);
            residual -= chunk;
@@ -325,10 +332,11 @@ inline FileGridMetaData::FileGridMetaData(uint64_t size, Codec c, const GridData
                   {0, 0, 0, 1}, // nodeCount[4]
                   {0, 0, 0}, // tileCount[3]
                   c, // codec
-                   0, // padding
+                   uint16_t(gridData.mBlindMetadataCount), // number of blind meta data
                   Version()}// version
    , gridName(gridData.gridName())
 {
+    NANOVDB_ASSERT(gridData.mBlindMetadataCount <= uint32_t(1 << 16));// due to uint32_t -> uin16_t conversion
    auto &treeData = *reinterpret_cast<const TreeData*>(gridData.treePtr());
    nameKey = stringHash(gridName);
    voxelCount = treeData.mVoxelCount;
@@ -722,9 +730,13 @@ inline uint64_t stringHash(const char* c_str)

 } // namespace io ======================================================================

+} // namespace nanovdb ===================================================================
+
+// the following stream specializations should not be namespaced!
+
 template<typename T>
 inline std::ostream&
-operator<<(std::ostream& os, const math::BBox<math::Vec3<T>>& b)
+operator<<(std::ostream& os, const nanovdb::math::BBox<nanovdb::math::Vec3<T>>& b)
 {
    os << "(" << b[0][0] << "," << b[0][1] << "," << b[0][2] << ") -> "
       << "(" << b[1][0] << "," << b[1][1] << "," << b[1][2] << ")";
@@ -732,7 +744,7 @@ operator<<(std::ostream& os, const math::BBox<math::Vec3<T>>& b)
 }

 inline std::ostream&
-operator<<(std::ostream& os, const CoordBBox& b)
+operator<<(std::ostream& os, const nanovdb::CoordBBox& b)
 {
    os << "(" << b[0][0] << "," << b[0][1] << "," << b[0][2] << ") -> "
       << "(" << b[1][0] << "," << b[1][1] << "," << b[1][2] << ")";
@@ -740,7 +752,7 @@ operator<<(std::ostream& os, const CoordBBox& b)
 }

 inline std::ostream&
-operator<<(std::ostream& os, const Coord& ijk)
+operator<<(std::ostream& os, const nanovdb::Coord& ijk)
 {
    os << "(" << ijk[0] << "," << ijk[1] << "," << ijk[2] << ")";
    return os;
@@ -748,7 +760,7 @@ operator<<(std::ostream& os, const Coord& ijk)

 template<typename T>
 inline std::ostream&
-operator<<(std::ostream& os, const math::Vec3<T>& v)
+operator<<(std::ostream& os, const nanovdb::math::Vec3<T>& v)
 {
    os << "(" << v[0] << "," << v[1] << "," << v[2] << ")";
    return os;
@@ -756,12 +768,10 @@ operator<<(std::ostream& os, const math::Vec3<T>& v)

 template<typename T>
 inline std::ostream&
-operator<<(std::ostream& os, const math::Vec4<T>& v)
+operator<<(std::ostream& os, const nanovdb::math::Vec4<T>& v)
 {
    os << "(" << v[0] << "," << v[1] << "," << v[2] << "," << v[3] << ")";
    return os;
 }

-} // namespace nanovdb ===================================================================
-
 #endif // NANOVDB_IO_H_HAS_BEEN_INCLUDED
@@ -74,9 +74,23 @@ public:
    {
        if (mDim == dim)
            return false;
+
+        // compute valid voxel range
+        Coord voxelMax = (mVoxel + Coord(mDim - 1)) & (~(dim - 1));
+        Coord voxelMin = mVoxel & (~(dim - 1));
+
        mDim = dim;
        const Vec3T &pos = ray(mT0), &inv = ray.invDir();
        mVoxel = RoundDown<CoordT>(pos) & (~(dim - 1));
+
+        // clamp mVoxel to valid range
+        mVoxel[0] = nanovdb::math::Min(mVoxel[0], voxelMax[0]);
+        mVoxel[1] = nanovdb::math::Min(mVoxel[1], voxelMax[1]);
+        mVoxel[2] = nanovdb::math::Min(mVoxel[2], voxelMax[2]);
+        mVoxel[0] = nanovdb::math::Max(mVoxel[0], voxelMin[0]);
+        mVoxel[1] = nanovdb::math::Max(mVoxel[1], voxelMin[1]);
+        mVoxel[2] = nanovdb::math::Max(mVoxel[2], voxelMin[2]);
+
        for (int axis = 0; axis < 3; ++axis) {
            if (mStep[axis] == 0)
                continue;
@@ -17,6 +17,10 @@

 #include <nanovdb/util/Util.h>// for __hostdev__ and lots of other utility functions

+#if defined(__CUDA_ARCH__)
+#include <cuda/std/limits>// for ::cuda::std::numeric_limits
+#endif
+
 namespace nanovdb {// =================================================================

 namespace math {// =============================================================
@@ -83,7 +87,13 @@ struct Delta<double>
 /// Maximum floating-point values
 template<typename T>
 struct Maximum;
-#if defined(__CUDA_ARCH__) || defined(__HIP__)
+#if defined(__CUDA_ARCH__)
+template<typename T>
+struct Maximum
+{
+    __hostdev__ static T value() { return ::cuda::std::numeric_limits<T>::max(); }
+};
+#elif defined(__HIP__)
 template<>
 struct Maximum<int>
 {
@@ -576,6 +586,774 @@ public:
    __hostdev__ inline Coord round() const { return *this; }
 }; // Coord class

+
+/// @brief Type alias for Coord so we have a consistent naming convention
+using Coord3 = Coord;
+
+
+template <typename T>
+class Vec2;
+
+/// @brief Signed (i, j) 32-bit integer coordinate class, similar to openvdb::math::Coord
+class Coord2
+{
+    int32_t mVec[2]; // private member data - three signed index coordinates
+public:
+    using ValueType = int32_t;
+    using IndexType = uint32_t;
+
+    /// @brief Initialize all coordinates to zero.
+    __hostdev__ Coord2()
+        : mVec{0, 0}
+    {
+    }
+
+    /// @brief Initializes all coordinates to the given signed integer.
+    __hostdev__ explicit Coord2(ValueType n)
+        : mVec{n, n}
+    {
+    }
+
+    /// @brief Initializes coordinate to the given signed integers.
+    __hostdev__ Coord2(ValueType i, ValueType j)
+        : mVec{i, j}
+    {
+    }
+
+    __hostdev__ Coord2(ValueType* ptr)
+        : mVec{ptr[0], ptr[1]}
+    {
+    }
+
+    __hostdev__ int32_t x() const { return mVec[0]; }
+    __hostdev__ int32_t y() const { return mVec[1]; }
+
+    __hostdev__ int32_t& x() { return mVec[0]; }
+    __hostdev__ int32_t& y() { return mVec[1]; }
+
+    __hostdev__ static Coord2 max() { return Coord2(int32_t((1u << 31) - 1)); }
+
+    __hostdev__ static Coord2 min() { return Coord2(-int32_t((1u << 31) - 1) - 1); }
+
+    __hostdev__ static size_t memUsage() { return sizeof(Coord2); }
+
+    /// @brief Return a const reference to the given Coord component.
+    /// @warning The argument is assumed to be 0 or 1,
+    __hostdev__ const ValueType& operator[](IndexType i) const { return mVec[i]; }
+
+    /// @brief Return a non-const reference to the given Coord component.
+    /// @warning The argument is assumed to be 0 or 1.
+    __hostdev__ ValueType& operator[](IndexType i) { return mVec[i]; }
+
+    /// @brief Assignment operator that works with openvdb::Coord
+    template<typename CoordT>
+    __hostdev__ Coord2& operator=(const CoordT& other)
+    {
+        static_assert(sizeof(Coord2) == sizeof(CoordT), "Mis-matched sizeof");
+        mVec[0] = other[0];
+        mVec[1] = other[1];
+        return *this;
+    }
+
+    /// @brief Return a new instance with coordinates masked by the given unsigned integer.
+    __hostdev__ Coord2 operator&(IndexType n) const { return Coord2(mVec[0] & n, mVec[1] & n); }
+
+    // @brief Return a new instance with coordinates left-shifted by the given unsigned integer.
+    __hostdev__ Coord2 operator<<(IndexType n) const { return Coord2(mVec[0] << n, mVec[1] << n); }
+
+    // @brief Return a new instance with coordinates right-shifted by the given unsigned integer.
+    __hostdev__ Coord2 operator>>(IndexType n) const { return Coord2(mVec[0] >> n, mVec[1] >> n); }
+
+    /// @brief Return true if this Coord is lexicographically less than the given Coord.
+    __hostdev__ bool operator<(const Coord2& rhs) const
+    {
+        return mVec[0] < rhs[0] ? true
+             : mVec[0] > rhs[0] ? false
+             : mVec[1] < rhs[1] ? true : false;
+    }
+
+    /// @brief Return true if this Coord is lexicographically less or equal to the given Coord.
+    __hostdev__ bool operator<=(const Coord2& rhs) const
+    {
+        return mVec[0] < rhs[0] ? true
+             : mVec[0] > rhs[0] ? false
+             : mVec[1] <= rhs[1] ? true : false;
+    }
+
+    // @brief Return true if this Coord is lexicographically greater than the given Coord.
+    __hostdev__ bool operator>(const Coord2& rhs) const
+    {
+        return mVec[0] > rhs[0] ? true
+             : mVec[0] < rhs[0] ? false
+             : mVec[1] > rhs[1] ? true : false;
+    }
+
+    // @brief Return true if this Coord is lexicographically greater or equal to the given Coord.
+    __hostdev__ bool operator>=(const Coord2& rhs) const
+    {
+        return mVec[0] > rhs[0] ? true
+             : mVec[0] < rhs[0] ? false
+             : mVec[1] >= rhs[1] ? true : false;
+    }
+
+    // @brief Return true if the Coord components are identical.
+    __hostdev__ bool   operator==(const Coord2& rhs) const { return mVec[0] == rhs[0] && mVec[1] == rhs[1]; }
+    __hostdev__ bool   operator!=(const Coord2& rhs) const { return mVec[0] != rhs[0] || mVec[1] != rhs[1]; }
+    __hostdev__ Coord2& operator&=(int n)
+    {
+        mVec[0] &= n;
+        mVec[1] &= n;
+        return *this;
+    }
+    __hostdev__ Coord2& operator<<=(uint32_t n)
+    {
+        mVec[0] <<= n;
+        mVec[1] <<= n;
+        return *this;
+    }
+    __hostdev__ Coord2& operator>>=(uint32_t n)
+    {
+        mVec[0] >>= n;
+        mVec[1] >>= n;
+        return *this;
+    }
+    __hostdev__ Coord2& operator+=(int n)
+    {
+        mVec[0] += n;
+        mVec[1] += n;
+        return *this;
+    }
+    __hostdev__ Coord2  operator+(const Coord2& rhs) const { return Coord2(mVec[0] + rhs[0], mVec[1] + rhs[1]); }
+    __hostdev__ Coord2  operator-(const Coord2& rhs) const { return Coord2(mVec[0] - rhs[0], mVec[1] - rhs[1]); }
+    __hostdev__ Coord2  operator-() const { return Coord2(-mVec[0], -mVec[1]); }
+    __hostdev__ Coord2& operator+=(const Coord2& rhs)
+    {
+        mVec[0] += rhs[0];
+        mVec[1] += rhs[1];
+        return *this;
+    }
+    __hostdev__ Coord2& operator-=(const Coord2& rhs)
+    {
+        mVec[0] -= rhs[0];
+        mVec[1] -= rhs[1];
+        return *this;
+    }
+
+    /// @brief Perform a component-wise minimum with the other Coord.
+    __hostdev__ Coord2& minComponent(const Coord2& other)
+    {
+        if (other[0] < mVec[0])
+            mVec[0] = other[0];
+        if (other[1] < mVec[1])
+            mVec[1] = other[1];
+        return *this;
+    }
+
+    /// @brief Perform a component-wise maximum with the other Coord.
+    __hostdev__ Coord2& maxComponent(const Coord2& other)
+    {
+        if (other[0] > mVec[0])
+            mVec[0] = other[0];
+        if (other[1] > mVec[1])
+            mVec[1] = other[1];
+        return *this;
+    }
+#if defined(__CUDACC__) // the following functions only run on the GPU!
+    __device__ inline Coord2& minComponentAtomic(const Coord2& other)
+    {
+        atomicMin(&mVec[0], other[0]);
+        atomicMin(&mVec[1], other[1]);
+        return *this;
+    }
+    __device__ inline Coord2& maxComponentAtomic(const Coord2& other)
+    {
+        atomicMax(&mVec[0], other[0]);
+        atomicMax(&mVec[1], other[1]);
+        return *this;
+    }
+#endif
+
+    __hostdev__ Coord2 offsetBy(ValueType dx, ValueType dy) const
+    {
+        return Coord2(mVec[0] + dx, mVec[1] + dy);
+    }
+
+    __hostdev__ Coord2 offsetBy(ValueType n) const { return this->offsetBy(n, n); }
+
+    /// Return true if any of the components of @a a are smaller than the
+    /// corresponding components of @a b.
+    __hostdev__ static inline bool lessThan(const Coord2& a, const Coord2& b)
+    {
+        return (a[0] < b[0] || a[1] < b[1]);
+    }
+
+    /// @brief Return the largest integer coordinates that are not greater
+    /// than @a xyz (node centered conversion).
+    template<typename Vec2T>
+    __hostdev__ static Coord2 Floor(const Vec2T& xy) { return Coord2(math::Floor(xy[0]), math::Floor(xy[1])); }
+
+    /// @brief Return a single precision floating-point vector of this coordinate
+    __hostdev__ inline Vec2<float> asVec2s() const;
+
+    /// @brief Return a double precision floating-point vector of this coordinate
+    __hostdev__ inline Vec2<double> asVec2d() const;
+
+    // returns a copy of itself, so it mimics the behaviour of Vec3<T>::round()
+    __hostdev__ inline Coord2 round() const { return *this; }
+}; // Coord2 class
+
+// ----------------------------> Vec2 <--------------------------------------
+
+/// @brief A simple vector class with three components, similar to openvdb::math::Vec3
+template<typename T>
+class Vec2
+{
+    T mVec[2];
+
+public:
+    static const int SIZE = 2;
+    static const int size = 2; // in openvdb::math::Tuple
+    using ValueType = T;
+    Vec2() = default;
+    __hostdev__ explicit Vec2(T x)
+        : mVec{x, x}
+    {
+    }
+    __hostdev__ Vec2(T x, T y)
+        : mVec{x, y}
+    {
+    }
+    template<template<class> class Vec2T, class T2>
+    __hostdev__ Vec2(const Vec2T<T2>& v)
+        : mVec{T(v[0]), T(v[1])}
+    {
+        static_assert(Vec2T<T2>::size == size, "expected Vec2T::size==2!");
+    }
+    template<typename T2>
+    __hostdev__ explicit Vec2(const Vec2<T2>& v)
+        : mVec{T(v[0]), T(v[1])}
+    {
+    }
+    __hostdev__ explicit Vec2(const Coord2& ijk)
+        : mVec{T(ijk[0]), T(ijk[1])}
+    {
+    }
+    __hostdev__ bool operator==(const Vec2& rhs) const { return mVec[0] == rhs[0] && mVec[1] == rhs[1]; }
+    __hostdev__ bool operator!=(const Vec2& rhs) const { return mVec[0] != rhs[0] || mVec[1] != rhs[1]; }
+    template<template<class> class Vec2T, class T2>
+    __hostdev__ Vec2& operator=(const Vec2T<T2>& rhs)
+    {
+        static_assert(Vec2T<T2>::size == size, "expected Vec2T::size==2!");
+        mVec[0] = rhs[0];
+        mVec[1] = rhs[1];
+        return *this;
+    }
+    __hostdev__ const T& operator[](int i) const { return mVec[i]; }
+    __hostdev__ T&       operator[](int i) { return mVec[i]; }
+    template<typename Vec2T>
+    __hostdev__ T dot(const Vec2T& v) const { return mVec[0] * v[0] + mVec[1] * v[1]; }
+    __hostdev__ T lengthSqr() const
+    {
+        return mVec[0] * mVec[0] + mVec[1] * mVec[1]; // 3 flops
+    }
+    __hostdev__ T     length() const { return Sqrt(this->lengthSqr()); }
+    __hostdev__ Vec2  operator-() const { return Vec2(-mVec[0], -mVec[1]); }
+    __hostdev__ Vec2  operator*(const Vec2& v) const { return Vec2(mVec[0] * v[0], mVec[1] * v[1]); }
+    __hostdev__ Vec2  operator/(const Vec2& v) const { return Vec2(mVec[0] / v[0], mVec[1] / v[1]); }
+    __hostdev__ Vec2  operator+(const Vec2& v) const { return Vec2(mVec[0] + v[0], mVec[1] + v[1]); }
+    __hostdev__ Vec2  operator-(const Vec2& v) const { return Vec2(mVec[0] - v[0], mVec[1] - v[1]); }
+    __hostdev__ Vec2  operator+(const Coord& ijk) const { return Vec2(mVec[0] + ijk[0], mVec[1] + ijk[1]); }
+    __hostdev__ Vec2  operator-(const Coord& ijk) const { return Vec2(mVec[0] - ijk[0], mVec[1] - ijk[1]); }
+    __hostdev__ Vec2  operator*(const T& s) const { return Vec2(s * mVec[0], s * mVec[1]); }
+    __hostdev__ Vec2  operator/(const T& s) const { return (T(1) / s) * (*this); }
+    __hostdev__ Vec2& operator+=(const Vec2& v)
+    {
+        mVec[0] += v[0];
+        mVec[1] += v[1];
+        return *this;
+    }
+    __hostdev__ Vec2& operator+=(const Coord& ijk)
+    {
+        mVec[0] += T(ijk[0]);
+        mVec[1] += T(ijk[1]);
+        return *this;
+    }
+    __hostdev__ Vec2& operator-=(const Vec2& v)
+    {
+        mVec[0] -= v[0];
+        mVec[1] -= v[1];
+        return *this;
+    }
+    __hostdev__ Vec2& operator-=(const Coord& ijk)
+    {
+        mVec[0] -= T(ijk[0]);
+        mVec[1] -= T(ijk[1]);
+        return *this;
+    }
+    __hostdev__ Vec2& operator*=(const T& s)
+    {
+        mVec[0] *= s;
+        mVec[1] *= s;
+        return *this;
+    }
+    __hostdev__ Vec2& operator/=(const T& s) { return (*this) *= T(1) / s; }
+    __hostdev__ Vec2& normalize() { return (*this) /= this->length(); }
+    /// @brief Perform a component-wise minimum with the other Coord.
+    __hostdev__ Vec2& minComponent(const Vec2& other)
+    {
+        if (other[0] < mVec[0])
+            mVec[0] = other[0];
+        if (other[1] < mVec[1])
+            mVec[1] = other[1];
+        return *this;
+    }
+
+    /// @brief Perform a component-wise maximum with the other Coord.
+    __hostdev__ Vec2& maxComponent(const Vec2& other)
+    {
+        if (other[0] > mVec[0])
+            mVec[0] = other[0];
+        if (other[1] > mVec[1])
+            mVec[1] = other[1];
+        return *this;
+    }
+    /// @brief Return the smallest vector component
+    __hostdev__ ValueType min() const
+    {
+        return mVec[0] < mVec[1] ? mVec[0] : mVec[1];
+    }
+    /// @brief Return the largest vector component
+    __hostdev__ ValueType max() const
+    {
+        return mVec[0] > mVec[1] ? mVec[0] : mVec[1];
+    }
+    /// @brief Round each component if this Vec<T> up to its integer value
+    /// @return Return an integer Coord
+    __hostdev__ Coord2 floor() const { return Coord2(Floor(mVec[0]), Floor(mVec[1])); }
+    /// @brief Round each component if this Vec<T> down to its integer value
+    /// @return Return an integer Coord
+    __hostdev__ Coord2 ceil() const { return Coord2(Ceil(mVec[0]), Ceil(mVec[1])); }
+    /// @brief Round each component if this Vec<T> to its closest integer value
+    /// @return Return an integer Coord
+    __hostdev__ Coord2 round() const
+    {
+        if constexpr(util::is_same<T, float>::value) {
+            return Coord2(Floor(mVec[0] + 0.5f), Floor(mVec[1] + 0.5f));
+        } else if constexpr(util::is_same<T, int>::value) {
+            return Coord2(mVec[0], mVec[1]);
+        } else {
+            return Coord2(Floor(mVec[0] + 0.5), Floor(mVec[1] + 0.5));
+        }
+    }
+
+    /// @brief return a non-const raw constant pointer to array of three vector components
+    __hostdev__ T* asPointer() { return mVec; }
+    /// @brief return a const raw constant pointer to array of three vector components
+    __hostdev__ const T* asPointer() const { return mVec; }
+}; // Vec2<T>
+
+template<typename T1, typename T2>
+__hostdev__ inline Vec2<T2> operator*(T1 scalar, const Vec2<T2>& vec)
+{
+    return Vec2<T2>(scalar * vec[0], scalar * vec[1]);
+}
+template<typename T1, typename T2>
+__hostdev__ inline Vec2<T2> operator/(T1 scalar, const Vec2<T2>& vec)
+{
+    return Vec2<T2>(scalar / vec[0], scalar / vec[1]);
+}
+
+/// @brief Return a single precision floating-point vector of this coordinate
+__hostdev__ inline Vec2<float> Coord2::asVec2s() const
+{
+    return Vec2<float>(float(mVec[0]), float(mVec[1]));
+}
+
+/// @brief Return a double precision floating-point vector of this coordinate
+__hostdev__ inline Vec2<double> Coord2::asVec2d() const
+{
+    return Vec2<double>(double(mVec[0]), double(mVec[1]));
+}
+
+
+// Matrix base class
+template<typename T, int ROWS, int COLS>
+class MatBase {
+protected:
+    T mData[ROWS * COLS];  // 1D array storage
+
+    static constexpr int rows() { return ROWS; }
+    static constexpr int cols() { return COLS; }
+    static constexpr int size() { return ROWS * COLS; }
+
+public:
+    MatBase() = default;
+
+    template<typename S>
+    __hostdev__ MatBase(S* array) {
+        for (int i = 0; i < ROWS * COLS; ++i) {
+            mData[i] = static_cast<T>(array[i]);
+        }
+    }
+
+    // 2D array access
+    __hostdev__ T* operator[](int row) {
+        return &mData[row * COLS];
+    }
+    __hostdev__ const T* operator[](int row) const {
+        return &mData[row * COLS];
+    }
+};
+
+// Forward declarations
+template<typename T> class Mat2;
+template<typename T> class Mat2x3;
+template<typename T> class Mat3x2;
+template<typename T> class Mat3;
+template<typename T> class Mat4;
+
+
+
+template <typename T>
+class Mat2 : public MatBase<T, 2, 2> {
+    using Base = MatBase<T, 2, 2>;
+public:
+    Mat2() = default;
+    /// @brief Constructor given individual array elements, the ordering is in row major form:
+    /** @verbatim
+        a b
+        c d
+        @endverbatim */
+    __hostdev__ Mat2(T a, T b, T c, T d) {
+        this->mData[0] = a; this->mData[1] = b;
+        this->mData[2] = c; this->mData[3] = d;
+    }
+
+    /// @brief Constructor given array of elements, the ordering is in row major form
+    template<typename Source>
+    __hostdev__ Mat2(Source* array) : Base(array) {}
+
+    __hostdev__ Mat2  operator-() const { return Mat2(-(*this)[0][0], -(*this)[0][1], -(*this)[1][0], -(*this)[1][1]); }
+
+    /// @brief Multiply by 2x2 matrix @a m and return the resulting matrix.
+    __hostdev__ Mat2<T> operator*(const Mat2<T>& m) const {
+        return Mat2<T>(
+            (*this)[0][0] * m[0][0] + (*this)[0][1] * m[1][0],
+            (*this)[0][0] * m[0][1] + (*this)[0][1] * m[1][1],
+            (*this)[1][0] * m[0][0] + (*this)[1][1] * m[1][0],
+            (*this)[1][0] * m[0][1] + (*this)[1][1] * m[1][1]
+        );
+    }
+
+    /// @brief Add each element of the given matrix to the corresponding element of this matrix.
+    __hostdev__ Mat2<T>& operator+=(const Mat2<T>& m) {
+        (*this)[0][0] += m[0][0];
+        (*this)[0][1] += m[0][1];
+        (*this)[1][0] += m[1][0];
+        (*this)[1][1] += m[1][1];
+        return *this;
+    }
+
+    /// @brief returns transpose of this
+    __hostdev__ Mat2<T> transpose() const {
+        return Mat2<T>((*this)[0][0], (*this)[1][0], (*this)[0][1], (*this)[1][1]);
+    }
+
+    /// @brief returns inverse of this
+    __hostdev__ Mat2<T> inverse() const {
+        T det = (*this)[0][0] * (*this)[1][1] - (*this)[0][1] * (*this)[1][0];
+        if (isApproxZero(det)) {
+            return Mat2<T>();
+        }
+        T invDet   = 1.f / det;
+        return Mat2<T>((*this)[1][1] * invDet, -(*this)[0][1] * invDet, -(*this)[1][0] * invDet, (*this)[0][0] * invDet);
+    }
+};
+
+template <typename T>
+class Mat2x3 : public MatBase<T, 2, 3> {
+    using Base = MatBase<T, 2, 3>;
+public:
+    Mat2x3() = default;
+    /// @brief Constructor given individual array elements, the ordering is in row major form:
+    /** @verbatim
+        a b c
+        d e f
+        @endverbatim */
+    __hostdev__ Mat2x3(T a, T b, T c, T d, T e, T f) {
+        this->mData[0] = a; this->mData[1] = b; this->mData[2] = c;
+        this->mData[3] = d; this->mData[4] = e; this->mData[5] = f;
+    }
+
+    /// @brief Constructor given array of elements, the ordering is in row major form
+    template<typename Source>
+    __hostdev__ Mat2x3(Source* array) : Base(array) {}
+
+
+    /// @brief Add two matrices and return the resulting matrix.
+    __hostdev__ Mat2x3<T> operator+(const Mat2x3<T>& m) const {
+        return Mat2x3<T>(
+            (*this)[0][0] + m[0][0], (*this)[0][1] + m[0][1], (*this)[0][2] + m[0][2],
+            (*this)[1][0] + m[1][0], (*this)[1][1] + m[1][1], (*this)[1][2] + m[1][2]
+        );
+    }
+
+    /// @brief Add a 2x3 matrix to this matrix.
+    __hostdev__ Mat2x3<T>& operator+=(const Mat2x3<T>& m) {
+        (*this)[0][0] += m[0][0]; (*this)[0][1] += m[0][1]; (*this)[0][2] += m[0][2];
+        (*this)[1][0] += m[1][0]; (*this)[1][1] += m[1][1]; (*this)[1][2] += m[1][2];
+        return *this;
+    }
+
+    /// @brief returns transpose of this
+    __hostdev__ Mat3x2<T> transpose() const {
+        return Mat3x2<T>(
+            (*this)[0][0], (*this)[1][0],  // First row
+            (*this)[0][1], (*this)[1][1],  // Second row
+            (*this)[0][2], (*this)[1][2]   // Third row
+        );
+    }
+};
+
+template <typename T>
+class Mat3x2: public MatBase<T, 3, 2> {
+    using Base = MatBase<T, 3, 2>;
+public:
+    Mat3x2() = default;
+
+    /// @brief Constructor given individual array elements, the ordering is in row major form:
+    /** @verbatim
+        a b
+        c d
+        e f
+        @endverbatim */
+    template<typename Source>
+    __hostdev__ Mat3x2(Source a, Source b, Source c, Source d, Source e, Source f)
+    {
+        this->mData[0] = a; this->mData[1] = b;
+        this->mData[2] = c; this->mData[3] = d;
+        this->mData[4] = e; this->mData[5] = f;
+    }
+
+    /// @brief Constructor given array of elements, the ordering is in row major form
+    template<typename Source>
+    __hostdev__ Mat3x2(Source *a): Base(a) {}
+
+    /// @brief returns transpose of this
+    __hostdev__ Mat2x3<T> transpose() const {
+        return Mat2x3<T>((*this)[0][0], (*this)[1][0], (*this)[2][0], // First row
+                   (*this)[0][1], (*this)[1][1], (*this)[2][1]); // Second row
+    }
+
+};
+
+template <typename T>
+class Mat3 : public MatBase<T, 3, 3> {
+    using Base = MatBase<T, 3, 3>;
+public:
+    Mat3() = default;
+
+    /// @brief Constructor given individual array elements, the ordering is in row major form:
+    /** @verbatim
+        a b c
+        d e f
+        g h i
+        @endverbatim */
+    template<typename Source>
+    __hostdev__ Mat3(Source a, Source b, Source c,
+         Source d, Source e, Source f,
+         Source g, Source h, Source i)
+    {
+        this->mData[0] = a; this->mData[1] = b; this->mData[2] = c;
+        this->mData[3] = d; this->mData[4] = e; this->mData[5] = f;
+        this->mData[6] = g; this->mData[7] = h; this->mData[8] = i;
+    }
+
+    /// @brief Constructor given array of elements, the ordering is in row major form
+    template<typename Source>
+    __hostdev__ Mat3(Source *a): Base(a) {}
+
+
+    /// @brief Add two matrices and return the resulting matrix.
+    __hostdev__ Mat3<T> operator+(const Mat3<T>& m) const {
+        return Mat3<T>(
+            (*this)[0][0] + m[0][0], (*this)[0][1] + m[0][1], (*this)[0][2] + m[0][2],
+            (*this)[1][0] + m[1][0], (*this)[1][1] + m[1][1], (*this)[1][2] + m[1][2],
+            (*this)[2][0] + m[2][0], (*this)[2][1] + m[2][1], (*this)[2][2] + m[2][2]
+        );
+    }
+
+    /// @brief Multiply by @a v and return the resulting vector.
+    __hostdev__ Vec3<T> operator*(const Vec3<T>& v) const {
+        return Vec3<T>(
+            (*this)[0][0] * v[0] + (*this)[0][1] * v[1] + (*this)[0][2] * v[2],
+            (*this)[1][0] * v[0] + (*this)[1][1] * v[1] + (*this)[1][2] * v[2],
+            (*this)[2][0] * v[0] + (*this)[2][1] * v[1] + (*this)[2][2] * v[2]
+        );
+    }
+
+    /// @brief Multiply by 3x3 matrix @a m and return the resulting matrix.
+    __hostdev__ Mat3<T> operator*(const Mat3<T>& m) const {
+        return Mat3<T>(
+            (*this)[0][0] * m[0][0] + (*this)[0][1] * m[1][0] + (*this)[0][2] * m[2][0],
+            (*this)[0][0] * m[0][1] + (*this)[0][1] * m[1][1] + (*this)[0][2] * m[2][1],
+            (*this)[0][0] * m[0][2] + (*this)[0][1] * m[1][2] + (*this)[0][2] * m[2][2],
+            (*this)[1][0] * m[0][0] + (*this)[1][1] * m[1][0] + (*this)[1][2] * m[2][0],
+            (*this)[1][0] * m[0][1] + (*this)[1][1] * m[1][1] + (*this)[1][2] * m[2][1],
+            (*this)[1][0] * m[0][2] + (*this)[1][1] * m[1][2] + (*this)[1][2] * m[2][2],
+            (*this)[2][0] * m[0][0] + (*this)[2][1] * m[1][0] + (*this)[2][2] * m[2][0],
+            (*this)[2][0] * m[0][1] + (*this)[2][1] * m[1][1] + (*this)[2][2] * m[2][1],
+            (*this)[2][0] * m[0][2] + (*this)[2][1] * m[1][2] + (*this)[2][2] * m[2][2]
+        );
+    }
+
+    /// @brief Add each element of the given matrix to the corresponding element of this matrix.
+    __hostdev__ Mat3<T>& operator+=(const Mat3<T>& m) {
+        (*this)[0][0] += m[0][0]; (*this)[0][1] += m[0][1]; (*this)[0][2] += m[0][2];
+        (*this)[1][0] += m[1][0]; (*this)[1][1] += m[1][1]; (*this)[1][2] += m[1][2];
+        (*this)[2][0] += m[2][0]; (*this)[2][1] += m[2][1]; (*this)[2][2] += m[2][2];
+        return *this;
+    }
+
+    /// @brief returns transpose of this
+    __hostdev__ Mat3 transpose() const {
+        return Mat3((*this)[0][0], (*this)[1][0], (*this)[2][0],
+                   (*this)[0][1], (*this)[1][1], (*this)[2][1],
+                   (*this)[0][2], (*this)[1][2], (*this)[2][2]);
+    }
+};
+
+template <typename T>
+class Mat4 : public MatBase<T, 4, 4> {
+    using Base = MatBase<T, 4, 4>;
+public:
+    Mat4() = default;
+
+    /// @brief Constructor given individual array elements, the ordering is in row major form:
+    /** @verbatim
+        a b c d
+        e f g h
+        i j k l
+        m n o p
+        @endverbatim */
+    template<typename Source>
+    __hostdev__ Mat4(Source a, Source b, Source c, Source d,
+         Source e, Source f, Source g, Source h,
+         Source i, Source j, Source k, Source l,
+         Source m, Source n, Source o, Source p)
+    {
+        this->mData[0] = a; this->mData[1] = b; this->mData[2] = c; this->mData[3] = d;
+        this->mData[4] = e; this->mData[5] = f; this->mData[6] = g; this->mData[7] = h;
+        this->mData[8] = i; this->mData[9] = j; this->mData[10] = k; this->mData[11] = l;
+        this->mData[12] = m; this->mData[13] = n; this->mData[14] = o; this->mData[15] = p;
+    }
+
+    /// @brief Constructor given array of elements, the ordering is in row major form
+    template<typename Source>
+    __hostdev__ Mat4(Source *a): Base(a) {}
+
+    /// @brief returns transpose of this
+    __hostdev__ Mat4 transpose() const {
+        return Mat4((*this)[0][0], (*this)[1][0], (*this)[2][0], (*this)[3][0],
+                   (*this)[0][1], (*this)[1][1], (*this)[2][1], (*this)[3][1],
+                   (*this)[0][2], (*this)[1][2], (*this)[2][2], (*this)[3][2],
+                   (*this)[0][3], (*this)[1][3], (*this)[2][3], (*this)[3][3]);
+    }
+};
+
+/// @brief Multiply a scalar by a 2x2 matrix, result is a 2x2 matrix
+template<typename T>
+__hostdev__ Mat2<T> operator*(const T& s, const Mat2<T>& m) {
+    return Mat2<T>(m[0][0] * s, m[0][1] * s, m[1][0] * s, m[1][1] * s);
+}
+
+/// @brief Multiply a 2x3 matrix by a 3x2 matrix, result is a 2x2 matrix
+template<typename T>
+__hostdev__ Mat2<T> operator*(const Mat2x3<T>& lhs, const Mat3x2<T>& rhs) {
+    return Mat2<T>(
+        // First row
+        lhs[0][0] * rhs[0][0] + lhs[0][1] * rhs[1][0] + lhs[0][2] * rhs[2][0],    // [0][0]
+        lhs[0][0] * rhs[0][1] + lhs[0][1] * rhs[1][1] + lhs[0][2] * rhs[2][1],    // [0][1]
+
+        // Second row
+        lhs[1][0] * rhs[0][0] + lhs[1][1] * rhs[1][0] + lhs[1][2] * rhs[2][0],    // [1][0]
+        lhs[1][0] * rhs[0][1] + lhs[1][1] * rhs[1][1] + lhs[1][2] * rhs[2][1]     // [1][1]
+    );
+}
+/// @brief Multiply a 3x3 matrix by a 2x3 matrix, result is a 2x3 matrix
+template<typename T>
+__hostdev__ Mat2x3<T> operator*(const Mat3<T>& lhs, const Mat2x3<T>& rhs) {
+    return Mat2x3<T>(
+        // First row
+        lhs[0][0] * rhs[0][0] + lhs[0][1] * rhs[1][0],
+        lhs[0][0] * rhs[0][1] + lhs[0][1] * rhs[1][1],
+        lhs[0][0] * rhs[0][2] + lhs[0][1] * rhs[1][2],
+
+        // Second row
+        lhs[1][0] * rhs[0][0] + lhs[1][1] * rhs[1][0],
+        lhs[1][0] * rhs[0][1] + lhs[1][1] * rhs[1][1],
+        lhs[1][0] * rhs[0][2] + lhs[1][1] * rhs[1][2]
+    );
+}
+/// @brief Multiply a 2x3 matrix by a 3x3 matrix, result is a 2x3 matrix
+template<typename T>
+__hostdev__ Mat2x3<T> operator*(const Mat2x3<T>& lhs, const Mat3<T>& rhs) {
+    return Mat2x3<T>(
+        // First row (3 elements)
+        lhs[0][0] * rhs[0][0] + lhs[0][1] * rhs[1][0] + lhs[0][2] * rhs[2][0],
+        lhs[0][0] * rhs[0][1] + lhs[0][1] * rhs[1][1] + lhs[0][2] * rhs[2][1],
+        lhs[0][0] * rhs[0][2] + lhs[0][1] * rhs[1][2] + lhs[0][2] * rhs[2][2],
+
+        // Second row (3 elements)
+        lhs[1][0] * rhs[0][0] + lhs[1][1] * rhs[1][0] + lhs[1][2] * rhs[2][0],
+        lhs[1][0] * rhs[0][1] + lhs[1][1] * rhs[1][1] + lhs[1][2] * rhs[2][1],
+        lhs[1][0] * rhs[0][2] + lhs[1][1] * rhs[1][2] + lhs[1][2] * rhs[2][2]
+    );
+}
+/// @brief Multiply a 3x2 matrix by a 2x2 matrix, result is a 3x2 matrix
+template<typename T>
+__hostdev__ Mat3x2<T> operator*(const Mat3x2<T>& lhs, const Mat2<T>& rhs) {
+    return Mat3x2<T>(
+        lhs[0][0] * rhs[0][0] + lhs[0][1] * rhs[1][0],
+        lhs[0][0] * rhs[0][1] + lhs[0][1] * rhs[1][1],
+        lhs[1][0] * rhs[0][0] + lhs[1][1] * rhs[1][0],
+        lhs[1][0] * rhs[0][1] + lhs[1][1] * rhs[1][1],
+        lhs[2][0] * rhs[0][0] + lhs[2][1] * rhs[1][0],
+        lhs[2][0] * rhs[0][1] + lhs[2][1] * rhs[1][1]
+    );
+}
+/// @brief Multiply a 2x2 matrix by a 2x3 matrix, result is a 2x3 matrix
+template<typename T>
+__hostdev__ Mat2x3<T> operator*(const Mat2<T>& lhs, const Mat2x3<T>& rhs) {
+    return Mat2x3<T>(
+        // First row (3 elements)
+        lhs[0][0] * rhs[0][0] + lhs[0][1] * rhs[1][0],
+        lhs[0][0] * rhs[0][1] + lhs[0][1] * rhs[1][1],
+        lhs[0][0] * rhs[0][2] + lhs[0][1] * rhs[1][2],
+
+        // Second row (3 elements)
+        lhs[1][0] * rhs[0][0] + lhs[1][1] * rhs[1][0],
+        lhs[1][0] * rhs[0][1] + lhs[1][1] * rhs[1][1],
+        lhs[1][0] * rhs[0][2] + lhs[1][1] * rhs[1][2]
+    );
+}
+/// @brief Multiply a 3x2 matrix by a 2x3 matrix, result is a 3x3 matrix
+template<typename T>
+__hostdev__ Mat3<T> operator*(const Mat3x2<T>& lhs, const Mat2x3<T>& rhs) {
+    return Mat3<T>(
+        lhs[0][0] * rhs[0][0] + lhs[0][1] * rhs[1][0],
+        lhs[0][0] * rhs[0][1] + lhs[0][1] * rhs[1][1],
+        lhs[0][0] * rhs[0][2] + lhs[0][1] * rhs[1][2],
+
+        lhs[1][0] * rhs[0][0] + lhs[1][1] * rhs[1][0],
+        lhs[1][0] * rhs[0][1] + lhs[1][1] * rhs[1][1],
+        lhs[1][0] * rhs[0][2] + lhs[1][1] * rhs[1][2],
+
+        lhs[2][0] * rhs[0][0] + lhs[2][1] * rhs[1][0],
+        lhs[2][0] * rhs[0][1] + lhs[2][1] * rhs[1][1],
+        lhs[2][0] * rhs[0][2] + lhs[2][1] * rhs[1][2]
+    );
+}
 // ----------------------------> Vec3 <--------------------------------------

 /// @brief A simple vector class with three components, similar to openvdb::math::Vec3
@@ -634,6 +1412,14 @@ public:
                    mVec[2] * v[0] - mVec[0] * v[2],
                    mVec[0] * v[1] - mVec[1] * v[0]);
    }
+    /// @brief Outer product of a 3x1 vector and a 1x3 vector, result is a 3x3 matrix
+    template<typename Vec3T>
+    __hostdev__ Mat3<ValueType> outer(const Vec3T& v) const
+    {
+        return Mat3<ValueType>(mVec[0] * v[0], mVec[0] * v[1], mVec[0] * v[2],
+                    mVec[1] * v[0], mVec[1] * v[1], mVec[1] * v[2],
+                    mVec[2] * v[0], mVec[2] * v[1], mVec[2] * v[2]);
+    }
    __hostdev__ T lengthSqr() const
    {
        return mVec[0] * mVec[0] + mVec[1] * mVec[1] + mVec[2] * mVec[2]; // 5 flops
@@ -892,6 +1678,19 @@ __hostdev__ inline Vec4<T2> operator/(T1 scalar, const Vec4<T2>& vec)
 {
    return Vec4<T2>(scalar / vec[0], scalar / vec[1], scalar / vec[2], scalar / vec[3]);
 }
+/// @brief Return the matrix vector product of a 4x4 matrix and a 4d vector
+/// @param m 4x4 matrix
+/// @param v 4d vector
+/// @return result of matrix-vector multiplication, i.e. m x v
+template <typename T>
+__hostdev__ inline Vec4<T> operator*(const Mat4<T>& m, const Vec4<T>& v) {
+    return Vec4<T>(
+        m[0][0] * v[0] + m[0][1] * v[1] + m[0][2] * v[2] + m[0][3] * v[3],
+        m[1][0] * v[0] + m[1][1] * v[1] + m[1][2] * v[2] + m[1][3] * v[3],
+        m[2][0] * v[0] + m[2][1] * v[1] + m[2][2] * v[2] + m[2][3] * v[3],
+        m[3][0] * v[0] + m[3][1] * v[1] + m[3][2] * v[2] + m[3][3] * v[3]
+    );
+}

 // ----------------------------> matMult <--------------------------------------

@@ -1042,10 +1841,10 @@ struct BaseBBox
        return *this;
    }

-    //__hostdev__ BaseBBox expandBy(typename Vec3T::ValueType padding) const
-    //{
-    //    return BaseBBox(mCoord[0].offsetBy(-padding),mCoord[1].offsetBy(padding));
-    //}
+//    __hostdev__ BaseBBox expandBy(typename Vec3T::ValueType padding) const
+//    {
+//        return BaseBBox(mCoord[0].offsetBy(-padding),mCoord[1].offsetBy(padding));
+//    }
    __hostdev__ bool isInside(const Vec3T& xyz)
    {
        if (xyz[0] < mCoord[0][0] || xyz[1] < mCoord[0][1] || xyz[2] < mCoord[0][2])
@@ -0,0 +1,781 @@
+// Copyright Contributors to the OpenVDB Project
+// SPDX-License-Identifier: Apache-2.0
+
+/*!
+    \file   nanovdb/putil/IO.h
+
+    \author Andrew Reidmeyer
+
+    \brief  This file contains IO utilities for PNanoVDB.
+*/
+
+#ifndef NANOVDB_PUTILS_IO_H_HAS_BEEN_INCLUDED
+#define NANOVDB_PUTILS_IO_H_HAS_BEEN_INCLUDED
+
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#if defined(__CUDACC__)
+#define PNANOVDB_IO_INLINE static __host__ __device__
+#elif defined(_WIN32)
+#define PNANOVDB_IO_INLINE static inline
+#else
+#define PNANOVDB_IO_INLINE static inline
+#endif
+
+// needs to match NANOVDB_DATA_ALIGNMENT from nanovdb/NanoVDB.h
+#define PNANOVDB_DATA_ALIGNMENT 32
+PNANOVDB_IO_INLINE void* pnanovdb_aligned_malloc(size_t alignment, size_t size)
+{
+    size_t aligned_size = size + alignment + sizeof(void*);
+    void* alloc_ptr = malloc(aligned_size);
+    void* ptr = 0;
+    if (alloc_ptr)
+    {
+        ptr = (void*)(alignment * (((size_t)alloc_ptr + sizeof(void*) + alignment - 1u) / alignment));
+        void** header_ptr = (void**)((size_t)ptr - sizeof(void*));
+        *header_ptr = alloc_ptr;
+    }
+    return ptr;
+}
+PNANOVDB_IO_INLINE void  pnanovdb_aligned_free(void* ptr)
+{
+    if (ptr)
+    {
+        void** header_ptr = (void**)((size_t)ptr - sizeof(void*));
+        free(*header_ptr);
+    }
+}
+
+// abstraction for file-like read/write
+typedef struct pnanovdb_io_buf_t
+{
+    void* file;
+    void(*read)(void* file, uint64_t offset, void* dst_data, uint64_t size);
+    void(*write)(void* file, uint64_t offset, void* src_data, uint64_t size);
+}pnanovdb_io_buf_t;
+
+// implementation for fread/fwrite
+PNANOVDB_IO_INLINE void pnanovdb_io_buf_fread(void* file, uint64_t offset, void* dst_data, uint64_t size)
+{
+    fseek((FILE*)file, offset, SEEK_SET);
+    fread(dst_data, size, 1u, (FILE*)file);
+}
+PNANOVDB_IO_INLINE void pnanovdb_io_buf_fwrite(void* file, uint64_t offset, void* src_data, uint64_t size)
+{
+    fseek((FILE*)file, offset, SEEK_SET);
+    fwrite(src_data, size, 1u, (FILE*)file);
+}
+
+// implementation for dynamic buffer
+typedef struct pnanovdb_io_dynamic_buf_t
+{
+    void* data;
+    uint64_t size;
+    uint64_t capacity;
+}pnanovdb_io_dynamic_buf_t;
+PNANOVDB_IO_INLINE void pnanovdb_io_dynamic_buf_read(void* file, uint64_t offset, void* dst_data, uint64_t size)
+{
+    pnanovdb_io_dynamic_buf_t* ptr = (pnanovdb_io_dynamic_buf_t*)file;
+    uint8_t* dst8 = (uint8_t*)dst_data;
+    uint8_t* src8 = (uint8_t*)ptr->data;
+    uint64_t size_in_bounds = size;
+    if (offset >= ptr->size)
+    {
+        size_in_bounds = 0;
+    }
+    else if (offset + size > ptr->size)
+    {
+        size_in_bounds = ptr->size - offset;
+    }
+    memcpy(dst8, src8 + offset, size_in_bounds);
+    memset(dst8 + size_in_bounds, 0, size - size_in_bounds);
+}
+PNANOVDB_IO_INLINE void pnanovdb_io_dynamic_buf_write(void* file, uint64_t offset, void* src_data, uint64_t size)
+{
+    pnanovdb_io_dynamic_buf_t* ptr = (pnanovdb_io_dynamic_buf_t*)file;
+    // grow capacity as needed
+    if (offset + size > ptr->capacity)
+    {
+        uint64_t new_capacity = 1024u;
+        while (new_capacity < offset + size)
+        {
+            new_capacity = (new_capacity << 1u) - (new_capacity >> 1u);
+        }
+        void* new_data = pnanovdb_aligned_malloc(PNANOVDB_DATA_ALIGNMENT, new_capacity);
+        memcpy(new_data, ptr->data, ptr->size);
+        if (ptr->capacity != 0u)
+        {
+            pnanovdb_aligned_free(ptr->data);
+        }
+        ptr->data = new_data;
+        ptr->capacity = new_capacity;
+    }
+    // expand size and zero initialize as needed
+    if (offset + size > ptr->size)
+    {
+        uint8_t* data8 = (uint8_t*)ptr->data;
+        memset(data8 + ptr->size, 0, offset + size - ptr->size);
+        ptr->size = offset + size;
+    }
+    // write data
+    uint8_t* src8 = (uint8_t*)src_data;
+    uint8_t* dst8 = (uint8_t*)ptr->data;
+    memcpy(dst8 + offset, src8, size);
+}
+
+#if defined(PNANOVDB_IO_BUF_CUSTOM)
+#define PNANOVDB_BUF_CUSTOM
+
+#define pnanovdb_buf_t pnanovdb_io_buf_t*
+PNANOVDB_IO_INLINE uint32_t pnanovdb_buf_read_uint32(pnanovdb_buf_t buf, uint64_t byte_offset)
+{
+    uint32_t ret = 0u;
+    buf->read(buf->file, byte_offset, &ret, 4u);
+    return ret;
+}
+PNANOVDB_IO_INLINE uint64_t pnanovdb_buf_read_uint64(pnanovdb_buf_t buf, uint64_t byte_offset)
+{
+    uint64_t ret = 0u;
+    buf->read(buf->file, byte_offset, &ret, 8u);
+    return ret;
+}
+PNANOVDB_IO_INLINE void pnanovdb_buf_write_uint32(pnanovdb_buf_t buf, uint64_t byte_offset, uint32_t value)
+{
+    buf->write(buf->file, byte_offset, &value, 4u);
+}
+PNANOVDB_IO_INLINE void pnanovdb_buf_write_uint64(pnanovdb_buf_t buf, uint64_t byte_offset, uint64_t value)
+{
+    buf->write(buf->file, byte_offset, &value, 8u);
+}
+#define pnanovdb_grid_type_t uint32_t
+#define PNANOVDB_GRID_TYPE_GET(grid_typeIn, nameIn) pnanovdb_grid_type_constants[grid_typeIn].nameIn
+
+#endif
+
+#ifndef PNANOVDB_C
+#define PNANOVDB_C
+#endif
+#include <nanovdb/PNanoVDB.h>
+
+typedef struct pnanovdb_io_fileheader_t
+{
+    pnanovdb_uint64_t magic;                 // 8 bytes,        0
+    pnanovdb_uint32_t version;                 // 4 bytes,        8
+    pnanovdb_uint32_t grid_count_and_codec; // 4 bytes,        12
+}pnanovdb_io_fileheader_t;
+
+#define PNANOVDB_IO_FILEHEADER_SIZE 16
+
+typedef struct pnanovdb_io_filemetadata_t
+{
+    pnanovdb_uint64_t grid_size;            // 8 bytes,     0
+    pnanovdb_uint64_t file_size;            // 8 bytes,     8
+    pnanovdb_uint64_t name_key;                // 8 bytes,     16
+    pnanovdb_uint64_t voxel_count;            // 8 bytes,     24
+    pnanovdb_uint32_t grid_type;            // 4 bytes,     32
+    pnanovdb_uint32_t grid_class;            // 4 bytes,     36
+    double world_bbox[6u];                    // 48 bytes,     40
+    pnanovdb_coord_t index_bbox_min;        // 12 bytes,    88
+    pnanovdb_coord_t index_bbox_max;        // 12 bytes,    100
+    double voxel_size[3u];                    // 24 bytes,    112
+    pnanovdb_uint32_t name_size;            // 4 bytes,        136
+    pnanovdb_uint32_t node_count_leaf;        // 4 bytes,        140
+    pnanovdb_uint32_t node_count_lower;        // 4 bytes,        144
+    pnanovdb_uint32_t node_count_upper;        // 4 bytes,        148
+    pnanovdb_uint32_t node_count_root;        // 4 bytes,        152
+    pnanovdb_uint32_t tile_count_lower;        // 4 bytes,        156
+    pnanovdb_uint32_t tile_count_upper;        // 4 bytes,        160
+    pnanovdb_uint32_t tile_count_root;        // 4 bytes,        164
+    pnanovdb_uint32_t codec_and_padding;    // 4 bytes,        168
+    pnanovdb_uint32_t version;                // 4 bytes,        172
+}pnanovdb_io_filemetadata_t;
+
+#define PNANOVDB_IO_FILEMETADATA_SIZE 176
+
+#if defined(PNANOVDB_IO_BUF_CUSTOM)
+
+PNANOVDB_IO_INLINE pnanovdb_grid_handle_t pnanovdb_io_fread_fileheader(
+    pnanovdb_buf_t file,
+    pnanovdb_io_fileheader_t* dst_header,
+    pnanovdb_io_filemetadata_t* dst_metadatas,
+    pnanovdb_uint64_t dst_metadata_count)
+{
+    pnanovdb_grid_t local_grid = {};
+    file->read(file->file, 0u, &local_grid, PNANOVDB_GRID_SIZE);
+    pnanovdb_bool_t is_raw_grid = PNANOVDB_FALSE;
+    if (local_grid.magic == PNANOVDB_MAGIC_GRID ||
+        local_grid.data2 == PNANOVDB_MAGIC_GRID)
+    {
+        is_raw_grid = PNANOVDB_TRUE;
+    }
+    if (is_raw_grid)
+    {
+        pnanovdb_grid_handle_t raw_grid = {pnanovdb_address_null()};
+        return raw_grid;
+    }
+
+    pnanovdb_uint64_t offset = 0u;
+    pnanovdb_io_fileheader_t local_header = {};
+    file->read(file->file, offset, &local_header, PNANOVDB_IO_FILEHEADER_SIZE);
+    if (dst_header)
+    {
+        *dst_header = local_header;
+    }
+    offset += PNANOVDB_IO_FILEHEADER_SIZE;
+    pnanovdb_uint32_t grid_count = local_header.grid_count_and_codec & 0xFFFF;
+    for (uint32_t idx = 0u; idx < grid_count; idx++)
+    {
+        pnanovdb_io_filemetadata_t local_metadata = {};
+        file->read(file->file, offset, &local_metadata, PNANOVDB_IO_FILEMETADATA_SIZE);
+        if (idx < dst_metadata_count)
+        {
+            dst_metadatas[idx] = local_metadata;
+        }
+        offset += PNANOVDB_IO_FILEMETADATA_SIZE;
+        // skip past name
+        offset += local_metadata.name_size;
+    }
+    pnanovdb_grid_handle_t first_grid = {pnanovdb_address_offset64(pnanovdb_address_null(), offset)};
+    return first_grid;
+}
+
+PNANOVDB_IO_INLINE void pnanovdb_io_memcpy(
+    pnanovdb_buf_t dst_buf, pnanovdb_address_t dst_addr,
+    pnanovdb_buf_t src_buf, pnanovdb_address_t src_addr,
+    pnanovdb_uint64_t num_bytes)
+{
+    pnanovdb_uint64_t num_words = (num_bytes >> 2u);
+    for (pnanovdb_uint64_t word_idx = 0u; word_idx < num_words; word_idx++)
+    {
+        pnanovdb_uint32_t val = pnanovdb_read_uint32(src_buf, pnanovdb_address_offset_product(src_addr, word_idx, 4u));
+        pnanovdb_write_uint32(dst_buf, pnanovdb_address_offset_product(dst_addr, word_idx, 4u), val);
+    }
+}
+
+PNANOVDB_IO_INLINE pnanovdb_uint32_t pnanovdb_io_get_leaf_size(pnanovdb_uint32_t grid_type, pnanovdb_buf_t buf, pnanovdb_leaf_handle_t leaf)
+{
+    pnanovdb_uint32_t size = PNANOVDB_GRID_TYPE_GET(grid_type, leaf_size);
+    if (grid_type == PNANOVDB_GRID_TYPE_FPN)
+    {
+        pnanovdb_uint32_t flags = pnanovdb_leaf_get_bbox_dif_and_flags(buf, leaf) >> 24u;
+        pnanovdb_uint32_t value_log_bits = flags >> 5;
+        pnanovdb_uint32_t value_bits = 1u << value_log_bits;
+        pnanovdb_uint32_t table_bytes = value_bits << (9u - 3u);
+        size += table_bytes;
+    }
+    return size;
+}
+
+PNANOVDB_IO_INLINE void pnanovdb_io_copy_subregion(
+    pnanovdb_buf_t src_buf,
+    pnanovdb_grid_handle_t src_grid,
+    pnanovdb_buf_t dst_buf,
+    pnanovdb_grid_handle_t dst_grid,
+    pnanovdb_coord_t ijk_min,
+    pnanovdb_coord_t ijk_max)
+{
+    // leaf precise, not voxel precise for now
+    ijk_min.x &= ~7;
+    ijk_min.y &= ~7;
+    ijk_min.z &= ~7;
+    ijk_max.x |= 7;
+    ijk_max.y |= 7;
+    ijk_max.z |= 7;
+
+    pnanovdb_buf_t buf = src_buf;
+    pnanovdb_grid_handle_t grid = src_grid;
+
+    // copy grid
+    pnanovdb_io_memcpy(dst_buf, dst_grid.address, buf, grid.address, PNANOVDB_GRID_SIZE);
+
+    pnanovdb_grid_type_t grid_type = pnanovdb_grid_get_grid_type(buf, grid);
+
+    // copy tree
+    pnanovdb_tree_handle_t tree = pnanovdb_grid_get_tree(buf, grid);
+    pnanovdb_tree_handle_t dst_tree = pnanovdb_grid_get_tree(dst_buf, dst_grid);
+    pnanovdb_io_memcpy(dst_buf, dst_tree.address, buf, tree.address, PNANOVDB_TREE_SIZE);
+
+    // copy root
+    pnanovdb_root_handle_t root = pnanovdb_tree_get_root(buf, tree);
+    pnanovdb_root_handle_t dst_root = pnanovdb_tree_get_root(dst_buf, dst_tree);
+    pnanovdb_io_memcpy(dst_buf, dst_root.address, buf, root.address, PNANOVDB_GRID_TYPE_GET(grid_type, root_size));
+
+    // set new root to clipped bounds
+    pnanovdb_root_set_bbox_min(dst_buf, dst_root, &ijk_min);
+    pnanovdb_root_set_bbox_max(dst_buf, dst_root, &ijk_max);
+
+    // copy root tiles
+    pnanovdb_uint32_t tile_count = pnanovdb_root_get_tile_count(buf, root);
+    for (pnanovdb_uint32_t tile_idx = 0u; tile_idx < tile_count; tile_idx++)
+    {
+        pnanovdb_root_tile_handle_t root_tile = pnanovdb_root_get_tile(grid_type, root, tile_idx);
+        pnanovdb_root_tile_handle_t dst_root_tile = pnanovdb_root_get_tile(grid_type, dst_root, tile_idx);
+        pnanovdb_io_memcpy(dst_buf, dst_root_tile.address, buf, root_tile.address, PNANOVDB_GRID_TYPE_GET(grid_type, root_tile_size));
+    }
+
+    // copy upper
+    for (pnanovdb_uint32_t tile_idx = 0u; tile_idx < tile_count; tile_idx++)
+    {
+        pnanovdb_root_tile_handle_t root_tile = pnanovdb_root_get_tile(grid_type, root, tile_idx);
+        pnanovdb_root_tile_handle_t dst_root_tile = pnanovdb_root_get_tile(grid_type, dst_root, tile_idx);
+        if (pnanovdb_root_tile_get_child_mask(buf, root_tile))
+        {
+            pnanovdb_upper_handle_t upper = pnanovdb_root_get_child(grid_type, buf, root, root_tile);
+            pnanovdb_upper_handle_t dst_upper = pnanovdb_root_get_child(grid_type, dst_buf, dst_root, dst_root_tile);
+            pnanovdb_io_memcpy(dst_buf, dst_upper.address, buf, upper.address, PNANOVDB_GRID_TYPE_GET(grid_type, upper_size));
+        }
+    }
+
+    // copy lower
+    for (pnanovdb_uint32_t tile_idx = 0u; tile_idx < tile_count; tile_idx++)
+    {
+        pnanovdb_root_tile_handle_t root_tile = pnanovdb_root_get_tile(grid_type, root, tile_idx);
+        pnanovdb_root_tile_handle_t dst_root_tile = pnanovdb_root_get_tile(grid_type, dst_root, tile_idx);
+        if (pnanovdb_root_tile_get_child_mask(buf, root_tile))
+        {
+            pnanovdb_upper_handle_t upper = pnanovdb_root_get_child(grid_type, buf, root, root_tile);
+            pnanovdb_upper_handle_t dst_upper = pnanovdb_root_get_child(grid_type, dst_buf, dst_root, dst_root_tile);
+            for (pnanovdb_uint32_t upper_n = 0u; upper_n < PNANOVDB_UPPER_TABLE_COUNT; upper_n++)
+            {
+                if (pnanovdb_upper_get_child_mask(buf, upper, upper_n))
+                {
+                    pnanovdb_lower_handle_t lower = pnanovdb_upper_get_child(grid_type, buf, upper, upper_n);
+                    pnanovdb_lower_handle_t dst_lower = pnanovdb_upper_get_child(grid_type, dst_buf, dst_upper, upper_n);
+                    pnanovdb_io_memcpy(dst_buf, dst_lower.address, buf, lower.address, PNANOVDB_GRID_TYPE_GET(grid_type, lower_size));
+                }
+            }
+        }
+    }
+
+    // copy leaf
+    pnanovdb_uint32_t dst_node_count_leaf = 0u;
+    pnanovdb_leaf_handle_t dst_current_leaf = {
+        pnanovdb_address_offset(dst_tree.address, pnanovdb_tree_get_node_offset_leaf(dst_buf, dst_tree))};
+    for (pnanovdb_uint32_t tile_idx = 0u; tile_idx < tile_count; tile_idx++)
+    {
+        pnanovdb_root_tile_handle_t root_tile = pnanovdb_root_get_tile(grid_type, root, tile_idx);
+        pnanovdb_root_tile_handle_t dst_root_tile = pnanovdb_root_get_tile(grid_type, dst_root, tile_idx);
+        if (pnanovdb_root_tile_get_child_mask(buf, root_tile))
+        {
+            pnanovdb_upper_handle_t upper = pnanovdb_root_get_child(grid_type, buf, root, root_tile);
+            pnanovdb_upper_handle_t dst_upper = pnanovdb_root_get_child(grid_type, dst_buf, dst_root, dst_root_tile);
+            for (pnanovdb_uint32_t upper_n = 0u; upper_n < PNANOVDB_UPPER_TABLE_COUNT; upper_n++)
+            {
+                if (pnanovdb_upper_get_child_mask(buf, upper, upper_n))
+                {
+                    pnanovdb_lower_handle_t lower = pnanovdb_upper_get_child(grid_type, buf, upper, upper_n);
+                    pnanovdb_lower_handle_t dst_lower = pnanovdb_upper_get_child(grid_type, dst_buf, dst_upper, upper_n);
+                    for (pnanovdb_uint32_t lower_n = 0u; lower_n < PNANOVDB_LOWER_TABLE_COUNT; lower_n++)
+                    {
+                        if (pnanovdb_lower_get_child_mask(buf, lower, lower_n))
+                        {
+                            pnanovdb_leaf_handle_t leaf = pnanovdb_lower_get_child(grid_type, buf, lower, lower_n);
+
+                            pnanovdb_coord_t leaf_ijk_min = pnanovdb_leaf_get_bbox_min(buf, leaf);
+                            pnanovdb_uint32_t dif_and_flags = pnanovdb_leaf_get_bbox_dif_and_flags(buf, leaf);
+                            pnanovdb_coord_t leaf_ijk_max = {
+                                pnanovdb_uint32_as_int32(dif_and_flags & 255u) + leaf_ijk_min.x,
+                                pnanovdb_uint32_as_int32((dif_and_flags >> 8u) & 255u) + leaf_ijk_min.y,
+                                pnanovdb_uint32_as_int32((dif_and_flags >> 16u) & 255u) + leaf_ijk_min.z
+                            };
+
+                            // leaf precise, not voxel precise for now
+                            leaf_ijk_min.x &= ~7;
+                            leaf_ijk_min.y &= ~7;
+                            leaf_ijk_min.z &= ~7;
+                            leaf_ijk_max.x |= 7;
+                            leaf_ijk_max.y |= 7;
+                            leaf_ijk_max.z |= 7;
+
+                            pnanovdb_bool_t pred = !(
+                                leaf_ijk_max.x < ijk_min.x || leaf_ijk_min.x > ijk_max.x ||
+                                leaf_ijk_max.y < ijk_min.y || leaf_ijk_min.y > ijk_max.y ||
+                                leaf_ijk_max.z < ijk_min.z || leaf_ijk_min.z > ijk_max.z );
+                            if (pred)
+                            {
+                                pnanovdb_uint32_t leaf_size = pnanovdb_io_get_leaf_size(grid_type, buf, leaf);
+                                pnanovdb_io_memcpy(dst_buf, dst_current_leaf.address, buf, leaf.address, leaf_size);
+
+                                pnanovdb_lower_set_child(grid_type, dst_buf, dst_lower, lower_n, dst_current_leaf);
+
+                                // voxel precise
+                                #if 0
+                                for (pnanovdb_uint32_t leaf_n = 0u; leaf_n < PNANOVDB_LEAF_TABLE_COUNT; leaf_n++)
+                                {
+                                    pnanovdb_coord_t voxel_ijk = {
+                                        (leaf_ijk_min.x & ~7) + (pnanovdb_uint32_as_int32(leaf_n >> 6u) & 7),
+                                        (leaf_ijk_min.y & ~7) + (pnanovdb_uint32_as_int32(leaf_n >> 3u) & 7),
+                                        (leaf_ijk_min.z & ~7) + (pnanovdb_uint32_as_int32(leaf_n >> 0u) & 7)
+                                    };
+
+                                    pnanovdb_bool_t voxel_pred = !(
+                                        voxel_ijk.x < ijk_min.x || voxel_ijk.x > ijk_max.x ||
+                                        voxel_ijk.y < ijk_min.y || voxel_ijk.y > ijk_max.y ||
+                                        voxel_ijk.z < ijk_min.z || voxel_ijk.z > ijk_max.z );
+                                    if (!voxel_pred)
+                                    {
+                                        pnanovdb_leaf_set_value_mask(dst_buf, dst_current_leaf, leaf_n, PNANOVDB_FALSE);
+                                        pnanovdb_address_t val_addr =
+                                            pnanovdb_leaf_get_table_address(grid_type, dst_buf, dst_current_leaf, leaf_n);
+                                        pnanovdb_address_t background_addr = pnanovdb_root_get_background_address(grid_type, buf, root);
+
+                                        pnanovdb_io_memcpy(dst_buf, val_addr, buf, background_addr, (PNANOVDB_GRID_TYPE_GET(grid_type, value_stride_bits) >> 3u));
+                                    }
+                                }
+                                #endif
+
+                                dst_current_leaf.address = pnanovdb_address_offset(dst_current_leaf.address, leaf_size);
+                                dst_node_count_leaf++;
+                            }
+                            else
+                            {
+                                pnanovdb_address_t background_addr = pnanovdb_root_get_background_address(grid_type, buf, root);
+                                pnanovdb_uint64_t background_index = pnanovdb_read_uint64(buf, background_addr);
+                                pnanovdb_lower_set_table_child(grid_type, dst_buf, dst_lower, lower_n, pnanovdb_uint64_as_int64(background_index));
+                                pnanovdb_lower_set_child_mask(dst_buf, dst_lower, lower_n, PNANOVDB_FALSE);
+                                pnanovdb_lower_set_value_mask(dst_buf, dst_lower, lower_n, PNANOVDB_FALSE);
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    //pnanovdb_grid_set_blind_metadata_count(dst_buf, dst_grid, 0u);
+
+    // need to update tree for new reduced leaf range
+    pnanovdb_tree_set_node_count_leaf(dst_buf, dst_tree, dst_node_count_leaf);
+
+    pnanovdb_uint32_t blindmetadata_count = pnanovdb_grid_get_blind_metadata_count(buf, grid);
+
+    pnanovdb_uint64_t dst_blind_metadata_offset =
+        dst_current_leaf.address.byte_offset -
+        dst_grid.address.byte_offset;
+    pnanovdb_grid_set_blind_metadata_offset(dst_buf, dst_grid, dst_blind_metadata_offset);
+
+    // copy blindmetadata headers
+    for (pnanovdb_uint32_t metadata_idx = 0u; metadata_idx < blindmetadata_count; metadata_idx++)
+    {
+        pnanovdb_gridblindmetadata_handle_t metadata = pnanovdb_grid_get_gridblindmetadata(buf, grid, metadata_idx);
+        pnanovdb_gridblindmetadata_handle_t dst_metadata = pnanovdb_grid_get_gridblindmetadata(dst_buf, dst_grid, metadata_idx);
+
+        pnanovdb_io_memcpy(dst_buf, dst_metadata.address, buf, metadata.address, PNANOVDB_GRIDBLINDMETADATA_SIZE);
+    }
+
+#if 0
+    // copy blindmetadata
+    for (pnanovdb_uint32_t metadata_idx = 0u; metadata_idx < blindmetadata_count; metadata_idx++)
+    {
+        pnanovdb_gridblindmetadata_handle_t metadata = pnanovdb_grid_get_gridblindmetadata(buf, grid, metadata_idx);
+        pnanovdb_gridblindmetadata_handle_t dst_metadata = pnanovdb_grid_get_gridblindmetadata(dst_buf, dst_grid, metadata_idx);
+
+        pnanovdb_int64_t byte_offset = pnanovdb_gridblindmetadata_get_data_offset(buf, metadata);
+        pnanovdb_address_t address = pnanovdb_address_offset64(metadata.address, pnanovdb_int64_as_uint64(byte_offset));
+
+        pnanovdb_int64_t dst_byte_offset = pnanovdb_gridblindmetadata_get_data_offset(dst_buf, dst_metadata);
+        pnanovdb_address_t dst_address = pnanovdb_address_offset64(dst_metadata.address, pnanovdb_int64_as_uint64(dst_byte_offset));
+
+        pnanovdb_uint32_t value_size = pnanovdb_gridblindmetadata_get_value_size(buf, metadata);
+        pnanovdb_uint64_t value_count = pnanovdb_gridblindmetadata_get_value_count(buf, metadata);
+
+        pnanovdb_io_memcpy(dst_buf, dst_address, buf, address, value_count * value_size);
+    }
+#else
+    for (pnanovdb_uint32_t metadata_idx = 0u; metadata_idx < blindmetadata_count; metadata_idx++)
+    {
+        pnanovdb_gridblindmetadata_handle_t metadata = pnanovdb_grid_get_gridblindmetadata(buf, grid, metadata_idx);
+        pnanovdb_gridblindmetadata_handle_t dst_metadata = pnanovdb_grid_get_gridblindmetadata(dst_buf, dst_grid, metadata_idx);
+
+        pnanovdb_int64_t byte_offset = pnanovdb_gridblindmetadata_get_data_offset(buf, metadata);
+        pnanovdb_address_t address = pnanovdb_address_offset64(metadata.address, pnanovdb_int64_as_uint64(byte_offset));
+
+        pnanovdb_int64_t dst_byte_offset = pnanovdb_gridblindmetadata_get_data_offset(dst_buf, dst_metadata);
+        pnanovdb_address_t dst_address = pnanovdb_address_offset64(dst_metadata.address, pnanovdb_int64_as_uint64(dst_byte_offset));
+
+        pnanovdb_uint32_t value_size = pnanovdb_gridblindmetadata_get_value_size(buf, metadata);
+        pnanovdb_uint64_t value_count = pnanovdb_gridblindmetadata_get_value_count(buf, metadata);
+
+        pnanovdb_uint64_t dst_value_index = 0llu;
+        pnanovdb_uint64_t value_index = 0llu;
+
+        pnanovdb_address_t index_addr = pnanovdb_root_get_background_address(grid_type, buf, root);
+        value_index = pnanovdb_read_uint64(buf, index_addr);
+
+        pnanovdb_address_t dst_index_addr = pnanovdb_root_get_background_address(grid_type, dst_buf, dst_root);
+        pnanovdb_write_uint64(dst_buf, dst_index_addr, dst_value_index);
+
+        pnanovdb_uint64_t last_root_tile_index = value_index;
+        pnanovdb_uint64_t last_upper_tile_index = value_index;
+        pnanovdb_uint64_t last_lower_tile_index = value_index;
+        pnanovdb_uint64_t last_root_tile_dst_index = dst_value_index;
+        pnanovdb_uint64_t last_upper_tile_dst_index = dst_value_index;
+        pnanovdb_uint64_t last_lower_tile_dst_index = dst_value_index;
+
+        pnanovdb_io_memcpy(
+            dst_buf, pnanovdb_address_offset64_product(dst_address, dst_value_index, value_size),
+            buf, pnanovdb_address_offset64_product(address, value_index, value_size), value_size);
+        dst_value_index++;
+
+        // walk subregion, copy referenced blindmetadata
+        pnanovdb_uint32_t dst_tile_count = pnanovdb_root_get_tile_count(dst_buf, dst_root);
+        for (pnanovdb_uint32_t tile_idx = 0u; tile_idx < dst_tile_count; tile_idx++)
+        {
+            pnanovdb_root_tile_handle_t dst_root_tile = pnanovdb_root_get_tile(grid_type, dst_root, tile_idx);
+            pnanovdb_root_tile_handle_t root_tile = pnanovdb_root_get_tile(grid_type, root, tile_idx);
+
+            if (pnanovdb_root_tile_get_child_mask(dst_buf, dst_root_tile))
+            {
+                pnanovdb_upper_handle_t dst_upper = pnanovdb_root_get_child(grid_type, dst_buf, dst_root, dst_root_tile);
+                pnanovdb_upper_handle_t upper = pnanovdb_root_get_child(grid_type, buf, root, root_tile);
+                for (pnanovdb_uint32_t upper_n = 0u; upper_n < PNANOVDB_UPPER_TABLE_COUNT; upper_n++)
+                {
+                    if (pnanovdb_upper_get_child_mask(dst_buf, dst_upper, upper_n))
+                    {
+                        pnanovdb_lower_handle_t dst_lower = pnanovdb_upper_get_child(grid_type, dst_buf, dst_upper, upper_n);
+                        pnanovdb_lower_handle_t lower = pnanovdb_upper_get_child(grid_type, buf, upper, upper_n);
+                        for (pnanovdb_uint32_t lower_n = 0u; lower_n < PNANOVDB_LOWER_TABLE_COUNT; lower_n++)
+                        {
+                            if (pnanovdb_lower_get_child_mask(dst_buf, dst_lower, lower_n))
+                            {
+                                pnanovdb_leaf_handle_t dst_leaf = pnanovdb_lower_get_child(grid_type, dst_buf, dst_lower, lower_n);
+                                pnanovdb_leaf_handle_t leaf = pnanovdb_lower_get_child(grid_type, buf, lower, lower_n);
+
+                                pnanovdb_uint32_t leaf_value_count = pnanovdb_leaf_onindex_get_value_count(dst_buf, dst_leaf);
+
+                                dst_index_addr = pnanovdb_leaf_get_table_address(grid_type, dst_buf, dst_leaf, 0u);
+                                pnanovdb_write_uint64(dst_buf, dst_index_addr, dst_value_index);
+
+                                pnanovdb_address_t index_addr = pnanovdb_leaf_get_table_address(grid_type, buf, leaf, 0u);
+                                value_index = pnanovdb_read_uint64(buf, index_addr);
+
+                                pnanovdb_io_memcpy(
+                                    dst_buf, pnanovdb_address_offset64_product(dst_address, dst_value_index, value_size),
+                                    buf, pnanovdb_address_offset64_product(address, value_index, value_size), value_size * leaf_value_count);
+                                dst_value_index+=leaf_value_count;
+                            }
+                            else if(pnanovdb_lower_get_child_mask(buf, lower, lower_n))
+                            {
+                                // this covers leaves disabled by pred above
+                                pnanovdb_address_t background_addr = pnanovdb_root_get_background_address(grid_type, dst_buf, dst_root);
+                                pnanovdb_uint64_t background_index = pnanovdb_read_uint64(buf, background_addr);
+
+                                dst_index_addr = pnanovdb_lower_get_table_address(grid_type, dst_buf, dst_lower, lower_n);
+                                pnanovdb_write_uint64(dst_buf, dst_index_addr, background_index);
+                            }
+                            else
+                            {
+                                pnanovdb_address_t index_addr = pnanovdb_lower_get_table_address(grid_type, buf, lower, lower_n);
+                                value_index = pnanovdb_read_uint64(buf, index_addr);
+                                if (last_lower_tile_index != value_index)
+                                {
+                                    last_lower_tile_index = value_index;
+
+                                    dst_index_addr = pnanovdb_lower_get_table_address(grid_type, dst_buf, dst_lower, lower_n);
+                                    pnanovdb_write_uint64(dst_buf, dst_index_addr, dst_value_index);
+                                    last_lower_tile_dst_index = dst_value_index;
+
+                                    pnanovdb_io_memcpy(
+                                        dst_buf, pnanovdb_address_offset64_product(dst_address, dst_value_index, value_size),
+                                        buf, pnanovdb_address_offset64_product(address, value_index, value_size), value_size);
+                                    dst_value_index++;
+                                }
+                                else
+                                {
+                                    dst_index_addr = pnanovdb_lower_get_table_address(grid_type, dst_buf, dst_lower, lower_n);
+                                    pnanovdb_write_uint64(dst_buf, dst_index_addr, last_lower_tile_dst_index);
+                                }
+                            }
+                        }
+                    }
+                    else
+                    {
+                        pnanovdb_address_t index_addr = pnanovdb_upper_get_table_address(grid_type, buf, upper, upper_n);
+                        value_index = pnanovdb_read_uint64(buf, index_addr);
+                        if (last_upper_tile_index != value_index)
+                        {
+                            last_upper_tile_index = value_index;
+
+                            dst_index_addr = pnanovdb_upper_get_table_address(grid_type, dst_buf, dst_upper, upper_n);
+                            pnanovdb_write_uint64(dst_buf, dst_index_addr, dst_value_index);
+                            last_upper_tile_dst_index = dst_value_index;
+
+                            pnanovdb_io_memcpy(
+                                dst_buf, pnanovdb_address_offset64_product(dst_address, dst_value_index, value_size),
+                                buf, pnanovdb_address_offset64_product(address, value_index, value_size), value_size);
+                            dst_value_index++;
+                        }
+                        else
+                        {
+                            dst_index_addr = pnanovdb_upper_get_table_address(grid_type, dst_buf, dst_upper, upper_n);
+                            pnanovdb_write_uint64(dst_buf, dst_index_addr, last_upper_tile_dst_index);
+                        }
+                    }
+                }
+            }
+            else
+            {
+                pnanovdb_address_t index_addr = pnanovdb_root_tile_get_value_address(grid_type, buf, root_tile);
+                value_index = pnanovdb_read_uint64(buf, index_addr);
+                if (last_root_tile_index != value_index)
+                {
+                    last_root_tile_index = value_index;
+
+                    dst_index_addr = pnanovdb_root_tile_get_value_address(grid_type, dst_buf, dst_root_tile);
+                    pnanovdb_write_uint64(dst_buf, dst_index_addr, dst_value_index);
+                    last_root_tile_dst_index = dst_value_index;
+
+                    pnanovdb_io_memcpy(
+                        dst_buf, pnanovdb_address_offset64_product(dst_address, dst_value_index, value_size),
+                        buf, pnanovdb_address_offset64_product(address, value_index, value_size), value_size);
+                    dst_value_index++;
+                }
+                else
+                {
+                    dst_index_addr = pnanovdb_root_tile_get_value_address(grid_type, dst_buf, dst_root_tile);
+                    pnanovdb_write_uint64(dst_buf, dst_index_addr, last_root_tile_dst_index);
+                }
+            }
+        }
+
+        pnanovdb_gridblindmetadata_set_value_count(dst_buf, dst_metadata, dst_value_index);
+
+        pnanovdb_int64_t metadata_size_diff = pnanovdb_uint64_as_int64(dst_value_index - value_count) * value_size;
+        // compact later metadata ranges against new count
+        for (pnanovdb_uint32_t local_metadata_idx = metadata_idx + 1u; local_metadata_idx < blindmetadata_count; local_metadata_idx++)
+        {
+            pnanovdb_gridblindmetadata_handle_t local_dst_metadata = pnanovdb_grid_get_gridblindmetadata(dst_buf, dst_grid, metadata_idx);
+
+            pnanovdb_int64_t local_dst_byte_offset = pnanovdb_gridblindmetadata_get_data_offset(dst_buf, local_dst_metadata);
+            local_dst_byte_offset += metadata_size_diff;
+            pnanovdb_gridblindmetadata_set_data_offset(dst_buf, local_dst_metadata, local_dst_byte_offset);
+        }
+    }
+
+    // compute final grid size based on maximum of final leaf and final metadata
+    pnanovdb_uint64_t grid_size =
+        dst_current_leaf.address.byte_offset -
+        dst_grid.address.byte_offset;
+    for (pnanovdb_uint32_t metadata_idx = 0u; metadata_idx < blindmetadata_count; metadata_idx++)
+    {
+        pnanovdb_gridblindmetadata_handle_t dst_metadata = pnanovdb_grid_get_gridblindmetadata(dst_buf, dst_grid, metadata_idx);
+
+        pnanovdb_int64_t dst_byte_offset = pnanovdb_gridblindmetadata_get_data_offset(dst_buf, dst_metadata);
+        pnanovdb_address_t dst_address = pnanovdb_address_offset64(dst_metadata.address, pnanovdb_int64_as_uint64(dst_byte_offset));
+
+        pnanovdb_uint32_t value_size = pnanovdb_gridblindmetadata_get_value_size(dst_buf, dst_metadata);
+        pnanovdb_uint64_t value_count = pnanovdb_gridblindmetadata_get_value_count(dst_buf, dst_metadata);
+
+        pnanovdb_uint64_t cmp_grid_size =
+            dst_address.byte_offset + value_count * value_size -
+            dst_grid.address.byte_offset;
+        if (grid_size < cmp_grid_size)
+        {
+            grid_size = cmp_grid_size;
+        }
+    }
+    pnanovdb_grid_set_grid_size(dst_buf, dst_grid, grid_size);
+#endif
+}
+
+PNANOVDB_IO_INLINE void pnanovdb_io_print_slice(pnanovdb_io_buf_t* src_nanovdb, FILE* dst_bmp)
+{
+    pnanovdb_grid_handle_t grid = pnanovdb_io_fread_fileheader(src_nanovdb, 0, 0, 0u);
+
+    pnanovdb_buf_t buf = src_nanovdb;
+
+    // set grid at post header print offset
+    printf("pnanovdb_io_print_slice grid(%lld)\n", (long long int)grid.address.byte_offset);
+
+    printf("pnanovdb_io_print_slice magic(%llx)\n",
+        (unsigned long long int)pnanovdb_grid_get_magic(buf, grid));
+    printf("pnanovdb_io_print_slice version(%x)\n", pnanovdb_grid_get_version(buf, grid));
+
+    pnanovdb_tree_handle_t tree = pnanovdb_grid_get_tree(buf, grid);
+    printf("pnanovdb_io_print_slice tree(%lld)\n", (long long int)tree.address.byte_offset);
+
+    printf("pnanovdb_io_print_slice node_count_leaf(%d)\n", pnanovdb_tree_get_node_count_leaf(buf, tree));
+    printf("pnanovdb_io_print_slice node_count_lower(%d)\n", pnanovdb_tree_get_node_count_lower(buf, tree));
+    printf("pnanovdb_io_print_slice node_count_upper(%d)\n", pnanovdb_tree_get_node_count_upper(buf, tree));
+
+    pnanovdb_root_handle_t root = pnanovdb_tree_get_root(buf, tree);
+    pnanovdb_grid_type_t grid_type = pnanovdb_grid_get_grid_type(buf, grid);
+
+    pnanovdb_coord_t bbox_min = pnanovdb_root_get_bbox_min(buf, root);
+    pnanovdb_coord_t bbox_max = pnanovdb_root_get_bbox_max(buf, root);
+
+    printf("pnanovdb_io_print_slice range([(%d, %d, %d),(%d, %d, %d)])\n",
+        bbox_min.x, bbox_min.y, bbox_min.z,
+        bbox_max.x, bbox_max.y, bbox_max.z);
+
+    pnanovdb_int32_t image_width = bbox_max.x - bbox_min.x;
+    pnanovdb_int32_t image_height = bbox_max.y - bbox_min.y;
+    pnanovdb_int32_t k = (bbox_max.z + bbox_min.z) / 2;
+
+    char headerField0 = 'B';
+    char headerField1 = 'M';
+    uint32_t size = 54 + image_width * image_height * 4u;
+    uint16_t reserved1 = 0;
+    uint16_t reserved2 = 0;
+    uint32_t offset = 54;
+    uint32_t headerSize = 40;
+    uint32_t width = image_width;
+    uint32_t height = image_height;
+    uint16_t colorPlanes = 1;
+    uint16_t bitsPerPixel = 32;
+    uint32_t compressionMethod = 0;
+    uint32_t imageSize = image_width * image_height * 4u;
+    uint32_t hRes = 2000;
+    uint32_t vRes = 2000;
+    uint32_t numColors = 0;
+    uint32_t numImportantColors = 0;
+
+    fwrite(&headerField0, 1, 1, dst_bmp);
+    fwrite(&headerField1, 1, 1, dst_bmp);
+    fwrite(&size, 4, 1, dst_bmp);
+    fwrite(&reserved1, 2, 1, dst_bmp);
+    fwrite(&reserved2, 2, 1, dst_bmp);
+    fwrite(&offset, 4, 1, dst_bmp);
+    fwrite(&headerSize, 4, 1, dst_bmp);
+    fwrite(&width, 4, 1, dst_bmp);
+    fwrite(&height, 4, 1, dst_bmp);
+    fwrite(&colorPlanes, 2, 1, dst_bmp);
+    fwrite(&bitsPerPixel, 2, 1, dst_bmp);
+    fwrite(&compressionMethod, 4, 1, dst_bmp);
+    fwrite(&imageSize, 4, 1, dst_bmp);
+    fwrite(&hRes, 4, 1, dst_bmp);
+    fwrite(&vRes, 4, 1, dst_bmp);
+    fwrite(&numColors, 4, 1, dst_bmp);
+    fwrite(&numImportantColors, 4, 1, dst_bmp);
+
+    for (pnanovdb_int32_t j = image_height - 1; j >= 0; j--)
+    {
+        for (pnanovdb_int32_t i = 0; i < image_width; i++)
+        {
+            pnanovdb_coord_t ijk = {
+                i + bbox_min.x,
+                j + bbox_min.y,
+                k
+            };
+
+            pnanovdb_address_t addr = pnanovdb_root_get_value_address(grid_type, buf, root, &ijk);
+
+            float density_value = 1.f;
+            float color_value = pnanovdb_read_float(buf, addr);
+
+            pnanovdb_uint32_t raw_value = pnanovdb_uint32_t(255.f * pnanovdb_max(0.f, pnanovdb_min(1.f, color_value))) |
+                (pnanovdb_uint32_t(255.f * pnanovdb_max(0.f, pnanovdb_min(1.f, color_value))) << 8u) |
+                (pnanovdb_uint32_t(255.f * pnanovdb_max(0.f, pnanovdb_min(1.f, color_value))) << 16u) |
+                (pnanovdb_uint32_t(255.f * pnanovdb_max(0.f, pnanovdb_min(1.f, density_value))) << 24u);
+
+            fwrite(&raw_value, 4u, 1u, dst_bmp);
+        }
+    }
+}
+
+#endif
+
+#endif // end of NANOVDB_PUTILS_IO_H_HAS_BEEN_INCLUDED
@@ -1,3 +1,5 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
 option(NANOVDB_BUILD_PYTHON_UNITTESTS [=[
  "Include the NanoVDB Python unit test. Requires a python interpreter]=]
 ${NANOVDB_BUILD_UNITTESTS})
@@ -1,3 +1,5 @@
+// Copyright Contributors to the OpenVDB Project
+// SPDX-License-Identifier: Apache-2.0
 #include <nanobind/nanobind.h>
 #include <nanobind/operators.h>
 #include <nanobind/stl/string.h>
@@ -321,8 +323,6 @@ NB_MODULE(nanovdb, m)
        .value("Vec4d", GridType::Vec4d)
        .value("Index", GridType::Index)
        .value("OnIndex", GridType::OnIndex)
-        .value("IndexMask", GridType::IndexMask)
-        .value("OnIndexMask", GridType::OnIndexMask)
        .value("PointIndex", GridType::PointIndex)
        .value("Vec3u8", GridType::Vec3u8)
        .value("Vec3u16", GridType::Vec3u16)
@@ -1,3 +1,5 @@
+// Copyright Contributors to the OpenVDB Project
+// SPDX-License-Identifier: Apache-2.0
 #include "PyCreateNanoGrid.h"

 #include <nanobind/stl/string.h>
@@ -1,3 +1,5 @@
+// Copyright Contributors to the OpenVDB Project
+// SPDX-License-Identifier: Apache-2.0
 #ifndef NANOVDB_PYCREATENANOGRID_HAS_BEEN_INCLUDED
 #define NANOVDB_PYCREATENANOGRID_HAS_BEEN_INCLUDED

@@ -1,3 +1,5 @@
+// Copyright Contributors to the OpenVDB Project
+// SPDX-License-Identifier: Apache-2.0
 #include "PyGridChecksum.h"

 #include <nanovdb/tools/GridChecksum.h>
--- a/Show More
+++ b/Show More