Test only one Windows job

Signed-off-by: Dan Bailey <danbailey@ilm.com>
Remove numpy installation from Windows CI
2025-10-31 10:35:28 -07:00 · 2025-10-31 10:21:52 -07:00 · 2025-10-31 08:09:02 +13:00 · 2025-10-30 09:58:36 -07:00 · 2025-10-30 08:07:47 -07:00 · 2025-10-29 23:13:15 -07:00
945 changed files with 47519 additions and 18719 deletions
@@ -55,7 +55,7 @@ jobs:
      github.event_name != 'workflow_dispatch' ||
      github.event.inputs.type == 'all' ||
      github.event.inputs.type == 'linux'
-    runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'ubuntu-20.04-8c-32g-300h') || 'ubuntu-latest' }}
+    runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'ubuntu-22.04-8c-32g-300h') || 'ubuntu-latest' }}
    name: >
      linux-ax:${{ matrix.config.image }}-cxx:${{ matrix.config.cxx }}-${{ matrix.config.build }}
    container:
@@ -67,21 +67,22 @@ jobs:
      matrix:
        # Only test unified builds (core+ax). weekly CI should test standalone
        config:
-          - { image: '2023-clang15', cxx: 'clang++', build: 'Release', cmake: '' }
-          - { image: '2023-clang15', cxx: 'g++',     build: 'Release', cmake: '' }
-          - { image: '2023-clang15', cxx: 'clang++', build: 'Debug',   cmake: '' }
-          - { image: '2022-clang11', cxx: 'clang++', build: 'Release', cmake: '' }
-          - { image: '2022-clang11', cxx: 'g++',     build: 'Release', cmake: '' }
+          - { image: '2026',           cxx: 'clang++', build: 'Release', cmake: '' }
+          - { image: '2026',           cxx: 'g++',     build: 'Release', cmake: '' }
+          - { image: '2026',           cxx: 'clang++', build: 'Debug',   cmake: '' }
+          - { image: '2025-clang19.1', cxx: 'clang++', build: 'Release', cmake: '' }
+          - { image: '2024-clang16.2', cxx: 'clang++', build: 'Release', cmake: '' }
+          # Needed to test LLVM 15.X
+          - { image: '2023-clang15',   cxx: 'clang++', build: 'Release', cmake: '-DDISABLE_DEPENDENCY_VERSION_CHECKS=ON' }
      fail-fast: false
    steps:
-      - name: Enable Node 16
-        if: contains(matrix.config.image, '2022')
-        run: |
-          echo "ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION=true" >> $GITHUB_ENV
      - uses: actions/checkout@v3
-      - name: pybind11
-        #if: contains(matrix.config.image, '2023') == false
-        run: ./ci/install_pybind11.sh 2.10.0
+      - name: nanobind
+        run: ./ci/install_nanobind.sh 2.5.0
+        # 2026 images are broken with gtest/glfw installs
+      - name: deps
+        if: contains(matrix.config.image, '2026')
+        run: ./ci/install_gtest.sh latest
      - name: timestamp
        id: timestamp
        run: echo "timestamp=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
@@ -98,12 +99,11 @@ jobs:
        run: >
          ./ci/build.sh -v
          --build-type=${{ matrix.config.build }}
-          --components='core,python,bin,axcore,axbin,axtest'
+          --components='core,python,axcore,axbin,axtest'
          --cargs=\"
          ${{ matrix.config.cmake }}
          -DOPENVDB_AX_TEST_CMD_DOWNLOADS=ON
          -DUSE_EXPLICIT_INSTANTIATION=OFF
-          -DOPENVDB_CXX_STRICT=ON
          \"
      - name: test
        run: cd build && ctest -V
@@ -126,10 +126,13 @@ jobs:
    strategy:
      matrix:
        config:
-          #@note llvm10 never got its own brew formula...
-          # Last macos runner befor M1 (macos-14)
-          - { runner: 'macos-13', cxx: 'clang++', build: 'Release', llvm: '12' }
-          - { runner: 'macos-13', cxx: 'clang++', build: 'Release', llvm: '13' }
+          - { runner: 'macos-15', cxx: 'clang++', build: 'Release', llvm: '21' }
+          - { runner: 'macos-14', cxx: 'clang++', build: 'Release', llvm: '21' }
+          - { runner: 'macos-14', cxx: 'clang++', build: 'Release', llvm: '20' }
+          - { runner: 'macos-14', cxx: 'clang++', build: 'Release', llvm: '19' }
+          - { runner: 'macos-14', cxx: 'clang++', build: 'Release', llvm: '18' }
+          - { runner: 'macos-14', cxx: 'clang++', build: 'Release', llvm: '17' }
+          - { runner: 'macos-14', cxx: 'clang++', build: 'Release', llvm: '16' }
      fail-fast: false
    steps:
      - uses: actions/checkout@v3
@@ -141,7 +144,7 @@ jobs:
        run: >
          ./ci/build.sh -v
          --build-type=${{ matrix.config.build }}
-          --components="core,python,bin,axcore,axbin,axtest"
+          --components="core,python,axcore,axbin,axtest"
          --cargs=\"
          -DOPENVDB_AX_TEST_CMD_DOWNLOADS=ON
          -DUSE_EXPLICIT_INSTANTIATION=OFF
@@ -157,15 +160,15 @@ jobs:
    if: |
      github.event_name == 'workflow_dispatch' &&
      github.event.inputs.type == 'grammar'
-    runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'ubuntu-20.04-8c-32g-300h') || 'ubuntu-latest' }}
+    runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'ubuntu-22.04-8c-32g-300h') || 'ubuntu-latest' }}
    container:
-      image: aswf/ci-openvdb:2022-clang11
+      image: aswf/ci-openvdb:2025-clang19.1
    steps:
    - uses: actions/checkout@v3
    - name: build
      run: ./ci/build.sh -v --components=axgr --target=openvdb_ax_grammar --cargs=\"-DOPENVDB_AX_GRAMMAR_NO_LINES=ON\"
    - name: upload grammar
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
      with:
        name: ax_grammar
        path: ./build/openvdb_ax/openvdb_ax/openvdb_ax/grammar
@@ -33,11 +33,8 @@ on:
      - 'pendingchanges/**'
      - '**.md'
  schedule:
-    # run this workflow every day at 7am UTC except Monday
-    - cron:  '0 7 * * 0,2-6'
-    # run this workflow Monday 7am UTC
-    # warning: This pattern is checked in various places below
-    - cron:  '0 7 * * 1'
+    # run this workflow every day 7am UTC
+    - cron:  '0 7 * * *'
  workflow_dispatch:
    inputs:
      type:
@@ -62,12 +59,9 @@ jobs:
      github.event_name != 'workflow_dispatch' ||
      github.event.inputs.type == 'all' ||
      github.event.inputs.type == 'linux'
-    runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'ubuntu-20.04-8c-32g-300h') || 'ubuntu-latest' }}
+    runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'ubuntu-22.04-8c-32g-300h') || 'ubuntu-latest' }}
    name: >
-      linux-vfx:${{ matrix.config.image }}-
-      abi:${{ matrix.config.abi }}-
-      cxx:${{ matrix.config.cxx }}-
-      type:${{ matrix.config.build }}
+      linux:${{ matrix.config.image }}-abi:${{ matrix.config.abi }}-cxx:${{ matrix.config.cxx }}-type:${{ matrix.config.build }}
    container:
      image: aswf/ci-openvdb:${{ matrix.config.image }}
    env:
@@ -76,27 +70,22 @@ jobs:
    strategy:
      matrix:
        config:
-          - { cxx: clang++, image: '2024', abi: '11', build: 'Release', cmake: '' }
-          - { cxx: g++,     image: '2024', abi: '11', build: 'Release', cmake: '' }
-          - { cxx: clang++, image: '2024', abi: '11', build: 'Debug',   cmake: '' }
-          - { cxx: clang++, image: '2023', abi: '11', build: 'Release', cmake: '' }
-          - { cxx: g++,     image: '2023', abi: '11', build: 'Release', cmake: '' }
-          - { cxx: clang++, image: '2022', abi: '10', build: 'Release', cmake: '-DDISABLE_DEPENDENCY_VERSION_CHECKS=ON' }
-          - { cxx: g++,     image: '2022', abi: '10', build: 'Release', cmake: '-DDISABLE_DEPENDENCY_VERSION_CHECKS=ON' }
+          - { cxx: clang++, image: '2026',           abi: '13', build: 'Release', cmake: '-DOPENVDB_USE_FUTURE_ABI_13=ON' }
+          - { cxx: g++,     image: '2026',           abi: '13', build: 'Release', cmake: '-DOPENVDB_USE_FUTURE_ABI_13=ON' }
+          - { cxx: clang++, image: '2026',           abi: '13', build: 'Debug',   cmake: '-DOPENVDB_USE_FUTURE_ABI_13=ON' }
+          - { cxx: clang++, image: '2025-clang19.1', abi: '12', build: 'Release', cmake: '' }
+          - { cxx: clang++, image: '2024-clang17.2', abi: '11', build: 'Release', cmake: '-DDISABLE_DEPENDENCY_VERSION_CHECKS=ON' }
      fail-fast: false
    steps:
-    - name: Enable Node 16
-      # Solution taken from https://github.blog/changelog/2024-03-07-github-actions-all-actions-will-run-on-node20-instead-of-node16-by-default
-      if: contains(matrix.config.image, '2022')
-      run: |
-        echo "ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION=true" >> $GITHUB_ENV
    - uses: actions/checkout@v3
-    - name: pybind11
-      #if: contains(matrix.config.image, '2023') == false
-      run: ./ci/install_pybind11.sh 2.10.0
-    - name: glfw
-      if: contains(matrix.config.image, '2023') == true
-      run: ./ci/install_glfw.sh 3.3.10
+    - name: nanobind
+      run: ./ci/install_nanobind.sh 2.5.0
+      # 2026 images are broken with gtest/glfw installs
+    - name: deps
+      if: contains(matrix.config.image, '2026')
+      run: |
+        ./ci/install_gtest.sh latest
+        ./ci/install_glfw.sh latest
    - name: timestamp
      id: timestamp
      run: echo "timestamp=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
@@ -120,9 +109,8 @@ jobs:
        -DOPENVDB_ABI_VERSION_NUMBER=${{ matrix.config.abi }}
        \"
    - name: test
-      # Always run tests on weekly builds but skip Debug on commits as they take a while.
-      # https://github.community/t/distinct-job-for-each-schedule/17811/2
-      if: contains(github.event.schedule, '0 7 * * 1') || matrix.config.build == 'Release'
+      # Skip Debug on commits as they take a while.
+      if: github.event_name == 'schedule'|| matrix.config.build == 'Release'
      run: |
        cd build && ctest -V
        cd - && ./ci/test_install.sh
@@ -178,9 +166,15 @@ jobs:
      github.event_name != 'workflow_dispatch' ||
      github.event.inputs.type == 'all' ||
      github.event.inputs.type == 'mac'
-    runs-on: macos-13 # Last macos runner befor M1 (macos-14)
+    runs-on: ${{ matrix.config.image }}
+    name: ${{ matrix.config.image }}
    env:
      CXX: clang++
+    strategy:
+      matrix:
+        config:
+          - { image: macos-14 }
+          - { image: macos-15 }
    steps:
    - uses: actions/checkout@v3
    - name: install
@@ -197,3 +191,4 @@ jobs:
        \'
    - name: test
      run: cd build && ctest -V
+
@@ -38,28 +38,15 @@ jobs:
    env:
      CXX: g++
    container:
-      # @note We can't update this as epydoc doesn't support python3. We'll
-      # need to re-write the python docs to use sphinx
-      image: aswf/ci-openvdb:2022
+      image: aswf/ci-openvdb:2026
    steps:
-    - name: Enable Node 16
-      run: |
-        echo "ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION=true" >> $GITHUB_ENV
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
    - name: install_doxygen
      run: ./ci/install_doxygen.sh 1_8_11
-    - name: pybind11
-      #if: contains(matrix.config.image, '2023') == false
-      run: ./ci/install_pybind11.sh 2.10.0
-    # - name: install_epydoc
-    #   run: pip install epydoc
+    - name: nanobind
+      run: ./ci/install_nanobind.sh 2.5.0
    - name: install_latex
-      run: |
-        # Fix error: Cannot prepare internal mirrorlist: No URLs in mirrorlist. CentOS 8 reached EOL means need to replace the official mirror to vault.centos.org
-        # Comment out mirrorlist and replace #baseurl=...mirror.centos.org with baseurl=...vault.centos.org in files starting with CentOS- in /etc/yum.repos.d folder
-        sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-*
-        sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*
-        yum -y install texlive-latex-bin texlive-dvips texlive-collection-fontsrecommended texlive-collection-latexrecommended
+      run: ./ci/install_texlive.sh
    - name: build
      run: >
        ./ci/build.sh -v
@@ -72,9 +59,6 @@ jobs:
        -DUSE_EXPLICIT_INSTANTIATION=OFF
        -DDISABLE_DEPENDENCY_VERSION_CHECKS=ON
        \"
-    # - name: epydoc
-    #   run: |
-    #     epydoc --html -o /usr/local/share/doc/OpenVDB/html/python pyopenvdb
    - name: pre_deploy
      # Overwrite global SSH configuration
      #  https://github.com/peaceiris/actions-gh-pages/issues/719
@@ -92,7 +76,7 @@ jobs:
        github.event.inputs.deploy == 'docs'
      uses: peaceiris/actions-gh-pages@v3
      with:
-        deploy_key: ${{ secrets.ACTIONS_DEPLOY_KEY }}
+        deploy_key: ${{ secrets.PRIVATE_KEY }}
        publish_dir: /usr/local/share/doc/OpenVDB/html
        destination_dir: documentation/doxygen
        external_repository: AcademySoftwareFoundation/openvdb-website
@@ -113,7 +97,7 @@ jobs:
      github.event.inputs.deploy == 'coverage'
    runs-on: ubuntu-latest
    container:
-      image: aswf/ci-openvdb:2023
+      image: aswf/ci-openvdb:2025-clang19.1
    env:
      CXX: g++
    steps:
@@ -65,8 +65,8 @@ jobs:
    if: >
      ${{ needs.checksecret.outputs.HOUDINI_SECRETS == 'true' ||
          github.repository_owner == 'AcademySoftwareFoundation' }}
-    runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'ubuntu-20.04-8c-32g-300h') || 'ubuntu-latest' }}
-    name: hou:${{ matrix.config.hou_hash }}-vfx:${{ matrix.config.image }}-cxx:${{ matrix.config.cxx }}
+    runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'ubuntu-22.04-8c-32g-300h') || 'ubuntu-latest' }}
+    name: hou:${{ matrix.config.hou_hash }}-vfx:${{ matrix.config.image }}-cxx:${{ matrix.config.cxx }}-build:${{ matrix.config.build }}
    container:
      image: aswf/ci-base:${{ matrix.config.image }}
    env:
@@ -75,32 +75,18 @@ jobs:
    strategy:
      matrix:
        config:
-          - { cxx: clang++, image: '2024', hou_hash: '20_5', build: 'Release', components: 'core,hou,bin,view,render,python,test' }
-          - { cxx: clang++, image: '2023', hou_hash: '20_5', build: 'Debug',   components: 'core,hou,bin,view,render,python,test,axcore,axbin,axtest' }
-          - { cxx: g++,     image: '2023', hou_hash: '20_5', build: 'Release', components: 'core,hou,bin,view,render,python,test,axcore,axbin,axtest' }
-          - { cxx: clang++, image: '2023', hou_hash: '20_0-newabi', build: 'Release', components: 'core,hou,bin,view,render,python,test,axcore,axbin,axtest' }
-          - { cxx: g++,     image: '2023', hou_hash: '20_0-newabi', build: 'Release', components: 'core,hou,bin,view,render,python,test,axcore,axbin,axtest' }
-          - { cxx: clang++, image: '2022',   hou_hash: '20_0-oldabi', build: 'Release', components: 'core,hou' }
-          - { cxx: g++,     image: '2022',   hou_hash: '20_0-oldabi', build: 'Release', components: 'core,hou' }
+          - { cxx: clang++, image: '2026', hou_hash: '21_0', build: 'Release', components: 'core,hou,bin,view,render,python,test,axcore,axbin,axtest' }
+          - { cxx: clang++, image: '2026', hou_hash: '21_0', build: 'Debug',   components: 'core,hou,bin,view,render,python,test,axcore,axbin,axtest' }
+          - { cxx: g++,     image: '2026', hou_hash: '21_0', build: 'Release', components: 'core,hou,bin,view,render,python,test,axcore,axbin,axtest' }
+          - { cxx: clang++, image: '2024', hou_hash: '20_5', build: 'Release', components: 'core,hou,bin,view,render,python,test,axcore,axbin,axtest' }
+          - { cxx: g++,     image: '2024', hou_hash: '20_5', build: 'Release', components: 'core,hou,bin,view,render,python,test,axcore,axbin,axtest' }
      fail-fast: false
    steps:
-      # See note on this step in the Houdini weekly.yml job
-      # We can remove this when we no longer use < 2023 images
-    - name: Enable Node 16
-      run: |
-        echo "ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION=true" >> $GITHUB_ENV
-    - name: remove zstd
-      run: yum -y remove zstd
    - uses: actions/checkout@v3
-    - name: pybind11
-      #if: contains(matrix.config.image, '2023') == false
-      run: ./ci/install_pybind11.sh 2.10.0
-    - name: glfw
-      if: contains(matrix.config.image, '2023') == true
-      run: ./ci/install_glfw.sh 3.3.10
-    - name: cppunit
-      if: contains(matrix.config.image, '2022') == false
-      run: ./ci/install_cppunit.sh 1.15.1
+    - name: nanobind
+      run: ./ci/install_nanobind.sh 2.5.0
+    - name: install_gtest
+      run: ./ci/install_gtest.sh latest
    - name: timestamp
      id: timestamp
      run: echo "timestamp=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
@@ -131,7 +117,7 @@ jobs:
    - name: build
      run: |
        ./ci/build.sh -v \
-          --build-type=Release \
+          --build-type="${{ matrix.config.build }}" \
          --components="${{ matrix.config.components }}" \
          --cargs=\" \
            -DHOUDINI_ROOT=$HOME/houdini_install/hou \
@@ -141,6 +127,7 @@ jobs:
            -DDISABLE_DEPENDENCY_VERSION_CHECKS=ON \
            \"
    - name: test
+      if: matrix.config.build == 'Release'
      run: cd build && ctest -V
    # Keep ccache light by stripping out any caches not accessed in the last day
    - name: ccache_clean
@@ -171,7 +158,7 @@ jobs:
        cp hou/hou.tar.gz $HOME/houdini_install/hou.tar.gz
        cd $HOME/houdini_install && tar -xzf hou.tar.gz && cd -
    - name: install_deps
-      run: ./ci/install_macos.sh 15
+      run: ./ci/install_macos.sh 20
    - name: build
      run: |
        ./ci/build.sh -v \
@@ -184,8 +171,7 @@ jobs:
            -DDISABLE_CMAKE_SEARCH_PATHS=ON \
            -DDISABLE_DEPENDENCY_VERSION_CHECKS=ON \
            -DUSE_EXPLICIT_INSTANTIATION=OFF \
-            -DTbb_INCLUDE_DIR=$HOME/houdini_install/hou/Frameworks/Houdini.framework/Versions/Current/Resources/toolkit/include/tbb \
-            -DLLVM_DIR=/opt/homebrew/opt/llvm@15/lib/cmake/llvm \
+            -DLLVM_DIR=/opt/homebrew/opt/llvm@20/lib/cmake/llvm \
            -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/install \
            \"
    - name: test
@@ -51,7 +51,7 @@ jobs:
      github.event_name != 'workflow_dispatch' ||
      github.event.inputs.type == 'all' ||
      github.event.inputs.type == 'linux'
-    runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'ubuntu-20.04-8c-32g-300h') || 'ubuntu-latest' }}
+    runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'ubuntu-22.04-8c-32g-300h') || 'ubuntu-latest' }}
    name: >
      linux-nanovdb:cxx:${{ matrix.config.cxx }}-${{ matrix.config.build }}
    container:
@@ -61,30 +61,20 @@ jobs:
    strategy:
      matrix:
        config:
-          - { cxx: g++,     image: '2022-clang11', build: 'Release' }
-          - { cxx: g++,     image: '2022-clang11', build: 'Debug' }
-          - { cxx: clang++, image: '2022-clang11', build: 'Release' }
-          - { cxx: clang++, image: '2022-clang11', build: 'Debug' }
+          - { cxx: g++,     image: '2024-clang17.2', build: 'Release' }
+          - { cxx: g++,     image: '2024-clang17.2', build: 'Debug' }
+          - { cxx: clang++, image: '2024-clang17.2', build: 'Release' }
+          - { cxx: clang++, image: '2024-clang17.2', build: 'Debug' }
      fail-fast: false
    steps:
-      - name: Enable Node 16
-        if: contains(matrix.config.image, '2022')
-        run: |
-          echo "ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION=true" >> $GITHUB_ENV
      - uses: actions/checkout@v3
-      - name: install_cuda_11
+      - name: setup_cuda_12
        run: |
-          # Fix error: Cannot prepare internal mirrorlist: No URLs in mirrorlist. CentOS 8 reached EOL means need to replace the official mirror to vault.centos.org
-          # Comment out mirrorlist and replace #baseurl=...mirror.centos.org with baseurl=...vault.centos.org in files starting with CentOS- in /etc/yum.repos.d folder
-          sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-*
-          sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*
-          yum -y install yum-utils
-          yum-config-manager --add-repo http://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/cuda-rhel7.repo
-          echo "Installing cuda toolkit"
-          yum --enablerepo=epel -y install cuda-toolkit-11-6
-          echo "/usr/local/cuda-11.6/bin" >> $GITHUB_PATH
-          echo "LD_LIBRARY_PATH=/usr/local/cuda-11.6/lib64:$LD_LIBRARY_PATH" >> $GITHUB_ENV
+          echo "/usr/local/cuda-12/bin" >> $GITHUB_PATH
+          echo "LD_LIBRARY_PATH=/usr/local/cuda-12/lib64:$LD_LIBRARY_PATH" >> $GITHUB_ENV
      - name: build
+      # NOTE: CMAKE_POSITION_INDEPENDENT_CODE set to fix default behaviour change in clang 14
+      #   https://github.com/AcademySoftwareFoundation/aswf-docker/issues/228
        run: >
          ./ci/build.sh -v
          --build-type=${{ matrix.config.build }}
@@ -92,12 +82,14 @@ jobs:
          --cargs=\'
          -DUSE_EXPLICIT_INSTANTIATION=OFF
          -DNANOVDB_USE_CUDA=ON
+          -DCMAKE_CUDA_ARCHITECTURES="80"
          -DNANOVDB_USE_OPENVDB=ON
          -DCMAKE_INSTALL_PREFIX=`pwd`
          -DUSE_BLOSC=OFF
+          -DCMAKE_POSITION_INDEPENDENT_CODE=ON
          \'
      - name: test
-        run: cd build && sudo ctest -V -E ".*cuda.*"
+        run: cd build && sudo ctest -V -E ".*cuda.*|.*mgpu.*"

  windows-nanovdb:
    if: |
@@ -125,6 +117,7 @@ jobs:
      shell: powershell
      run: .\ci\install_windows.ps1
    - name: build
+    # nvcc doesn't set _WIN32 when run in bash so we need to set it manually
      shell: bash
      run: >
        ./ci/build.sh -v
@@ -135,13 +128,15 @@ jobs:
        -DMSVC_COMPRESS_PDB=ON
        -DUSE_EXPLICIT_INSTANTIATION=OFF
        -DNANOVDB_USE_CUDA=ON
+        -DCMAKE_CUDA_ARCHITECTURES="80"
+        -DCMAKE_CUDA_FLAGS="-D_WIN32"
        -DNANOVDB_USE_OPENVDB=ON
        -DVCPKG_TARGET_TRIPLET=${VCPKG_DEFAULT_TRIPLET}
        -DCMAKE_TOOLCHAIN_FILE=\"${VCPKG_INSTALLATION_ROOT}\\scripts\\buildsystems\\vcpkg.cmake\"
        \'
    - name: test
      shell: bash
-      run: cd build && ctest -V -E ".*cuda.*"
+      run: cd build && ctest -V -E ".*cuda.*|.*mgpu.*"

  macos-nanovdb:
    if: |
@@ -154,8 +149,8 @@ jobs:
    strategy:
      matrix:
        config:
-          - { runner: 'macos-12', cxx: 'clang++', build: 'Release' }
-          - { runner: 'macos-12', cxx: 'clang++', build: 'Debug'   }
+          - { runner: 'macos-14', cxx: 'clang++', build: 'Release' }
+          - { runner: 'macos-14', cxx: 'clang++', build: 'Debug'   }
      fail-fast: false
    steps:
      - uses: actions/checkout@v3
@@ -168,22 +163,22 @@ jobs:
          ./ci/build.sh -v
          --build-type=${{ matrix.config.build }}
          --components=core,nano,nanotest,nanoexam,nanobench,nanotool
-          --cargs=\'-DUSE_EXPLICIT_INSTANTIATION=OFF -DNANOVDB_USE_CUDA=OFF -DNANOVDB_USE_OPENVDB=ON\'
+          --cargs=\'-DUSE_EXPLICIT_INSTANTIATION=OFF -DNANOVDB_USE_CUDA=OFF -DNANOVDB_USE_OPENVDB=ON -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/install \'
      - name: test
-        run: cd build && ctest -V -E ".*cuda.*"
+        run: cd build && ctest -V -E ".*cuda.*|.*mgpu.*"

  nanovdb-lite:
    if: |
      github.event_name != 'workflow_dispatch' ||
      github.event.inputs.type == 'all' ||
      github.event.inputs.type == 'linux'
-    runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'ubuntu-20.04-8c-32g-300h') || 'ubuntu-latest' }}
+    runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'ubuntu-22.04-8c-32g-300h') || 'ubuntu-latest' }}
+    container:
+      image: aswf/ci-openvdb:2024-clang17.2
    steps:
      - uses: actions/checkout@v3
      - name: install_gtest
-        run: |
-          sudo apt-get update
-          sudo apt-get -q install -y libgtest-dev
+        run: ./ci/install_gtest.sh latest
      - name: build_and_test
        run: |
          cd nanovdb/nanovdb
@@ -61,7 +61,7 @@ jobs:
      (github.event_name != 'workflow_dispatch' ||
       github.event.inputs.type == 'all' ||
       github.event.inputs.type == 'houdini')
-    runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'ubuntu-20.04-8c-32g-300h') || 'ubuntu-latest' }}
+    runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'ubuntu-22.04-8c-32g-300h') || 'ubuntu-latest' }}
    name: linux-houdini:${{ matrix.config.hou_hash }}
    env:
      CXX: clang++
@@ -70,28 +70,13 @@ jobs:
    strategy:
      matrix:
        config:
-          - { houdini_version: '19.5', platform: 'linux_x86_64_gcc9.3',  hou_hash: '19_5' }
-          - { houdini_version: '20.0', platform: 'linux_x86_64_gcc9.3',  hou_hash: '20_0-oldabi' }
-          - { houdini_version: '20.0', platform: 'linux_x86_64_gcc11.2', hou_hash: '20_0-newabi' }
+          - { houdini_version: '21.0', platform: 'linux_x86_64_gcc11.2', hou_hash: '21_0' }
          - { houdini_version: '20.5', platform: 'linux_x86_64_gcc11.2', hou_hash: '20_5' }
      fail-fast: false
    container:
      image: aswf/ci-base:2024
    steps:
-    - name: Enable Node 16
-      run: |
-        echo "ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION=true" >> $GITHUB_ENV
    - uses: actions/checkout@v3
-    # We bumped from the 2021 CI image to 2023 here to fix some OpenSSL issues
-    # with the Houdini download script. In so doing we broke some of the caching
-    # between this job and the jobs in houdini.yml which _don't_ use the 2023
-    # image yet. The issue is that the cache action will use zstd if it's
-    # available to zip the cache and this causes it to be inserted with a unique
-    # hash which images without zstd (i.e. the 2021/2022 images don't have
-    # access to). For now, uninstall zstd here instead of installing it
-    # everywhere and ask the LF to add zstd to the older base images.
-    - name: remove zstd
-      run: yum -y remove zstd
    - name: timestamp
      id: timestamp
      run: echo "timestamp=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
@@ -118,7 +103,7 @@ jobs:
    # Note that macos-14 (current macos-latest) switches to M1. We could instead test
    # the arm build here instead of the x86 one.
    runs-on: macos-latest
-    name: macos-houdini-20
+    name: macos-houdini
    env:
      HOUDINI_CLIENT_ID: ${{ secrets.HOUDINI_CLIENT_ID }}
      HOUDINI_SECRET_KEY: ${{ secrets.HOUDINI_SECRET_KEY }}
@@ -128,7 +113,7 @@ jobs:
      id: timestamp
      run: echo "timestamp=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
    - name: download_houdini
-      run: ./ci/download_houdini.sh 20.0 macosx_arm64_clang14.0_13 --prod
+      run: ./ci/download_houdini.sh 21.0 macosx_arm64_clang15.0_14 --prod
    - name: install_houdini
      run: |
        mkdir $HOME/houdini_install
@@ -151,34 +136,38 @@ jobs:
      github.event_name != 'workflow_dispatch' ||
      github.event.inputs.type == 'all' ||
      github.event.inputs.type == 'extra'
-    runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'ubuntu-20.04-8c-32g-300h') || 'ubuntu-latest' }}
+    runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'ubuntu-22.04-8c-32g-300h') || 'ubuntu-latest' }}
    name: linux-extra:${{ matrix.config.name }}
    container:
-      image: aswf/ci-openvdb:2024
+      image: aswf/ci-openvdb:${{ matrix.config.image }}
    env:
      CXX: clang++
    strategy:
      matrix:
        config:
-          - { name: 'all',   build: 'Release', components: 'core,python,bin,view,render,test',               cmake: '-DUSE_BLOSC=ON  -DUSE_ZLIB=ON  -DUSE_EXR=ON  -DUSE_PNG=ON'  }
-          - { name: 'lite',  build: 'Release', components: 'core,python,bin,view,render,test',               cmake: '-DUSE_BLOSC=OFF -DUSE_ZLIB=OFF -DUSE_EXR=OFF -DUSE_PNG=OFF -DOPENVDB_USE_DELAYED_LOADING=OFF' }
-          - { name: 'half',  build: 'Release', components: 'core,python,bin,view,render,test',               cmake: '-DUSE_BLOSC=OFF -DUSE_IMATH_HALF=ON' }
-          - { name: 'sse',   build: 'Release', components: 'core,python,bin,view,render,test',               cmake: '-DOPENVDB_SIMD=SSE42' }
-          - { name: 'avx',   build: 'Release', components: 'core,python,bin,view,render,test',               cmake: '-DOPENVDB_SIMD=AVX' }
-          - { name: 'numpy', build: 'Release', components: 'core,python,bin,view,render,test',               cmake: '-DUSE_NUMPY=ON -DOPENVDB_PYTHON_WRAP_ALL_GRID_TYPES=ON' }
-          - { name: 'asan',  build: 'asan',    components: 'core,test,axcore,axtest',                        cmake: '-DNANOVDB_USE_OPENVDB=ON -DOPENVDB_AX_STATIC=OFF -DOPENVDB_CORE_STATIC=OFF -DUSE_BLOSC=OFF' } # We never called blosc_destroy(), so disable blosc to silence these errors
-          - { name: 'ubsan', build: 'ubsan',   components: 'core,test,axcore,axtest',                        cmake: '-DCMAKE_CXX_FLAGS="-Wno-deprecated-declarations" ' }
-          - { name: 'c++20', build: 'Release', components: 'core,test,axcore,axtest',                        cmake: '-DCMAKE_CXX_STANDARD=20' }
-          - { name: 'conf',  build: 'Release', components: 'core,python,bin,view,render,test,axcore,axtest', cmake: '-DCMAKE_FIND_PACKAGE_PREFER_CONFIG=ON' }
+          - { name: 'all',    image: '2026',           build: 'Release', components: 'core,python,bin,view,render,test', cmake: '-DUSE_BLOSC=ON  -DUSE_ZLIB=ON  -DUSE_EXR=ON  -DUSE_PNG=ON'  }
+          - { name: 'lite',   image: '2026',           build: 'Release', components: 'core,python,bin,view,render,test', cmake: '-DUSE_BLOSC=OFF -DUSE_ZLIB=OFF -DUSE_EXR=OFF -DUSE_PNG=OFF -DOPENVDB_USE_DELAYED_LOADING=OFF' }
+          - { name: 'half',   image: '2026',           build: 'Release', components: 'core,python,bin,view,render,test', cmake: '-DUSE_BLOSC=OFF -DUSE_IMATH_HALF=ON' }
+          - { name: 'sse',    image: '2026',           build: 'Release', components: 'core,python,bin,view,render,test', cmake: '-DOPENVDB_SIMD=SSE42' }
+          - { name: 'avx',    image: '2026',           build: 'Release', components: 'core,python,bin,view,render,test', cmake: '-DOPENVDB_SIMD=AVX' }
+          - { name: 'pygrid', image: '2026',           build: 'Release', components: 'core,python,bin,view,render,test', cmake: '-DOPENVDB_PYTHON_WRAP_ALL_GRID_TYPES=ON' }
+          - { name: 'asan',   image: '2026',           build: 'asan',    components: 'core,test',                        cmake: '-DDISABLE_DEPENDENCY_VERSION_CHECKS=ON -DNANOVDB_USE_OPENVDB=ON -DOPENVDB_AX_STATIC=OFF -DOPENVDB_CORE_STATIC=OFF -DUSE_BLOSC=OFF' } # We never called blosc_destroy(), so disable blosc to silence these errors
+          - { name: 'ubsan',  image: '2026',           build: 'ubsan',   components: 'core,test',                        cmake: '-DDISABLE_DEPENDENCY_VERSION_CHECKS=ON -DCMAKE_CXX_FLAGS="-Wno-deprecated-declarations" ' }
+          - { name: 'c++20',  image: '2026',           build: 'Release', components: 'core,test',                        cmake: '-DCMAKE_CXX_STANDARD=20' }
+          - { name: 'conf',   image: '2026',           build: 'Release', components: 'core,python,bin,view,render,test', cmake: '-DCMAKE_FIND_PACKAGE_PREFER_CONFIG=ON' }
      fail-fast: false
    steps:
-    - name: Enable Node 16
-      run: |
-        echo "ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION=true" >> $GITHUB_ENV
    - uses: actions/checkout@v3
-    - name: pybind11
-      #if: contains(container.image, '2023') == false
-      run: ./ci/install_pybind11.sh 2.10.0
+    - name: nanobind
+      run: ./ci/install_nanobind.sh 2.5.0
+      # 2026 images are broken with gtest/glfw/exr installs
+    - name: deps
+      if: contains(matrix.config.image, '2026')
+      run: |
+        ./ci/install_gtest.sh latest
+        ./ci/install_glfw.sh latest
+        ./ci/install_imath.sh v3.2.2
+        ./ci/install_exr.sh v3.2.2
    - name: build
      run: >
        ./ci/build.sh -v
@@ -200,20 +189,22 @@ jobs:
    strategy:
      matrix:
        config:
-          - { runson: ubuntu-latest, cxx: g++,     cmake: '' }
+          # llvm-17 is currently the default which has symbol issues with the python module
+          - { runson: ubuntu-latest, cxx: g++, cmake: '-DOPENVDB_PYTHON_USE_AX=OFF' }
          # Disable the clang job for now. See https://github.com/actions/runner-images/issues/8659
          # - { runson: ubuntu-latest, cxx: clang++, cmake: '' }
          # @todo gcc on macos
-          - { runson: macos-latest,  cxx: '',      cmake: '-DCMAKE_CXX_COMPILER=/opt/homebrew/opt/llvm@15/bin/clang++ -DLLVM_DIR=/opt/homebrew/opt/llvm@15/lib/cmake/llvm' }
+          - { runson: macos-latest, cxx: '', cmake: '-DLLVM_DIR=/opt/homebrew/opt/llvm@20/lib/cmake/llvm' }
      fail-fast: false
    steps:
      - uses: actions/checkout@v3
      - name: install_deps
        run: |
          if [ "$RUNNER_OS" == "Linux" ]; then
-            sudo apt-get -q install -y libboost-dev libboost-iostreams-dev libtbb-dev libblosc-dev llvm-dev libgtest-dev libcppunit-dev pybind11-dev
+            sudo apt-get -q install -y libboost-dev libboost-iostreams-dev libtbb-dev libblosc-dev libgtest-dev libgmock-dev python3-numpy
+            ./ci/install_nanobind.sh 2.5.0
          elif [ "$RUNNER_OS" == "macOS" ]; then
-            ./ci/install_macos.sh 15
+            ./ci/install_macos.sh 20
            ./ci/install_tbb_macos.sh
          else
            echo "$RUNNER_OS not supported"; exit 1
@@ -223,7 +214,12 @@ jobs:
          ./ci/build.sh -v
          --build-type=Release
          --components=\"core,axcore,python,bin,render,test,axbin\"
-          --cargs=\"-DCMAKE_CXX_STANDARD=20 -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/install ${{ matrix.config.cmake }}\"
+          --cargs=\'
+          -DCMAKE_CXX_STANDARD=20
+          -DOPENVDB_USE_DELAYED_LOADING=OFF
+          -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/install
+          ${{ matrix.config.cmake }}
+          \'
      - name: test
        run: cd build && ctest -V

@@ -245,9 +241,9 @@ jobs:
          # Boost as both shared and static libs are installed.
          # USE_EXPLICIT_INSTANTIATION is disabled for debug static libraries
          # due to disk space constraints
-          - { vc: 'x64-windows-static', components: 'core,bin,view,render,test',       build: 'Release', cmake: '-A x64 -G \"Visual Studio 17 2022\" -DOPENVDB_CORE_SHARED=OFF -DUSE_STATIC_DEPENDENCIES=ON -DBLOSC_USE_EXTERNAL_SOURCES=ON' }
+          # - { vc: 'x64-windows-static', components: 'core,bin,view,render,test',       build: 'Release', cmake: '-A x64 -G \"Visual Studio 17 2022\" -DOPENVDB_CORE_SHARED=OFF -DUSE_STATIC_DEPENDENCIES=ON -DBLOSC_USE_EXTERNAL_SOURCES=ON' }
          - { vc: 'x64-windows',        components: 'core,bin,view,render,python,test', build: 'Release', cmake: '-A x64 -G \"Visual Studio 17 2022\" -DOPENVDB_CORE_STATIC=OFF' }
-          - { vc: 'x64-windows',        components: 'core,bin,view,render,python,test', build: 'Debug',   cmake: '-A x64 -G \"Visual Studio 17 2022\" -DOPENVDB_CORE_STATIC=OFF' }
+          # - { vc: 'x64-windows',        components: 'core,bin,view,render,python,test', build: 'Debug',   cmake: '-A x64 -G \"Visual Studio 17 2022\" -DOPENVDB_CORE_STATIC=OFF' }
      fail-fast: false
    steps:
    - uses: actions/checkout@v3
@@ -291,7 +287,7 @@ jobs:
      github.event_name != 'workflow_dispatch' ||
      github.event.inputs.type == 'all' ||
      github.event.inputs.type == 'ax'
-    runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'ubuntu-20.04-8c-32g-300h') || 'ubuntu-latest' }}
+    runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'ubuntu-22.04-8c-32g-300h') || 'ubuntu-latest' }}
    name: >
      linux-ax:${{ matrix.config.image }}-cxx:${{ matrix.config.cxx }}-${{ matrix.config.build }}
    container:
@@ -301,25 +297,14 @@ jobs:
    strategy:
      matrix:
        config:
-          # Unified
-          - { image: '2023-clang14', cxx: 'clang++', build: 'Release', components: 'core,bin,axcore,axbin,axtest', cmake: '' }
-          - { image: '2023-clang14', cxx: 'g++',     build: 'Release', components: 'core,bin,axcore,axbin,axtest', cmake: '' }
-          - { image: '2022-clang13', cxx: 'clang++', build: 'Release', components: 'core,bin,axcore,axbin,axtest', cmake: '' }
-          - { image: '2022-clang13', cxx: 'g++',     build: 'Release', components: 'core,bin,axcore,axbin,axtest', cmake: '' }
-          # Standalone
-          - { image: '2022-clang11', cxx: 'clang++', build: 'Debug',   components: 'core', cmake: '' }
-          - { image: '2022-clang11', cxx: 'clang++', build: 'Release', components: 'core', cmake: '' }
-          - { image: '2022-clang11', cxx: 'g++',     build: 'Release', components: 'core', cmake: '' }
+          - { image: '2025-clang18.1', cxx: 'clang++', build: 'Release', components: 'core,axcore,axbin,axtest', cmake: '' }
+          # @note  LLVM 17 builds are incompatible with the vdb python plugin on linux
+          - { image: '2024-clang17.2', cxx: 'clang++', build: 'Release', components: 'core,axcore,axbin,axtest', cmake: '-DOPENVDB_PYTHON_USE_AX=OFF' }
      fail-fast: false
    steps:
-      - name: Enable Node 16
-        if: contains(matrix.config.image, '2022')
-        run: |
-          echo "ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION=true" >> $GITHUB_ENV
      - uses: actions/checkout@v3
-      - name: pybind11
-        #f: contains(matrix.config.image, '2023') == false
-        run: ./ci/install_pybind11.sh 2.10.0
+      - name: nanobind
+        run: ./ci/install_nanobind.sh 2.5.0
      - name: build
        run: >
          ./ci/build.sh -v
@@ -330,6 +315,7 @@ jobs:
          -DOPENVDB_AX_TEST_CMD_DOWNLOADS=ON
          -DUSE_EXPLICIT_INSTANTIATION=OFF
          -DOPENVDB_CXX_STRICT=ON
+          -DOPENVDB_BUILD_VDB_TOOL=OFF
          \"
      - name: clean
        if: matrix.config.components == 'core'
@@ -345,6 +331,7 @@ jobs:
          -DOPENVDB_AX_TEST_CMD_DOWNLOADS=ON
          -DUSE_EXPLICIT_INSTANTIATION=OFF
          -DOPENVDB_CXX_STRICT=ON
+          -DOPENVDB_BUILD_VDB_TOOL=OFF
          \"
      - name: test
        run: cd build && ctest -V
@@ -356,16 +343,14 @@ jobs:
      github.event_name != 'workflow_dispatch' ||
      github.event.inputs.type == 'all' ||
      github.event.inputs.type == 'ax'
-    runs-on: macos-13
+    runs-on: ${{ matrix.config.runner }}
    name: macos-cxx:${{ matrix.config.cxx }}-llvm:${{ matrix.config.llvm }}-${{ matrix.config.build }}
    env:
      CXX: ${{ matrix.config.cxx }}
    strategy:
      matrix:
        config:
-          - { cxx: 'clang++', build: 'Release', llvm: '14' }
-          - { cxx: 'clang++', build: 'Release', llvm: '15' }
-          #- { cxx: 'clang++', build: 'Release', llvm: '16' } - not supported yet
+          - { runner: 'macos-latest', cxx: 'clang++', build: 'Release', llvm: '20' }
      fail-fast: false
    steps:
      - uses: actions/checkout@v3
@@ -395,7 +380,7 @@ jobs:
      github.event.inputs.type == 'all' ||
      github.event.inputs.type == 'ax'
    runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'windows-2022-8c-32g-300h') || 'windows-latest' }}
-    name: windows-vc:${{ matrix.config.vc }}-type:${{ matrix.config.build }}
+    name: ${{ matrix.config.vc }}-${{ matrix.config.crt }}-${{ matrix.config.build }}-llvm${{ matrix.config.llvm }}
    env:
      VCPKG_DEFAULT_TRIPLET: ${{ matrix.config.vc }}
      # Export this with '' avoid bash treating \ as escape
@@ -409,17 +394,22 @@ jobs:
          # @todo  We don't currently run the axtests with shared builds of ax
          # due to symbol issues using LLVM as a static lib (which is the only
          # option on Windows).
-          - { vc: 'x64-windows',        crt: 'MD',  components: 'core,bin,axcore,axbin,python', build: 'Release', cmake: '-DOPENVDB_CORE_STATIC=OFF -DOPENVDB_AX_STATIC=OFF' }
-          - { vc: 'x64-windows-static', crt: 'MT',  components: 'core,bin,axcore,axbin,axtest', build: 'Release', cmake: '-DOPENVDB_CORE_SHARED=OFF -DOPENVDB_AX_SHARED=OFF -DUSE_STATIC_DEPENDENCIES=ON -DBLOSC_USE_EXTERNAL_SOURCES=ON' }
-          - { vc: 'x64-windows-static', crt: 'MTd', components: 'core,bin,axcore,axbin,axtest', build: 'Debug',   cmake: '-DOPENVDB_CORE_SHARED=OFF -DOPENVDB_AX_SHARED=OFF -DUSE_STATIC_DEPENDENCIES=ON -DBLOSC_USE_EXTERNAL_SOURCES=ON' }
+          # @note  currently only test static builds with MT as building LLVM as
+          # a shared lib on windows is a bit dicey
+          - { vc: 'x64-windows-static', llvm: '18.1.0', crt: 'MultiThreaded',      components: 'core,axcore,axbin,axtest', build: 'Release', cmake: '-DOPENVDB_CORE_SHARED=OFF -DOPENVDB_AX_SHARED=OFF -DUSE_STATIC_DEPENDENCIES=ON -DBLOSC_USE_EXTERNAL_SOURCES=ON' }
+          - { vc: 'x64-windows-static', llvm: '18.1.0', crt: 'MultiThreadedDebug', components: 'core,axcore,axbin,axtest', build: 'Debug',   cmake: '-DOPENVDB_CORE_SHARED=OFF -DOPENVDB_AX_SHARED=OFF -DUSE_STATIC_DEPENDENCIES=ON -DBLOSC_USE_EXTERNAL_SOURCES=ON' }
+          - { vc: 'x64-windows',        llvm: '18.1.0', crt: 'MultiThreadedDLL',   components: 'core,axcore,axbin',        build: 'Release', cmake: '-DOPENVDB_CORE_STATIC=OFF -DOPENVDB_AX_STATIC=OFF' }
+          - { vc: 'x64-windows-static', llvm: '15.0.0', crt: 'MultiThreaded',      components: 'core,axcore,axbin,axtest', build: 'Release', cmake: '-DOPENVDB_CORE_SHARED=OFF -DOPENVDB_AX_SHARED=OFF -DUSE_STATIC_DEPENDENCIES=ON -DBLOSC_USE_EXTERNAL_SOURCES=ON' }
      fail-fast: false
    steps:
    - uses: actions/checkout@v3
    - name: llvm
-      run: ./ci/install_llvm_windows.sh ${{ matrix.config.crt }}
+      run: ./ci/install_llvm_windows.sh ${{ matrix.config.llvm }} ${{ matrix.config.crt }}
    - name: install
      shell: powershell
-      run: .\ci\install_windows.ps1
+      run: |
+        vcpkg update
+        vcpkg install tbb gtest
    - name: build
      run: >
        ./ci/build.sh -v
@@ -427,10 +417,13 @@ jobs:
        --components="${{ matrix.config.components }}"
        --cargs=\'
        -A x64 -G \"Visual Studio 17 2022\"
+        -DOPENVDB_USE_DELAYED_LOADING=OFF
+        -DUSE_BLOSC=OFF \
+        -DUSE_ZLIB=OFF \
        -DVCPKG_TARGET_TRIPLET=${VCPKG_DEFAULT_TRIPLET}
        -DCMAKE_TOOLCHAIN_FILE=\"${VCPKG_INSTALLATION_ROOT}\\scripts\\buildsystems\\vcpkg.cmake\"
        -DMSVC_COMPRESS_PDB=ON
-        -DOPENVDB_AX_TEST_CMD_DOWNLOADS=ON
+        -DOPENVDB_AX_TEST_CMD_DOWNLOADS=OFF
        -DUSE_EXPLICIT_INSTANTIATION=OFF
        -DLLVM_DIR=\"${HOME}\\llvm_install\\lib\\cmake\\llvm\"
        -DCMAKE_INSTALL_PREFIX=\"${VDB_INSTALL_PREFIX}\"
@@ -453,7 +446,7 @@ jobs:
    if: |
      github.event_name != 'workflow_dispatch' ||
      github.event.inputs.type == 'all' ||
-      github.event.inputs.type == 'win'
+      github.event.inputs.type == 'winnano'
    runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'windows-2022-8c-32g-300h') || 'windows-latest' }}
    env:
      VCPKG_DEFAULT_TRIPLET: ${{ matrix.config.vc }}
@@ -495,6 +488,7 @@ jobs:
        -DMSVC_COMPRESS_PDB=ON
        -DUSE_EXPLICIT_INSTANTIATION=OFF
        -DNANOVDB_USE_CUDA=ON
+        -DCMAKE_CUDA_ARCHITECTURES="80"
        -DNANOVDB_USE_OPENVDB=ON
        -DVCPKG_TARGET_TRIPLET=${VCPKG_DEFAULT_TRIPLET}
        -DCMAKE_TOOLCHAIN_FILE=\"${VCPKG_INSTALLATION_ROOT}\\scripts\\buildsystems\\vcpkg.cmake\"
@@ -512,18 +506,15 @@ jobs:
      github.event_name != 'workflow_dispatch' ||
      github.event.inputs.type == 'all' ||
      github.event.inputs.type == 'blosc'
-    runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'ubuntu-20.04-8c-32g-300h') || 'ubuntu-latest' }}
+    runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'ubuntu-22.04-8c-32g-300h') || 'ubuntu-latest' }}
    name: linux-blosc:${{ matrix.blosc }}
    container:
-      image: aswf/ci-base:2023
+      image: aswf/ci-openvdb:2025-clang19.1
    strategy:
      matrix:
-        blosc: ['1.18.0','1.19.0','1.20.0','1.21.0']
+        blosc: ['1.21.0', 'latest']
      fail-fast: false
    steps:
-    - name: Enable Node 16
-      run: |
-        echo "ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION=true" >> $GITHUB_ENV
    - uses: actions/checkout@v3
    - name: install_blosc
      run: sudo ./ci/install_blosc.sh ${{ matrix.blosc }}
@@ -611,7 +602,7 @@ jobs:
        -strict
        -extended
    - name: upload_report
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
      if: always()
      with:
        name: abi_report
@@ -1,7 +1,385 @@
 OpenVDB Version History
 =======================

-Version 11.0.1 - In development
+Version 12.1.2 - In development
+
+
+Version 12.1.1 - September 30, 2025
+
+  OpenVDB AX:
+    Fixes:
+    - Fixed an issue which was causing host specific optimization passes to be
+      omitted when using LLVM 16 and higher.
+
+  Houdini:
+    - Fixed a compilation issue with the OpenVDB Activate SOP with Houdini 21
+    [Reported by @callen128]
+
+
+Version 12.1.0 - August 8, 2025
+
+  OpenVDB:
+    New features:
+    - New level set constructor tools::createLevelSetCapsule creates a grid
+      containing a narrow-band level set representation of a capsule (tube
+      with constant radius and sphere caps).
+    - New level set constructor tools::createLevelSetTaperedCapsule creates a
+      grid containing a narrow-band level set representation of a tapered
+      capsule (tube with sphere caps and different radii at both ends, or
+      equivalently the convex hull of two spheres with possibly different
+      centers and radii).
+    - New level set constructor tools::createLevelSetTubeComplex creates a grid
+      containing a narrow-band level set representation of a tube complex (a
+      collection of capsules or tapered capsules defined by endpoint coordinates
+      and segment indices).
+    - New level set constructor tools::createLevelSetDilatedMesh creates a grid
+      containing a narrow-band level set representation of a dilated surface
+      mesh (dilated by a radius in all directions).
+    - New class tools::ConvexVoxelizer, used by the above methods, which can be
+      inherited to implement a convex level set constructor.
+    - Introduced anisotropic surfacing tools in points/PointRasterizeSDF.h and
+      principal component analysis (PCA) methods in points/PrincipalComponentAnalysis.h.
+      The latter tool analyses point neighborhoods and computes affine
+      transformations representing elliptical distributions. The prior tool
+      takes rotational and stretch point attributes (optionally computed from
+      the prior PCA methods) and generates anisotropic surfaces.
+
+    Improvements:
+    - Significantly improved the performance of all LeafNode ValueIterators,
+      up to 5x on some platforms and up to 10x when delay loading is enabled.
+      Construction of a ValueIterator from a leaf node now requests the leaf
+      buffers ahead of iteration to avoid potentially expensive API calls.
+    - Improved the performance of PointRasterizeSDF.h tools with larger radii
+      with better bound computation approximations.
+    - Significant performance improvement to delay loaded files - the warning
+      logic which checks the underlying mapped file for the last modified time
+      has been removed, resulting in performance gains upwards of 10x when
+      loading in node data buffers. Note that the check that was removed is
+      entirely redundant on Windows and would only print a warning on other
+      system.
+      [Contributed by jjtParadox]
+    - Added OPENVDB_HAS_INVOKABLE_MEMBER_FUNCTION and OPENVDB_HAS_MEMBER_FUNCTION
+      macros to help determine the presence and compatibility of interface
+      member functions at compile time.
+    - Added OPENVDB_ASSUME macros to mimic builtin assume and C++23 assume
+      attributes.
+    - Improved read and write of PLY files with vdb_tool so they support Vec3d
+      and 64 bit attributes. Changed vdb_tool version number from 10.6.1 to
+      10.7.0.
+
+    API Changes:
+    - Mat3 and Mat4 functions have been marked [[nodiscard]] where it may be
+      confusing if they are operating in place or returning a result, in
+      particular tranpose, invert, adjoint, and timesDiagonal.
+    - Re-worked the API in PointRasterizeSDF.h with the addition of anisotropic
+      surfacing kernels. The new API exposes more concise free functions which
+      take structs of parameters. The old API remains supported but is
+      deprecated.
+    - Deprecated providing a filter or interrupter as arguments to
+      points::rasterize. These can instead be implemented on user transfer
+      schemes.
+    - Transfer schemes provided to points::rasterize can now optionally
+      implement a rasterizePoints callback that takes a range of points per
+      voxel.
+
+    Fixes:
+    - Fixed a bug in points::rasterizeSpheres and points::rasterizeSmoothSpheres
+      which could cause sections of the generated surface to incorrectly be
+      marked as negative interior values.
+    - Fixed a bug in points::rasterizeSmoothSpheres which could result in
+      incomplete surface reconstruction with significantly larger search
+      distances.
+    - Fixed some precision issues in various Matrix methods.
+    - Fixed bugs in vdb_tool that prevented build with newer namespaces in
+      nanovdb.
+
+  OpenVDB AX:
+    Improvements:
+    - Added support for LLVM versions greater than 15. Upgraded CI to test with
+      LLVM versions 16, 17, 18, 19 and 20.
+    - As of this release, LLVM 15 is the minimum required version for OpenVDB
+      AX. Note that there is an issue with LLVM 17 symbols which makes it
+      incompatible with the OpenVDB Python plugin. AS such, LLVM version 18 and
+      greater are the recommended LLVM versions to use, with LLVM 18 being the
+      new future minimum version.
+    - Significant performance improvement to the compilation of AX kernels when
+      using LLVM 18 and greater.
+    - Significant backend refactoring to AX IR generation to support newer
+      versions of LLVM. In particular, parts of the codegen::Function and
+      codegen::FunctionBuilder API require new usage from LLVM 16 due to
+      the LLVM opaque pointer changes. If you're not using the OpenVDB AX
+      Function codegen API then no client changes should be necessary.
+    - Migrated OpenVDB AX unit tests from cppunit to gtest
+      [Contributed by Tim Straubinger]
+
+    Fixes:
+    - Fixed a case of undefined behaviour when explicitly casting variables to
+      the same type as their storage type e.g:
+        double a; double b = double(a);
+    - Fixed an AX compiler error which would produce invalid IR when using 64bit
+      integer variables for matrix array offsetting e.g:
+        int64 a = 0; mat4f b = 1; float c = b[a,a];
+
+  NanoVDB:
+    - Added nanovdb::openToIndexVDB, which can convert an OpenVDB grid into a
+      NanoVDB IndexGrid.
+    - Added new options "--index" and "--onIndex" to the command-line tool
+      nanovdb_convert so it can produce IndexGrids.
+    - Fixed memory leak in DeviceBuffer.
+    - Added constructor for DeviceBuffer from an existing HostBuffer.
+    - Added VoxelBlockManager, an acceleration structure for sequential access
+      and stencil ops over active voxels.
+    - Added helper functions for kernel launches with shared memory.
+    - Added support for CUDA 13.0.
+    - Fix warnings and compilation errors for GCC and Clang.
+    - Fixed inconsistent usage of util::cuda::mallocAsync/freeAsync that
+      affected vGPUs.
+    - Added support for custom memory resources in PointsToGrid.
+    - Fixed race condition in split/mergeGridHandles.
+    - Fixed offset bug in nanovdb::tools::cuda::IndexToGrid related to non-POD
+      build types.
+      [Contributed by Andreas Henne]
+
+  Houdini:
+    - Define OP_InputIdx and OP_OutputIdx for Houdini 21.0 compatibility
+    - Add Houdini 21 style camera interface to support LOP cameras
+    - Include env flag for avoiding stream writing (was backported to 19.5)
+
+
+Version 12.0.1 - April 3, 2025
+
+  OpenVDB:
+    New features:
+    - Added read and write support for OFF (Object File Format) files to
+      vdb_tool
+
+    Build:
+    - Fixed a build issue with vdb_render on Windows with newer versions of
+      glew not including glu.h by default
+    - Fixed issues with newer versions of clang refusing to accept older
+      ::template syntax
+
+  NanoVDB:
+    New features:
+    - Added support for multiple GPUs to DeviceBuffer
+    - Added a UnifiedBuffer class that wraps CUDA unified memory
+    - Added example for multiGPU sparse convolution
+    - Added CUDA utility functions for device queries
+    - Added functions to independently stop and compute the elapsed time for
+      timer classes
+
+    Improvements:
+    - Replaced CUB's CachingDeviceAllocator with the default asynchronous
+      stream ordered allocator in PointsToGrid for improved performance
+
+    Bug fix:
+    - Map is now properly transposed when converting NanoVDB to OpenVDB.
+    - Fixed ostream specializations being hidden within the nanovdb namespace
+    - Fixed a call to a non-existent method in GridBuild.h
+
+
+Version 12.0.0 - October 31, 2024
+
+      OpenVDB is now licensed under the Apache 2.0 license, instead of the MPL 2.0 license.
+
+      This version introduces ABI changes relative to older major releases,
+      so to preserve ABI compatibility it might be necessary to define the
+      macro OPENVDB_ABI_VERSION_NUMBER=N, where, for example, N is 10 for
+      Houdini 20.0 and 11 for Houdini 20.5.
+
+      GCC 9 is no longer supported.
+
+  OpenVDB:
+    New features:
+    - Added fillet() method in tools::LevelSetFilter to round off concave edges
+      to create smoother transition between surfaces.
+
+    Improvements:
+    - Added openvdb::assertAbort to replace cassert and a
+      OPENVDB_ENABLE_ASSERTS cmake argument/compile define to toggle
+      assertions in OpenVDB code, independantly of NDEBUG. Asserts are
+      no longer enabled by default in when NDEBUG is absent (e.g.
+      Debug builds).
+    - Removed last traces of Boost when OPENVDB_USE_DELAYED_LOADING is OFF
+      [Reported by Brian McKinnon]
+    - RootNode code cleanup to eliminate redundant key conversion and
+      to create map values in-place.
+    - Add RootNode::deleteChildOrTile() to delete a child or tile of
+      the root node.
+    - ValueAccessors are now defined and created in the Tree class
+      instead of in the Grid class so that custom Tree implementations
+      may define and create their own ValueAccessors if desired.
+    - Added support for PDAL to vdb_tool [Contributed by Tom Matterson]
+    - LeafManager and NodeManager now use Index64 for leaf counts internally.
+    - Added RootNode::probeChild() const.
+    - Added RootNode::probeChild() and RootNode::probeConstChild().
+    - Added RootNode::probe() and RootNode::probeConst() to query key presence,
+      child node, value and active state.
+    - Added InternalNode::probeChild() const.
+    - Added InternalNode::probeChild() and probeChildConst() with coord access
+      and optionally value and active state.
+    - Added InternalNode::probeChild() and probeChildConst() with index access
+      and optionally value and active state.
+    - Added InternalNode::isValueOff(), LeafNode::isValueOff(),
+      LeafNodeBool::isValueOff(), LeafNodeMask::isValueOff().
+    - Added LeafNodeMask::probeValue(Index,val), LeafNodeBool::probeValue(Index,val).
+    - Added RootNode::getValueUnsafe(), RootNode::getChildUnsafe(),
+      RootNode::getConstChildUnsafe().
+    - Added InternalNode::getValueUnsafe(), InternalNode::getChildUnsafe(),
+      InternalNode::getConstChildUnsafe().
+    - Added InternalNode::setActiveStateUnsafe(), InternalNode::setValueOnlyUnsafe(),
+      InternalNode::setValueOnUnsafe(), InternalNode::setValueOffUnsafe().
+    - Added InternalNode::setChildUnsafe(), InternalNode::resetChildUnsafe(),
+      InternalNode::stealChildUnsafe(), InternalNode::deleteChildUnsafe().
+    - For LeafNode, LeafNodeBool and LeafNodeMask - added
+      LeafNode::getValueUnsafe(), LeafNode::setActiveStateunsafe(),
+      LeafNode::setValueOnlyUnsafe(), LeafNode::setValueOnUnsafe(),
+      LeafNode::setValueOffUnsafe().
+
+    ABI changes:
+    - Tree::leafCount(), Tree::unallocatedLeafCount(),
+      Tree::nonLeafCount() and Tree::nodeCount() now use Index64 in their
+      return types instead of Index32.
+
+    API Changes:
+    - RootNode::tileCount(), RootNode::activeTileCount() and
+      RootNode::inactiveTileCount() are now public.
+    - RootNode::hasKey() and RootNode::coordToKey() are now public.
+    - RootNode::leafCount(), RootNode::nonLeafCount() and RootNode::nodeCount()
+      now use Index64 instead of Index32. The Index32 variant is deprecated.
+    - InternalNode::leafCount(), InternalNode::nonLeafCount() and
+      InternalNode::nodeCount() now use Index64 instead of Index32. The Index32
+      variant is deprecated.
+    - LeafNode::leafCount() and LeafNode::nonLeafCount() now use Index64 instead
+      of Index32. The Index32 variant is deprecated.
+
+    Bug Fixes:
+    - Fix potential crash reading corrupt .vdb files with invalid
+      blosc or zip chunks.
+      [Fix thanks to Matthias Ueberheide]
+    - Fix a bug in RootNode::setOrigin() where the origin was updated
+      before the error was thrown potentially leaving the root in an
+      invalid state.
+    - Fixed a thread sanitizer issue which could cause undefined
+      behaviour in VolumeToSpheres::fillWithSpheres
+      [Reported by Jérémie Dumas]
+    - Fixed an occurance of undefined behaviour in tools::activate
+      (though this would typically not have manifested with any
+      unintended behaviour)
+
+  NanoVDB:
+    Bug fix:
+    - nanovdb::readGrids works with raw grid buffer.
+
+    Improvements:
+    - Restructure files location and namespace to be more align with
+      OpenVDB. The namespaces touched by the restructuring are: io,
+      cuda, util, tools, and math.
+    - Add two scripts updateFiles.sh and updateFiles.py to update the
+      files using NanoVDB. The script updateFiles.py works on both
+      Windows and Linux.  For a more complete list of changes, see API
+      Changes (details).
+
+    - cuda::PointsToGrid supports target density.
+    - Add support for NanoVDB Grid of type UInt8.
+    - Add ability to use externally managed CUDA buffer.
+    - Add create methods for CudaDeviceBuffer and exceptions.
+    - Improve GridValidator logic, e.g. include check for grid count.
+    - Add operator > and >= for class Coord according to lexicographical order.
+    - Add toCodec to convert string to Codec enumeration type.
+    - Add nanovdb::strlen<GridType>().
+    - Add strncpy util.
+    - Add NANOVDB_DISABLE_SYNC_CUDA_MALLOC that maps cudaMallocAsync
+      and cudaFreeAsync to cudaMalloc and cudaFree respectively.
+    - Add guard to UINT64_C.
+    - Remove use of cudaMallocAsync in PointsToGrid.cuh.
+    - Align PNanoVDB blind metadata to NanoVDB.
+
+    API Changes:
+    - Change mapToGridType to toGridType.
+    - Change mapToMagic to toMagic.
+    - Change CpuTimer.h to Timer.h.
+
+    API Changes (details):
+    - These APIs are now under the math namespace: Ray, DDA, HDDA,
+      Vec3, Vec4, BBox, ZeroCrossing, TreeMarcher, PointTreeMarcher,
+      BoxStencil, CurvatureStencil, GradStencil, WenoStencil, AlignUp,
+      Min, Max, Abs, Clamp, Sqrt, Sign, Maximum, Delta, RoundDown, pi,
+      isApproxZero, Round, createSampler, SampleFromVoxels.
+
+    - These APIs are now under the tools namespace: createNanoGrid,
+      StatsMode, createLevelSetSphere, createFogVolumeSphere,
+      createFogVolumeSphere, createFogVolumeSphere,
+      createFogVolumeTorus, createLevelSetBox, CreateNanoGrid,
+      updateGridStats, evalChecksum, validateChecksum, checkGrid,
+      Extrema.
+    - These APIs are now under the util namespace: is_floating_point,
+      findLowestOn, findHighestOn, Range, streq, strcpy, strcat,
+      empty, Split, invoke, forEach, reduce, prefixSum, is_same,
+      is_specialization, PtrAdd, PtrDiff.
+
+    - Move nanovdb::build to nanovdb::tools::build.
+    - Rename nanovdb::BBoxR to nanovdb::Vec3dBBox.
+    - Rename nanovdb::BBox<nanovdb::Vec3d> to nanovdb::Vec3dBbox.
+    - Move nanovdb::cudaCreateNodeManager to nanovdb::cuda::createNodeManager.
+    - Move and rename nanovdb::cudaVoxelsToGrid to nanovdb::cuda::voxelsToGrid.
+    - Move and rename nanovdb::cudaPointsToGrid to nanovdb::cuda::pointsToGrid.
+    - Move nanovdb::DitherLUT to nanovdb::math::DitherLUT.
+    - Move and rename nanovdb::PackedRGBA8 to nanovdb::math::Rgba8.
+    - Move nanovdb::Rgba8 to nanovdb::math::Rgba8.
+    - Move and rename nanovdb::CpuTimer to nanovdb::util::Timer.
+    - Move nanovdb::GpuTimer to nanovdb::util::cuda::Timer.
+    - Move and rename nanovdb::CountOn to nanovdb::util::countOn.
+
+    - Move util/GridHandle.h to GridHandle.h.
+    - Move util/BuildGrid.h to tools/GridBuilder.h.
+    - Move util/GridBuilder.h to tools/GridBuilder.h.
+    - Move util/IO.h to io/IO.h.
+    - Move util/CSampleFromVoxels.h to math/CSampleFromVoxels.h.
+    - Move util/DitherLUT.h to math/DitherLUT.h.
+    - Move util/HDDA.h to math/HDDA.h.
+    - Move util/Ray.h to math/Ray.h.
+    - Move util/SampleFromVoxels.h to math/SampleFromVoxels.h.
+    - Move util/Stencils.h to nanovdb/math/Stencils.h.
+    - Move util/CreateNanoGrid.h to tools/CreateNanoGrid.h.
+    - Move and rename util/Primitives.h to tools/CreatePrimitives.h.
+    - Move util/GridChecksum.h to tools/GridChecksum.h.
+    - Move util/GridStats.h to tools/GridStats.h.
+    - Move util/GridChecksum.h to tools/GridChecksum.h.
+    - Move util/GridValidator.h to tools/GridValidator.h.
+    - Move util/NanoToOpenVDB.h to tools/NanoToOpenVDB.h.
+    - Move util/cuda/CudaGridChecksum.cuh to tools/cuda/CudaGridChecksum.cuh.
+    - Move util/cuda/CudaGridStats.cuh to tools/cuda/CudaGridStats.cuh.
+    - Move util/cuda/CudaGridValidator.cuh to tools/cuda/CudaGridValidator.cuh.
+    - Move util/cuda/CudaIndexToGrid.cuh to tools/cuda/CudaIndexToGrid.cuh.
+    - Move and rename util/cuda/CudaPointsToGrid.cuh to tools/cuda/PointsToGrid.cuh.
+    - Move util/cuda/CudaSignedFloodFill.cuh to tools/cuda/CudaSignedFloodFill.cuh.
+    - Move and rename util/cuda/CudaDeviceBuffer.h to cuda/DeviceBuffer.h.
+    - Move and rename util/cuda/CudaGridHandle.cuh to cuda/GridHandle.cuh.
+    - Move and rename util/cuda/CudaUtils.h to util/cuda/Util.h.
+    - Move and consolidate util/cuda/GpuTimer.h to util/cuda/Timer.h.
+
+  Houdini:
+    - When OPENVDB_ENABLE_RPATH is ON, the location of
+      libopenvdb_houdini is now added to the rpath of all Houdini
+      dsos.
+
+  Python:
+    - OpenVDB Python bindings are now implemented using nanobind
+      instead of pybind11
+    - The OpenVDB Python module has been changed from pyopenvdb to openvdb
+    - Added Python bindings for NanoVDB
+
+  Build:
+    - Fixed an issue with OpenVDB AX's CMake on Windows where the
+      static and shared library targets would have the same name
+      [Reported by Nicholas Yue]
+    - USE_EXPLICIT_INSTANTIATION is now disabled on Windows by default
+      due to OOM linker issues.
+    - Jemalloc is now the preferred allocator of choice on all
+      platforms when CONCURRENT_MALLOC is set to Auto.
+    - Fixed an issue with the Blosc CMake FindPackage for the OpenVDB
+      Windows static library.


 Version 11.0.0 - November 1, 2023
@@ -399,7 +777,6 @@ Version 9.1.0 - June 9, 2022
      Positional arguments as input files are deprecated.
    - Added tools::minMax() which supports multithreaded evaluation of active
      minimum and maximum values. Grid::evalMinMax() has been deprecated.
-      [Contributed by Greg Hurst]
    - Significant performance improvements to AX point kernels, primarily due to
      providing AX access to attribute buffers for superior code generation.
    - vdb_print now prints both the in-core memory and total memory usage for
@@ -1,5 +1,5 @@
 # Copyright Contributors to the OpenVDB Project
-# SPDX-License-Identifier: MPL-2.0
+# SPDX-License-Identifier: Apache-2.0
 #
 #[=======================================================================[

@@ -42,7 +42,7 @@

 # note: cmake_minimum_required must be called before project commands to
 #  ensure policy scope is set up correctly
-cmake_minimum_required(VERSION 3.18)
+cmake_minimum_required(VERSION 3.24)

 # CMP0091 allows for MSVC ABI targetting via CMAKE_MSVC_RUNTIME_LIBRARY
 # from CMake 3.15 and above. Must come before project().
@@ -52,9 +52,9 @@ endif()

 ###### Version

-set(OpenVDB_MAJOR_VERSION 11)
-set(OpenVDB_MINOR_VERSION 0)
-set(OpenVDB_PATCH_VERSION 1)
+set(OpenVDB_MAJOR_VERSION 12)
+set(OpenVDB_MINOR_VERSION 1)
+set(OpenVDB_PATCH_VERSION 2)
 set(OpenVDB_VERSION "${OpenVDB_MAJOR_VERSION}.${OpenVDB_MINOR_VERSION}.${OpenVDB_PATCH_VERSION}")

 project(OpenVDB LANGUAGES CXX VERSION ${OpenVDB_VERSION})
@@ -268,6 +268,13 @@ mark_as_advanced(

 include(cmake/config/OpenVDBVersions.cmake)

+if(OPENVDB_FUTURE_DEPRECATION AND FUTURE_MINIMUM_CMAKE_VERSION)
+  if(CMAKE_VERSION VERSION_LESS ${FUTURE_MINIMUM_CMAKE_VERSION})
+    message(DEPRECATION "Support for CMake versions < ${FUTURE_MINIMUM_CMAKE_VERSION} "
+      "is deprecated and will be removed.")
+  endif()
+endif()
+
 foreach(DEPRECATED_ABI ${OPENVDB_DEPRECATED_ABI_LIST})
  option(OPENVDB_USE_DEPRECATED_ABI_${DEPRECATED_ABI} "Bypass minimum ABI requirement checks" OFF)
  mark_as_advanced(OPENVDB_USE_DEPRECATED_ABI_${DEPRECATED_ABI})
@@ -285,6 +292,30 @@ list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake")

 # Add cmake modules to installation command

+if(SKBUILD)
+  set(OPENVDB_INSTALL_BINDIR openvdb/${CMAKE_INSTALL_BINDIR})
+  set(OPENVDB_INSTALL_LIBDIR openvdb/${CMAKE_INSTALL_LIBDIR})
+  set(OPENVDB_INSTALL_INCLUDEDIR openvdb/${CMAKE_INSTALL_INCLUDEDIR})
+
+  if(OPENVDB_BUILD_NANOVDB)
+    set(NANOVDB_INSTALL_BINDIR nanovdb/${CMAKE_INSTALL_BINDIR})
+    set(NANOVDB_INSTALL_LIBDIR nanovdb/${CMAKE_INSTALL_LIBDIR})
+    set(NANOVDB_INSTALL_INCLUDEDIR nanovdb/${CMAKE_INSTALL_INCLUDEDIR})
+    set(NANOVDB_INSTALL_DOCDIR nanovdb/${CMAKE_INSTALL_DOCDIR})
+  endif()
+else()
+  set(OPENVDB_INSTALL_BINDIR ${CMAKE_INSTALL_BINDIR})
+  set(OPENVDB_INSTALL_LIBDIR ${CMAKE_INSTALL_LIBDIR})
+  set(OPENVDB_INSTALL_INCLUDEDIR ${CMAKE_INSTALL_INCLUDEDIR})
+
+  if(OPENVDB_BUILD_NANOVDB)
+    set(NANOVDB_INSTALL_BINDIR ${CMAKE_INSTALL_BINDIR})
+    set(NANOVDB_INSTALL_LIBDIR ${CMAKE_INSTALL_LIBDIR})
+    set(NANOVDB_INSTALL_INCLUDEDIR ${CMAKE_INSTALL_INCLUDEDIR})
+    set(NANOVDB_INSTALL_DOCDIR ${CMAKE_INSTALL_DOCDIR})
+  endif()
+endif()
+
 if(OPENVDB_INSTALL_CMAKE_MODULES)
  set(OPENVDB_CMAKE_MODULES
    cmake/FindBlosc.cmake
@@ -298,7 +329,7 @@ if(OPENVDB_INSTALL_CMAKE_MODULES)
    cmake/OpenVDBMayaSetup.cmake
    cmake/OpenVDBUtils.cmake
  )
-  install(FILES ${OPENVDB_CMAKE_MODULES} DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/OpenVDB)
+  install(FILES ${OPENVDB_CMAKE_MODULES} DESTINATION ${OPENVDB_INSTALL_LIBDIR}/cmake/OpenVDB)
 endif()

 # Configure component dependencies by loading the Houdini/Maya setup
@@ -451,6 +482,44 @@ if(NOT OPENVDB_BUILD_NANOVDB AND
  find_package(OpenVDB REQUIRED COMPONENTS nanovdb)
 endif()

+# Locate Python and nanobind if necessary
+if(OPENVDB_BUILD_PYTHON_MODULE OR (OPENVDB_BUILD_NANOVDB AND NANOVDB_BUILD_PYTHON_MODULE))
+
+  find_package(Python ${MINIMUM_PYTHON_VERSION} REQUIRED COMPONENTS Development Interpreter)
+  if(OPENVDB_FUTURE_DEPRECATION AND FUTURE_MINIMUM_PYTHON_VERSION)
+    if(Python_VERSION VERSION_LESS ${FUTURE_MINIMUM_PYTHON_VERSION})
+      message(DEPRECATION "Support for Python versions < ${FUTURE_MINIMUM_GLFW_VERSION} "
+        "is deprecated and will be removed.")
+    endif()
+  endif()
+
+  find_package(nanobind REQUIRED CONFIG)
+  include(FindPackageHandleStandardArgs)
+  find_package_handle_standard_args(nanobind
+    FOUND_VAR nanobind_FOUND
+    REQUIRED_VARS NB_DIR
+    VERSION_VAR nanobind_VERSION)
+
+  if(OPENVDB_FUTURE_DEPRECATION AND FUTURE_MINIMUM_NANOBIND_VERSION)
+    if(nanobind_VERSION VERSION_LESS ${FUTURE_MINIMUM_NANOBIND_VERSION})
+      message(DEPRECATION "Support for nanobind versions < ${FUTURE_MINIMUM_NANOBIND_VERSION} "
+        "is deprecated and will be removed.")
+    endif()
+  endif()
+
+  if(NOT DEFINED VDB_PYTHON_INSTALL_DIRECTORY)
+    if(DEFINED Python_SITELIB)
+      get_filename_component(Python_PACKAGES_DIR ${Python_SITELIB} NAME)
+    else()
+      set(Python_PACKAGES_DIR "site-packages")
+    endif()
+    set(VDB_PYTHON_INSTALL_DIRECTORY
+      ${CMAKE_INSTALL_LIBDIR}/python${Python_VERSION_MAJOR}.${Python_VERSION_MINOR}/${Python_PACKAGES_DIR}
+      CACHE STRING "The directory to install the openvdb and nanovdb Python modules."
+    )
+  endif()
+endif()
+

 # Validate the OpenVDB ABI Version. If OpenVDB_ABI is not set, we're either building
 # the core library OR the ABI hasn't been deduced from a VDB installation. Use the
@@ -22,7 +22,7 @@ open source software best practice policies of the Linux Foundation.

 #### License

-OpenVDB is licensed under the [Mozilla Public License, version 2.0](LICENSE.md)
+OpenVDB is licensed under the [Apache License, version 2.0](LICENSE.md)
 license. Contributions to OpenVDB should abide by that standard
 license.

@@ -54,7 +54,7 @@ Every commit must be signed off.  That is, every commit log message
 must include a “`Signed-off-by`” line (generated, for example, with
 “`git commit --signoff`”), indicating that the committer wrote the
 code and has the right to release it under the
-[Mozilla Public License, version 2.0](LICENSE.md)
+[Apache License, version 2.0](LICENSE.md)
 license. See the [TAC documentation on contribution sign off](https://github.com/AcademySoftwareFoundation/tac/blob/master/process/contributing.md#contribution-sign-off) for more information on this requirement.

 ## Maintainer
@@ -1,373 +1,176 @@
-Mozilla Public License Version 2.0
-==================================
-
-1. Definitions
--------------
-
-1.1. "Contributor"
-    means each individual or legal entity that creates, contributes to
-    the creation of, or owns Covered Software.
-
-1.2. "Contributor Version"
-    means the combination of the Contributions of others (if any) used
-    by a Contributor and that particular Contributor's Contribution.
-
-1.3. "Contribution"
-    means Covered Software of a particular Contributor.
-
-1.4. "Covered Software"
-    means Source Code Form to which the initial Contributor has attached
-    the notice in Exhibit A, the Executable Form of such Source Code
-    Form, and Modifications of such Source Code Form, in each case
-    including portions thereof.
-
-1.5. "Incompatible With Secondary Licenses"
-    means
-
-    (a) that the initial Contributor has attached the notice described
-        in Exhibit B to the Covered Software; or
-
-    (b) that the Covered Software was made available under the terms of
-        version 1.1 or earlier of the License, but not also under the
-        terms of a Secondary License.
-
-1.6. "Executable Form"
-    means any form of the work other than Source Code Form.
-
-1.7. "Larger Work"
-    means a work that combines Covered Software with other material, in
-    a separate file or files, that is not Covered Software.
-
-1.8. "License"
-    means this document.
-
-1.9. "Licensable"
-    means having the right to grant, to the maximum extent possible,
-    whether at the time of the initial grant or subsequently, any and
-    all of the rights conveyed by this License.
-
-1.10. "Modifications"
-    means any of the following:
-
-    (a) any file in Source Code Form that results from an addition to,
-        deletion from, or modification of the contents of Covered
-        Software; or
-
-    (b) any new file in Source Code Form that contains any Covered
-        Software.
-
-1.11. "Patent Claims" of a Contributor
-    means any patent claim(s), including without limitation, method,
-    process, and apparatus claims, in any patent Licensable by such
-    Contributor that would be infringed, but for the grant of the
-    License, by the making, using, selling, offering for sale, having
-    made, import, or transfer of either its Contributions or its
-    Contributor Version.
-
-1.12. "Secondary License"
-    means either the GNU General Public License, Version 2.0, the GNU
-    Lesser General Public License, Version 2.1, the GNU Affero General
-    Public License, Version 3.0, or any later versions of those
-    licenses.
-
-1.13. "Source Code Form"
-    means the form of the work preferred for making modifications.
-
-1.14. "You" (or "Your")
-    means an individual or a legal entity exercising rights under this
-    License. For legal entities, "You" includes any entity that
-    controls, is controlled by, or is under common control with You. For
-    purposes of this definition, "control" means (a) the power, direct
-    or indirect, to cause the direction or management of such entity,
-    whether by contract or otherwise, or (b) ownership of more than
-    fifty percent (50%) of the outstanding shares or beneficial
-    ownership of such entity.
-
-2. License Grants and Conditions
--------------------------------
-
-2.1. Grants
-
-Each Contributor hereby grants You a world-wide, royalty-free,
-non-exclusive license:
-
-(a) under intellectual property rights (other than patent or trademark)
-    Licensable by such Contributor to use, reproduce, make available,
-    modify, display, perform, distribute, and otherwise exploit its
-    Contributions, either on an unmodified basis, with Modifications, or
-    as part of a Larger Work; and
-
-(b) under Patent Claims of such Contributor to make, use, sell, offer
-    for sale, have made, import, and otherwise transfer either its
-    Contributions or its Contributor Version.
-
-2.2. Effective Date
-
-The licenses granted in Section 2.1 with respect to any Contribution
-become effective for each Contribution on the date the Contributor first
-distributes such Contribution.
-
-2.3. Limitations on Grant Scope
-
-The licenses granted in this Section 2 are the only rights granted under
-this License. No additional rights or licenses will be implied from the
-distribution or licensing of Covered Software under this License.
-Notwithstanding Section 2.1(b) above, no patent license is granted by a
-Contributor:
-
-(a) for any code that a Contributor has removed from Covered Software;
-    or
-
-(b) for infringements caused by: (i) Your and any other third party's
-    modifications of Covered Software, or (ii) the combination of its
-    Contributions with other software (except as part of its Contributor
-    Version); or
-
-(c) under Patent Claims infringed by Covered Software in the absence of
-    its Contributions.
-
-This License does not grant any rights in the trademarks, service marks,
-or logos of any Contributor (except as may be necessary to comply with
-the notice requirements in Section 3.4).
-
-2.4. Subsequent Licenses
-
-No Contributor makes additional grants as a result of Your choice to
-distribute the Covered Software under a subsequent version of this
-License (see Section 10.2) or under the terms of a Secondary License (if
-permitted under the terms of Section 3.3).
-
-2.5. Representation
-
-Each Contributor represents that the Contributor believes its
-Contributions are its original creation(s) or it has sufficient rights
-to grant the rights to its Contributions conveyed by this License.
-
-2.6. Fair Use
-
-This License is not intended to limit any rights You have under
-applicable copyright doctrines of fair use, fair dealing, or other
-equivalents.
-
-2.7. Conditions
-
-Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted
-in Section 2.1.
-
-3. Responsibilities
-------------------
-
-3.1. Distribution of Source Form
-
-All distribution of Covered Software in Source Code Form, including any
-Modifications that You create or to which You contribute, must be under
-the terms of this License. You must inform recipients that the Source
-Code Form of the Covered Software is governed by the terms of this
-License, and how they can obtain a copy of this License. You may not
-attempt to alter or restrict the recipients' rights in the Source Code
-Form.
-
-3.2. Distribution of Executable Form
-
-If You distribute Covered Software in Executable Form then:
-
-(a) such Covered Software must also be made available in Source Code
-    Form, as described in Section 3.1, and You must inform recipients of
-    the Executable Form how they can obtain a copy of such Source Code
-    Form by reasonable means in a timely manner, at a charge no more
-    than the cost of distribution to the recipient; and
-
-(b) You may distribute such Executable Form under the terms of this
-    License, or sublicense it under different terms, provided that the
-    license for the Executable Form does not attempt to limit or alter
-    the recipients' rights in the Source Code Form under this License.
-
-3.3. Distribution of a Larger Work
-
-You may create and distribute a Larger Work under terms of Your choice,
-provided that You also comply with the requirements of this License for
-the Covered Software. If the Larger Work is a combination of Covered
-Software with a work governed by one or more Secondary Licenses, and the
-Covered Software is not Incompatible With Secondary Licenses, this
-License permits You to additionally distribute such Covered Software
-under the terms of such Secondary License(s), so that the recipient of
-the Larger Work may, at their option, further distribute the Covered
-Software under the terms of either this License or such Secondary
-License(s).
-
-3.4. Notices
-
-You may not remove or alter the substance of any license notices
-(including copyright notices, patent notices, disclaimers of warranty,
-or limitations of liability) contained within the Source Code Form of
-the Covered Software, except that You may alter any license notices to
-the extent required to remedy known factual inaccuracies.
-
-3.5. Application of Additional Terms
-
-You may choose to offer, and to charge a fee for, warranty, support,
-indemnity or liability obligations to one or more recipients of Covered
-Software. However, You may do so only on Your own behalf, and not on
-behalf of any Contributor. You must make it absolutely clear that any
-such warranty, support, indemnity, or liability obligation is offered by
-You alone, and You hereby agree to indemnify every Contributor for any
-liability incurred by such Contributor as a result of warranty, support,
-indemnity or liability terms You offer. You may include additional
-disclaimers of warranty and limitations of liability specific to any
-jurisdiction.
-
-4. Inability to Comply Due to Statute or Regulation
---------------------------------------------------
-
-If it is impossible for You to comply with any of the terms of this
-License with respect to some or all of the Covered Software due to
-statute, judicial order, or regulation then You must: (a) comply with
-the terms of this License to the maximum extent possible; and (b)
-describe the limitations and the code they affect. Such description must
-be placed in a text file included with all distributions of the Covered
-Software under this License. Except to the extent prohibited by statute
-or regulation, such description must be sufficiently detailed for a
-recipient of ordinary skill to be able to understand it.
-
-5. Termination
--------------
-
-5.1. The rights granted under this License will terminate automatically
-if You fail to comply with any of its terms. However, if You become
-compliant, then the rights granted under this License from a particular
-Contributor are reinstated (a) provisionally, unless and until such
-Contributor explicitly and finally terminates Your grants, and (b) on an
-ongoing basis, if such Contributor fails to notify You of the
-non-compliance by some reasonable means prior to 60 days after You have
-come back into compliance. Moreover, Your grants from a particular
-Contributor are reinstated on an ongoing basis if such Contributor
-notifies You of the non-compliance by some reasonable means, this is the
-first time You have received notice of non-compliance with this License
-from such Contributor, and You become compliant prior to 30 days after
-Your receipt of the notice.
-
-5.2. If You initiate litigation against any entity by asserting a patent
-infringement claim (excluding declaratory judgment actions,
-counter-claims, and cross-claims) alleging that a Contributor Version
-directly or indirectly infringes any patent, then the rights granted to
-You by any and all Contributors for the Covered Software under Section
-2.1 of this License shall terminate.
-
-5.3. In the event of termination under Sections 5.1 or 5.2 above, all
-end user license agreements (excluding distributors and resellers) which
-have been validly granted by You or Your distributors under this License
-prior to termination shall survive termination.
-
-************************************************************************
-*                                                                      *
-*  6. Disclaimer of Warranty                                           *
-*  -------------------------                                           *
-*                                                                      *
-*  Covered Software is provided under this License on an "as is"       *
-*  basis, without warranty of any kind, either expressed, implied, or  *
-*  statutory, including, without limitation, warranties that the       *
-*  Covered Software is free of defects, merchantable, fit for a        *
-*  particular purpose or non-infringing. The entire risk as to the     *
-*  quality and performance of the Covered Software is with You.        *
-*  Should any Covered Software prove defective in any respect, You     *
-*  (not any Contributor) assume the cost of any necessary servicing,   *
-*  repair, or correction. This disclaimer of warranty constitutes an   *
-*  essential part of this License. No use of any Covered Software is   *
-*  authorized under this License except under this disclaimer.         *
-*                                                                      *
-************************************************************************
-
-************************************************************************
-*                                                                      *
-*  7. Limitation of Liability                                          *
-*  --------------------------                                          *
-*                                                                      *
-*  Under no circumstances and under no legal theory, whether tort      *
-*  (including negligence), contract, or otherwise, shall any           *
-*  Contributor, or anyone who distributes Covered Software as          *
-*  permitted above, be liable to You for any direct, indirect,         *
-*  special, incidental, or consequential damages of any character      *
-*  including, without limitation, damages for lost profits, loss of    *
-*  goodwill, work stoppage, computer failure or malfunction, or any    *
-*  and all other commercial damages or losses, even if such party      *
-*  shall have been informed of the possibility of such damages. This   *
-*  limitation of liability shall not apply to liability for death or   *
-*  personal injury resulting from such party's negligence to the       *
-*  extent applicable law prohibits such limitation. Some               *
-*  jurisdictions do not allow the exclusion or limitation of           *
-*  incidental or consequential damages, so this exclusion and          *
-*  limitation may not apply to You.                                    *
-*                                                                      *
-************************************************************************
-
-8. Litigation
-------------
-
-Any litigation relating to this License may be brought only in the
-courts of a jurisdiction where the defendant maintains its principal
-place of business and such litigation shall be governed by laws of that
-jurisdiction, without reference to its conflict-of-law provisions.
-Nothing in this Section shall prevent a party's ability to bring
-cross-claims or counter-claims.
-
-9. Miscellaneous
----------------
-
-This License represents the complete agreement concerning the subject
-matter hereof. If any provision of this License is held to be
-unenforceable, such provision shall be reformed only to the extent
-necessary to make it enforceable. Any law or regulation which provides
-that the language of a contract shall be construed against the drafter
-shall not be used to construe this License against a Contributor.
-
-10. Versions of the License
---------------------------
-
-10.1. New Versions
-
-Mozilla Foundation is the license steward. Except as provided in Section
-10.3, no one other than the license steward has the right to modify or
-publish new versions of this License. Each version will be given a
-distinguishing version number.
-
-10.2. Effect of New Versions
-
-You may distribute the Covered Software under the terms of the version
-of the License under which You originally received the Covered Software,
-or under the terms of any subsequent version published by the license
-steward.
-
-10.3. Modified Versions
-
-If you create software not governed by this License, and you want to
-create a new license for such software, you may create and use a
-modified version of this License if you rename the license and remove
-any references to the name of the license steward (except to note that
-such modified license differs from this License).
-
-10.4. Distributing Source Code Form that is Incompatible With Secondary
-Licenses
-
-If You choose to distribute Source Code Form that is Incompatible With
-Secondary Licenses under the terms of this version of the License, the
-notice described in Exhibit B of this License must be attached.
-
-Exhibit A - Source Code Form License Notice
-------------------------------------------
-
-  This Source Code Form is subject to the terms of the Mozilla Public
-  License, v. 2.0. If a copy of the MPL was not distributed with this
-  file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-If it is not possible or desirable to put the notice in a particular
-file, then You may include the notice in a location (such as a LICENSE
-file in a relevant directory) where a recipient would be likely to look
-for such a notice.
-
-You may add additional accurate notices of copyright ownership.
-
-Exhibit B - "Incompatible With Secondary Licenses" Notice
---------------------------------------------------------
-
-  This Source Code Form is "Incompatible With Secondary Licenses", as
-  defined by the Mozilla Public License, v. 2.0.
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
@@ -10,7 +10,7 @@
 [Discussion Forum](https://github.com/AcademySoftwareFoundation/openvdb/discussions) |
 [Documentation](https://www.openvdb.org/documentation/doxygen) |
 [Releases](https://github.com/AcademySoftwareFoundation/openvdb/releases) |
-[License](https://www.mozilla.org/MPL/2.0) |
+[License](https://www.apache.org/licenses/LICENSE-2.0) |
 [Slack](https://slack.aswf.io/)

 OpenVDB is an open source C++ library comprising a novel hierarchical data
@@ -28,14 +28,14 @@ generally less stable than the [production releases](https://github.com/AcademyS

 ### License

-OpenVDB is released under the [Mozilla Public License Version 2.0](https://www.mozilla.org/MPL/2.0/),
+OpenVDB is released under the [Apache License, Version 2.0](https://www.apache.org/licenses/LICENSE-2.0),
 which is a free, open source software license developed and maintained by the
-Mozilla Foundation.
+Apache Software Foundation.

 The trademarks of any contributor to this project may not be used in
 association with the project without the contributor's express permission.

-NOTE: OpenVDB is in the process of changing its license from [Mozilla Public License Version 2.0](https://www.mozilla.org/MPL/2.0/) to [Apache License, Version 2.0](https://www.apache.org/licenses/LICENSE-2.0)! Please see the file RE-LICENSE_NOTE.txt for more details.
+NOTE: OpenVDB has completed re-licensing from [Mozilla Public License Version 2.0](https://www.mozilla.org/MPL/2.0/) to [Apache License, Version 2.0](https://www.apache.org/licenses/LICENSE-2.0). Please see the file RE-LICENSE_NOTE.txt for more details.

 ### Contributing

@@ -1,4 +1,6 @@
 #!/usr/bin/env bash
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0

 set -e

@@ -31,8 +33,13 @@ PARMS[--target]=install
 PARMS[--build-dir]=build

 if [[ $RUNNER_NAME == *"8c-32g-300h"* ]]; then
-    # ASWF github actions runners have 8 threads
-    PARMS[-j]=8
+    if [[ $CXX == "g++" ]]; then
+        # GCC hits memory limits on runners, reduce threads
+        PARMS[-j]=6
+    else
+        # ASWF github actions runners have 8 threads
+        PARMS[-j]=8
+    fi
 else
    # Github actions runners have 2 threads
    # https://help.github.com/en/actions/reference/virtual-environments-for-github-hosted-runners
@@ -49,7 +56,8 @@ declare -A COMPONENTS
 COMPONENTS['core']='OPENVDB_BUILD_CORE'
 COMPONENTS['python']='OPENVDB_BUILD_PYTHON_MODULE'
 COMPONENTS['test']='OPENVDB_BUILD_UNITTESTS'
-COMPONENTS['bin']='OPENVDB_BUILD_BINARIES'
+# @todo split out vdb tool from this...
+COMPONENTS['bin']='OPENVDB_BUILD_VDB_PRINT,OPENVDB_BUILD_VDB_LOD,OPENVDB_BUILD_VDB_TOOL,OPENVDB_BUILD_VDB_TOOL_UNITTESTS'
 COMPONENTS['view']='OPENVDB_BUILD_VDB_VIEW'
 COMPONENTS['render']='OPENVDB_BUILD_VDB_RENDER'
 COMPONENTS['hou']='OPENVDB_BUILD_HOUDINI_PLUGIN'
@@ -148,7 +156,11 @@ for comp in "${!COMPONENTS[@]}"; do
            setting="ON"; break
        fi
    done
-    CMAKE_EXTRA+=("-D${COMPONENTS[$comp]}=$setting")
+    OPENVDB_TOGGLES="${COMPONENTS[$comp]}"
+    OPENVDB_TOGGLES=(${OPENVDB_TOGGLES//,/ })
+    for toggle in "${!OPENVDB_TOGGLES[@]}"; do
+        CMAKE_EXTRA+=("-D${OPENVDB_TOGGLES[$toggle]}=$setting")
+    done
 done

 ################################################
@@ -180,15 +192,14 @@ cd ${BUILD_DIR}
 set -x

 # Note:
-# - print and lod binary options are always on and can be toggles with: OPENVDB_BUILD_BINARIES=ON/OFF
+# - OPENVDB_BUILD_BINARIES is always ON, but each binary will be toggled depending on the provided comps
 # - always enabled the python tests with OPENVDB_BUILD_PYTHON_UNITTESTS if the python module is in use,
 #   regardless of the 'test' component being enabled or not (see the OPENVDB_BUILD_PYTHON_UNITTESTS option).
 cmake \
    -DOPENVDB_USE_DEPRECATED_ABI_10=ON \
-    -DOPENVDB_BUILD_VDB_PRINT=ON \
-    -DOPENVDB_BUILD_VDB_LOD=ON \
-    -DOPENVDB_BUILD_VDB_TOOL=ON \
+    -DOPENVDB_USE_DEPRECATED_ABI_11=ON \
    -DOPENVDB_TOOL_USE_NANO=OFF \
+    -DOPENVDB_BUILD_BINARIES=ON \
    -DOPENVDB_BUILD_PYTHON_UNITTESTS=ON \
    -DMSVC_MP_THREAD_COUNT=${PARMS[-j]} \
    "${CMAKE_EXTRA[@]}" \
@@ -1,4 +1,6 @@
 #!/usr/bin/env bash
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0

 set -ex

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 #
 # Copyright Contributors to the OpenVDB Project
-# SPDX-License-Identifier: MPL-2.0
+# SPDX-License-Identifier: Apache-2.0
 #
 # Python script to download the latest Houdini builds
 # using the SideFX download API:
@@ -1,4 +1,6 @@
 #!/usr/bin/env bash
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0

 set -ex

@@ -6,8 +8,14 @@ HOUDINI_MAJOR="$1"
 PLATFORM="$2"
 OTHER_ARGS="$3"

-pip install --user requests
-python ci/download_houdini.py $HOUDINI_MAJOR $PLATFORM $OTHER_ARGS
+if [[ $PLATFORM =~ "linux" ]]; then
+    pip3 install --user requests
+elif [[ $PLATFORM =~ "macos" ]]; then
+    # Tell the homebrew install of pip3 that we don't care about installing directly through pip3
+    pip3 install --user requests --break-system-packages
+fi
+
+python3 ci/download_houdini.py $HOUDINI_MAJOR $PLATFORM $OTHER_ARGS

 if [[ $PLATFORM =~ "linux" ]]; then
    # create dir hierarchy
@@ -29,20 +37,20 @@ if [[ $PLATFORM =~ "linux" ]]; then
    ls -al dsolib/

    # copy required libraries
-    cp -r toolkit/cmake ../hou/toolkit/.
-    cp -r toolkit/include ../hou/toolkit/.
-    cp -r dsolib/libHoudini* ../hou/dsolib/.
-    cp -r dsolib/libopenvdb_sesi* ../hou/dsolib/.
-    cp -r dsolib/libblosc* ../hou/dsolib/.
-    cp -r dsolib/libhboost* ../hou/dsolib/.
-    cp -r dsolib/libz* ../hou/dsolib/.
-    cp -r dsolib/libbz2* ../hou/dsolib/.
-    cp -r dsolib/libtbb* ../hou/dsolib/.
-    cp -r dsolib/libjemalloc* ../hou/dsolib/.
-    cp -r dsolib/liblzma* ../hou/dsolib/.
-    cp -r dsolib/libIex* ../hou/dsolib/.
-    cp -r dsolib/libImath* ../hou/dsolib/.
-    cp -r dsolib/libIlmThread* ../hou/dsolib/.
+    mv toolkit/cmake ../hou/toolkit/.
+    mv toolkit/include ../hou/toolkit/.
+    mv dsolib/libHoudini* ../hou/dsolib/.
+    mv dsolib/libopenvdb_sesi* ../hou/dsolib/.
+    mv dsolib/libblosc* ../hou/dsolib/.
+    mv dsolib/libhboost* ../hou/dsolib/.
+    mv dsolib/libz* ../hou/dsolib/.
+    mv dsolib/libbz2* ../hou/dsolib/.
+    mv dsolib/libtbb* ../hou/dsolib/.
+    mv dsolib/libjemalloc* ../hou/dsolib/.
+    mv dsolib/liblzma* ../hou/dsolib/.
+    mv dsolib/libIex* ../hou/dsolib/.
+    mv dsolib/libImath* ../hou/dsolib/.
+    mv dsolib/libIlmThread* ../hou/dsolib/.
    cd ..

 elif [[ $PLATFORM =~ "macos" ]]; then
@@ -1,4 +1,6 @@
 #!/usr/bin/env python3
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0

 import os
 import sys
@@ -1,4 +1,6 @@
 #!/usr/bin/env bash
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
 #################################################################################
 # This script extracts all code blocks from AX documentation which are NOT      #
 # marked as cpp/sh/unparsed and attempts to parse or compile them through the   #
@@ -1,4 +1,6 @@
 #!/usr/bin/env bash
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0

 set -ex

@@ -1,4 +1,6 @@
 #!/usr/bin/env bash
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0

 set -ex
 CURL_VERSION="$1"
@@ -1,4 +1,6 @@
 #!/usr/bin/env bash
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0

 set -ex

@@ -13,6 +15,6 @@ fi

 mkdir build
 cd build
-cmake ../.
+cmake -DCMAKE_POLICY_VERSION_MINIMUM=3.5 ../.
 make -j8
 make install
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+
+set -ex
+EXR_VERSION="$1"
+
+git clone https://github.com/AcademySoftwareFoundation/openexr.git
+cd openexr
+
+if [ "$EXR_VERSION" != "latest" ]; then
+    git checkout tags/${EXR_VERSION} -b ${EXR_VERSION}
+fi
+
+mkdir build
+cd build
+
+cmake ..
+
+make -j$(nproc)
+sudo make install
@@ -1,4 +1,6 @@
 #!/usr/bin/env bash
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0

 set -ex
 GLFW_VERSION="$1"
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+
+set -ex
+GTEST_VERSION="$1"
+
+git clone https://github.com/google/googletest.git
+cd googletest
+
+if [ "$GTEST_VERSION" != "latest" ]; then
+    git checkout tags/${GTEST_VERSION} -b ${GTEST_VERSION}
+fi
+
+mkdir build
+cd build
+
+cmake ..
+
+make -j$(nproc)
+sudo make install
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+
+set -ex
+IMATH_VERSION="$1"
+
+git clone https://github.com/AcademySoftwareFoundation/Imath.git
+cd Imath
+
+if [ "$IMATH_VERSION" != "latest" ]; then
+    git checkout tags/${IMATH_VERSION} -b ${IMATH_VERSION}
+fi
+
+mkdir build
+cd build
+
+cmake ..
+
+make -j$(nproc)
+sudo make install
@@ -1,27 +1,58 @@
 #!/usr/bin/env bash
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0

 set -ex

-LLVM_CRT="$1"
+LLVM_VER="$1"
+LLVM_CRT="$2"

-git clone -b llvmorg-12.0.0 --depth 1 https://github.com/llvm/llvm-project.git llvm
+# Legacy define for older versions of LLVM
+LLVM_LEGACY_CRT_DEFINE=""
+if [[ "${LLVM_CRT}" == "MultiThreaded" ]]; then
+    LLVM_LEGACY_CRT_DEFINE="MT"
+elif [[ "${LLVM_CRT}" == "MultiThreadedDebug" ]]; then
+    LLVM_LEGACY_CRT_DEFINE="MTd"
+elif [[ "${LLVM_CRT}" == "MultiThreadedDLL" ]]; then
+    LLVM_LEGACY_CRT_DEFINE="MD"
+elif [[ "${LLVM_CRT}" == "MultiThreadedDebugDLL" ]]; then
+    LLVM_LEGACY_CRT_DEFINE="MDd"
+fi
+
+git clone -b llvmorg-${LLVM_VER} --depth 1 https://github.com/llvm/llvm-project.git llvm
 cd llvm

 mkdir .build
 cd .build

-# @note  currently only test static builds with MT
 cmake -B . -S ../llvm -A x64 -G "Visual Studio 17 2022" -Thost=x64 \
    -DCMAKE_INSTALL_PREFIX="${HOME}/llvm_install" \
+    -DCMAKE_BUILD_TYPE=Release \
+    -DLLVM_ENABLE_ASSERTIONS=OFF \
+    -DLLVM_ENABLE_PROJECTS="" \
+    -DLLVM_ENABLE_RUNTIMES="" \
+    -DLLVM_ENABLE_UNWIND_TABLES=OFF \
    -DLLVM_TARGETS_TO_BUILD="host" \
-    -DLLVM_BUILD_TOOLS=OFF \
    -DLLVM_INCLUDE_BENCHMARKS=OFF \
    -DLLVM_INCLUDE_EXAMPLES=OFF \
    -DLLVM_INCLUDE_TESTS=OFF \
+    -DLLVM_INCLUDE_DOCS=OFF \
    -DLLVM_INCLUDE_TOOLS=OFF \
-    -DLLVM_USE_CRT_RELEASE=${LLVM_CRT}
+    -DLLVM_INCLUDE_UTILS=OFF \
+    -DLLVM_BUILD_BENCHMARKS=OFF \
+    -DLLVM_BUILD_EXAMPLES=OFF \
+    -DLLVM_BUILD_TESTS=OFF \
+    -DLLVM_BUILD_DOCS=OFF \
+    -DLLVM_BUILD_TOOLS=OFF \
+    -DLLVM_BUILD_UTILS=OFF \
+    -DLLVM_ENABLE_BINDINGS=OFF \
+    -DLLVM_ENABLE_OCAMLDOC=OFF \
+    -DLLVM_PARALLEL_COMPILE_JOBS=4 \
+    -DLLVM_PARALLEL_LINK_JOBS=4 \
+    -DCMAKE_MSVC_RUNTIME_LIBRARY=${LLVM_CRT} \
+    -DLLVM_USE_CRT_RELEASE=${LLVM_LEGACY_CRT_DEFINE}

-cmake --build . --config Release --target install
+cmake --build . --parallel 4 --config Release --target install

 cd ..

@@ -1,4 +1,6 @@
 #!/usr/bin/env bash
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0

 set -x

@@ -18,12 +20,14 @@ brew install glfw
 brew install googletest
 brew install jq # for trivial parsing of brew json
 brew install openexr
-brew install pybind11 # also installs the dependent python version
+brew install nanobind # also installs the dependent python version
+brew install robin-map # required for nanobind
+brew install numpy # only for python tests
 brew install zlib
 brew install jemalloc

-# Alias python version installed by pybind11 to path
-py_version=$(brew info pybind11 --json | \
+# Alias python version installed by nanobind to path
+py_version=$(brew info nanobind --json | \
    jq -cr '.[].dependencies[] | select(. | startswith("python"))')
 echo "Using python $py_version"
 # export for subsequent action steps (note, not exported for this env)
@@ -42,9 +46,7 @@ LLVM_VERSION=$1
 if [ ! -z "$LLVM_VERSION" ]; then
    if [ "$LLVM_VERSION" == "latest" ]; then
        brew install llvm
-        brew install cppunit
    else
        brew install llvm@$LLVM_VERSION
-        brew install cppunit
    fi
 fi
@@ -1,4 +1,6 @@
 #!/usr/bin/env bash
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0

 set -ex

@@ -9,8 +11,8 @@ if [ ! -z "${INSTALL_ROOT}" ]; then
    CMAKE_EXTRA+=("-DCMAKE_INSTALL_PREFIX=${INSTALL_ROOT}")
 fi

-git clone https://github.com/pybind/pybind11.git
-cd pybind11
+git clone --recurse-submodules https://github.com/wjakob/nanobind.git
+cd nanobind

 if [ "$PYBIND11_VERSION" != "latest" ]; then
    git checkout tags/v${PYBIND11_VERSION} -b v${PYBIND11_VERSION}
@@ -20,9 +22,9 @@ mkdir build
 cd build

 cmake \
-    -DPYBIND11_TEST=OFF \
+    -DNB_TEST=OFF \
    "${CMAKE_EXTRA[@]}" \
    ..

 make -j8
-make install
+sudo make install
@@ -1,4 +1,6 @@
 #!/usr/bin/env bash
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0

 set -x

@@ -0,0 +1,28 @@
+#!/usr/bin/env bash
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
+
+set -ex
+
+# Download the TeX Live installer
+wget https://mirror.ctan.org/systems/texlive/tlnet/install-tl-unx.tar.gz
+
+# Extract
+tar xzf install-tl-unx.tar.gz
+rm install-tl-unx.tar.gz
+cd install-tl-*
+
+# Run the installer with no docs or source installation
+./install-tl --no-interaction --no-doc-install --no-src-install --scheme=minimal
+
+# Export the texlive PATH to GITHUB_PATH
+TEXLIVE_PATH=/usr/local/texlive/2025/bin/x86_64-linux
+export PATH=${TEXLIVE_PATH}:${PATH}
+echo "${TEXLIVE_PATH}" >> $GITHUB_PATH
+
+# Install the required packages
+tlmgr install latex-bin dvips collection-fontsrecommended collection-latexrecommended
+
+# Validate that installing dvips and latex has been successful
+dvips --version
+latex --version
@@ -18,7 +18,7 @@ $vcpkgPackages = @(
    "boost-iostreams",
    "boost-interprocess",
    "boost-algorithm",
-    "pybind11"
+    "nanobind"
 )

 # Update vcpkg
@@ -37,4 +37,4 @@ try {
    vcpkg install $vcpkgPackages
 }

-Write-Host "vcpkg install completed successfully"
+Write-Host "vcpkg install completed successfully"
@@ -1,4 +1,6 @@
 #!/usr/bin/env bash
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
 set -e

 # Various tests to test the FindOpenVDB CMake modules and
@@ -8,7 +10,7 @@ set -e
 # the expected VDB installation

 cmakelists="
-cmake_minimum_required(VERSION 3.18)
+cmake_minimum_required(VERSION 3.24)
 project(TestInstall LANGUAGES CXX)
 find_package(OpenVDB REQUIRED COMPONENTS openvdb)
 add_executable(test_vdb_print \"../openvdb_cmd/vdb_print/main.cc\")
@@ -1,4 +1,6 @@
 #!/usr/bin/env bash
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0

 set -ex

@@ -0,0 +1,24 @@
+# SPDX-License-Identifier: MIT
+#
+# SPDX-FileCopyrightText: Copyright (c) 2019-2023 Lars Melchior and contributors
+
+set(CPM_DOWNLOAD_VERSION 0.40.2)
+set(CPM_HASH_SUM "c8cdc32c03816538ce22781ed72964dc864b2a34a310d3b7104812a5ca2d835d")
+
+if(CPM_SOURCE_CACHE)
+  set(CPM_DOWNLOAD_LOCATION "${CPM_SOURCE_CACHE}/cpm/CPM_${CPM_DOWNLOAD_VERSION}.cmake")
+elseif(DEFINED ENV{CPM_SOURCE_CACHE})
+  set(CPM_DOWNLOAD_LOCATION "$ENV{CPM_SOURCE_CACHE}/cpm/CPM_${CPM_DOWNLOAD_VERSION}.cmake")
+else()
+  set(CPM_DOWNLOAD_LOCATION "${CMAKE_BINARY_DIR}/cmake/CPM_${CPM_DOWNLOAD_VERSION}.cmake")
+endif()
+
+# Expand relative path. This is important if the provided path contains a tilde (~)
+get_filename_component(CPM_DOWNLOAD_LOCATION ${CPM_DOWNLOAD_LOCATION} ABSOLUTE)
+
+file(DOWNLOAD
+     https://github.com/cpm-cmake/CPM.cmake/releases/download/v${CPM_DOWNLOAD_VERSION}/CPM.cmake
+     ${CPM_DOWNLOAD_LOCATION} EXPECTED_HASH SHA256=${CPM_HASH_SUM}
+)
+
+include(${CPM_DOWNLOAD_LOCATION})
@@ -1,5 +1,5 @@
 # Copyright Contributors to the OpenVDB Project
-# SPDX-License-Identifier: MPL-2.0
+# SPDX-License-Identifier: Apache-2.0
 #
 #[=======================================================================[.rst:

@@ -88,7 +88,7 @@ may be provided to tell this module where to look.

 #]=======================================================================]

-cmake_minimum_required(VERSION 3.18)
+cmake_minimum_required(VERSION 3.24)
 include(GNUInstallDirs)

 mark_as_advanced(
@@ -1,320 +0,0 @@
-# Copyright Contributors to the OpenVDB Project
-# SPDX-License-Identifier: MPL-2.0
-#
-#[=======================================================================[.rst:
-
-FindCppUnit
-----------
-
-Find CppUnit include dirs and libraries
-
-Use this module by invoking find_package with the form::
-
-  find_package(CppUnit
-    [version] [EXACT]      # Minimum or EXACT version
-    [REQUIRED]             # Fail with error if CppUnit is not found
-    )
-
-
-IMPORTED Targets
-^^^^^^^^^^^^^^^^
-
-``CppUnit::cppunit``
-  This module defines IMPORTED target CppUnit::cppunit, if CppUnit has been
-  found.
-
-Result Variables
-^^^^^^^^^^^^^^^^
-
-This will define the following variables:
-
-``CppUnit_FOUND``
-  True if the system has the CppUnit library.
-``CppUnit_VERSION``
-  The version of the CppUnit library which was found.
-``CppUnit_INCLUDE_DIRS``
-  Include directories needed to use CppUnit.
-``CppUnit_RELEASE_LIBRARIES``
-  Libraries needed to link to the release version of CppUnit.
-``CppUnit_RELEASE_LIBRARY_DIRS``
-  CppUnit release library directories.
-``CppUnit_DEBUG_LIBRARIES``
-  Libraries needed to link to the debug version of CppUnit.
-``CppUnit_DEBUG_LIBRARY_DIRS``
-  CppUnit debug library directories.
-
-Deprecated - use [RELEASE|DEBUG] variants:
-
-``CppUnit_LIBRARIES``
-  Libraries needed to link to CppUnit.
-``CppUnit_LIBRARY_DIRS``
-  CppUnit library directories.
-
-Cache Variables
-^^^^^^^^^^^^^^^
-
-The following cache variables may also be set:
-
-``CppUnit_INCLUDE_DIR``
-  The directory containing ``cppunit/Portability.h``.
-``CppUnit_LIBRARY``
-  The path to the CppUnit library.
-
-Hints
-^^^^^
-
-Instead of explicitly setting the cache variables, the following variables
-may be provided to tell this module where to look.
-
-``CppUnit_ROOT``
-  Preferred installation prefix.
-``CPPUNIT_INCLUDEDIR``
-  Preferred include directory e.g. <prefix>/include
-``CPPUNIT_LIBRARYDIR``
-  Preferred library directory e.g. <prefix>/lib
-``CPPUNIT_DEBUG_SUFFIX``
-  Suffix of the debug version of CppUnit libs. Defaults to "_d".
-``SYSTEM_LIBRARY_PATHS``
-  Global list of library paths intended to be searched by and find_xxx call
-``CPPUNIT_USE_STATIC_LIBS``
-  Only search for static cppunit libraries
-``DISABLE_CMAKE_SEARCH_PATHS``
-  Disable CMakes default search paths for find_xxx calls in this module
-
-#]=======================================================================]
-
-cmake_minimum_required(VERSION 3.18)
-include(GNUInstallDirs)
-
-
-mark_as_advanced(
-  CppUnit_INCLUDE_DIR
-  CppUnit_LIBRARY
-)
-
-set(_FIND_CPPUNIT_ADDITIONAL_OPTIONS "")
-if(DISABLE_CMAKE_SEARCH_PATHS)
-  set(_FIND_CPPUNIT_ADDITIONAL_OPTIONS NO_DEFAULT_PATH)
-endif()
-
-# Set _CPPUNIT_ROOT based on a user provided root var. Xxx_ROOT and ENV{Xxx_ROOT}
-# are prioritised over the legacy capitalized XXX_ROOT variables for matching
-# CMake 3.12 behaviour
-# @todo  deprecate -D and ENV CPPUNIT_ROOT from CMake 3.12
-if(CppUnit_ROOT)
-  set(_CPPUNIT_ROOT ${CppUnit_ROOT})
-elseif(DEFINED ENV{CppUnit_ROOT})
-  set(_CPPUNIT_ROOT $ENV{CppUnit_ROOT})
-elseif(CPPUNIT_ROOT)
-  set(_CPPUNIT_ROOT ${CPPUNIT_ROOT})
-elseif(DEFINED ENV{CPPUNIT_ROOT})
-  set(_CPPUNIT_ROOT $ENV{CPPUNIT_ROOT})
-endif()
-
-# Additionally try and use pkconfig to find cppunit
-if(USE_PKGCONFIG)
-  if(NOT DEFINED PKG_CONFIG_FOUND)
-    find_package(PkgConfig)
-  endif()
-  pkg_check_modules(PC_CppUnit QUIET cppunit)
-endif()
-
-# ------------------------------------------------------------------------
-#  Search for CppUnit include DIR
-# ------------------------------------------------------------------------
-
-set(_CPPUNIT_INCLUDE_SEARCH_DIRS "")
-list(APPEND _CPPUNIT_INCLUDE_SEARCH_DIRS
-  ${CPPUNIT_INCLUDEDIR}
-  ${_CPPUNIT_ROOT}
-  ${PC_CppUnit_INCLUDEDIR}
-  ${SYSTEM_LIBRARY_PATHS}
-)
-
-# Look for a standard cppunit header file.
-find_path(CppUnit_INCLUDE_DIR cppunit/Portability.h
-  ${_FIND_CPPUNIT_ADDITIONAL_OPTIONS}
-  PATHS ${_CPPUNIT_INCLUDE_SEARCH_DIRS}
-  PATH_SUFFIXES ${CMAKE_INSTALL_INCLUDEDIR} include
-)
-
-if(EXISTS "${CppUnit_INCLUDE_DIR}/cppunit/Portability.h")
-  file(STRINGS "${CppUnit_INCLUDE_DIR}/cppunit/Portability.h"
-    _cppunit_version_string REGEX "#define CPPUNIT_VERSION "
-  )
-  string(REGEX REPLACE "#define CPPUNIT_VERSION +\"(.+)\".*$" "\\1"
-    _cppunit_version_string "${_cppunit_version_string}"
-  )
-  string(STRIP "${_cppunit_version_string}" CppUnit_VERSION)
-  unset(_cppunit_version_string )
-endif()
-
-# ------------------------------------------------------------------------
-#  Search for CppUnit lib DIR
-# ------------------------------------------------------------------------
-
-if(NOT DEFINED CPPUNIT_DEBUG_SUFFIX)
-  set(CPPUNIT_DEBUG_SUFFIX "d")
-endif()
-
-set(_CPPUNIT_LIBRARYDIR_SEARCH_DIRS "")
-list(APPEND _CPPUNIT_LIBRARYDIR_SEARCH_DIRS
-  ${CPPUNIT_LIBRARYDIR}
-  ${_CPPUNIT_ROOT}
-  ${PC_CppUnit_LIBDIR}
-  ${SYSTEM_LIBRARY_PATHS}
-)
-
-# Library suffix handling
-
-set(_CPPUNIT_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES})
-
-if(MSVC)
-  if(CPPUNIT_USE_STATIC_LIBS)
-    set(CMAKE_FIND_LIBRARY_SUFFIXES ".lib")
-  else()
-    list(APPEND CMAKE_FIND_LIBRARY_SUFFIXES "_dll.lib")
-  endif()
-else()
-  if(CPPUNIT_USE_STATIC_LIBS)
-    set(CMAKE_FIND_LIBRARY_SUFFIXES ".a")
-  endif()
-endif()
-
-list(APPEND CPPUNIT_BUILD_TYPES RELEASE DEBUG)
-
-foreach(BUILD_TYPE ${CPPUNIT_BUILD_TYPES})
-  set(_CPPUNIT_LIB_NAME cppunit)
-  if(BUILD_TYPE STREQUAL DEBUG)
-    set(_CPPUNIT_LIB_NAME "${_CPPUNIT_LIB_NAME}${CPPUNIT_DEBUG_SUFFIX}")
-  endif()
-
-  # Find the lib
-  find_library(CppUnit_LIBRARY_${BUILD_TYPE} ${_CPPUNIT_LIB_NAME}
-    ${_FIND_CPPUNIT_ADDITIONAL_OPTIONS}
-    PATHS ${_CPPUNIT_LIBRARYDIR_SEARCH_DIRS}
-    PATH_SUFFIXES ${CMAKE_INSTALL_LIBDIR} lib64 lib
-  )
-endforeach()
-
-# Reset library suffix
-
-set(CMAKE_FIND_LIBRARY_SUFFIXES ${_CPPUNIT_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES})
-unset(_CPPUNIT_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES)
-
-if(CppUnit_LIBRARY_DEBUG AND CppUnit_LIBRARY_RELEASE)
-  # if the generator is multi-config or if CMAKE_BUILD_TYPE is set for
-  # single-config generators, set optimized and debug libraries
-  get_property(_isMultiConfig GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG)
-  if(_isMultiConfig OR CMAKE_BUILD_TYPE)
-    set(CppUnit_LIBRARY optimized ${CppUnit_LIBRARY_RELEASE} debug ${CppUnit_LIBRARY_DEBUG})
-  else()
-    # For single-config generators where CMAKE_BUILD_TYPE has no value,
-    # just use the release libraries
-    set(CppUnit_LIBRARY ${CppUnit_LIBRARY_RELEASE})
-  endif()
-  # FIXME: This probably should be set for both cases
-  set(CppUnit_LIBRARIES optimized ${CppUnit_LIBRARY_RELEASE} debug ${CppUnit_LIBRARY_DEBUG})
-endif()
-
-# if only the release version was found, set the debug variable also to the release version
-if(CppUnit_LIBRARY_RELEASE AND NOT CppUnit_LIBRARY_DEBUG)
-  set(CppUnit_LIBRARY_DEBUG ${CppUnit_LIBRARY_RELEASE})
-  set(CppUnit_LIBRARY       ${CppUnit_LIBRARY_RELEASE})
-  set(CppUnit_LIBRARIES     ${CppUnit_LIBRARY_RELEASE})
-endif()
-
-# if only the debug version was found, set the release variable also to the debug version
-if(CppUnit_LIBRARY_DEBUG AND NOT CppUnit_LIBRARY_RELEASE)
-  set(CppUnit_LIBRARY_RELEASE ${CppUnit_LIBRARY_DEBUG})
-  set(CppUnit_LIBRARY         ${CppUnit_LIBRARY_DEBUG})
-  set(CppUnit_LIBRARIES       ${CppUnit_LIBRARY_DEBUG})
-endif()
-
-# If the debug & release library ends up being the same, omit the keywords
-if("${CppUnit_LIBRARY_RELEASE}" STREQUAL "${CppUnit_LIBRARY_DEBUG}")
-  set(CppUnit_LIBRARY   ${CppUnit_LIBRARY_RELEASE} )
-  set(CppUnit_LIBRARIES ${CppUnit_LIBRARY_RELEASE} )
-endif()
-
-if(CppUnit_LIBRARY)
-  set(CppUnit_FOUND TRUE)
-else()
-  set(CppUnit_FOUND FALSE)
-endif()
-
-# ------------------------------------------------------------------------
-#  Cache and set CppUnit_FOUND
-# ------------------------------------------------------------------------
-
-include(FindPackageHandleStandardArgs)
-find_package_handle_standard_args(CppUnit
-  FOUND_VAR CppUnit_FOUND
-  REQUIRED_VARS
-    CppUnit_LIBRARY
-    CppUnit_INCLUDE_DIR
-  VERSION_VAR CppUnit_VERSION
-)
-
-if(NOT CppUnit_FOUND)
-  if(CppUnit_FIND_REQUIRED)
-    message(FATAL_ERROR "Unable to find CppUnit")
-  endif()
-  return()
-endif()
-
-# Partition release/debug lib vars
-
-set(CppUnit_RELEASE_LIBRARIES ${CppUnit_LIBRARY_RELEASE})
-get_filename_component(CppUnit_RELEASE_LIBRARY_DIRS ${CppUnit_RELEASE_LIBRARIES} DIRECTORY)
-set(CppUnit_DEBUG_LIBRARIES ${CppUnit_LIBRARY_DEBUG})
-get_filename_component(CppUnit_DEBUG_LIBRARY_DIRS ${CppUnit_DEBUG_LIBRARIES} DIRECTORY)
-set(CppUnit_LIBRARIES ${CppUnit_RELEASE_LIBRARIES})
-set(CppUnit_LIBRARY_DIRS ${CppUnit_RELEASE_LIBRARY_DIRS})
-set(CppUnit_INCLUDE_DIRS ${CppUnit_INCLUDE_DIR})
-
-# Configure lib type. If XXX_USE_STATIC_LIBS, we always assume a static
-# lib is in use. If win32, we can't mark the import .libs as shared, so
-# these are always marked as UNKNOWN. Otherwise, infer from extension.
-set(CPPUNIT_LIB_TYPE UNKNOWN)
-if(CPPUNIT_USE_STATIC_LIBS)
-  set(CPPUNIT_LIB_TYPE STATIC)
-elseif(UNIX)
-  get_filename_component(_CPPUNIT_EXT ${CppUnit_LIBRARY_RELEASE} EXT)
-  if(_CPPUNIT_EXT STREQUAL ".a")
-    set(CPPUNIT_LIB_TYPE STATIC)
-  elseif(_CPPUNIT_EXT STREQUAL ".so" OR
-         _CPPUNIT_EXT STREQUAL ".dylib")
-    set(CPPUNIT_LIB_TYPE SHARED)
-  endif()
-endif()
-
-if(NOT TARGET CppUnit::cppunit)
-  add_library(CppUnit::cppunit ${CPPUNIT_LIB_TYPE} IMPORTED)
-  set_target_properties(CppUnit::cppunit PROPERTIES
-    INTERFACE_COMPILE_OPTIONS "${PC_CppUnit_CFLAGS_OTHER}"
-    INTERFACE_INCLUDE_DIRECTORIES "${CppUnit_INCLUDE_DIRS}")
-
-  # Standard location
-  set_target_properties(CppUnit::cppunit PROPERTIES
-    IMPORTED_LINK_INTERFACE_LANGUAGES "CXX"
-    IMPORTED_LOCATION "${CppUnit_LIBRARY}")
-
-  # Release location
-  if(EXISTS "${CppUnit_LIBRARY_RELEASE}")
-    set_property(TARGET CppUnit::cppunit APPEND PROPERTY
-      IMPORTED_CONFIGURATIONS RELEASE)
-    set_target_properties(CppUnit::cppunit PROPERTIES
-      IMPORTED_LINK_INTERFACE_LANGUAGES_RELEASE "CXX"
-      IMPORTED_LOCATION_RELEASE "${CppUnit_LIBRARY_RELEASE}")
-  endif()
-
-  # Debug location
-  if(EXISTS "${CppUnit_LIBRARY_DEBUG}")
-    set_property(TARGET CppUnit::cppunit APPEND PROPERTY
-      IMPORTED_CONFIGURATIONS DEBUG)
-    set_target_properties(CppUnit::cppunit PROPERTIES
-      IMPORTED_LINK_INTERFACE_LANGUAGES_DEBUG "CXX"
-      IMPORTED_LOCATION_DEBUG "${CppUnit_LIBRARY_DEBUG}")
-  endif()
-endif()
@@ -1,5 +1,5 @@
 # Copyright Contributors to the OpenVDB Project
-# SPDX-License-Identifier: MPL-2.0
+# SPDX-License-Identifier: Apache-2.0
 #
 #[=======================================================================[.rst:

@@ -64,7 +64,7 @@ may be provided to tell this module where to look.

 #]=======================================================================]

-cmake_minimum_required(VERSION 3.18)
+cmake_minimum_required(VERSION 3.24)
 include(GNUInstallDirs)


@@ -1,5 +1,5 @@
 # Copyright Contributors to the OpenVDB Project
-# SPDX-License-Identifier: MPL-2.0
+# SPDX-License-Identifier: Apache-2.0
 #
 #[=======================================================================[.rst:

@@ -87,7 +87,7 @@ may be provided to tell this module where to look.

 #]=======================================================================]

-cmake_minimum_required(VERSION 3.18)
+cmake_minimum_required(VERSION 3.24)
 include(GNUInstallDirs)


@@ -0,0 +1,44 @@
+include (FindPackageHandleStandardArgs)
+
+if(NCCL_ROOT)
+  set(_NCCL_ROOT ${NCCL_ROOT})
+elseif(DEFINED ENV{NCCL_ROOT})
+  set(_NCCL_ROOT $ENV{NCCL_ROOT})
+endif()
+
+set(_NCCL_INCLUDE_SEARCH_DIRS "")
+list(APPEND _NCCL_INCLUDE_SEARCH_DIRS
+  ${_NCCL_ROOT}/include
+)
+find_path(NCCL_INCLUDE_DIR
+    NAMES nccl.h
+    PATHS ${_NCCL_INCLUDE_SEARCH_DIRS}
+)
+
+set(_NCCL_LIBRARY_SEARCH_DIRS "")
+list(APPEND _NCCL_LIBRARY_SEARCH_DIRS
+  ${_NCCL_ROOT}/bin
+)
+find_library(NCCL_LIBRARY
+    NAMES nccl
+    PATHS ${_NCCL_LIBRARY_SEARCH_DIRS}
+)
+
+find_package_handle_standard_args(NCCL REQUIRED_VARS NCCL_INCLUDE_DIR NCCL_LIBRARY)
+
+if (NCCL_FOUND)
+    mark_as_advanced(NCCL_INCLUDE_DIR)
+    mark_as_advanced(NCCL_LIBRARY)
+endif()
+
+if (NCCL_FOUND AND NOT TARGET NCCL::NCCL)
+    add_library(NCCL::NCCL SHARED IMPORTED)
+    if(MSVC)
+      string(REPLACE ".lib" ".dll" NCCL_DLL ${NCCL_LIBRARY})
+      set_property(TARGET NCCL::NCCL PROPERTY IMPORTED_LOCATION ${NCCL_DLL})
+      set_property(TARGET NCCL::NCCL PROPERTY IMPORTED_IMPLIB ${NCCL_LIBRARY})
+    else()
+      set_property(TARGET NCCL::NCCL PROPERTY IMPORTED_LOCATION ${NCCL_LIBRARY})
+    endif()
+    target_include_directories(NCCL::NCCL INTERFACE ${NCCL_INCLUDE_DIR})
+endif()
@@ -1,5 +1,5 @@
 # Copyright Contributors to the OpenVDB Project
-# SPDX-License-Identifier: MPL-2.0
+# SPDX-License-Identifier: Apache-2.0
 #
 #[=======================================================================[.rst:

@@ -93,7 +93,7 @@ may be provided to tell this module where to look.

 #]=======================================================================]

-cmake_minimum_required(VERSION 3.18)
+cmake_minimum_required(VERSION 3.24)
 include(GNUInstallDirs)


@@ -1,5 +1,5 @@
 # Copyright Contributors to the OpenVDB Project
-# SPDX-License-Identifier: MPL-2.0
+# SPDX-License-Identifier: Apache-2.0
 #
 #[=======================================================================[.rst:

@@ -109,7 +109,7 @@ may be provided to tell this module where to look.

 #]=======================================================================]

-cmake_minimum_required(VERSION 3.18)
+cmake_minimum_required(VERSION 3.24)
 include(GNUInstallDirs)


@@ -334,10 +334,8 @@ set(_OPENVDB_ORIG_CMAKE_FIND_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES})

 set(OPENVDB_PYTHON_PATH_SUFFIXES
  lib64/python
-  lib64/python2.7
  lib64/python3
  lib/python
-  lib/python2.7
  lib/python3
 )

@@ -1,5 +1,5 @@
 # Copyright Contributors to the OpenVDB Project
-# SPDX-License-Identifier: MPL-2.0
+# SPDX-License-Identifier: Apache-2.0
 #
 #[=======================================================================[.rst:

@@ -93,7 +93,7 @@ may be provided to tell this module where to look.

 #]=======================================================================]

-cmake_minimum_required(VERSION 3.18)
+cmake_minimum_required(VERSION 3.24)
 include(GNUInstallDirs)


@@ -1,5 +1,5 @@
 # Copyright Contributors to the OpenVDB Project
-# SPDX-License-Identifier: MPL-2.0
+# SPDX-License-Identifier: Apache-2.0
 #
 #[=======================================================================[.rst:

@@ -47,7 +47,7 @@ The following variables may be provided to tell this module where to look.
 # Find the glfw3 installation and use glfw's CMake to initialize
 # the glfw lib

-cmake_minimum_required(VERSION 3.18)
+cmake_minimum_required(VERSION 3.24)

 set(_FIND_GLFW3_ADDITIONAL_OPTIONS "")
 if(DISABLE_CMAKE_SEARCH_PATHS)
@@ -109,18 +109,6 @@ endif()
 set(glfw3_FIND_VERSION ${MINIMUM_GLFW_VERSION})
 find_package(glfw3 ${MINIMUM_GLFW_VERSION} REQUIRED)

-# We only use find_package_handle_standard_args to verify and print
-# appropriate messages. This now explicitly errors in 3.19...
-# @todo Improve this entire GLFW3 search
-#  https://gitlab.kitware.com/cmake/cmake/-/issues/21505
-if(${CMAKE_VERSION} VERSION_LESS 3.19)
-  find_package(PackageHandleStandardArgs)
-  find_package_handle_standard_args(glfw3
-    REQUIRED_VARS glfw3_DIR glfw3_FOUND
-    VERSION_VAR glfw3_VERSION
-  )
-endif()
-
 if(OPENVDB_FUTURE_DEPRECATION AND FUTURE_MINIMUM_GLFW_VERSION)
  if(glfw3_VERSION VERSION_LESS ${FUTURE_MINIMUM_GLFW_VERSION})
    message(DEPRECATION "Support for GLFW versions < ${FUTURE_MINIMUM_GLFW_VERSION} "
@@ -1,5 +1,5 @@
 # Copyright Contributors to the OpenVDB Project
-# SPDX-License-Identifier: MPL-2.0
+# SPDX-License-Identifier: Apache-2.0
 #
 #[=======================================================================[.rst:

@@ -73,7 +73,7 @@ may be provided to tell this module where to look.
 # Find the Houdini installation and use Houdini's CMake to initialize
 # the Houdini lib

-cmake_minimum_required(VERSION 3.18)
+cmake_minimum_required(VERSION 3.24)

 # Include utility functions for version information
 include(${CMAKE_CURRENT_LIST_DIR}/OpenVDBUtils.cmake)
@@ -1,5 +1,5 @@
 # Copyright Contributors to the OpenVDB Project
-# SPDX-License-Identifier: MPL-2.0
+# SPDX-License-Identifier: Apache-2.0
 #
 #[=======================================================================[.rst:

@@ -51,7 +51,7 @@ variables may be provided to tell this module where to look.
 # Find the Maya installation and use Maya's CMake to initialize
 # the Maya lib

-cmake_minimum_required(VERSION 3.18)
+cmake_minimum_required(VERSION 3.24)


 set(_FIND_MAYA_ADDITIONAL_OPTIONS "")
@@ -1,5 +1,5 @@
 # Copyright Contributors to the OpenVDB Project
-# SPDX-License-Identifier: MPL-2.0
+# SPDX-License-Identifier: Apache-2.0
 #
 #[=======================================================================[.rst:

@@ -57,7 +57,7 @@ The following functions are provided:

 #]=======================================================================]

-cmake_minimum_required(VERSION 3.18)
+cmake_minimum_required(VERSION 3.24)


 function(OPENVDB_GET_VERSION_DEFINE HEADER KEY VALUE)
@@ -1,5 +1,5 @@
 # Copyright Contributors to the OpenVDB Project
-# SPDX-License-Identifier: MPL-2.0
+# SPDX-License-Identifier: Apache-2.0
 #
 #[=======================================================================[.rst:

@@ -16,7 +16,7 @@ existing from a previous run of cmake.

 #]=======================================================================]

-cmake_minimum_required(VERSION 3.18)
+cmake_minimum_required(VERSION 3.24)

 set(MANIFEST "${CMAKE_CURRENT_BINARY_DIR}/install_manifest.txt")

@@ -1,5 +1,5 @@
 # Copyright Contributors to the OpenVDB Project
-# SPDX-License-Identifier: MPL-2.0
+# SPDX-License-Identifier: Apache-2.0
 #
 #[=======================================================================[

@@ -7,7 +7,7 @@

 #]=======================================================================]

-cmake_minimum_required(VERSION 3.18)
+cmake_minimum_required(VERSION 3.24)

 ###############################################################################

@@ -184,20 +184,23 @@ if(OPENVDB_CXX_STRICT)
  add_compile_options("$<$<AND:$<CXX_COMPILER_ID:GNU>,$<VERSION_GREATER_EQUAL:$<CXX_COMPILER_VERSION>,9.3.0>>:-Wimplicit-fallthrough>")
  # Only check global constructors for libraries (we should really check for
  # executables too but gtest relies on these types of constructors for its
-  # framework).
-  add_compile_options("$<$<AND:$<NOT:$<STREQUAL:$<TARGET_PROPERTY:TYPE>,EXECUTABLE>>,$<COMPILE_LANG_AND_ID:CXX,Clang,AppleClang>>:-Wglobal-constructors>")
+  # framework). nanobind also incorporates these constructors so skip it as well.
+  add_compile_options("$<$<AND:$<NOT:$<OR:$<STREQUAL:$<TARGET_PROPERTY:TYPE>,EXECUTABLE>,$<STREQUAL:$<TARGET_PROPERTY:NAME>,nanobind-static>>>,$<COMPILE_LANG_AND_ID:CXX,Clang,AppleClang>>:-Wglobal-constructors>")
  add_compile_options("$<$<COMPILE_LANG_AND_ID:CXX,Clang,AppleClang>:-Wno-sign-conversion>")
  # GNU
  add_compile_options("$<$<COMPILE_LANG_AND_ID:CXX,GNU>:-Werror>")
  add_compile_options("$<$<COMPILE_LANG_AND_ID:CXX,GNU>:-Wall>")
  add_compile_options("$<$<COMPILE_LANG_AND_ID:CXX,GNU>:-Wextra>")
-  add_compile_options("$<$<COMPILE_LANG_AND_ID:CXX,GNU>:-pedantic>")
  add_compile_options("$<$<COMPILE_LANG_AND_ID:CXX,GNU>:-Wcast-align>")
-  add_compile_options("$<$<COMPILE_LANG_AND_ID:CXX,GNU>:-Wcast-qual>")
-  add_compile_options("$<$<COMPILE_LANG_AND_ID:CXX,GNU>:-Wconversion>")
  add_compile_options("$<$<COMPILE_LANG_AND_ID:CXX,GNU>:-Wdisabled-optimization>")
  add_compile_options("$<$<COMPILE_LANG_AND_ID:CXX,GNU>:-Woverloaded-virtual>")
  add_compile_options("$<$<COMPILE_LANG_AND_ID:CXX,GNU>:-Wnon-virtual-dtor>")
+  # Disable select warnings for the nanobind static library
+  add_compile_options("$<$<AND:$<NOT:$<OR:$<STREQUAL:$<TARGET_PROPERTY:NAME>,nanobind-static>,$<STREQUAL:$<TARGET_PROPERTY:NAME>,openvdb_python>,$<STREQUAL:$<TARGET_PROPERTY:NAME>,nanovdb_python>>>,$<COMPILE_LANG_AND_ID:CXX,GNU>>:-pedantic>")
+  add_compile_options("$<$<AND:$<NOT:$<OR:$<STREQUAL:$<TARGET_PROPERTY:NAME>,nanobind-static>,$<STREQUAL:$<TARGET_PROPERTY:NAME>,openvdb_python>,$<STREQUAL:$<TARGET_PROPERTY:NAME>,nanovdb_python>>>,$<COMPILE_LANG_AND_ID:CXX,GNU>>:-Wcast-qual>")
+  add_compile_options("$<$<AND:$<NOT:$<OR:$<STREQUAL:$<TARGET_PROPERTY:NAME>,nanobind-static>,$<STREQUAL:$<TARGET_PROPERTY:NAME>,openvdb_python>,$<STREQUAL:$<TARGET_PROPERTY:NAME>,nanovdb_python>>>,$<COMPILE_LANG_AND_ID:CXX,GNU>>:-Wconversion>")
+  add_compile_options("$<$<AND:$<OR:$<STREQUAL:$<TARGET_PROPERTY:NAME>,nanobind-static>,$<STREQUAL:$<TARGET_PROPERTY:NAME>,openvdb_python>,$<STREQUAL:$<TARGET_PROPERTY:NAME>,nanovdb_python>>,$<COMPILE_LANG_AND_ID:CXX,GNU>>:-Wno-unused-variable>")
+  add_compile_options("$<$<AND:$<OR:$<STREQUAL:$<TARGET_PROPERTY:NAME>,nanobind-static>,$<STREQUAL:$<TARGET_PROPERTY:NAME>,openvdb_python>,$<STREQUAL:$<TARGET_PROPERTY:NAME>,nanovdb_python>>,$<COMPILE_LANG_AND_ID:CXX,GNU>>:-Wno-unused-but-set-parameter>")
 else()
  # NO OPENVDB_CXX_STRICT, suppress some warnings
  if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
@@ -1,5 +1,5 @@
 # Copyright Contributors to the OpenVDB Project
-# SPDX-License-Identifier: MPL-2.0
+# SPDX-License-Identifier: Apache-2.0
 #
 #[=======================================================================[

@@ -8,7 +8,7 @@

 #]=======================================================================]

-cmake_minimum_required(VERSION 3.18)
+cmake_minimum_required(VERSION 3.24)

 ###############################################################################

@@ -27,59 +27,59 @@ set(OPENVDB_DEPRECATED_ABI_LIST ${MINIMUM_OPENVDB_ABI_VERSION} ${_PREV_ABI})
 unset(_PREV_ABI)

 if(NOT DISABLE_DEPENDENCY_VERSION_CHECKS)
-  # @note  Currently tracking CY2022 of the VFX platform where available
+  # @note  Currently tracking CY2024 of the VFX platform where available
+  # (that is, our minimum requirements are for the current VFX platform)
+  # minus 2 years to be able to build with legacy ABI)

  # @Note  Compiler versions are not really a hard and fast rule, you just need
  # a compiler with complete support for our MINIMUM_CXX_STANDARD (currently 17).
  # What's usually important is that the version of libstdc++ and glibc in use
  # matches. Compilers other than GCC should provide options to ensure this
  # targetting e.g. --gcc-toolchain and -fgnuc-version with Clang.
-  set(MINIMUM_GCC_VERSION 9.3.1)
+  set(MINIMUM_GCC_VERSION 11.2.1)
  set(MINIMUM_CLANG_VERSION 5.0)
  set(MINIMUM_ICC_VERSION 19)
-  set(MINIMUM_MSVC_VERSION 19.28) # 1928 (Visual Studio 2019 Version 16.8 + 16.9)
+  set(MINIMUM_MSVC_VERSION 19.30) # 1928 (Visual Studio 2019 Version 16.8 + 16.9)

-  # Should be 1.76 for VFX 22, but only version in apt is 1.73
-  set(MINIMUM_BOOST_VERSION 1.73)
-  set(MINIMUM_PYBIND_VERSION 2.9.1)
-  set(MINIMUM_IMATH_VERSION 3.1)
-  set(MINIMUM_OPENEXR_VERSION 3.1)
+  set(MINIMUM_BOOST_VERSION 1.82)
+  set(MINIMUM_OPENEXR_VERSION 3.2)
+  set(MINIMUM_IMATH_VERSION 3.2)
  set(MINIMUM_ZLIB_VERSION 1.2.7)
  set(MINIMUM_TBB_VERSION 2020.3)
-  set(MINIMUM_LLVM_VERSION 10.0.0)
+  set(MINIMUM_LLVM_VERSION 15.0.0)
  set(MINIMUM_BLOSC_VERSION 1.17.0)
-  set(MINIMUM_GLFW_VERSION 3.1)
+  set(MINIMUM_GLFW_VERSION 3.3)

-  set(MINIMUM_PYTHON_VERSION 3.9.1)
-  set(MINIMUM_NUMPY_VERSION 1.20.0)
+  set(MINIMUM_PYTHON_VERSION 3.11)

  set(MINIMUM_GOOGLETEST_VERSION 1.10)
-  set(MINIMUM_LOG4CPLUS_VERSION 1.1.2)
-  set(MINIMUM_HOUDINI_VERSION 19.5)
+  set(MINIMUM_LOG4CPLUS_VERSION 2.0)
+  set(MINIMUM_HOUDINI_VERSION 20.5)

  # These always promote warnings rather than errors
  set(MINIMUM_MAYA_VERSION 2017)
  set(MINIMUM_DOXYGEN_VERSION 1.8.8)
 endif()

-# VFX 23 deprecations to transition to MINIMUM_* variables in OpenVDB 12.0.0
+# VFX 24 deprecations to transition to MINIMUM_* variables in OpenVDB 14.0.0
 # @note  At the time of writing, any variables that are commented out don't
 #   have target transitional versions.

-set(FUTURE_MINIMUM_GCC_VERSION 11.2.1)
-set(FUTURE_MINIMUM_MSVC_VERSION 19.30) # 1930 (Visual Studio 2022)
+# set(FUTURE_MINIMUM_GCC_VERSION 11.2.1)
+# set(FUTURE_MINIMUM_MSVC_VERSION 19.30)
 # set(FUTURE_MINIMUM_ICC_VERSION 19)

 # set(FUTURE_MINIMUM_CXX_STANDARD 20)
-set(FUTURE_MINIMUM_CMAKE_VERSION 3.20)
-# set(FUTURE_MINIMUM_OPENEXR_VERSION 3.1)
-set(FUTURE_MINIMUM_BOOST_VERSION 1.80)
-set(FUTURE_MINIMUM_GLFW_VERSION 3.3)
-set(FUTURE_MINIMUM_LOG4CPLUS_VERSION 2.0)
+set(FUTURE_MINIMUM_CMAKE_VERSION 4.0)
+set(FUTURE_MINIMUM_OPENEXR_VERSION 3.3)
+set(FUTURE_MINIMUM_IMATH_VERSION 3.3)
+set(FUTURE_MINIMUM_TBB_VERSION 2021)
+set(FUTURE_MINIMUM_BOOST_VERSION 1.85)
+#set(FUTURE_MINIMUM_GLFW_VERSION 3.3)
+#set(FUTURE_MINIMUM_LOG4CPLUS_VERSION 2.0)
+set(FUTURE_MINIMUM_NANOBIND_VERSION 2.5.0)

 # set(FUTURE_MINIMUM_BLOSC_VERSION 1.17.0)
-# set(FUTURE_MINIMUM_TBB_VERSION 2020.3)
-set(FUTURE_MINIMUM_PYTHON_VERSION 3.10)
-set(FUTURE_MINIMUM_NUMPY_VERSION 1.23.0)
+# set(FUTURE_MINIMUM_PYTHON_VERSION 3.11)
 # set(FUTURE_MINIMUM_HOUDINI_VERSION 20.0)
-set(FUTURE_MINIMUM_LLVM_VERSION 13.0.0)
+set(FUTURE_MINIMUM_LLVM_VERSION 18.0.0)
@@ -1,5 +1,5 @@
 # Copyright Contributors to the OpenVDB Project
-# SPDX-License-Identifier: MPL-2.0
+# SPDX-License-Identifier: Apache-2.0
 #
 #[=======================================================================[

@@ -7,7 +7,7 @@

 #]=======================================================================]

-cmake_minimum_required(VERSION 3.18)
+cmake_minimum_required(VERSION 3.24)
 project(OpenVDBDocs LANGUAGES NONE)

 include(GNUInstallDirs)
@@ -44,6 +44,7 @@ set(DOXY_FILES
  openvdb/openvdb/points/impl
  openvdb/openvdb/python/pyopenvdb.h
  openvdb/openvdb/tools
+  openvdb/openvdb/tools/impl
  openvdb/openvdb/tree
  openvdb/openvdb/util
  doc/build.txt
@@ -238,10 +238,8 @@ it's a good idea to read the above section on

 DCC      | Supported Version | OpenVDB ABI |
 -------- | ----------------- | ----------- |
-Houdini  | 19.5              | 9           |
-Houdini  | 20.0              | 10          |
-Maya     | 2018              | Any         |
-Maya     | 2019              | Any         |
+Houdini  | 20.5              | 11          |
+Houdini  | 21.0              | 12          |

@subsection buildBuildHou Building Against Houdini

@@ -259,12 +257,10 @@ CMake          | Cross-platform family of tools designed to help build software
 C++17 Compiler | Matching Houdini compiler and version                           | All                |
 Boost          | Components: iostreams                                           | All                |
 GoogleTest     | A unit testing framework module for C++                         | Unit Tests         |
-CppUnit        | A unit testing framework module for C++                         | Unit Tests (AX)    |
 GLFW           | Simple API for OpenGL development                               | OpenVDB View       |
 Doxygen        | Documentation generation from C++                               | Documentation      |
 Log4cplus      | An optional dependency for improved OpenVDB Logging             | Optional (All)     |
-PyBind11       | C++/python bindings                                             | Optional (Python)  |
-NumPy          | Scientific computing with Python                                | Optional (Python)  |
+nanobind       | C++/python bindings                                             | Optional (Python)  |
 LLVM           | Target-independent code generation                              | OpenVDB AX         |

 At a minimum a matching C++17 compiler and CMake will be required. See
@@ -351,14 +347,12 @@ Boost          | Components: iostreams
 ZLIB           | Compression library for disk serialization compression            | All                              |
 Blosc          | Recommended dependency for improved disk compression              | All*                             |
 GoogleTest     | A unit testing framework module for C++                           | Unit Tests                       |
-CppUnit        | A unit testing framework module for C++                           | Unit Tests (AX)                  |
 GLFW           | Simple API for OpenGL development                                 | OpenVDB View                     |
 Doxygen        | Documentation generation from C++                                 | Documentation                    |
 OpenEXR        | EXR serialization support                                         | OpenVDB Render                   |
 Imath          | Used half precision floating points and EXR serialization support | Optional (All)                   |
 Log4cplus      | An optional dependency for improved OpenVDB Logging               | Optional (All)                   |
-PyBind11       | C++/python bindings                                               | Optional (Python)                |
-NumPy          | Scientific computing with Python                                  | Optional (Python)                |
+nanobind       | C++/python bindings                                               | Optional (Python)                |
 LLVM           | Target-independent code generation                                | OpenVDB AX                       |


@@ -514,7 +508,7 @@ You can add the below CMake snippet to your main `CMakeLists.txt` to bring in
 OpenVDB as a dependency:

@code{.cmake}
-cmake_minimum_required(VERSION 3.18)
+cmake_minimum_required(VERSION 3.24)
 list(APPEND CMAKE_MODULE_PATH "/location/of/openvdb/install/lib/cmake/OpenVDB")
 find_package(OpenVDB REQUIRED)
 target_link_libraries(myapp OpenVDB::openvdb)
@@ -607,17 +601,12 @@ OpenVDB uses [imported targets](https://cmake.org/cmake/help/latest/command/add_
 for all its dependencies. For imported Boost compatibility, the following
 versions of CMake are required:

-  - Boost 1.73 requires CMake 3.17.2 or newer.
-  - Boost 1.74 requires CMake 3.19 or newer.
-  - Boost 1.75 requires CMake 3.19.5 or newer.
-  - Boost 1.76 requires CMake 3.20.3 or newer.
-  - Boost 1.77 requires CMake 3.21.3 or newer.
-  - Boost 1.78 requires CMake 3.22.2 or newer.
-  - Boost 1.79 requires CMake 3.23.2 or newer.
-  - Boost 1.80 requires CMake 3.24.2 or newer.
-  - Boost 1.81 requires CMake 3.25.2 or newer.
  - Boost 1.82 requires CMake 3.27.0 or newer.
  - Boost 1.83 requires CMake 3.27.4 or newer.
+  - Boost 1.84 requires CMake 3.28.2 or newer.
+  - Boost 1.85 requires CMake 3.29.3 or newer.
+  - Boost 1.86 requires CMake 3.30.6, 3.31.1, or newer.
+  - Boost 1.87 requires CMake 3.31.4 or newer.

 Use the above list to identify the version of CMake you require for your version
 of Boost. Note that this may be implemented into the OpenVDB CMake in the future.
@@ -2,6 +2,415 @@

@page changes Release Notes

+@htmlonly <a name="v12_1_2_changes"></a>@endhtmlonly
+@par
+<B>Version 12.1.2</B> - <I>In development</I>
+
+
+@htmlonly <a name="v12_1_1_changes"></a>@endhtmlonly
+@par
+<B>Version 12.1.1</B> - <I>September 30, 2025</I>
+
+@par
+OpenVDB AX:
+- Fixes:
+  - Fixed an issue which was causing host specific optimization passes to be
+    omitted when using LLVM 16 and higher.
+
+@par
+Houdini:
+  - Fixed a compilation issue with the OpenVDB Activate SOP with Houdini 21
+  [Reported by @callen128]
+
+
+@htmlonly <a name="v12_1_0_changes"></a>@endhtmlonly
+@par
+<B>Version 12.1.0</B> - <I>August 8, 2025</I>
+
+@par
+OpenVDB:
+- New features:
+  - New level set constructor tools::createLevelSetCapsule creates a grid
+    containing a narrow-band level set representation of a capsule (tube
+    with constant radius and sphere caps).
+  - New level set constructor tools::createLevelSetTaperedCapsule creates a
+    grid containing a narrow-band level set representation of a tapered
+    capsule (tube with sphere caps and different radii at both ends, or
+    equivalently the convex hull of two spheres with possibly different
+    centers and radii).
+  - New level set constructor tools::createLevelSetTubeComplex creates a grid
+    containing a narrow-band level set representation of a tube complex (a
+    collection of capsules or tapered capsules defined by endpoint coordinates
+    and segment indices).
+  - New level set constructor tools::createLevelSetDilatedMesh creates a grid
+    containing a narrow-band level set representation of a dilated surface
+    mesh (dilated by a radius in all directions).
+  - New class tools::ConvexVoxelizer, used by the above methods, which can be
+    inherited to implement a convex level set constructor.
+  - Introduced anisotropic surfacing tools in points/PointRasterizeSDF.h and
+    principal component analysis (PCA) methods in points/PrincipalComponentAnalysis.h.
+    The latter tool analyses point neighborhoods and computes affine
+    transformations representing elliptical distributions. The prior tool
+    takes rotational and stretch point attributes (optionally computed from
+      the prior PCA methods) and generates anisotropic surfaces.
+
+- Improvements:
+  - Significantly improved the performance of all LeafNode ValueIterators,
+    up to 5x on some platforms and up to 10x when delay loading is enabled.
+    Construction of a ValueIterator from a leaf node now requests the leaf
+    buffers ahead of iteration to avoid potentially expensive API calls.
+  - Improved the performance of PointRasterizeSDF.h tools with larger radii
+    with better bound computation approximations.
+  - Significant performance improvement to delay loaded files - the warning
+    logic which checks the underlying mapped file for the last modified time
+    has been removed, resulting in performance gains upwards of 10x when
+    loading in node data buffers. Note that the check that was removed is
+    entirely redundant on Windows and would only print a warning on other
+    system.
+    [Contributed by jjtParadox]
+  - Added OPENVDB_HAS_INVOKABLE_MEMBER_FUNCTION and OPENVDB_HAS_MEMBER_FUNCTION
+    macros to help determine the presence and compatibility of interface
+    member functions at compile time.
+  - Added OPENVDB_ASSUME macros to mimic builtin assume and C++23 assume
+    attributes.
+  - Improved read and write of PLY files with vdb_tool so they support Vec3d
+    and 64 bit attributes. Changed vdb_tool version number from 10.6.1 to
+    10.7.0.
+
+- API Changes:
+  - Mat3 and Mat4 functions have been marked [[nodiscard]] where it may be
+    confusing if they are operating in place or returning a result, in
+    particular tranpose, invert, adjoint, and timesDiagonal.
+  - Re-worked the API in PointRasterizeSDF.h with the addition of anisotropic
+    surfacing kernels. The new API exposes more concise free functions which
+    take structs of parameters. The old API remains supported but is
+    deprecated.
+  - Deprecated providing a filter or interrupter as arguments to
+    points::rasterize. These can instead be implemented on user transfer
+    schemes.
+  - Transfer schemes provided to points::rasterize can now optionally
+    implement a rasterizePoints callback that takes a range of points per
+    voxel.
+
+- Fixes:
+  - Fixed a bug in points::rasterizeSpheres and points::rasterizeSmoothSpheres
+    which could cause sections of the generated surface to incorrectly be
+    marked as negative interior values.
+  - Fixed a bug in points::rasterizeSmoothSpheres which could result in
+    incomplete surface reconstruction with significantly larger search
+    distances.
+  - Fixed some precision issues in various Matrix methods.
+  - Fixed bugs in vdb_tool that prevented build with newer namespaces in
+    nanovdb.
+
+@par
+OpenVDB AX:
+- Improvements:
+  - Added support for LLVM versions greater than 15. Upgraded CI to test with
+    LLVM versions 16, 17, 18, 19 and 20.
+  - As of this release, LLVM 15 is the minimum required version for OpenVDB
+    AX. Note that there is an issue with LLVM 17 symbols which makes it
+    incompatible with the OpenVDB Python plugin. AS such, LLVM version 18 and
+    greater are the recommended LLVM versions to use, with LLVM 18 being the
+    new future minimum version.
+  - Significant performance improvement to the compilation of AX kernels when
+    using LLVM 18 and greater.
+  - Significant backend refactoring to AX IR generation to support newer
+    versions of LLVM. In particular, parts of the codegen::Function and
+    codegen::FunctionBuilder API require new usage from LLVM 16 due to
+    the LLVM opaque pointer changes. If you're not using the OpenVDB AX
+    Function codegen API then no client changes should be necessary.
+  - Migrated OpenVDB AX unit tests from cppunit to gtest
+    [Contributed by Tim Straubinger]
+
+- Fixes:
+  - Fixed a case of undefined behaviour when explicitly casting variables to
+    the same type as their storage type e.g:
+      double a; double b = double(a);
+  - Fixed an AX compiler error which would produce invalid IR when using 64bit
+    integer variables for matrix array offsetting e.g:
+      int64 a = 0; mat4f b = 1; float c = b[a,a];
+
+@par
+NanoVDB:
+  - Added nanovdb::openToIndexVDB, which can convert an OpenVDB grid into a
+    NanoVDB IndexGrid.
+  - Added new options "--index" and "--onIndex" to the command-line tool
+    nanovdb_convert so it can produce IndexGrids.
+  - Fixed memory leak in DeviceBuffer.
+  - Added constructor for DeviceBuffer from an existing HostBuffer.
+  - Added VoxelBlockManager, an acceleration structure for sequential access
+    and stencil ops over active voxels.
+  - Added helper functions for kernel launches with shared memory.
+  - Added support for CUDA 13.0.
+  - Fix warnings and compilation errors for GCC and Clang.
+  - Fixed inconsistent usage of util::cuda::mallocAsync/freeAsync that
+    affected vGPUs.
+  - Added support for custom memory resources in PointsToGrid.
+  - Fixed race condition in split/mergeGridHandles.
+  - Fixed offset bug in nanovdb::tools::cuda::IndexToGrid related to non-POD
+    build types.
+    [Contributed by Andreas Henne]
+
+@par
+Houdini:
+  - Define OP_InputIdx and OP_OutputIdx for Houdini 21.0 compatibility
+  - Add Houdini 21 style camera interface to support LOP cameras
+  - Include env flag for avoiding stream writing (was backported to 19.5)
+
+
+@htmlonly <a name="v12_0_1_changes"></a>@endhtmlonly
+@par
+<B>Version 12.0.1</B> - <I>April 3, 2025</I>
+
+@par
+OpenVDB:
+- New features:
+  - Added read and write support for OFF (Object File Format) files to vdb_tool
+
+- Build:
+  - Fixed a build issue with vdb_render on Windows with newer versions of glew not including glu.h by default
+  - Fixed issues with newer versions of clang refusing to accept older ::template syntax
+
+@par
+NanoVDB:
+- New features:
+  - Added support for multiple GPUs to DeviceBuffer
+  - Added a UnifiedBuffer class that wraps CUDA unified memory
+  - Added example for multiGPU sparse convolution
+  - Added CUDA utility functions for device queries
+  - Added functions to independently stop and compute the elapsed time for timer classes
+
+- Improvements:
+  - Replaced CUB's CachingDeviceAllocator with the default asynchronous stream ordered allocator in PointsToGrid for improved performance
+
+- Bug fix:
+  - Map is now properly transposed when converting NanoVDB to OpenVDB.
+  - Fixed ostream specializations being hidden within the nanovdb namespace
+  - Fixed a call to a non-existent method in GridBuild.h
+
+
+@htmlonly <a name="v1_changes"></a>@endhtmlonly
+@par
+<B>Version 12.0.0</B> - <I>October 31, 2024</I>
+
+@par
+<BLOCKQUOTE>
+OpenVDB is now licensed under the Apache&nbsp;2.0 license, instead of the MPL&nbsp;2.0 license.
+</BLOCKQUOTE>
+
+@par
+<BLOCKQUOTE>
+This version introduces ABI changes relative to older major releases, so to
+preserve ABI compatibility it might be necessary to define the macro
+<TT>OPENVDB_ABI_VERSION_NUMBER=</TT><I>N</I>, where, for example,
+<I>N</I> is 10 for Houdini&nbsp;20.0 and 11 for Houdini&nbsp;20.5.
+</BLOCKQUOTE>
+
+@par
+<BLOCKQUOTE>
+GCC 9 is no longer supported.
+</BLOCKQUOTE>
+
+@par
+OpenVDB:
+- New features:
+  - Added tools::LevelSetFilter::fillet() method to round off concave edges
+    to create smoother transition between surfaces.
+
+- Improvements:
+  - Added openvdb::assertAbort to replace cassert and a
+    @c OPENVDB_ENABLE_ASSERTS cmake argument/compile define to toggle
+    assertions in OpenVDB code, independantly of NDEBUG. Asserts are
+    no longer enabled by default in when NDEBUG is absent (e.g.
+    Debug builds).
+  - Removed last traces of Boost when @c OPENVDB_USE_DELAYED_LOADING is OFF
+    <I>[Reported&nbsp;by&nbsp;Brian&nbsp;McKinnon]</I>
+  - RootNode code cleanup to eliminate redundant key conversion and
+    to create map values in-place.
+  - Add RootNode::deleteChildOrTile() to delete a child or tile of
+    the root node.
+  - ValueAccessors are now defined and created in the Tree class
+    instead of in the Grid class so that custom Tree implementations
+    may define and create their own ValueAccessors if desired.
+  - Added support for PDAL to vdb_tool [Contributed by Tom Matterson]
+  - LeafManager and NodeManager now use Index64 for leaf counts internally.
+  - Added RootNode::probeChild() const.
+  - Added RootNode::probeChild() and RootNode::probeConstChild().
+  - Added RootNode::probe() and RootNode::probeConst() to query key presence,
+    child node, value and active state.
+  - Added InternalNode::probeChild() const.
+  - Added InternalNode::probeChild() and probeChildConst() with coord access
+    and optionally value and active state.
+  - Added InternalNode::probeChild() and probeChildConst() with index access
+    and optionally value and active state.
+  - Added InternalNode::isValueOff(), LeafNode::isValueOff(),
+    LeafNodeBool::isValueOff(), LeafNodeMask::isValueOff().
+  - Added LeafNodeMask::probeValue(Index,val), LeafNodeBool::probeValue(Index,val).
+  - Added RootNode::getValueUnsafe(), RootNode::getChildUnsafe(),
+    RootNode::getConstChildUnsafe().
+  - Added InternalNode::getValueUnsafe(), InternalNode::getChildUnsafe(),
+    InternalNode::getConstChildUnsafe().
+  - Added InternalNode::setActiveStateUnsafe(), InternalNode::setValueOnlyUnsafe(),
+    InternalNode::setValueOnUnsafe(), InternalNode::setValueOffUnsafe().
+  - Added InternalNode::setChildUnsafe(), InternalNode::resetChildUnsafe(),
+    InternalNode::stealChildUnsafe(), InternalNode::deleteChildUnsafe().
+  - For LeafNode, LeafNodeBool and LeafNodeMask - added
+    LeafNode::getValueUnsafe(), LeafNode::setActiveStateunsafe(),
+    LeafNode::setValueOnlyUnsafe(), LeafNode::setValueOnUnsafe(),
+    LeafNode::setValueOffUnsafe().
+
+- ABI changes:
+  - Tree::leafCount(), Tree::unallocatedLeafCount(),
+    Tree::nonLeafCount() and Tree::nodeCount() now use Index64 in their
+    return types instead of Index32.
+
+- API Changes:
+  - RootNode::tileCount(), RootNode::activeTileCount() and
+    RootNode::inactiveTileCount() are now public.
+  - RootNode::hasKey() and RootNode::coordToKey() are now public.
+  - RootNode::leafCount(), RootNode::nonLeafCount() and RootNode::nodeCount()
+    now use Index64 instead of Index32. The Index32 variant is deprecated.
+  - InternalNode::leafCount(), InternalNode::nonLeafCount() and
+    InternalNode::nodeCount() now use Index64 instead of Index32. The Index32
+    variant is deprecated.
+  - LeafNode::leafCount() and LeafNode::nonLeafCount() now use Index64 instead
+    of Index32. The Index32 variant is deprecated.
+
+- Bug Fixes:
+  - Fix potential crash reading corrupt .vdb files with invalid
+    blosc or zip chunks.
+    <I>[Reported&nbsp;by&nbsp;Matthias&nbsp;Ueberheide]</I>
+
+  - Fix a bug in RootNode::setOrigin() where the origin was updated
+    before the error was thrown potentially leaving the root in an
+    invalid state.
+  - Fixed a thread sanitizer issue which could cause undefined
+    behaviour in VolumeToSpheres::fillWithSpheres
+    <I>[Reported&nbsp;by&nbsp;Jérémie&nbsp;Dumas]</I>
+  - Fixed an occurance of undefined behaviour in tools::activate
+    (though this would typically not have manifested with any
+    unintended behaviour)
+
+@par
+NanoVDB:
+- Bug fix:
+  - nanovdb::readGrids works with raw grid buffer.
+
+- Improvements:
+  - Restructure files location and namespace to be more align with
+    OpenVDB. The namespaces touched by the restructuring are: io,
+    cuda, util, tools, and math.
+  - Add two scripts updateFiles.sh and updateFiles.py to update the
+    files using NanoVDB. The script updateFiles.py works on both
+    Windows and Linux.  For a more complete list of changes, see API
+    Changes (details).
+
+  - cuda::PointsToGrid supports target density.
+  - Add support for NanoVDB Grid of type UInt8.
+  - Add ability to use externally managed CUDA buffer.
+  - Add create methods for CudaDeviceBuffer and exceptions.
+  - Improve GridValidator logic, e.g. include check for grid count.
+  - Add operator > and >= for class Coord according to lexicographical order.
+  - Add toCodec to convert string to Codec enumeration type.
+  - Add nanovdb::strlen<GridType>().
+  - Add strncpy util.
+  - Add @c NANOVDB_DISABLE_SYNC_CUDA_MALLOC that maps cudaMallocAsync
+    and cudaFreeAsync to cudaMalloc and cudaFree respectively.
+  - Add guard to @c UINT64_C.
+  - Remove use of cudaMallocAsync in PointsToGrid.cuh.
+  - Align PNanoVDB blind metadata to NanoVDB.
+
+- API Changes:
+  - Change mapToGridType to toGridType.
+  - Change mapToMagic to toMagic.
+  - Change <TT>CpuTimer.h</TT> to <TT>Timer.h</TT>.
+
+- API Changes (details):
+  - These APIs are now under the math namespace: Ray, DDA, HDDA,
+    Vec3, Vec4, BBox, ZeroCrossing, TreeMarcher, PointTreeMarcher,
+    BoxStencil, CurvatureStencil, GradStencil, WenoStencil, AlignUp,
+    Min, Max, Abs, Clamp, Sqrt, Sign, Maximum, Delta, RoundDown, pi,
+    isApproxZero, Round, createSampler, SampleFromVoxels.
+
+  - These APIs are now under the tools namespace: createNanoGrid,
+    StatsMode, createLevelSetSphere, createFogVolumeSphere,
+    createFogVolumeSphere, createFogVolumeSphere,
+    createFogVolumeTorus, createLevelSetBox, CreateNanoGrid,
+    updateGridStats, evalChecksum, validateChecksum, checkGrid,
+    Extrema.
+  - These APIs are now under the util namespace: is_floating_point,
+    findLowestOn, findHighestOn, Range, streq, strcpy, strcat,
+    empty, Split, invoke, forEach, reduce, prefixSum, is_same,
+    is_specialization, PtrAdd, PtrDiff.
+
+  - Move nanovdb::build to nanovdb::tools::build.
+  - Rename nanovdb::BBoxR to nanovdb::Vec3dBBox.
+  - Rename nanovdb::BBox<nanovdb::Vec3d> to nanovdb::Vec3dBbox.
+  - Move nanovdb::cudaCreateNodeManager to nanovdb::cuda::createNodeManager.
+  - Move and rename nanovdb::cudaVoxelsToGrid to nanovdb::cuda::voxelsToGrid.
+  - Move and rename nanovdb::cudaPointsToGrid to nanovdb::cuda::pointsToGrid.
+  - Move nanovdb::DitherLUT to nanovdb::math::DitherLUT.
+  - Move and rename nanovdb::PackedRGBA8 to nanovdb::math::Rgba8.
+  - Move nanovdb::Rgba8 to nanovdb::math::Rgba8.
+  - Move and rename nanovdb::CpuTimer to nanovdb::util::Timer.
+  - Move nanovdb::GpuTimer to nanovdb::util::cuda::Timer.
+  - Move and rename nanovdb::CountOn to nanovdb::util::countOn.
+
+  - Move <TT>util/GridHandle.h</TT> to <TT>GridHandle.h</TT>.
+  - Move <TT>util/BuildGrid.h</TT> to <TT>tools/GridBuilder.h</TT>.
+  - Move <TT>util/GridBuilder.h</TT> to <TT>tools/GridBuilder.h</TT>.
+  - Move <TT>util/IO.h</TT> to <TT>io/IO.h</TT>.
+  - Move <TT>util/CSampleFromVoxels.h</TT> to <TT>math/CSampleFromVoxels.h</TT>.
+  - Move <TT>util/DitherLUT.h</TT> to <TT>math/DitherLUT.h</TT>.
+  - Move <TT>util/HDDA.h</TT> to <TT>math/HDDA.h</TT>.
+  - Move <TT>util/Ray.h</TT> to <TT>math/Ray.h</TT>.
+  - Move <TT>util/SampleFromVoxels.h</TT> to <TT>math/SampleFromVoxels.h</TT>.
+  - Move <TT>util/Stencils.h</TT> to <TT>nanovdb/math/Stencils.h</TT>.
+  - Move <TT>util/CreateNanoGrid.h</TT> to <TT>tools/CreateNanoGrid.h</TT>.
+  - Move and rename <TT>util/Primitives.h</TT> to <TT>tools/CreatePrimitives.h</TT>.
+  - Move <TT>util/GridChecksum.h</TT> to <TT>tools/GridChecksum.h</TT>.
+  - Move <TT>util/GridStats.h</TT> to <TT>tools/GridStats.h</TT>.
+  - Move <TT>util/GridChecksum.h</TT> to <TT>tools/GridChecksum.h</TT>.
+  - Move <TT>util/GridValidator.h</TT> to <TT>tools/GridValidator.h</TT>.
+  - Move <TT>util/NanoToOpenVDB.h</TT> to <TT>tools/NanoToOpenVDB.h</TT>.
+  - Move <TT>util/cuda/CudaGridChecksum.cuh</TT> to <TT>tools/cuda/CudaGridChecksum.cuh</TT>.
+  - Move <TT>util/cuda/CudaGridStats.cuh</TT> to <TT>tools/cuda/CudaGridStats.cuh</TT>.
+  - Move <TT>util/cuda/CudaGridValidator.cuh</TT> to <TT>tools/cuda/CudaGridValidator.cuh</TT>.
+  - Move <TT>util/cuda/CudaIndexToGrid.cuh</TT> to <TT>tools/cuda/CudaIndexToGrid.cuh</TT>.
+  - Move and rename <TT>util/cuda/CudaPointsToGrid.cuh</TT> to <TT>tools/cuda/PointsToGrid.cuh</TT>.
+  - Move <TT>util/cuda/CudaSignedFloodFill.cuh</TT> to <TT>tools/cuda/CudaSignedFloodFill.cuh</TT>.
+  - Move and rename <TT>util/cuda/CudaDeviceBuffer.h</TT> to <TT>cuda/DeviceBuffer.h</TT>.
+  - Move and rename <TT>util/cuda/CudaGridHandle.cuh</TT> to <TT>cuda/GridHandle.cuh</TT>.
+  - Move and rename <TT>util/cuda/CudaUtils.h</TT> to <TT>util/cuda/Util.h</TT>.
+  - Move and consolidate <TT>util/cuda/GpuTimer.h</TT> to <TT>util/cuda/Timer.h</TT>.
+
+@par
+Python:
+  - OpenVDB Python bindings are now implemented using nanobind
+    instead of pybind11
+  - The OpenVDB Python module has been changed from pyopenvdb to openvdb
+  - Added Python bindings for NanoVDB
+
+@par
+Houdini:
+  - When @c OPENVDB_ENABLE_RPATH is @c ON, the location of
+    libopenvdb_houdini is now added to the rpath of all Houdini
+    dsos.
+
+@par
+Build:
+  - Fixed an issue with OpenVDB AX's CMake on Windows where the
+    static and shared library targets would have the same name
+    <I>[Reported&nbsp;by&nbsp;Nicholas&nbsp;Yue]</I>
+  - @c USE_EXPLICIT_INSTANTIATION is now disabled on Windows by default
+    due to OOM linker issues.
+  - Jemalloc is now the preferred allocator of choice on all
+    platforms when @c CONCURRENT_MALLOC is set to Auto.
+  - Fixed an issue with the Blosc CMake FindPackage for the OpenVDB
+    Windows static library.
+
+
@htmlonly <a name="v11_0_0_changes"></a>@endhtmlonly
@par
 <B>Version 11.0.0</B> - <I>November 1, 2023</I>
@@ -7,7 +7,7 @@
 This document details the coding practices that are used in the OpenVDB
 codebase.  Contributed code should conform to these guidelines to maintain
 consistency and maintainability.  If there is a rule that you would like
-clarified, changed, or added, please send a note to openvdb@gmail.com.
+clarified, changed, or added, please send a note to openvdb-dev@lists.aswf.io.


@section sStyleContents Contents
@@ -41,14 +41,14 @@ OpenVDB Print           | Core Library dependencies                       | -
 OpenVDB LOD             | Core Library dependencies                       | -
 OpenVDB Render          | Core Library dependencies                       | OpenEXR, Imath::Imath, libpng
 OpenVDB View            | Core Library dependencies, OpenGL, GLFW3, GLEW* | -
-OpenVDB Python          | Core Library dependencies, Python, PyBind11     | NumPy
+OpenVDB Python          | Core Library dependencies, Python, nanobind     | -
 OpenVDB AX              | Core Library dependencies, LLVM                 | Bison, Flex
 NanoVDB                 | -                                               | Core Library, CUDA, TBB, Blosc, ZLib
-OpenVDB UnitTests       | Core Library dependencies, GoogleTest           | CppUnit**
+OpenVDB UnitTests       | Core Library dependencies, GoogleTest, numpy*   | -
 OpenVDB Documentation   | Doxygen                                         | -

 - @b * GLEW is only required for building the vdb_view binary on Windows.
- - @b ** CppUnit is only required for building the AX unit tests.
+ - @b * numpy only required for python binding unit tests
 - @b Note - Bison and Flex are only required for re-generation of the AX
 grammar which is not typically necessary for the majority of users.

@@ -56,41 +56,39 @@ OpenVDB Documentation   | Doxygen                                         | -

 Package        | Minimum | Recommended | Description                                                       | apt-get | Homebrew | Source
 -------------- | ------- | ----------- | ----------------------------------------------------------------- | ------- | -------- | ------
-CMake          | 3.18    | Latest      | Cross-platform family of tools designed to help build software    | Y       | Y        | https://cmake.org
-GCC            | 9.3.1   | 11.2.1      | C++ 17 Compiler: The GNU Compiler Collection                      | Y       | Y        | https://www.gnu.org/software/gcc
+CMake          | 3.24    | Latest      | Cross-platform family of tools designed to help build software    | Y       | Y        | https://cmake.org
+GCC            | 11.2.1  | 11.2.1      | C++ 17 Compiler: The GNU Compiler Collection                      | Y       | Y        | https://www.gnu.org/software/gcc
 Clang          | 5.0     | Latest      | C++ 17 Compiler: A C language family frontend for LLVM            | Y       | Y        | https://clang.llvm.org
 Intel ICC      | 19      | Latest      | C++ 17 Compiler: Intels C++ Compiler                              | Y       | Y        | https://software.intel.com/en-us/c-compilers
-MSVC           | 19.28   | 19.30       | C++ 17 Compiler: Microsoft Visual C++ Compiler                    | Y       | Y        | https://visualstudio.microsoft.com/vs
-Imath          | 3.1     | Latest      | Half precision floating points                                    | Y       | Y        | http://www.openexr.com
-OpenEXR        | 3.1     | Latest      | EXR serialization support                                         | Y       | Y        | http://www.openexr.com
-TBB            | 2020.2  | 2020.3      | Threading Building Blocks - template library for task parallelism | Y       | Y        | https://www.threadingbuildingblocks.org
+MSVC           | 19.30   | 19.30       | C++ 17 Compiler: Microsoft Visual C++ Compiler                    | Y       | Y        | https://visualstudio.microsoft.com/vs
+Imath          | 3.2     | >= 3.3      | Half precision floating points                                    | Y       | Y        | http://www.openexr.com
+OpenEXR        | 3.2     | >= 3.3      | EXR serialization support                                         | Y       | Y        | http://www.openexr.com
+TBB            | 2020.3  | >= 2021     | Threading Building Blocks - template library for task parallelism | Y       | Y        | https://www.threadingbuildingblocks.org
 ZLIB           | 1.2.7   | Latest      | Compression library for disk serialization compression            | Y       | Y        | https://www.zlib.net
-Boost          | 1.73    | 1.80        | Components: iostreams                                             | Y       | Y        | https://www.boost.org
-LLVM           | 10.0.0  | 13.0.0*     | Target-independent code generation                                | Y       | Y        | https://llvm.org/
-Bison          | 3.0.0   | 3.7.0       | General-purpose parser generator                                  | Y       | Y        | https://www.gnu.org/software/gcc
-Flex           | 2.6.0   | 2.6.4       | Fast lexical analyzer generator                                   | Y       | Y        | https://github.com/westes/flex
-Python         | 3.9.1   | 3.10        | The python interpreter and libraries                              | Y       | Y        | https://www.python.org
-PyBind11       | 2.9.1   | Latest      | C++/python bindings                                               | Y       | Y        | https://pybind11.readthedocs.io
-NumPy          | 1.20.0  | 1.23.0      | Scientific computing with Python                                  | Y       | Y        | http://www.numpy.org
+Boost          | 1.82    | >= 1.85     | Components: iostreams                                             | Y       | Y        | https://www.boost.org
+LLVM           | 15.0.0* | 18.0.0      | Target-independent code generation                                | Y       | Y        | https://llvm.org/
+Bison          | 3.7.0   | 3.7.0       | General-purpose parser generator                                  | Y       | Y        | https://www.gnu.org/software/gcc
+Flex           | 2.6.4   | 2.6.4       | Fast lexical analyzer generator                                   | Y       | Y        | https://github.com/westes/flex
+Python         | 3.10    | 3.11        | The python interpreter and libraries                              | Y       | Y        | https://www.python.org
+nanobind**     | 2.0.0   | >= 2.5.0    | C++/python bindings                                               | Y       | Y        | https://nanobind.readthedocs.io
 GoogleTest     | 1.10    | Latest      | A unit testing framework module for C++                           | Y       | Y        | https://github.com/google/googletest
-CppUnit        | 1.10    | Latest      | A unit testing framework module for C++                           | N       | Y        | https://freedesktop.org/wiki/Software/cppunit
-Blosc          | 1.17.0* | 1.17.0      | Recommended dependency for improved disk compression              | Y       | Y        | https://github.com/Blosc/c-blosc/releases
+Blosc          | 1.17.0* | Latest      | Recommended dependency for improved disk compression              | Y       | Y        | https://github.com/Blosc/c-blosc/releases
 Log4cplus      | 1.1.2   | Latest      | An optional dependency for improved OpenVDB Logging               | Y       | Y        | https://github.com/log4cplus/log4cplus
 libpng         | -       | Latest      | Library for manipulating PNG images                               | Y       | Y        | http://www.libpng.org/pub/png/libpng.html
-GLFW           | 3.1     | > 3.3       | Simple API for OpenGL development                                 | Y       | Y        | https://www.glfw.org
+GLFW           | 3.3     | Latest      | Simple API for OpenGL development                                 | Y       | Y        | https://www.glfw.org
 OpenGL         | 3.2     | Latest      | Environment for developing portable graphics applications         | Y       | Y        | https://www.opengl.org
 GLEW           | 1.0.0   | Latest      | A cross-platform OpenGL extension loading library.                | Y       | Y        | http://glew.sourceforge.net
 CUDA           | -       | Latest      | Parallel computing platform for graphical processing units.       | Y       | N        | https://developer.nvidia.com/cuda-downloads
 Doxygen        | 1.8.8   | 1.8.11      | Documentation generation from C++                                 | Y       | Y        | http://www.doxygen.nl

- - @b * See the note in [known issues](@ref depKnownIssues) regarding supported blosc/llvm versions.
+ - @b *  See the note in [known issues](@ref depKnownIssues) regarding supported blosc/llvm versions.
+ - @b ** See the note in [known issues](@ref depKnownIssues) regarding supported nanobind build issues.

@subsection depKnownIssues Known Issues

- - @b LLVM >= 15. LLVM has recently changed their IR pointer data types to be
- completely opaque. Some support is provided in LLVM 15 but tests with AX show
- that this is extremely unstable. AX currently only supports LLVM versions up
- until 14.X.X. See https://llvm.org/docs/OpaquePointers.html for more details.
+ - @b On Linux, LLVM == 17 has symbol issues which makes it incompatible when
+ used with the OpenVDB Python plugin. LLVM versions greater than 18 should be
+ preferred.

 - @b GLFW: Crash with GLFW 3.2.1 when used with EGL on Linux. OpenVDB View may
 segfault on closure. Fixed in GLFW 3.3
@@ -112,6 +110,11 @@ Doxygen        | 1.8.8   | 1.8.11      | Documentation generation from C++
 - https://github.com/Blosc/c-blosc/issues/260
 - https://github.com/Blosc/c-blosc/issues/261

+ - @b nanobind: Nanobind requires a fast-hash-map implementation library called
+ tsl-robin-map. If you're installing nanobind through homebrew, this library
+ may not be installed correctly and OpenVDB's CMake will only produce a warning.
+ Make sure to also: `brew install robin-map`.
+
@section depInstallingDependencies Installing Dependencies

 This section details a number of ways to get you up and running with the
@@ -145,17 +148,15 @@ apt-get install zlibc                   # zlib
 apt-get install libboost-iostreams-dev  # Boost::iostream
 apt-get install libblosc-dev            # Blosc
 # AX
-apt-get install llvm-10-dev             # LLVM
+apt-get install llvm-15-dev             # LLVM
 # Python
-apt-get install pybind11-dev            # Python
 apt-get install python-dev              # Python
-apt-get install python-numpy            # NumPy
+pip install nanobind                    # nanobind
 # Optional
 apt-get install libpng-dev              # libpng
 apt-get install libopenexr-dev          # OpenEXR
 apt-get install liblog4cplus-dev        # Log4cplus
 apt-get install googletest              # GoogleTest
-apt-get install libcppunit-dev          # CppUnit
 # vdb_view
 apt-get install libglfw3-dev            # GLFW
 # Documentation
@@ -180,12 +181,10 @@ brew install c-blosc                   # blosc
 # AX
 brew install llvm                      # LLVM
 # Python
-brew install pybind11                  # PyBind11
 brew install python                    # Python
-brew install numpy                     # NumPy
+brew install nanobind robin-map        # nanobind
 # Unit Tests
 brew install gtest                     # GoogleTest
-brew install cppunit                   # CppUnit
 # vdb_view
 brew install glfw                      # GLFW
 # vdb_render
@@ -40,10 +40,10 @@ details of VDB are described in the paper
 <a href="http://www.museth.org/Ken/Publications.html" target="_blank">"VDB: High-Resolution Sparse Volumes with Dynamic Topology"</a>.

@section sWhatLicense What license is OpenVDB distributed under?
-OpenVDB is released under the Mozilla Public License Version 2.0, which is a
-free, open source software license developed and maintained by the Mozilla
-Foundation. For more information about this license, see the
-<a href="https://www.mozilla.org/en-US/MPL/" target="_blank">Mozilla FAQ</a>.
+OpenVDB is released under the Apache License Version 2.0, which is a
+free, open source software license developed and maintained by the Apache Software
+Foundation. For more information about this license, see
+<a href="https://www.apache.org/licenses/LICENSE-2.0" target="_blank">Apache License, Version 2</a>.

@section sWhatCLA Is there a Contributor License Agreement for OpenVDB?
 Yes, developers who wish to contribute code to be considered for inclusion
@@ -441,11 +441,9 @@ not on their active states or other attributes.

 Large data sets are often handled in Python using
 <a href="http://www.numpy.org/">NumPy</a>.
-The OpenVDB Python module can optionally be compiled with NumPy support.
-With NumPy enabled, the @b copyFromArray and @b copyToArray grid methods
-can be used to exchange data efficiently between scalar-valued grids
-and three-dimensional NumPy arrays and between vector-valued grids and
-four-dimensional NumPy arrays.
+The @b copyFromArray and @b copyToArray grid methods can be used to exchange
+data efficiently between scalar-valued grids and three-dimensional NumPy
+arrays and between vector-valued grids and four-dimensional NumPy arrays.
@code{.py}
 >>> import pyopenvdb as vdb
 >>> import numpy
@@ -1,5 +1,5 @@
 # Copyright Contributors to the OpenVDB Project
-# SPDX-License-Identifier: MPL-2.0
+# SPDX-License-Identifier: Apache-2.0
 #
 #[=======================================================================[

@@ -7,7 +7,7 @@

 #]=======================================================================]

-cmake_minimum_required(VERSION 3.18)
+cmake_minimum_required(VERSION 3.24)
 project(NanoVDB LANGUAGES C CXX)

 include(GNUInstallDirs)
@@ -26,6 +26,7 @@ option(NANOVDB_BUILD_TOOLS "Build command-line tools" ON)
 option(NANOVDB_BUILD_UNITTESTS "Build Unit tests" OFF)
 option(NANOVDB_BUILD_EXAMPLES "Build examples" OFF)
 #option(NANOVDB_BUILD_BENCHMARK "Build benchmark in examples" OFF)
+option(NANOVDB_BUILD_PYTHON_MODULE "Build the nanovdb Python module" OFF)

 option(NANOVDB_USE_INTRINSICS "Build with hardware intrinsics support" OFF)
 option(NANOVDB_USE_CUDA "Build with CUDA support" OFF)
@@ -73,18 +74,16 @@ endif()

 #if(NANOVDB_BUILD_UNITTESTS OR NANOVDB_BUILD_BENCHMARK)
 if(NANOVDB_BUILD_UNITTESTS)
-  find_package(GTest REQUIRED)
+  find_package(GTest ${MINIMUM_GOOGLETEST_VERSION} REQUIRED)
 endif()

 if(NANOVDB_USE_CUDA)
  set(CMAKE_CUDA_STANDARD 17)
  set(CMAKE_CUDA_STANDARD_REQUIRED ON)

-  if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.18)
-    # Allow the user to provide CMAKE_CUDA_ARCHITECTURES
-    if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES)
-      set(CMAKE_CUDA_ARCHITECTURES 75)
-    endif()
+  # Allow the user to provide CMAKE_CUDA_ARCHITECTURES
+  if(NOT DEFINED CMAKE_CUDA_ARCHITECTURES)
+    set(CMAKE_CUDA_ARCHITECTURES 75)
  endif()

  enable_language(CUDA)
@@ -114,6 +113,20 @@ if(NANOVDB_USE_CUDA)
      get_target_property(VDB_MSVC_RUNTIME_SELECTION openvdb MSVC_RUNTIME_LIBRARY)
    endif()
  endif()
+
+  find_package(CUDAToolkit)
+
+
+  include(../../cmake/CPM.cmake)
+
+  # This will automatically clone CCCL from GitHub and make the exported cmake targets available
+  CPMAddPackage(
+      NAME CCCL
+      GITHUB_REPOSITORY nvidia/cccl
+      GIT_TAG main # Fetches the latest commit on the main branch
+  )
+
+  find_package(NCCL)
 endif()

 if(NANOVDB_USE_OPENVDB)
@@ -123,15 +136,15 @@ if(NANOVDB_USE_OPENVDB)
 endif()

 if(NANOVDB_USE_TBB AND NOT TARGET TBB::tbb)
-  find_package(TBB REQUIRED)
+  find_package(TBB ${MINIMUM_TBB_VERSION} REQUIRED)
 endif()

 if(NANOVDB_USE_BLOSC AND NOT TARGET Blosc::blosc)
-  find_package(Blosc REQUIRED)
+  find_package(Blosc ${MINIMUM_BLOSC_VERSION} REQUIRED)
 endif()

 if(NANOVDB_USE_ZLIB AND NOT TARGET ZLIB::ZLIB)
-  find_package(ZLIB REQUIRED)
+  find_package(ZLIB ${MINIMUM_ZLIB_VERSION} REQUIRED)
 endif()

 if(NANOVDB_USE_MAGICAVOXEL)
@@ -170,8 +183,13 @@ set(NANOVDB_INCLUDE_FILES
 # NanoVDB cuda header files
 set(NANOVDB_INCLUDE_CUDA_FILES
  cuda/DeviceBuffer.h
+  cuda/DeviceMesh.h
+  cuda/DeviceResource.h
+  cuda/DeviceStreamMap.h
  cuda/GridHandle.cuh
  cuda/NodeManager.cuh
+  cuda/TempPool.h
+  cuda/UnifiedBuffer.h
 )

 # NanoVDB io header files
@@ -199,6 +217,7 @@ set(NANOVDB_INCLUDE_TOOLS_FILES
  tools/GridStats.h
  tools/GridValidator.h
  tools/NanoToOpenVDB.h
+  tools/VoxelBlockManager.h
 )

 # NanoVDB tools/cuda header files
@@ -210,6 +229,14 @@ set(NANOVDB_INCLUDE_TOOLS_CUDA_FILES
  tools/cuda/IndexToGrid.cuh
  tools/cuda/PointsToGrid.cuh
  tools/cuda/SignedFloodFill.cuh
+  tools/cuda/MergeGrids.cuh
+  tools/cuda/TopologyBuilder.cuh
+  tools/cuda/RefineGrid.cuh
+  tools/cuda/PruneGrid.cuh
+  tools/cuda/DistributedPointsToGrid.cuh
+  tools/cuda/DilateGrid.cuh
+  tools/cuda/CoarsenGrid.cuh
+  tools/cuda/VoxelBlockManager.cuh
 )

 # NanoVDB util header files
@@ -238,6 +265,7 @@ set(NANOVDB_INCLUDE_UTIL_FILES
  util/Stencils.h
  util/Timer.h
  util/Util.h
+  util/MorphologyHelpers.h
 )

 # NanoVDB util/cuda header files
@@ -256,6 +284,10 @@ set(NANOVDB_INCLUDE_UTIL_CUDA_FILES
  util/cuda/CudaGridValidator.cuh
  util/cuda/CudaPointsToGrid.cuh
  util/cuda/GpuTimer.h
+  util/cuda/Morphology.cuh
+  util/cuda/MorphologyHelpers.cuh
+  util/cuda/DeviceGridTraits.cuh
+  util/cuda/Injection.cuh
 )

 add_library(nanovdb INTERFACE)
@@ -308,8 +340,12 @@ if(NANOVDB_USE_CUDA)
  endif()

  # Add CUDA includes to any C++ units which use NanoVDB
-  target_include_directories(nanovdb INTERFACE
-    ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
+  target_link_libraries(nanovdb INTERFACE CCCL::CCCL CUDA::cudart)
+
+  if(NCCL_FOUND)
+    target_compile_definitions(nanovdb INTERFACE -DNANOVDB_USE_NCCL)
+    target_link_libraries(nanovdb INTERFACE NCCL::NCCL)
+  endif()
 endif()

 if(NANOVDB_USE_MAGICAVOXEL)
@@ -321,16 +357,16 @@ if(TARGET Threads::Threads)
  target_link_libraries(nanovdb INTERFACE Threads::Threads)
 endif()

-set(NANOVDB_INSTALL_INCLUDE_DIR ${CMAKE_INSTALL_INCLUDEDIR}/nanovdb)
-set(NANOVDB_INSTALL_CUDA_DIR ${NANOVDB_INSTALL_INCLUDE_DIR}/cuda)
-set(NANOVDB_INSTALL_IO_DIR ${NANOVDB_INSTALL_INCLUDE_DIR}/io)
-set(NANOVDB_INSTALL_MATH_DIR ${NANOVDB_INSTALL_INCLUDE_DIR}/math)
-set(NANOVDB_INSTALL_TOOLS_DIR ${NANOVDB_INSTALL_INCLUDE_DIR}/tools)
+set(NANOVDB_INSTALL_ROOT_DIR ${NANOVDB_INSTALL_INCLUDEDIR}/nanovdb)
+set(NANOVDB_INSTALL_CUDA_DIR ${NANOVDB_INSTALL_ROOT_DIR}/cuda)
+set(NANOVDB_INSTALL_IO_DIR ${NANOVDB_INSTALL_ROOT_DIR}/io)
+set(NANOVDB_INSTALL_MATH_DIR ${NANOVDB_INSTALL_ROOT_DIR}/math)
+set(NANOVDB_INSTALL_TOOLS_DIR ${NANOVDB_INSTALL_ROOT_DIR}/tools)
 set(NANOVDB_INSTALL_TOOLS_CUDA_DIR ${NANOVDB_INSTALL_TOOLS_DIR}/cuda)
-set(NANOVDB_INSTALL_UTIL_DIR ${NANOVDB_INSTALL_INCLUDE_DIR}/util)
+set(NANOVDB_INSTALL_UTIL_DIR ${NANOVDB_INSTALL_ROOT_DIR}/util)
 set(NANOVDB_INSTALL_UTIL_CUDA_DIR ${NANOVDB_INSTALL_UTIL_DIR}/cuda)

-install(FILES ${NANOVDB_INCLUDE_FILES} DESTINATION ${NANOVDB_INSTALL_INCLUDE_DIR})
+install(FILES ${NANOVDB_INCLUDE_FILES} DESTINATION ${NANOVDB_INSTALL_ROOT_DIR})
 install(FILES ${NANOVDB_INCLUDE_CUDA_FILES} DESTINATION ${NANOVDB_INSTALL_CUDA_DIR})
 install(FILES ${NANOVDB_INCLUDE_IO_FILES} DESTINATION ${NANOVDB_INSTALL_IO_DIR})
 install(FILES ${NANOVDB_INCLUDE_MATH_FILES} DESTINATION ${NANOVDB_INSTALL_MATH_DIR})
@@ -354,3 +390,7 @@ endif()
 if(NANOVDB_BUILD_EXAMPLES)
  add_subdirectory(examples)
 endif()
+
+if(NANOVDB_BUILD_PYTHON_MODULE)
+  add_subdirectory(python)
+endif()
@@ -1,10 +1,13 @@
 // Copyright Contributors to the OpenVDB Project
-// SPDX-License-Identifier: MPL-2.0
+// SPDX-License-Identifier: Apache-2.0

 //
 // Simple C-wrapper for the nanovdb structure
 // Meant for systems where you lack a C++ compiler.
 //
+
+/// @warning This header file will soon be deprecated in favor of PNanoVDB.h
+
 #ifndef __CNANOVDB__
 #define __CNANOVDB__

@@ -1,5 +1,5 @@
 // Copyright Contributors to the OpenVDB Project
-// SPDX-License-Identifier: MPL-2.0
+// SPDX-License-Identifier: Apache-2.0

 /*!
    \file nanovdb/GridHandle.h
@@ -45,13 +45,14 @@ class GridHandle
 public:
    using BufferType = BufferT;

-    /// @brief  Move constructor from a host buffer
+    /// @brief  Move constructor from a dual host-device buffer
    /// @param buffer buffer containing one or more NanoGrids that will be moved into this GridHandle
    /// @throw Will throw and error with the buffer does not contain a valid NanoGrid!
+    /// @note The implementation of this template specialization is in nanovdb/cuda/GridHandle.cuh since it requires CUDA
    template<typename T = BufferT, typename util::enable_if<BufferTraits<T>::hasDeviceDual, int>::type = 0>
    GridHandle(T&& buffer);

-    /// @brief  Move constructor from a dual host-device buffer
+    /// @brief  Move constructor from a host buffer
    /// @param buffer buffer containing one or more NanoGrids that will be moved into this GridHandle
    /// @throw Will throw and error with the buffer does not contain a valid NanoGrid!
    template<typename T = BufferT, typename util::disable_if<BufferTraits<T>::hasDeviceDual, int>::type = 0>
@@ -110,19 +111,28 @@ public:
    typename util::enable_if<BufferTraits<U>::hasDeviceDual, const void*>::type
    deviceData() const { return mBuffer.deviceData(); }
    template<typename U = BufferT>
+    typename util::enable_if<BufferTraits<U>::hasDeviceDual, const void*>::type
+    deviceData(int device) const { return mBuffer.deviceData(device); }
+    template<typename U = BufferT>
    typename util::enable_if<BufferTraits<U>::hasDeviceDual, void*>::type
    deviceData() { return mBuffer.deviceData(); }
+    template<typename U = BufferT>
+    typename util::enable_if<BufferTraits<U>::hasDeviceDual, void*>::type
+    deviceData(int device) { return mBuffer.deviceData(device); }

+    //@{
    /// @brief Returns the size in bytes of the raw memory buffer managed by this GridHandle.
-    uint64_t size() const { return mBuffer.size(); }
+    [[deprecated("Use GridHandle::bufferSize instead.")]] uint64_t size() const { return mBuffer.size(); }
+    uint64_t bufferSize() const { return mBuffer.size(); }
+    //@}

    //@{
    /// @brief Return true if this handle is empty, i.e. has no allocated memory
-    bool empty()   const { return this->size() == 0; }
-    bool isEmpty() const { return this->size() == 0; }
+    bool empty()   const { return mBuffer.size() == 0; }
+    bool isEmpty() const { return mBuffer.size() == 0; }
    //@}

-    /// @brief Return true if this handle contains any grids
+    /// @brief Return true if this handle is not empty, i.e. contains at least one grid
    operator bool() const { return !this->empty(); }

    /// @brief Returns a const host pointer to the @a n'th NanoVDB grid encoded in this GridHandle.
@@ -152,7 +162,7 @@ public:

    /// @brief Return a const pointer to the @a n'th grid encoded in this GridHandle on the device, e.g. GPU
    /// @tparam ValueT Value type of the grid point to be returned
-    /// @param n Index if of the grid pointer to be returned
+    /// @param n Index of the grid pointer to be returned
    /// @param verbose if non-zero error messages will be printed in case something failed
    /// @warning Note that the return pointer can be NULL if the GridHandle was not initialized, @a n is invalid,
    ///          or if the template parameter does not match the specified grid.
@@ -164,13 +174,25 @@ public:
    /// @note This method is only available if the buffer supports devices
    template<typename U = BufferT>
    typename util::enable_if<BufferTraits<U>::hasDeviceDual, void>::type
-    deviceUpload(void* stream = nullptr, bool sync = true) { mBuffer.deviceUpload(stream, sync); }
+    deviceUpload(void* stream, bool sync = true) { mBuffer.deviceUpload(stream, sync); }
+
+    /// @brief Upload the host buffer to a specific device buffer. It device buffer doesn't exist it's created first
+    /// @param device Device to upload host data to
+    /// @param stream cuda stream
+    /// @param sync if false the memory copy is asynchronous
+    template<typename U = BufferT>
+    typename util::enable_if<BufferTraits<U>::hasDeviceDual, void>::type
+    deviceUpload(int device = 0, void* stream = nullptr, bool sync = true) { mBuffer.deviceUpload(device, stream, sync); }

    /// @brief Download the grid to from the device, e.g. from GPU to CPU
    /// @note This method is only available if the buffer supports devices
    template<typename U = BufferT>
    typename util::enable_if<BufferTraits<U>::hasDeviceDual, void>::type
-    deviceDownload(void* stream = nullptr, bool sync = true) { mBuffer.deviceDownload(stream, sync); }
+    deviceDownload(void* stream, bool sync = true) { mBuffer.deviceDownload(stream, sync); }
+
+    template<typename U = BufferT>
+    typename util::enable_if<BufferTraits<U>::hasDeviceDual, void>::type
+    deviceDownload(int device = 0, void* stream = nullptr, bool sync = true) { mBuffer.deviceDownload(device, stream, sync); }

    /// @brief Check if the buffer is this handle has any padding, i.e. if the buffer is larger than the combined size of all its grids
    /// @return true is the combined size of all grid is smaller than the buffer size
@@ -184,6 +206,23 @@ public:
    /// @return Return the byte size of the specified grid
    uint64_t gridSize(uint32_t n = 0) const {return mMetaData[n].size; }

+    /// @brief compute the total sum of memory footprints of all the grids in this buffer
+    /// @return the number of bytes occupied by all grids associated with this buffer
+    uint64_t totalGridSize() const {
+        uint64_t sum = 0;
+        for (auto &m : mMetaData) sum += m.size;
+        NANOVDB_ASSERT(sum <= mBuffer.size());
+        return sum;
+    }
+
+    /// @brief compute the size of unused storage in this buffer
+    /// @return the number of unused bytes in this buffer.
+    uint64_t freeSize() const {return mBuffer.size() - this->totalGridSize();}
+
+    /// @brief Test if this buffer has any unused storage left, i.e. memory not occupied by grids
+    /// @return true if there is no extra storage left in this buffer, i.e. empty or fully occupied with grids
+    bool isFull() const { return this->totalGridSize() == mBuffer.size(); }
+
    /// @brief Return the GridType of the @a n'th grid in this GridHandle
    /// @param n index of the grid (assumed to be less than gridCount())
    /// @return Return the GridType of the specified grid
@@ -250,12 +289,12 @@ public:
    /// @param is input stream containing a raw grid buffer
    /// @param gridName string name of the grid to be read
    /// @param pool optional pool from which to allocate the new grid buffer
-    /// @throw Will throw a std::logic_error if the stream does not contain a valid raw grid with the speficied name
+    /// @throw Will throw a std::logic_error if the stream does not contain a valid raw grid with the specified name
    void read(std::istream& is, const std::string &gridName, const BufferT& pool = BufferT());

    /// @brief Read a raw grid buffer from a file
    /// @param filename string name of the input file containing a raw grid buffer
-    /// @param pool optional pool from which to allocate the new grid buffe
+    /// @param pool optional pool from which to allocate the new grid buffer
    void read(const std::string &fileName, const BufferT& pool = BufferT()) {
        std::ifstream is(fileName, std::ios::in | std::ios::binary);
        if (!is.is_open()) throw std::ios_base::failure("Unable to open file named \"" + fileName + "\" for input");
@@ -315,6 +354,7 @@ inline __hostdev__ void cpyGridHandleMeta(const GridData *data, GridHandleMetaDa
    }
 }// void cpyGridHandleMeta(const GridData *data, GridHandleMetaData *meta)

+// template specialization of move constructor from a host buffer
 template<typename BufferT>
 template<typename T, typename util::disable_if<BufferTraits<T>::hasDeviceDual, int>::type>
 GridHandle<BufferT>::GridHandle(T&& buffer)
@@ -450,7 +490,7 @@ splitGrids(const GridHandle<BufferT> &handle, const BufferT* other = nullptr)
        h = HandleT(std::move(buffer));
        ptr = util::PtrAdd(ptr, src->mGridSize);
    }
-    return std::move(handles);
+    return handles;
 }// splitGrids

 /// @brief Combines (or merges) multiple GridHandles into a single GridHandle containing all grids
@@ -1,5 +1,5 @@
 // Copyright Contributors to the OpenVDB Project
-// SPDX-License-Identifier: MPL-2.0
+// SPDX-License-Identifier: Apache-2.0

 /*!
    @file nanovdb/HostBuffer.h
@@ -192,6 +192,13 @@ public:
    void* data() { return mData; }
    //@}

+    /// @brief Returns an offset pointer of a specific type from the allocated host memory
+    /// @tparam T Type of the pointer returned
+    /// @param count Numbers of elements of @c parameter type T to skip
+    /// @warning might return NULL
+    template <typename T>
+    T* data(ptrdiff_t count = 0) const {return mData ? reinterpret_cast<T*>(mData) + count : nullptr;}
+
    //@{
    /// @brief Returns the size in bytes associated with this buffer.
    uint64_t bufferSize() const { return mSize; }
@@ -316,7 +323,6 @@ struct HostBuffer::Pool
               << mSize << " bytes of which\n\t" << (util::PtrDiff(alignedFree, mData) - mPadding)
               << " bytes are used by " << mRegister.size() << " other buffer(s). "
               << "Pool is " << (mManaged ? "internally" : "externally") << " managed.\n";
-            //std::cerr << ss.str();
            throw std::runtime_error(ss.str());
        }
        buffer->mSize = size;
@@ -389,7 +395,6 @@ struct HostBuffer::Pool
            }

            for (HostBuffer* buffer : mRegister) { // update registered buffers
-                //buffer->mData = paddedData + ptrdiff_t(buffer->mData - (mData + mPadding));
                buffer->mData = util::PtrAdd(paddedData, util::PtrDiff(buffer->mData, util::PtrAdd(mData, mPadding)));
            }
            mFree = util::PtrAdd(paddedData, memUsage); // update the free pointer
@@ -414,12 +419,8 @@ private:

    static void* alloc(uint64_t size)
    {
-//#if (__cplusplus >= 201703L)
-//    return std::aligned_alloc(NANOVDB_DATA_ALIGNMENT, size);//C++17 or newer
-//#else
    // make sure we alloc enough space to align the result
    return std::malloc(size + NANOVDB_DATA_ALIGNMENT);
-//#endif
    }

    static void* realloc(void* const origData,
@@ -1,5 +1,5 @@
 // Copyright Contributors to the OpenVDB Project
-// SPDX-License-Identifier: MPL-2.0
+// SPDX-License-Identifier: Apache-2.0

 /*!
    \file nanovdb/NodeManager.h
@@ -42,8 +42,8 @@ NodeManagerHandle<BufferT> createNodeManager(const NanoGrid<BuildT> &grid,
                                             const BufferT& buffer = BufferT());

 struct NodeManagerData
-{// 48B = 6*8B
-    uint64_t        mMagic;// 8B
+{// 40B = 5*8B
+    __hostdev__ NodeManagerData(void *grid) : mPadding{0}, mGrid(grid), mPtr{0,0,0}{}
    union {int64_t  mPadding; uint8_t mLinear;};// 8B of which 1B is used for a binary flag
    void           *mGrid;//  8B pointer to either host or device grid
    union {int64_t *mPtr[3], mOff[3];};// 24B, use mOff if mLinear!=0
@@ -288,11 +288,7 @@ NodeManagerHandle<BufferT> createNodeManager(const NanoGrid<BuildT> &grid,
    auto *data = reinterpret_cast<NodeManagerData*>(handle.data());
    NANOVDB_ASSERT(data && isAligned(data));
    NANOVDB_ASSERT(toGridType<BuildT>() == grid.gridType());
-#ifdef NANOVDB_USE_NEW_MAGIC_NUMBERS
-    *data = NodeManagerData{NANOVDB_MAGIC_NODE, {0u}, (void*)&grid, {{0u,0u,0u}}};
-#else
-    *data = NodeManagerData{NANOVDB_MAGIC_NUMB, {0u}, (void*)&grid, {{0u,0u,0u}}};
-#endif
+    *data = NodeManagerData((void*)&grid);

    if (NodeManager<BuildT>::isLinear(grid)) {
        data->mLinear = uint8_t(1u);
@@ -316,7 +312,7 @@ NodeManagerHandle<BufferT> createNodeManager(const NanoGrid<BuildT> &grid,
    }

    return handle;// // is converted to r-value so return value is move constructed!
-}
+}// createNodeManager

 } // namespace nanovdb

@@ -1,6 +1,6 @@

 // Copyright Contributors to the OpenVDB Project
-// SPDX-License-Identifier: MPL-2.0
+// SPDX-License-Identifier: Apache-2.0

 /*!
    \file   nanovdb/PNanoVDB.h
@@ -198,47 +198,110 @@ PNANOVDB_BUF_FORCE_INLINE void pnanovdb_buf_write_uint64(pnanovdb_buf_t buf, uin
 typedef uint32_t pnanovdb_grid_type_t;
 #define PNANOVDB_GRID_TYPE_GET(grid_typeIn, nameIn) pnanovdb_grid_type_constants[grid_typeIn].nameIn
 #elif defined(PNANOVDB_BUF_HLSL)
-#if defined(PNANOVDB_ADDRESS_32)
+#if defined(PNANOVDB_BUF_HLSL_RW)
+#if defined(PNANOVDB_BUF_HLSL_64)
+#define pnanovdb_buf_t RWStructuredBuffer<uint2>
+#else
+#define pnanovdb_buf_t RWStructuredBuffer<uint>
+#endif
+#else
+#if defined(PNANOVDB_BUF_HLSL_64)
+#define pnanovdb_buf_t StructuredBuffer<uint2>
+#else
 #define pnanovdb_buf_t StructuredBuffer<uint>
+#endif
+#endif
+#if defined(PNANOVDB_ADDRESS_32)
 uint pnanovdb_buf_read_uint32(pnanovdb_buf_t buf, uint byte_offset)
 {
+#if defined(PNANOVDB_BUF_HLSL_64)
+    uint2 val = buf[(byte_offset >> 3u)];
+    return ((byte_offset & 4u) == 0u) ? val.x : val.y;
+#else
    return buf[(byte_offset >> 2u)];
+#endif
 }
 uint2 pnanovdb_buf_read_uint64(pnanovdb_buf_t buf, uint byte_offset)
 {
    uint2 ret;
+#if defined(PNANOVDB_BUF_HLSL_64)
+    ret = buf[(byte_offset >> 3u)];
+#else
    ret.x = pnanovdb_buf_read_uint32(buf, byte_offset + 0u);
    ret.y = pnanovdb_buf_read_uint32(buf, byte_offset + 4u);
+#endif
    return ret;
 }
 void pnanovdb_buf_write_uint32(pnanovdb_buf_t buf, uint byte_offset, uint value)
 {
    // NOP, by default no write in HLSL
+#if defined(PNANOVDB_BUF_HLSL_RW)
+#if defined(PNANOVDB_BUF_HLSL_64)
+    if ((byte_offset & 4u) == 0u) {buf[(byte_offset >> 3u)].x = value;}
+    else {buf[(byte_offset >> 3u)].y = value;}
+#else
+    buf[(byte_offset >> 2u)] = value;
+#endif
+#endif
 }
 void pnanovdb_buf_write_uint64(pnanovdb_buf_t buf, uint byte_offset, uint2 value)
 {
    // NOP, by default no write in HLSL
+#if defined(PNANOVDB_BUF_HLSL_RW)
+#if defined(PNANOVDB_BUF_HLSL_64)
+    buf[(byte_offset >> 3u)] = value;
+#else
+    pnanovdb_buf_write_uint32(buf, byte_offset + 0u, value.x);
+    pnanovdb_buf_write_uint32(buf, byte_offset + 4u, value.y);
+#endif
+#endif
 }
 #elif defined(PNANOVDB_ADDRESS_64)
-#define pnanovdb_buf_t StructuredBuffer<uint>
 uint pnanovdb_buf_read_uint32(pnanovdb_buf_t buf, uint64_t byte_offset)
 {
+#if defined(PNANOVDB_BUF_HLSL_64)
+    uint2 val = buf[uint(byte_offset >> 3u)];
+    return ((uint(byte_offset) & 4u) == 0u) ? val.x : val.y;
+#else
    return buf[uint(byte_offset >> 2u)];
+#endif
 }
 uint64_t pnanovdb_buf_read_uint64(pnanovdb_buf_t buf, uint64_t byte_offset)
 {
    uint64_t ret;
+#if defined(PNANOVDB_BUF_HLSL_64)
+    uint2 raw = buf[uint(byte_offset >> 3u)];
+    ret = uint64_t(raw.x) | (uint64_t(raw.y) << 32u);
+#else
    ret = pnanovdb_buf_read_uint32(buf, byte_offset + 0u);
-    ret = ret + (uint64_t(pnanovdb_buf_read_uint32(buf, byte_offset + 4u)) << 32u);
+    ret = ret | (uint64_t(pnanovdb_buf_read_uint32(buf, byte_offset + 4u)) << 32u);
+#endif
    return ret;
 }
 void pnanovdb_buf_write_uint32(pnanovdb_buf_t buf, uint64_t byte_offset, uint value)
 {
    // NOP, by default no write in HLSL
+#if defined(PNANOVDB_BUF_HLSL_RW)
+#if defined(PNANOVDB_BUF_HLSL_64)
+    if ((byte_offset & 4u) == 0u) {buf[uint(byte_offset >> 3u)].x = value;}
+    else {buf[uint(byte_offset >> 3u)].y = value;}
+#else
+    buf[uint(byte_offset >> 2u)] = value;
+#endif
+#endif
 }
 void pnanovdb_buf_write_uint64(pnanovdb_buf_t buf, uint64_t byte_offset, uint64_t value)
 {
    // NOP, by default no write in HLSL
+#if defined(PNANOVDB_BUF_HLSL_RW)
+#if defined(PNANOVDB_BUF_HLSL_64)
+    uint2 raw = uint2(uint(value), uint(value >> 32u));
+    buf[uint(byte_offset >> 3u)] = raw;
+#else
+    pnanovdb_buf_write_uint32(buf, byte_offset + 0u, uint(value));
+    pnanovdb_buf_write_uint32(buf, byte_offset + 4u, uint(value >> 32u));
+#endif
+#endif
 }
 #endif
 #define pnanovdb_grid_type_t uint
@@ -343,14 +406,26 @@ typedef struct pnanovdb_vec3_t
 {
    float x, y, z;
 }pnanovdb_vec3_t;
+typedef struct pnanovdb_uvec4_t
+{
+    pnanovdb_uint32_t x, y, z, w;
+}pnanovdb_uvec4_t;
+typedef struct pnanovdb_ivec4_t
+{
+    pnanovdb_int32_t x, y, z, w;
+}pnanovdb_ivec4_t;
+typedef struct pnanovdb_vec4_t
+{
+    float x, y, z, w;
+}pnanovdb_vec4_t;
 PNANOVDB_FORCE_INLINE pnanovdb_int32_t pnanovdb_uint32_as_int32(pnanovdb_uint32_t v) { return (pnanovdb_int32_t)v; }
 PNANOVDB_FORCE_INLINE pnanovdb_int64_t pnanovdb_uint64_as_int64(pnanovdb_uint64_t v) { return (pnanovdb_int64_t)v; }
 PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_int64_as_uint64(pnanovdb_int64_t v) { return (pnanovdb_uint64_t)v; }
 PNANOVDB_FORCE_INLINE pnanovdb_uint32_t pnanovdb_int32_as_uint32(pnanovdb_int32_t v) { return (pnanovdb_uint32_t)v; }
 PNANOVDB_FORCE_INLINE float pnanovdb_uint32_as_float(pnanovdb_uint32_t v) { float vf; pnanovdb_memcpy(&vf, &v, sizeof(vf)); return vf; }
-PNANOVDB_FORCE_INLINE pnanovdb_uint32_t pnanovdb_float_as_uint32(float v) { return *((pnanovdb_uint32_t*)(&v)); }
+PNANOVDB_FORCE_INLINE pnanovdb_uint32_t pnanovdb_float_as_uint32(float v) { pnanovdb_uint32_t vu; pnanovdb_memcpy(&vu, &v, sizeof(vu)); return vu; }
 PNANOVDB_FORCE_INLINE double pnanovdb_uint64_as_double(pnanovdb_uint64_t v) { double vf; pnanovdb_memcpy(&vf, &v, sizeof(vf)); return vf; }
-PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_double_as_uint64(double v) { return *((pnanovdb_uint64_t*)(&v)); }
+PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_double_as_uint64(double v) { pnanovdb_uint64_t vu; pnanovdb_memcpy(&vu, &v, sizeof(vu)); return vu; }
 PNANOVDB_FORCE_INLINE pnanovdb_uint32_t pnanovdb_uint64_low(pnanovdb_uint64_t v) { return (pnanovdb_uint32_t)v; }
 PNANOVDB_FORCE_INLINE pnanovdb_uint32_t pnanovdb_uint64_high(pnanovdb_uint64_t v) { return (pnanovdb_uint32_t)(v >> 32u); }
 PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_uint32_as_uint64(pnanovdb_uint32_t x, pnanovdb_uint32_t y) { return ((pnanovdb_uint64_t)x) | (((pnanovdb_uint64_t)y) << 32u); }
@@ -373,6 +448,9 @@ typedef bool pnanovdb_bool_t;
 #define PNANOVDB_TRUE true
 typedef int3 pnanovdb_coord_t;
 typedef float3 pnanovdb_vec3_t;
+typedef uint4 pnanovdb_uvec4_t;
+typedef int4 pnanovdb_ivec4_t;
+typedef float4 pnanovdb_vec4_t;
 pnanovdb_int32_t pnanovdb_uint32_as_int32(pnanovdb_uint32_t v) { return int(v); }
 pnanovdb_uint32_t pnanovdb_int32_as_uint32(pnanovdb_int32_t v) { return uint(v); }
 float pnanovdb_uint32_as_float(pnanovdb_uint32_t v) { return asfloat(v); }
@@ -420,6 +498,9 @@ bool pnanovdb_int64_is_zero(pnanovdb_int64_t a) { return a == 0; }
 #define pnanovdb_int64_t ivec2
 #define pnanovdb_coord_t ivec3
 #define pnanovdb_vec3_t vec3
+#define pnanovdb_uvec4_t uvec4
+#define pnanovdb_ivec4_t ivec4
+#define pnanovdb_vec4_t vec4
 pnanovdb_int32_t pnanovdb_uint32_as_int32(pnanovdb_uint32_t v) { return int(v); }
 pnanovdb_int64_t pnanovdb_uint64_as_int64(pnanovdb_uint64_t v) { return ivec2(v); }
 pnanovdb_uint64_t pnanovdb_int64_as_uint64(pnanovdb_int64_t v) { return uvec2(v); }
@@ -525,6 +606,35 @@ PNANOVDB_FORCE_INLINE pnanovdb_coord_t pnanovdb_coord_add(pnanovdb_coord_t a, pn
    v.z = a.z + b.z;
    return v;
 }
+PNANOVDB_FORCE_INLINE pnanovdb_uvec4_t pnanovdb_uvec4_add(pnanovdb_uvec4_t a, pnanovdb_uvec4_t b)
+{
+    pnanovdb_uvec4_t ret;
+    ret.x = a.x + b.x;
+    ret.y = a.y + b.y;
+    ret.z = a.z + b.z;
+    ret.w = a.w + b.w;
+    return ret;
+}
+
+PNANOVDB_FORCE_INLINE pnanovdb_uvec4_t pnanovdb_uvec4_sub(pnanovdb_uvec4_t a, pnanovdb_uvec4_t b)
+{
+    pnanovdb_uvec4_t ret;
+    ret.x = a.x - b.x;
+    ret.y = a.y - b.y;
+    ret.z = a.z - b.z;
+    ret.w = a.w - b.w;
+    return ret;
+}
+
+PNANOVDB_FORCE_INLINE pnanovdb_vec4_t pnanovdb_vec4_add(pnanovdb_vec4_t a, pnanovdb_vec4_t b)
+{
+    pnanovdb_vec4_t ret;
+    ret.x = a.x + b.x;
+    ret.y = a.y + b.y;
+    ret.z = a.z + b.z;
+    ret.w = a.w + b.w;
+    return ret;
+}
 #elif defined(PNANOVDB_HLSL)
 pnanovdb_vec3_t pnanovdb_vec3_uniform(float a) { return float3(a, a, a); }
 pnanovdb_vec3_t pnanovdb_vec3_add(pnanovdb_vec3_t a, pnanovdb_vec3_t b) { return a + b; }
@@ -536,6 +646,9 @@ pnanovdb_vec3_t pnanovdb_vec3_max(pnanovdb_vec3_t a, pnanovdb_vec3_t b) { return
 pnanovdb_vec3_t pnanovdb_coord_to_vec3(pnanovdb_coord_t coord) { return float3(coord); }
 pnanovdb_coord_t pnanovdb_coord_uniform(pnanovdb_int32_t a) { return int3(a, a, a); }
 pnanovdb_coord_t pnanovdb_coord_add(pnanovdb_coord_t a, pnanovdb_coord_t b) { return a + b; }
+pnanovdb_uvec4_t pnanovdb_uvec4_add(pnanovdb_uvec4_t a, pnanovdb_uvec4_t b) { return a + b; }
+pnanovdb_uvec4_t pnanovdb_uvec4_sub(pnanovdb_uvec4_t a, pnanovdb_uvec4_t b) { return a - b; }
+pnanovdb_vec4_t pnanovdb_vec4_add(pnanovdb_vec4_t a, pnanovdb_vec4_t b) { return a + b; }
 #elif defined(PNANOVDB_GLSL)
 pnanovdb_vec3_t pnanovdb_vec3_uniform(float a) { return vec3(a, a, a); }
 pnanovdb_vec3_t pnanovdb_vec3_add(pnanovdb_vec3_t a, pnanovdb_vec3_t b) { return a + b; }
@@ -547,6 +660,9 @@ pnanovdb_vec3_t pnanovdb_vec3_max(pnanovdb_vec3_t a, pnanovdb_vec3_t b) { return
 pnanovdb_vec3_t pnanovdb_coord_to_vec3(const pnanovdb_coord_t coord) { return vec3(coord); }
 pnanovdb_coord_t pnanovdb_coord_uniform(pnanovdb_int32_t a) { return ivec3(a, a, a); }
 pnanovdb_coord_t pnanovdb_coord_add(pnanovdb_coord_t a, pnanovdb_coord_t b) { return a + b; }
+pnanovdb_uvec4_t pnanovdb_uvec4_add(pnanovdb_uvec4_t a, pnanovdb_uvec4_t b) { return a + b; }
+pnanovdb_uvec4_t pnanovdb_uvec4_sub(pnanovdb_uvec4_t a, pnanovdb_uvec4_t b) { return a - b; }
+pnanovdb_vec4_t pnanovdb_vec4_add(pnanovdb_vec4_t a, pnanovdb_vec4_t b) { return a + b; }
 #endif

 // ------------------------------------------------ Uint64 Utils -----------------------------------------------------------
@@ -589,6 +705,19 @@ PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_uint64_offset(pnanovdb_uint64_t
    return pnanovdb_uint32_as_uint64(low, high);
 }

+PNANOVDB_FORCE_INLINE pnanovdb_int64_t pnanovdb_uint64_diff(pnanovdb_uint64_t a, pnanovdb_uint64_t b)
+{
+    pnanovdb_uint32_t low = pnanovdb_uint64_low(a);
+    pnanovdb_uint32_t high = pnanovdb_uint64_high(a);
+    low -= pnanovdb_uint64_low(b);
+    if (low > pnanovdb_uint64_low(a))
+    {
+        high -= 1u;
+    }
+    high -= pnanovdb_uint64_high(b);
+    return pnanovdb_uint64_as_int64(pnanovdb_uint32_as_uint64(low, high));
+}
+
 PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_uint64_dec(pnanovdb_uint64_t a)
 {
    pnanovdb_uint32_t low = pnanovdb_uint64_low(a);
@@ -636,6 +765,11 @@ PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_uint64_offset(pnanovdb_uint64_t
    return a + b;
 }

+PNANOVDB_FORCE_INLINE pnanovdb_int64_t pnanovdb_uint64_diff(pnanovdb_uint64_t a, pnanovdb_uint64_t b)
+{
+    return pnanovdb_uint64_as_int64(a - b);
+}
+
 PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_uint64_dec(pnanovdb_uint64_t a)
 {
    return a - 1u;
@@ -702,6 +836,10 @@ PNANOVDB_FORCE_INLINE pnanovdb_address_t pnanovdb_address_offset64_product(pnano
    ret.byte_offset += pnanovdb_uint64_low(byte_offset) * multiplier;
    return ret;
 }
+PNANOVDB_FORCE_INLINE pnanovdb_int64_t pnanovdb_address_diff(pnanovdb_address_t a, pnanovdb_address_t b)
+{
+    return pnanovdb_uint64_diff(pnanovdb_uint32_as_uint64_low(a.byte_offset),pnanovdb_uint32_as_uint64_low(b.byte_offset));
+}
 PNANOVDB_FORCE_INLINE pnanovdb_uint32_t pnanovdb_address_mask(pnanovdb_address_t address, pnanovdb_uint32_t mask)
 {
    return address.byte_offset & mask;
@@ -725,6 +863,12 @@ PNANOVDB_FORCE_INLINE pnanovdb_bool_t pnanovdb_address_in_interval(pnanovdb_addr
 {
    return address.byte_offset >= min_address.byte_offset && address.byte_offset < max_address.byte_offset;
 }
+PNANOVDB_FORCE_INLINE pnanovdb_address_t pnanovdb_address_add(pnanovdb_address_t a, pnanovdb_address_t b)
+{
+    pnanovdb_address_t sum = a;
+    sum.byte_offset += b.byte_offset;
+    return sum;
+}
 #elif defined(PNANOVDB_ADDRESS_64)
 struct pnanovdb_address_t
 {
@@ -762,6 +906,10 @@ PNANOVDB_FORCE_INLINE pnanovdb_address_t pnanovdb_address_offset64_product(pnano
    ret.byte_offset += byte_offset * pnanovdb_uint32_as_uint64_low(multiplier);
    return ret;
 }
+PNANOVDB_FORCE_INLINE pnanovdb_int64_t pnanovdb_address_diff(pnanovdb_address_t a, pnanovdb_address_t b)
+{
+    return pnanovdb_uint64_diff(a.byte_offset, b.byte_offset);
+}
 PNANOVDB_FORCE_INLINE pnanovdb_uint32_t pnanovdb_address_mask(pnanovdb_address_t address, pnanovdb_uint32_t mask)
 {
    return pnanovdb_uint64_low(address.byte_offset) & mask;
@@ -785,6 +933,12 @@ PNANOVDB_FORCE_INLINE pnanovdb_bool_t pnanovdb_address_in_interval(pnanovdb_addr
 {
    return address.byte_offset >= min_address.byte_offset && address.byte_offset < max_address.byte_offset;
 }
+PNANOVDB_FORCE_INLINE pnanovdb_address_t pnanovdb_address_add(pnanovdb_address_t a, pnanovdb_address_t b)
+{
+    pnanovdb_address_t sum = a;
+    sum.byte_offset += b.byte_offset;
+    return sum;
+}
 #endif

 // ------------------------------------------------ High Level Buffer Read -----------------------------------------------------------
@@ -933,7 +1087,7 @@ PNANOVDB_FORCE_INLINE void pnanovdb_write_vec3(pnanovdb_buf_t buf, pnanovdb_addr
 #define PNANOVDB_MAGIC_FILE   0x324244566f6e614eUL// "NanoVDB2" in hex - little endian (uint64_t)

 #define PNANOVDB_MAJOR_VERSION_NUMBER 32// reflects changes to the ABI
-#define PNANOVDB_MINOR_VERSION_NUMBER  7// reflects changes to the API but not ABI
+#define PNANOVDB_MINOR_VERSION_NUMBER  9// reflects changes to the API but not ABI
 #define PNANOVDB_PATCH_VERSION_NUMBER  0// reflects bug-fixes with no ABI or API changes

 #define PNANOVDB_GRID_TYPE_UNKNOWN 0
@@ -957,14 +1111,16 @@ PNANOVDB_FORCE_INLINE void pnanovdb_write_vec3(pnanovdb_buf_t buf, pnanovdb_addr
 #define PNANOVDB_GRID_TYPE_VEC4D 18
 #define PNANOVDB_GRID_TYPE_INDEX 19
 #define PNANOVDB_GRID_TYPE_ONINDEX 20
-#define PNANOVDB_GRID_TYPE_INDEXMASK 21
-#define PNANOVDB_GRID_TYPE_ONINDEXMASK 22
+//#define PNANOVDB_GRID_TYPE_INDEXMASK 21   // retired - available for future use
+//#define PNANOVDB_GRID_TYPE_ONINDEXMASK 22 // retired - available for future use
 #define PNANOVDB_GRID_TYPE_POINTINDEX 23
 #define PNANOVDB_GRID_TYPE_VEC3U8 24
 #define PNANOVDB_GRID_TYPE_VEC3U16 25
 #define PNANOVDB_GRID_TYPE_UINT8 26
 #define PNANOVDB_GRID_TYPE_END 27

+#define PNANOVDB_GRID_TYPE_CAP 32
+
 #define PNANOVDB_GRID_CLASS_UNKNOWN 0
 #define PNANOVDB_GRID_CLASS_LEVEL_SET 1     // narrow band level set, e.g. SDF
 #define PNANOVDB_GRID_CLASS_FOG_VOLUME 2    // fog volume, e.g. density
@@ -989,22 +1145,22 @@ PNANOVDB_FORCE_INLINE void pnanovdb_write_vec3(pnanovdb_buf_t buf, pnanovdb_addr
 #define PNANOVDB_LEAF_TYPE_LITE 1
 #define PNANOVDB_LEAF_TYPE_FP 2
 #define PNANOVDB_LEAF_TYPE_INDEX 3
-#define PNANOVDB_LEAF_TYPE_INDEXMASK 4
+//#define PNANOVDB_LEAF_TYPE_INDEXMASK 4 // retired - available for future use
 #define PNANOVDB_LEAF_TYPE_POINTINDEX 5

 // BuildType = Unknown, float, double, int16_t, int32_t, int64_t, Vec3f, Vec3d, Mask, ...
 // bit count of values in leaf nodes, i.e. 8*sizeof(*nanovdb::LeafNode<BuildType>::mValues) or zero if no values are stored
-PNANOVDB_STATIC_CONST pnanovdb_uint32_t pnanovdb_grid_type_value_strides_bits[PNANOVDB_GRID_TYPE_END]  = {  0, 32, 64, 16, 32, 64,  96, 192,  0, 16, 32,  1, 32,  4,  8, 16,  0, 128, 256,  0,  0,  0,  0, 16, 24, 48,  8 };
+PNANOVDB_STATIC_CONST pnanovdb_uint32_t pnanovdb_grid_type_value_strides_bits[PNANOVDB_GRID_TYPE_CAP]  = {  0, 32, 64, 16, 32, 64,  96, 192,  0, 16, 32,  1, 32,  4,  8, 16,  0, 128, 256,  0,  0,  0,  0, 16, 24, 48,  8,  0 };
 // bit count of the Tile union in InternalNodes, i.e. 8*sizeof(nanovdb::InternalData::Tile)
-PNANOVDB_STATIC_CONST pnanovdb_uint32_t pnanovdb_grid_type_table_strides_bits[PNANOVDB_GRID_TYPE_END]  = { 64, 64, 64, 64, 64, 64, 128, 192, 64, 64, 64, 64, 64, 64, 64, 64, 64, 128, 256, 64, 64, 64, 64, 64, 64, 64, 64 };
+PNANOVDB_STATIC_CONST pnanovdb_uint32_t pnanovdb_grid_type_table_strides_bits[PNANOVDB_GRID_TYPE_CAP]  = { 64, 64, 64, 64, 64, 64, 128, 192, 64, 64, 64, 64, 64, 64, 64, 64, 64, 128, 256, 64, 64, 64, 64, 64, 64, 64, 64, 64 };
 // bit count of min/max values, i.e. 8*sizeof(nanovdb::LeafData::mMinimum) or zero if no min/max exists
-PNANOVDB_STATIC_CONST pnanovdb_uint32_t pnanovdb_grid_type_minmax_strides_bits[PNANOVDB_GRID_TYPE_END] = {  0, 32, 64, 16, 32, 64,  96, 192,  8, 16, 32,  8, 32, 32, 32, 32, 32, 128, 256, 64, 64, 64, 64, 64, 24, 48,  8 };
+PNANOVDB_STATIC_CONST pnanovdb_uint32_t pnanovdb_grid_type_minmax_strides_bits[PNANOVDB_GRID_TYPE_CAP] = {  0, 32, 64, 16, 32, 64,  96, 192,  8, 16, 32,  8, 32, 32, 32, 32, 32, 128, 256, 64, 64,  0,  0, 64, 24, 48,  8,  0 };
 // bit alignment of the value type, controlled by the smallest native type, which is why it is always 0, 8, 16, 32, or 64, e.g. for Vec3f it is 32
-PNANOVDB_STATIC_CONST pnanovdb_uint32_t pnanovdb_grid_type_minmax_aligns_bits[PNANOVDB_GRID_TYPE_END]  = {  0, 32, 64, 16, 32, 64,  32,  64,  8, 16, 32,  8, 32, 32, 32, 32, 32,  32,  64, 64, 64, 64, 64, 64,  8, 16,  8 };
+PNANOVDB_STATIC_CONST pnanovdb_uint32_t pnanovdb_grid_type_minmax_aligns_bits[PNANOVDB_GRID_TYPE_CAP]  = {  0, 32, 64, 16, 32, 64,  32,  64,  8, 16, 32,  8, 32, 32, 32, 32, 32,  32,  64, 64, 64,  0,  0, 64,  8, 16,  8,  0 };
 // bit alignment of the stats (avg/std-dev) types, e.g. 8*sizeof(nanovdb::LeafData::mAverage)
-PNANOVDB_STATIC_CONST pnanovdb_uint32_t pnanovdb_grid_type_stat_strides_bits[PNANOVDB_GRID_TYPE_END]   = {  0, 32, 64, 32, 32, 64,  32,  64,  8, 32, 32,  8, 32, 32, 32, 32, 32,  32,  64, 64, 64, 64, 64, 64, 32, 32, 32 };
+PNANOVDB_STATIC_CONST pnanovdb_uint32_t pnanovdb_grid_type_stat_strides_bits[PNANOVDB_GRID_TYPE_CAP]   = {  0, 32, 64, 32, 32, 64,  32,  64,  8, 32, 32,  8, 32, 32, 32, 32, 32,  32,  64, 64, 64,  0,  0, 64, 32, 32, 32,  0 };
 // one of the 4 leaf types defined above, e.g. PNANOVDB_LEAF_TYPE_INDEX = 3
-PNANOVDB_STATIC_CONST pnanovdb_uint32_t pnanovdb_grid_type_leaf_type[PNANOVDB_GRID_TYPE_END]           = {  0,  0,  0,  0,  0,  0,  0,    0,  1,  0,  0,  1,  0,  2,  2,  2,  2,   0,   0,  3,  3,  4,  4,  5,  0,  0,  0 };
+PNANOVDB_STATIC_CONST pnanovdb_uint32_t pnanovdb_grid_type_leaf_type[PNANOVDB_GRID_TYPE_CAP]           = {  0,  0,  0,  0,  0,  0,  0,    0,  1,  0,  0,  1,  0,  2,  2,  2,  2,   0,   0,  3,  3,  0,  0,  5,  0,  0,  0,  0 };

 struct pnanovdb_map_t
 {
@@ -1099,7 +1255,9 @@ struct pnanovdb_grid_t
    pnanovdb_uint32_t grid_type;                // 4 bytes,     636
    pnanovdb_int64_t blind_metadata_offset;     // 8 bytes,     640
    pnanovdb_uint32_t blind_metadata_count;     // 4 bytes,     648
-    pnanovdb_uint32_t pad[5];                   // 20 bytes,    652
+    pnanovdb_uint32_t data0;                    // 4 bytes,     652
+    pnanovdb_uint64_t data1;                    // 8 bytes,     656
+    pnanovdb_uint64_t data2;                    // 8 bytes,     664
 };
 PNANOVDB_STRUCT_TYPEDEF(pnanovdb_grid_t)
 struct pnanovdb_grid_handle_t { pnanovdb_address_t address; };
@@ -1122,6 +1280,9 @@ PNANOVDB_STRUCT_TYPEDEF(pnanovdb_grid_handle_t)
 #define PNANOVDB_GRID_OFF_GRID_TYPE 636
 #define PNANOVDB_GRID_OFF_BLIND_METADATA_OFFSET 640
 #define PNANOVDB_GRID_OFF_BLIND_METADATA_COUNT 648
+#define PNANOVDB_GRID_OFF_DATA0 652
+#define PNANOVDB_GRID_OFF_DATA1 656
+#define PNANOVDB_GRID_OFF_DATA2 664

 PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_grid_get_magic(pnanovdb_buf_t buf, pnanovdb_grid_handle_t p) {
    return pnanovdb_read_uint64(buf, pnanovdb_address_offset(p.address, PNANOVDB_GRID_OFF_MAGIC));
@@ -1170,6 +1331,15 @@ PNANOVDB_FORCE_INLINE pnanovdb_int64_t pnanovdb_grid_get_blind_metadata_offset(p
 PNANOVDB_FORCE_INLINE pnanovdb_uint32_t pnanovdb_grid_get_blind_metadata_count(pnanovdb_buf_t buf, pnanovdb_grid_handle_t p) {
    return pnanovdb_read_uint32(buf, pnanovdb_address_offset(p.address, PNANOVDB_GRID_OFF_BLIND_METADATA_COUNT));
 }
+PNANOVDB_FORCE_INLINE pnanovdb_uint32_t pnanovdb_grid_get_data0(pnanovdb_buf_t buf, pnanovdb_grid_handle_t p) {
+    return pnanovdb_read_uint32(buf, pnanovdb_address_offset(p.address, PNANOVDB_GRID_OFF_DATA0));
+}
+PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_grid_get_data1(pnanovdb_buf_t buf, pnanovdb_grid_handle_t p) {
+    return pnanovdb_read_uint64(buf, pnanovdb_address_offset(p.address, PNANOVDB_GRID_OFF_DATA1));
+}
+PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_grid_get_data2(pnanovdb_buf_t buf, pnanovdb_grid_handle_t p) {
+    return pnanovdb_read_uint64(buf, pnanovdb_address_offset(p.address, PNANOVDB_GRID_OFF_DATA2));
+}

 PNANOVDB_FORCE_INLINE void pnanovdb_grid_set_magic(pnanovdb_buf_t buf, pnanovdb_grid_handle_t p, pnanovdb_uint64_t magic) {
    pnanovdb_write_uint64(buf, pnanovdb_address_offset(p.address, PNANOVDB_GRID_OFF_MAGIC), magic);
@@ -1213,6 +1383,15 @@ PNANOVDB_FORCE_INLINE void pnanovdb_grid_set_blind_metadata_offset(pnanovdb_buf_
 PNANOVDB_FORCE_INLINE void pnanovdb_grid_set_blind_metadata_count(pnanovdb_buf_t buf, pnanovdb_grid_handle_t p, pnanovdb_uint32_t metadata_count) {
    pnanovdb_write_uint32(buf, pnanovdb_address_offset(p.address, PNANOVDB_GRID_OFF_BLIND_METADATA_COUNT), metadata_count);
 }
+PNANOVDB_FORCE_INLINE void pnanovdb_grid_set_data0(pnanovdb_buf_t buf, pnanovdb_grid_handle_t p, pnanovdb_uint32_t data0) {
+    pnanovdb_write_uint32(buf, pnanovdb_address_offset(p.address, PNANOVDB_GRID_OFF_DATA0), data0);
+}
+PNANOVDB_FORCE_INLINE void pnanovdb_grid_set_data1(pnanovdb_buf_t buf, pnanovdb_grid_handle_t p, pnanovdb_uint64_t data1) {
+    pnanovdb_write_uint64(buf, pnanovdb_address_offset(p.address, PNANOVDB_GRID_OFF_DATA1), data1);
+}
+PNANOVDB_FORCE_INLINE void pnanovdb_grid_set_data2(pnanovdb_buf_t buf, pnanovdb_grid_handle_t p, pnanovdb_uint64_t data2) {
+    pnanovdb_write_uint64(buf, pnanovdb_address_offset(p.address, PNANOVDB_GRID_OFF_DATA2), data2);
+}

 PNANOVDB_FORCE_INLINE pnanovdb_uint32_t pnanovdb_make_version(pnanovdb_uint32_t major, pnanovdb_uint32_t minor, pnanovdb_uint32_t patch_num)
 {
@@ -1232,6 +1411,25 @@ PNANOVDB_FORCE_INLINE pnanovdb_uint32_t pnanovdb_version_get_patch(pnanovdb_uint
    return version & ((1u << 10u) - 1u);
 }

+#define PNANOVDB_GRIDBLINDMETADATA_CLASS_UNKNOWN 0
+#define PNANOVDB_GRIDBLINDMETADATA_CLASS_INDEX_ARRAY 1
+#define PNANOVDB_GRIDBLINDMETADATA_CLASS_ATTRIBUTE_ARRAY 2
+#define PNANOVDB_GRIDBLINDMETADATA_CLASS_GRID_NAME 3
+#define PNANOVDB_GRIDBLINDMETADATA_CLASS_CHANNEL_ARRAY 4
+#define PNANOVDB_GRIDBLINDMETADATA_CLASS_END 5
+
+#define PNANOVDB_GRIDBLINDMETADATA_SEMANTIC_UNKNOWN 0
+#define PNANOVDB_GRIDBLINDMETADATA_SEMANTIC_POINT_POSITION 1
+#define PNANOVDB_GRIDBLINDMETADATA_SEMANTIC_POINT_COLOR 2
+#define PNANOVDB_GRIDBLINDMETADATA_SEMANTIC_POINT_NORMAL 3
+#define PNANOVDB_GRIDBLINDMETADATA_SEMANTIC_POINT_RADIUS 4
+#define PNANOVDB_GRIDBLINDMETADATA_SEMANTIC_POINT_VELOCITY 5
+#define PNANOVDB_GRIDBLINDMETADATA_SEMANTIC_POINT_ID 6
+#define PNANOVDB_GRIDBLINDMETADATA_SEMANTIC_WORLD_COORDS 7
+#define PNANOVDB_GRIDBLINDMETADATA_SEMANTIC_GRID_COORDS 8
+#define PNANOVDB_GRIDBLINDMETADATA_SEMANTIC_VOXEL_COORDS 9
+#define PNANOVDB_GRIDBLINDMETADATA_SEMANTIC_END 10
+
 struct pnanovdb_gridblindmetadata_t
 {
    pnanovdb_int64_t data_offset;       // 8 bytes,     0
@@ -1278,6 +1476,28 @@ PNANOVDB_FORCE_INLINE pnanovdb_uint32_t pnanovdb_gridblindmetadata_get_name(pnan
    return pnanovdb_read_uint32(buf, pnanovdb_address_offset(p.address, PNANOVDB_GRIDBLINDMETADATA_OFF_NAME + 4u * index));
 }

+PNANOVDB_FORCE_INLINE void pnanovdb_gridblindmetadata_set_data_offset(pnanovdb_buf_t buf, pnanovdb_gridblindmetadata_handle_t p, pnanovdb_int64_t data_offset) {
+    pnanovdb_write_int64(buf, pnanovdb_address_offset(p.address, PNANOVDB_GRIDBLINDMETADATA_OFF_DATA_OFFSET), data_offset);
+}
+PNANOVDB_FORCE_INLINE void pnanovdb_gridblindmetadata_set_value_count(pnanovdb_buf_t buf, pnanovdb_gridblindmetadata_handle_t p, pnanovdb_uint64_t value_count) {
+    pnanovdb_write_uint64(buf, pnanovdb_address_offset(p.address, PNANOVDB_GRIDBLINDMETADATA_OFF_VALUE_COUNT), value_count);
+}
+PNANOVDB_FORCE_INLINE void pnanovdb_gridblindmetadata_set_value_size(pnanovdb_buf_t buf, pnanovdb_gridblindmetadata_handle_t p, pnanovdb_uint32_t value_size) {
+    pnanovdb_write_uint32(buf, pnanovdb_address_offset(p.address, PNANOVDB_GRIDBLINDMETADATA_OFF_VALUE_SIZE), value_size);
+}
+PNANOVDB_FORCE_INLINE void pnanovdb_gridblindmetadata_set_semantic(pnanovdb_buf_t buf, pnanovdb_gridblindmetadata_handle_t p, pnanovdb_uint32_t semantic) {
+    pnanovdb_write_uint32(buf, pnanovdb_address_offset(p.address, PNANOVDB_GRIDBLINDMETADATA_OFF_SEMANTIC), semantic);
+}
+PNANOVDB_FORCE_INLINE void pnanovdb_gridblindmetadata_set_data_class(pnanovdb_buf_t buf, pnanovdb_gridblindmetadata_handle_t p, pnanovdb_uint32_t data_class) {
+    pnanovdb_write_uint32(buf, pnanovdb_address_offset(p.address, PNANOVDB_GRIDBLINDMETADATA_OFF_DATA_CLASS), data_class);
+}
+PNANOVDB_FORCE_INLINE void pnanovdb_gridblindmetadata_set_data_type(pnanovdb_buf_t buf, pnanovdb_gridblindmetadata_handle_t p, pnanovdb_uint32_t data_type) {
+    pnanovdb_write_uint32(buf, pnanovdb_address_offset(p.address, PNANOVDB_GRIDBLINDMETADATA_OFF_DATA_TYPE), data_type);
+}
+PNANOVDB_FORCE_INLINE void pnanovdb_gridblindmetadata_set_name(pnanovdb_buf_t buf, pnanovdb_gridblindmetadata_handle_t p, pnanovdb_uint32_t index, pnanovdb_uint32_t name) {
+    pnanovdb_write_uint32(buf, pnanovdb_address_offset(p.address, PNANOVDB_GRIDBLINDMETADATA_OFF_NAME + 4u * index), name);
+}
+
 struct pnanovdb_tree_t
 {
    pnanovdb_uint64_t node_offset_leaf;
@@ -1287,9 +1507,9 @@ struct pnanovdb_tree_t
    pnanovdb_uint32_t node_count_leaf;
    pnanovdb_uint32_t node_count_lower;
    pnanovdb_uint32_t node_count_upper;
-    pnanovdb_uint32_t tile_count_leaf;
    pnanovdb_uint32_t tile_count_lower;
    pnanovdb_uint32_t tile_count_upper;
+    pnanovdb_uint32_t tile_count_root;
    pnanovdb_uint64_t voxel_count;
 };
 PNANOVDB_STRUCT_TYPEDEF(pnanovdb_tree_t)
@@ -1305,9 +1525,9 @@ PNANOVDB_STRUCT_TYPEDEF(pnanovdb_tree_handle_t)
 #define PNANOVDB_TREE_OFF_NODE_COUNT_LEAF 32
 #define PNANOVDB_TREE_OFF_NODE_COUNT_LOWER 36
 #define PNANOVDB_TREE_OFF_NODE_COUNT_UPPER 40
-#define PNANOVDB_TREE_OFF_TILE_COUNT_LEAF 44
-#define PNANOVDB_TREE_OFF_TILE_COUNT_LOWER 48
-#define PNANOVDB_TREE_OFF_TILE_COUNT_UPPER 52
+#define PNANOVDB_TREE_OFF_TILE_COUNT_LOWER 44
+#define PNANOVDB_TREE_OFF_TILE_COUNT_UPPER 48
+#define PNANOVDB_TREE_OFF_TILE_COUNT_ROOT 52
 #define PNANOVDB_TREE_OFF_VOXEL_COUNT 56

 PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_tree_get_node_offset_leaf(pnanovdb_buf_t buf, pnanovdb_tree_handle_t p) {
@@ -1331,15 +1551,15 @@ PNANOVDB_FORCE_INLINE pnanovdb_uint32_t pnanovdb_tree_get_node_count_lower(pnano
 PNANOVDB_FORCE_INLINE pnanovdb_uint32_t pnanovdb_tree_get_node_count_upper(pnanovdb_buf_t buf, pnanovdb_tree_handle_t p) {
    return pnanovdb_read_uint32(buf, pnanovdb_address_offset(p.address, PNANOVDB_TREE_OFF_NODE_COUNT_UPPER));
 }
-PNANOVDB_FORCE_INLINE pnanovdb_uint32_t pnanovdb_tree_get_tile_count_leaf(pnanovdb_buf_t buf, pnanovdb_tree_handle_t p) {
-    return pnanovdb_read_uint32(buf, pnanovdb_address_offset(p.address, PNANOVDB_TREE_OFF_TILE_COUNT_LEAF));
-}
 PNANOVDB_FORCE_INLINE pnanovdb_uint32_t pnanovdb_tree_get_tile_count_lower(pnanovdb_buf_t buf, pnanovdb_tree_handle_t p) {
    return pnanovdb_read_uint32(buf, pnanovdb_address_offset(p.address, PNANOVDB_TREE_OFF_TILE_COUNT_LOWER));
 }
 PNANOVDB_FORCE_INLINE pnanovdb_uint32_t pnanovdb_tree_get_tile_count_upper(pnanovdb_buf_t buf, pnanovdb_tree_handle_t p) {
    return pnanovdb_read_uint32(buf, pnanovdb_address_offset(p.address, PNANOVDB_TREE_OFF_TILE_COUNT_UPPER));
 }
+PNANOVDB_FORCE_INLINE pnanovdb_uint32_t pnanovdb_tree_get_tile_count_root(pnanovdb_buf_t buf, pnanovdb_tree_handle_t p) {
+    return pnanovdb_read_uint32(buf, pnanovdb_address_offset(p.address, PNANOVDB_TREE_OFF_TILE_COUNT_ROOT));
+}
 PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_tree_get_voxel_count(pnanovdb_buf_t buf, pnanovdb_tree_handle_t p) {
    return pnanovdb_read_uint64(buf, pnanovdb_address_offset(p.address, PNANOVDB_TREE_OFF_VOXEL_COUNT));
 }
@@ -1365,15 +1585,15 @@ PNANOVDB_FORCE_INLINE void pnanovdb_tree_set_node_count_lower(pnanovdb_buf_t buf
 PNANOVDB_FORCE_INLINE void pnanovdb_tree_set_node_count_upper(pnanovdb_buf_t buf, pnanovdb_tree_handle_t p, pnanovdb_uint32_t node_count_upper) {
    pnanovdb_write_uint32(buf, pnanovdb_address_offset(p.address, PNANOVDB_TREE_OFF_NODE_COUNT_UPPER), node_count_upper);
 }
-PNANOVDB_FORCE_INLINE void pnanovdb_tree_set_tile_count_leaf(pnanovdb_buf_t buf, pnanovdb_tree_handle_t p, pnanovdb_uint32_t tile_count_leaf) {
-    pnanovdb_write_uint32(buf, pnanovdb_address_offset(p.address, PNANOVDB_TREE_OFF_TILE_COUNT_LEAF), tile_count_leaf);
-}
 PNANOVDB_FORCE_INLINE void pnanovdb_tree_set_tile_count_lower(pnanovdb_buf_t buf, pnanovdb_tree_handle_t p, pnanovdb_uint32_t tile_count_lower) {
    pnanovdb_write_uint32(buf, pnanovdb_address_offset(p.address, PNANOVDB_TREE_OFF_TILE_COUNT_LOWER), tile_count_lower);
 }
 PNANOVDB_FORCE_INLINE void pnanovdb_tree_set_tile_count_upper(pnanovdb_buf_t buf, pnanovdb_tree_handle_t p, pnanovdb_uint32_t tile_count_upper) {
    pnanovdb_write_uint32(buf, pnanovdb_address_offset(p.address, PNANOVDB_TREE_OFF_TILE_COUNT_UPPER), tile_count_upper);
 }
+PNANOVDB_FORCE_INLINE void pnanovdb_tree_set_tile_count_root(pnanovdb_buf_t buf, pnanovdb_tree_handle_t p, pnanovdb_uint32_t tile_count_root) {
+    pnanovdb_write_uint32(buf, pnanovdb_address_offset(p.address, PNANOVDB_TREE_OFF_TILE_COUNT_ROOT), tile_count_root);
+}
 PNANOVDB_FORCE_INLINE void pnanovdb_tree_set_voxel_count(pnanovdb_buf_t buf, pnanovdb_tree_handle_t p, pnanovdb_uint64_t voxel_count) {
    pnanovdb_write_uint64(buf, pnanovdb_address_offset(p.address, PNANOVDB_TREE_OFF_VOXEL_COUNT), voxel_count);
 }
@@ -1501,6 +1721,13 @@ PNANOVDB_FORCE_INLINE void pnanovdb_upper_set_bbox_min(pnanovdb_buf_t buf, pnano
 PNANOVDB_FORCE_INLINE void pnanovdb_upper_set_bbox_max(pnanovdb_buf_t buf, pnanovdb_upper_handle_t p, PNANOVDB_IN(pnanovdb_coord_t) bbox_max) {
    pnanovdb_write_coord(buf, pnanovdb_address_offset(p.address, PNANOVDB_UPPER_OFF_BBOX_MAX), bbox_max);
 }
+PNANOVDB_FORCE_INLINE void pnanovdb_upper_set_value_mask(pnanovdb_buf_t buf, pnanovdb_upper_handle_t p, pnanovdb_uint32_t bit_index, pnanovdb_bool_t value) {
+    pnanovdb_address_t addr = pnanovdb_address_offset(p.address, PNANOVDB_UPPER_OFF_VALUE_MASK + 4u * (bit_index >> 5u));
+    pnanovdb_uint32_t valueMask = pnanovdb_read_uint32(buf, addr);
+    if (!value) { valueMask &= ~(1u << (bit_index & 31u)); }
+    if (value) valueMask |= (1u << (bit_index & 31u));
+    pnanovdb_write_uint32(buf, addr, valueMask);
+}
 PNANOVDB_FORCE_INLINE void pnanovdb_upper_set_child_mask(pnanovdb_buf_t buf, pnanovdb_upper_handle_t p, pnanovdb_uint32_t bit_index, pnanovdb_bool_t value) {
    pnanovdb_address_t addr = pnanovdb_address_offset(p.address, PNANOVDB_UPPER_OFF_CHILD_MASK + 4u * (bit_index >> 5u));
    pnanovdb_uint32_t valueMask = pnanovdb_read_uint32(buf, addr);
@@ -1556,6 +1783,13 @@ PNANOVDB_FORCE_INLINE void pnanovdb_lower_set_bbox_min(pnanovdb_buf_t buf, pnano
 PNANOVDB_FORCE_INLINE void pnanovdb_lower_set_bbox_max(pnanovdb_buf_t buf, pnanovdb_lower_handle_t p, PNANOVDB_IN(pnanovdb_coord_t) bbox_max) {
    pnanovdb_write_coord(buf, pnanovdb_address_offset(p.address, PNANOVDB_LOWER_OFF_BBOX_MAX), bbox_max);
 }
+PNANOVDB_FORCE_INLINE void pnanovdb_lower_set_value_mask(pnanovdb_buf_t buf, pnanovdb_lower_handle_t p, pnanovdb_uint32_t bit_index, pnanovdb_bool_t value) {
+    pnanovdb_address_t addr = pnanovdb_address_offset(p.address, PNANOVDB_LOWER_OFF_VALUE_MASK + 4u * (bit_index >> 5u));
+    pnanovdb_uint32_t valueMask = pnanovdb_read_uint32(buf, addr);
+    if (!value) { valueMask &= ~(1u << (bit_index & 31u)); }
+    if (value) valueMask |= (1u << (bit_index & 31u));
+    pnanovdb_write_uint32(buf, addr, valueMask);
+}
 PNANOVDB_FORCE_INLINE void pnanovdb_lower_set_child_mask(pnanovdb_buf_t buf, pnanovdb_lower_handle_t p, pnanovdb_uint32_t bit_index, pnanovdb_bool_t value) {
    pnanovdb_address_t addr = pnanovdb_address_offset(p.address, PNANOVDB_LOWER_OFF_CHILD_MASK + 4u * (bit_index >> 5u));
    pnanovdb_uint32_t valueMask = pnanovdb_read_uint32(buf, addr);
@@ -1604,6 +1838,13 @@ PNANOVDB_FORCE_INLINE void pnanovdb_leaf_set_bbox_min(pnanovdb_buf_t buf, pnanov
 PNANOVDB_FORCE_INLINE void pnanovdb_leaf_set_bbox_dif_and_flags(pnanovdb_buf_t buf, pnanovdb_leaf_handle_t p, pnanovdb_uint32_t bbox_dif_and_flags) {
    pnanovdb_write_uint32(buf, pnanovdb_address_offset(p.address, PNANOVDB_LEAF_OFF_BBOX_DIF_AND_FLAGS), bbox_dif_and_flags);
 }
+PNANOVDB_FORCE_INLINE void pnanovdb_leaf_set_value_mask(pnanovdb_buf_t buf, pnanovdb_leaf_handle_t p, pnanovdb_uint32_t bit_index, pnanovdb_bool_t value) {
+    pnanovdb_address_t addr = pnanovdb_address_offset(p.address, PNANOVDB_LEAF_OFF_VALUE_MASK + 4u * (bit_index >> 5u));
+    pnanovdb_uint32_t valueMask = pnanovdb_read_uint32(buf, addr);
+    if (!value) { valueMask &= ~(1u << (bit_index & 31u)); }
+    if (value) valueMask |= (1u << (bit_index & 31u));
+    pnanovdb_write_uint32(buf, addr, valueMask);
+}

 struct pnanovdb_grid_type_constants_t
 {
@@ -1639,7 +1880,7 @@ struct pnanovdb_grid_type_constants_t
 PNANOVDB_STRUCT_TYPEDEF(pnanovdb_grid_type_constants_t)

 // The following table with offsets will nedd to be updates as new GridTypes are added in NanoVDB.h
-PNANOVDB_STATIC_CONST pnanovdb_grid_type_constants_t pnanovdb_grid_type_constants[PNANOVDB_GRID_TYPE_END] =
+PNANOVDB_STATIC_CONST pnanovdb_grid_type_constants_t pnanovdb_grid_type_constants[PNANOVDB_GRID_TYPE_CAP] =
 {
 {28, 28, 28, 28, 28, 32,  0, 8, 20, 32,  8224, 8224, 8224, 8224, 8224, 270368,  1056, 1056, 1056, 1056, 1056, 33824,  80, 80, 80, 80, 96, 96},
 {28, 32, 36, 40, 44, 64,  32, 8, 20, 32,  8224, 8228, 8232, 8236, 8256, 270400,  1056, 1060, 1064, 1068, 1088, 33856,  80, 84, 88, 92, 96, 2144},
@@ -1662,12 +1903,13 @@ PNANOVDB_STATIC_CONST pnanovdb_grid_type_constants_t pnanovdb_grid_type_constant
 {32, 64, 96, 128, 136, 160,  256, 32, 24, 64,  8224, 8256, 8288, 8296, 8320, 1056896,  1056, 1088, 1120, 1128, 1152, 132224,  80, 112, 144, 152, 160, 16544},
 {32, 40, 48, 56, 64, 96,  0, 8, 24, 32,  8224, 8232, 8240, 8248, 8256, 270400,  1056, 1064, 1072, 1080, 1088, 33856,  80, 80, 80, 80, 80, 96},
 {32, 40, 48, 56, 64, 96,  0, 8, 24, 32,  8224, 8232, 8240, 8248, 8256, 270400,  1056, 1064, 1072, 1080, 1088, 33856,  80, 80, 80, 80, 80, 96},
-{32, 40, 48, 56, 64, 96,  0, 8, 24, 32,  8224, 8232, 8240, 8248, 8256, 270400,  1056, 1064, 1072, 1080, 1088, 33856,  80, 80, 80, 80, 80, 160},
-{32, 40, 48, 56, 64, 96,  0, 8, 24, 32,  8224, 8232, 8240, 8248, 8256, 270400,  1056, 1064, 1072, 1080, 1088, 33856,  80, 80, 80, 80, 80, 160},
+{28, 28, 28, 28, 28, 32,  0, 8, 20, 32,  8224, 8224, 8224, 8224, 8224, 270368,  1056, 1056, 1056, 1056, 1056, 33824,  80, 80, 80, 80, 96, 96},
+{28, 28, 28, 28, 28, 32,  0, 8, 20, 32,  8224, 8224, 8224, 8224, 8224, 270368,  1056, 1056, 1056, 1056, 1056, 33824,  80, 80, 80, 80, 96, 96},
 {32, 40, 48, 56, 64, 96,  16, 8, 24, 32,  8224, 8232, 8240, 8248, 8256, 270400,  1056, 1064, 1072, 1080, 1088, 33856,  80, 88, 96, 96, 96, 1120},
 {28, 31, 34, 40, 44, 64,  24, 8, 20, 32,  8224, 8227, 8232, 8236, 8256, 270400,  1056, 1059, 1064, 1068, 1088, 33856,  80, 83, 88, 92, 96, 1632},
 {28, 34, 40, 48, 52, 64,  48, 8, 20, 32,  8224, 8230, 8236, 8240, 8256, 270400,  1056, 1062, 1068, 1072, 1088, 33856,  80, 86, 92, 96, 128, 3200},
 {28, 29, 30, 32, 36, 64,  8, 8, 20, 32,  8224, 8225, 8228, 8232, 8256, 270400,  1056, 1057, 1060, 1064, 1088, 33856,  80, 81, 84, 88, 96, 608},
+{28, 28, 28, 28, 28, 32,  0, 8, 20, 32,  8224, 8224, 8224, 8224, 8224, 270368,  1056, 1056, 1056, 1056, 1056, 33824,  80, 80, 80, 80, 96, 96},
 };

 // ------------------------------------------------ Basic Lookup -----------------------------------------------------------
@@ -1719,6 +1961,11 @@ PNANOVDB_FORCE_INLINE pnanovdb_root_tile_handle_t pnanovdb_root_get_tile_zero(pn
    return tile;
 }

+PNANOVDB_FORCE_INLINE pnanovdb_bool_t pnanovdb_root_tile_get_child_mask(pnanovdb_buf_t buf, pnanovdb_root_tile_handle_t tile)
+{
+    return !pnanovdb_int64_is_zero(pnanovdb_root_tile_get_child(buf, tile));
+}
+
 PNANOVDB_FORCE_INLINE pnanovdb_upper_handle_t pnanovdb_root_get_child(pnanovdb_grid_type_t grid_type, pnanovdb_buf_t buf, pnanovdb_root_handle_t root, pnanovdb_root_tile_handle_t tile)
 {
    pnanovdb_upper_handle_t upper = { root.address };
@@ -1726,6 +1973,11 @@ PNANOVDB_FORCE_INLINE pnanovdb_upper_handle_t pnanovdb_root_get_child(pnanovdb_g
    return upper;
 }

+PNANOVDB_FORCE_INLINE void pnanovdb_root_set_child(pnanovdb_grid_type_t grid_type, pnanovdb_buf_t buf, pnanovdb_root_handle_t root, pnanovdb_root_tile_handle_t tile, pnanovdb_upper_handle_t upper)
+{
+    pnanovdb_root_tile_set_child(buf, tile, pnanovdb_address_diff(upper.address, root.address));
+}
+
 PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_coord_to_key(PNANOVDB_IN(pnanovdb_coord_t) ijk)
 {
 #if defined(PNANOVDB_NATIVE_64)
@@ -1760,6 +2012,27 @@ PNANOVDB_FORCE_INLINE pnanovdb_root_tile_handle_t pnanovdb_root_find_tile(pnanov
    return null_handle;
 }

+// ----------------------------- Grid Type Safe Set ---------------------------------------
+
+PNANOVDB_FORCE_INLINE void pnanovdb_grid_set_first_gridblindmetadata(pnanovdb_buf_t buf, pnanovdb_grid_handle_t p, pnanovdb_gridblindmetadata_handle_t gridblindmetadata) {
+    pnanovdb_grid_set_blind_metadata_offset(buf, p, pnanovdb_address_diff(gridblindmetadata.address, p.address));
+}
+
+// ----------------------------- Tree Type Safe Set ---------------------------------------
+
+PNANOVDB_FORCE_INLINE void pnanovdb_tree_set_first_leaf(pnanovdb_buf_t buf, pnanovdb_tree_handle_t p, pnanovdb_leaf_handle_t leaf) {
+    pnanovdb_tree_set_node_offset_leaf(buf, p, pnanovdb_address_diff(leaf.address, p.address));
+}
+PNANOVDB_FORCE_INLINE void pnanovdb_tree_set_first_lower(pnanovdb_buf_t buf, pnanovdb_tree_handle_t p, pnanovdb_lower_handle_t lower) {
+    pnanovdb_tree_set_node_offset_lower(buf, p, pnanovdb_address_diff(lower.address, p.address));
+}
+PNANOVDB_FORCE_INLINE void pnanovdb_tree_set_first_upper(pnanovdb_buf_t buf, pnanovdb_tree_handle_t p, pnanovdb_upper_handle_t upper) {
+    pnanovdb_tree_set_node_offset_upper(buf, p, pnanovdb_address_diff(upper.address, p.address));
+}
+PNANOVDB_FORCE_INLINE void pnanovdb_tree_set_first_root(pnanovdb_buf_t buf, pnanovdb_tree_handle_t p, pnanovdb_root_handle_t root) {
+    pnanovdb_tree_set_node_offset_root(buf, p, pnanovdb_address_diff(root.address, p.address));
+}
+
 // ----------------------------- Leaf Node ---------------------------------------

 PNANOVDB_FORCE_INLINE pnanovdb_uint32_t pnanovdb_leaf_coord_to_offset(PNANOVDB_IN(pnanovdb_coord_t) ijk)
@@ -1880,57 +2153,6 @@ PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_leaf_index_get_value_index(pnan
    return pnanovdb_uint64_offset(offset, n);
 }

-// ----------------------------- Leaf IndexMask Specialization ---------------------------------------
-
-PNANOVDB_FORCE_INLINE pnanovdb_bool_t pnanovdb_leaf_indexmask_has_stats(pnanovdb_buf_t buf, pnanovdb_leaf_handle_t leaf)
-{
-    return pnanovdb_leaf_index_has_stats(buf, leaf);
-}
-PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_leaf_indexmask_get_min_index(pnanovdb_buf_t buf, pnanovdb_address_t min_address)
-{
-    return pnanovdb_leaf_index_get_min_index(buf, min_address);
-}
-PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_leaf_indexmask_get_max_index(pnanovdb_buf_t buf, pnanovdb_address_t max_address)
-{
-    return pnanovdb_leaf_index_get_max_index(buf, max_address);
-}
-PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_leaf_indexmask_get_ave_index(pnanovdb_buf_t buf, pnanovdb_address_t ave_address)
-{
-    return pnanovdb_leaf_index_get_ave_index(buf, ave_address);
-}
-PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_leaf_indexmask_get_dev_index(pnanovdb_buf_t buf, pnanovdb_address_t dev_address)
-{
-    return pnanovdb_leaf_index_get_dev_index(buf, dev_address);
-}
-PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_leaf_indexmask_get_value_index(pnanovdb_buf_t buf, pnanovdb_address_t value_address, PNANOVDB_IN(pnanovdb_coord_t) ijk)
-{
-    return pnanovdb_leaf_index_get_value_index(buf, value_address, ijk);
-}
-PNANOVDB_FORCE_INLINE pnanovdb_bool_t pnanovdb_leaf_indexmask_get_mask_bit(pnanovdb_buf_t buf, pnanovdb_leaf_handle_t leaf, pnanovdb_uint32_t n)
-{
-    pnanovdb_uint32_t word_idx = n >> 5;
-    pnanovdb_uint32_t bit_idx = n & 31;
-    pnanovdb_uint32_t val_mask =
-        pnanovdb_read_uint32(buf, pnanovdb_address_offset(leaf.address, 96u + 4u * word_idx));
-    return (val_mask & (1u << bit_idx)) != 0u;
-}
-PNANOVDB_FORCE_INLINE void pnanovdb_leaf_indexmask_set_mask_bit(pnanovdb_buf_t buf, pnanovdb_leaf_handle_t leaf, pnanovdb_uint32_t n, pnanovdb_bool_t v)
-{
-    pnanovdb_uint32_t word_idx = n >> 5;
-    pnanovdb_uint32_t bit_idx = n & 31;
-    pnanovdb_uint32_t val_mask =
-        pnanovdb_read_uint32(buf, pnanovdb_address_offset(leaf.address, 96u + 4u * word_idx));
-    if (v)
-    {
-        val_mask = val_mask | (1u << bit_idx);
-    }
-    else
-    {
-        val_mask = val_mask & ~(1u << bit_idx);
-    }
-    pnanovdb_write_uint32(buf, pnanovdb_address_offset(leaf.address, 96u + 4u * word_idx), val_mask);
-}
-
 // ----------------------------- Leaf OnIndex Specialization ---------------------------------------

 PNANOVDB_FORCE_INLINE pnanovdb_uint32_t pnanovdb_leaf_onindex_get_value_count(pnanovdb_buf_t buf, pnanovdb_leaf_handle_t leaf)
@@ -2022,65 +2244,6 @@ PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_leaf_onindex_get_value_index(pn
    return value_index;
 }

-// ----------------------------- Leaf OnIndexMask Specialization ---------------------------------------
-
-PNANOVDB_FORCE_INLINE pnanovdb_uint32_t pnanovdb_leaf_onindexmask_get_value_count(pnanovdb_buf_t buf, pnanovdb_leaf_handle_t leaf)
-{
-    return pnanovdb_leaf_onindex_get_value_count(buf, leaf);
-}
-PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_leaf_onindexmask_get_last_offset(pnanovdb_buf_t buf, pnanovdb_leaf_handle_t leaf)
-{
-    return pnanovdb_leaf_onindex_get_last_offset(buf, leaf);
-}
-PNANOVDB_FORCE_INLINE pnanovdb_bool_t pnanovdb_leaf_onindexmask_has_stats(pnanovdb_buf_t buf, pnanovdb_leaf_handle_t leaf)
-{
-    return pnanovdb_leaf_onindex_has_stats(buf, leaf);
-}
-PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_leaf_onindexmask_get_min_index(pnanovdb_buf_t buf, pnanovdb_address_t min_address)
-{
-    return pnanovdb_leaf_onindex_get_min_index(buf, min_address);
-}
-PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_leaf_onindexmask_get_max_index(pnanovdb_buf_t buf, pnanovdb_address_t max_address)
-{
-    return pnanovdb_leaf_onindex_get_max_index(buf, max_address);
-}
-PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_leaf_onindexmask_get_ave_index(pnanovdb_buf_t buf, pnanovdb_address_t ave_address)
-{
-    return pnanovdb_leaf_onindex_get_ave_index(buf, ave_address);
-}
-PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_leaf_onindexmask_get_dev_index(pnanovdb_buf_t buf, pnanovdb_address_t dev_address)
-{
-    return pnanovdb_leaf_onindex_get_dev_index(buf, dev_address);
-}
-PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_leaf_onindexmask_get_value_index(pnanovdb_buf_t buf, pnanovdb_address_t value_address, PNANOVDB_IN(pnanovdb_coord_t) ijk)
-{
-    return pnanovdb_leaf_onindex_get_value_index(buf, value_address, ijk);
-}
-PNANOVDB_FORCE_INLINE pnanovdb_bool_t pnanovdb_leaf_onindexmask_get_mask_bit(pnanovdb_buf_t buf, pnanovdb_leaf_handle_t leaf, pnanovdb_uint32_t n)
-{
-    pnanovdb_uint32_t word_idx = n >> 5;
-    pnanovdb_uint32_t bit_idx = n & 31;
-    pnanovdb_uint32_t val_mask =
-        pnanovdb_read_uint32(buf, pnanovdb_address_offset(leaf.address, 96u + 4u * word_idx));
-    return (val_mask & (1u << bit_idx)) != 0u;
-}
-PNANOVDB_FORCE_INLINE void pnanovdb_leaf_onindexmask_set_mask_bit(pnanovdb_buf_t buf, pnanovdb_leaf_handle_t leaf, pnanovdb_uint32_t n, pnanovdb_bool_t v)
-{
-    pnanovdb_uint32_t word_idx = n >> 5;
-    pnanovdb_uint32_t bit_idx = n & 31;
-    pnanovdb_uint32_t val_mask =
-        pnanovdb_read_uint32(buf, pnanovdb_address_offset(leaf.address, 96u + 4u * word_idx));
-    if (v)
-    {
-        val_mask = val_mask | (1u << bit_idx);
-    }
-    else
-    {
-        val_mask = val_mask & ~(1u << bit_idx);
-    }
-    pnanovdb_write_uint32(buf, pnanovdb_address_offset(leaf.address, 96u + 4u * word_idx), val_mask);
-}
-
 // ----------------------------- Leaf PointIndex Specialization ---------------------------------------

 PNANOVDB_FORCE_INLINE pnanovdb_uint64_t pnanovdb_leaf_pointindex_get_offset(pnanovdb_buf_t buf, pnanovdb_leaf_handle_t leaf)
@@ -2210,6 +2373,17 @@ PNANOVDB_FORCE_INLINE pnanovdb_address_t pnanovdb_lower_get_value_address(pnanov
    return pnanovdb_lower_get_value_address_and_level(grid_type, buf, lower, ijk, PNANOVDB_REF(level));
 }

+PNANOVDB_FORCE_INLINE void pnanovdb_lower_set_table_child(pnanovdb_grid_type_t grid_type, pnanovdb_buf_t buf, pnanovdb_lower_handle_t node, pnanovdb_uint32_t n, pnanovdb_int64_t child)
+{
+    pnanovdb_address_t table_address = pnanovdb_lower_get_table_address(grid_type, buf, node, n);
+    pnanovdb_write_int64(buf, table_address, child);
+}
+
+PNANOVDB_FORCE_INLINE void pnanovdb_lower_set_child(pnanovdb_grid_type_t grid_type, pnanovdb_buf_t buf, pnanovdb_lower_handle_t lower, pnanovdb_uint32_t n, pnanovdb_leaf_handle_t leaf)
+{
+    pnanovdb_lower_set_table_child(grid_type, buf, lower, n, pnanovdb_address_diff(leaf.address, lower.address));
+}
+
 // ------------------------------------------------ Upper Node -----------------------------------------------------------

 PNANOVDB_FORCE_INLINE pnanovdb_uint32_t pnanovdb_upper_coord_to_offset(PNANOVDB_IN(pnanovdb_coord_t) ijk)
@@ -2291,8 +2465,19 @@ PNANOVDB_FORCE_INLINE void pnanovdb_upper_set_table_child(pnanovdb_grid_type_t g
    pnanovdb_write_int64(buf, bufAddress, child);
 }

+PNANOVDB_FORCE_INLINE void pnanovdb_upper_set_child(pnanovdb_grid_type_t grid_type, pnanovdb_buf_t buf, pnanovdb_upper_handle_t upper, pnanovdb_uint32_t n, pnanovdb_lower_handle_t lower)
+{
+    pnanovdb_upper_set_table_child(grid_type, buf, upper, n, pnanovdb_address_diff(lower.address, upper.address));
+}
+
 // ------------------------------------------------ Root -----------------------------------------------------------

+PNANOVDB_FORCE_INLINE pnanovdb_address_t pnanovdb_root_get_background_address(pnanovdb_grid_type_t grid_type, pnanovdb_buf_t buf, pnanovdb_root_handle_t root)
+{
+    pnanovdb_uint32_t byte_offset = PNANOVDB_GRID_TYPE_GET(grid_type, root_off_background);
+    return pnanovdb_address_offset(root.address, byte_offset);
+}
+
 PNANOVDB_FORCE_INLINE pnanovdb_address_t pnanovdb_root_get_min_address(pnanovdb_grid_type_t grid_type, pnanovdb_buf_t buf, pnanovdb_root_handle_t root)
 {
    pnanovdb_uint32_t byte_offset = PNANOVDB_GRID_TYPE_GET(grid_type, root_off_min);
@@ -2591,12 +2776,6 @@ PNANOVDB_FORCE_INLINE pnanovdb_address_t pnanovdb_lower_get_value_address_and_ca
    return pnanovdb_lower_get_value_address_and_level_and_cache(grid_type, buf, lower, ijk, acc, PNANOVDB_REF(level));
 }

-PNANOVDB_FORCE_INLINE void pnanovdb_lower_set_table_child(pnanovdb_grid_type_t grid_type, pnanovdb_buf_t buf, pnanovdb_lower_handle_t node, pnanovdb_uint32_t n, pnanovdb_int64_t child)
-{
-    pnanovdb_address_t table_address = pnanovdb_lower_get_table_address(grid_type, buf, node, n);
-    pnanovdb_write_int64(buf, table_address, child);
-}
-
 PNANOVDB_FORCE_INLINE pnanovdb_address_t pnanovdb_upper_get_value_address_and_level_and_cache(pnanovdb_grid_type_t grid_type, pnanovdb_buf_t buf, pnanovdb_upper_handle_t upper, PNANOVDB_IN(pnanovdb_coord_t) ijk, PNANOVDB_INOUT(pnanovdb_readaccessor_t) acc, PNANOVDB_INOUT(pnanovdb_uint32_t) level)
 {
    pnanovdb_uint32_t n = pnanovdb_upper_coord_to_offset(ijk);
@@ -3227,6 +3406,19 @@ PNANOVDB_FORCE_INLINE pnanovdb_bool_t pnanovdb_hdda_update(PNANOVDB_INOUT(pnanov
    {
        return PNANOVDB_FALSE;
    }
+
+    // compute valid voxel range
+    pnanovdb_coord_t voxel_max = {
+        (PNANOVDB_DEREF(hdda).voxel.x + PNANOVDB_DEREF(hdda).dim - 1) & (~(dim - 1)),
+        (PNANOVDB_DEREF(hdda).voxel.y + PNANOVDB_DEREF(hdda).dim - 1) & (~(dim - 1)),
+        (PNANOVDB_DEREF(hdda).voxel.z + PNANOVDB_DEREF(hdda).dim - 1) & (~(dim - 1))
+    };
+    pnanovdb_coord_t voxel_min = {
+        PNANOVDB_DEREF(hdda).voxel.x & (~(dim - 1)),
+        PNANOVDB_DEREF(hdda).voxel.y & (~(dim - 1)),
+        PNANOVDB_DEREF(hdda).voxel.z & (~(dim - 1))
+    };
+
    PNANOVDB_DEREF(hdda).dim = dim;

    pnanovdb_vec3_t pos = pnanovdb_vec3_add(
@@ -3237,6 +3429,14 @@ PNANOVDB_FORCE_INLINE pnanovdb_bool_t pnanovdb_hdda_update(PNANOVDB_INOUT(pnanov

    PNANOVDB_DEREF(hdda).voxel = pnanovdb_hdda_pos_to_voxel(PNANOVDB_REF(pos), dim);

+    // clamp voxel to valid range
+    PNANOVDB_DEREF(hdda).voxel.x = PNANOVDB_DEREF(hdda).voxel.x < voxel_max.x ? PNANOVDB_DEREF(hdda).voxel.x : voxel_max.x;
+    PNANOVDB_DEREF(hdda).voxel.y = PNANOVDB_DEREF(hdda).voxel.y < voxel_max.y ? PNANOVDB_DEREF(hdda).voxel.y : voxel_max.y;
+    PNANOVDB_DEREF(hdda).voxel.z = PNANOVDB_DEREF(hdda).voxel.z < voxel_max.z ? PNANOVDB_DEREF(hdda).voxel.z : voxel_max.z;
+    PNANOVDB_DEREF(hdda).voxel.x = PNANOVDB_DEREF(hdda).voxel.x > voxel_min.x ? PNANOVDB_DEREF(hdda).voxel.x : voxel_min.x;
+    PNANOVDB_DEREF(hdda).voxel.y = PNANOVDB_DEREF(hdda).voxel.y > voxel_min.y ? PNANOVDB_DEREF(hdda).voxel.y : voxel_min.y;
+    PNANOVDB_DEREF(hdda).voxel.z = PNANOVDB_DEREF(hdda).voxel.z > voxel_min.z ? PNANOVDB_DEREF(hdda).voxel.z : voxel_min.z;
+
    if (PNANOVDB_DEREF(hdda).step.x != 0)
    {
        PNANOVDB_DEREF(hdda).next.x = PNANOVDB_DEREF(hdda).tmin + (PNANOVDB_DEREF(hdda).voxel.x - pos.x) * dir_inv.x;
@@ -9,6 +9,6 @@ A lightweight GPU friendly version of VDB initially targeting rendering applicat
 * [Examples](../../doc/nanovdb/HelloWorld.md)

 ### Copyright Contributors to the OpenVDB Project
-### SPDX-License-Identifier: MPL-2.0
+### SPDX-License-Identifier: Apache-2.0

 [![CircleCI](https://circleci.com/gh/NVIDIA-Omniverse/NanoVDB.svg?style=svg&circle-token=6c3b9bb166841033a0e0c1a9ee5ac49f73d89152)](https://circleci.com/gh/NVIDIA-Omniverse/NanoVDB)
@@ -1,5 +1,5 @@
 # Copyright Contributors to the OpenVDB Project
-# SPDX-License-Identifier: MPL-2.0
+# SPDX-License-Identifier: Apache-2.0
 #
 #[=======================================================================[

@@ -7,7 +7,7 @@

 #]=======================================================================]

-cmake_minimum_required(VERSION 3.18)
+cmake_minimum_required(VERSION 3.24)
 project(NanoVDBExamples LANGUAGES CXX)

 include(GNUInstallDirs)
@@ -32,17 +32,17 @@ if(NOT NANOVDB_USE_OPENVDB)
 else()
  add_executable(nanovdb_convert convert/nanovdb_convert.cc)
  target_link_libraries(nanovdb_convert PRIVATE nanovdb)
-  install(TARGETS nanovdb_convert RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
+  install(TARGETS nanovdb_convert RUNTIME DESTINATION ${NANOVDB_INSTALL_BINDIR})
 endif()

 # -----------------------------------------------------------------------------
 # nanovdb_print
 add_executable(nanovdb_print print/nanovdb_print.cc)
 target_link_libraries(nanovdb_print PRIVATE nanovdb)
-install(TARGETS nanovdb_print RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
+install(TARGETS nanovdb_print RUNTIME DESTINATION ${NANOVDB_INSTALL_BINDIR})

 # -----------------------------------------------------------------------------
 # nanovdb_validate
 add_executable(nanovdb_validate validate/nanovdb_validate.cc)
 target_link_libraries(nanovdb_validate PRIVATE nanovdb)
-install(TARGETS nanovdb_validate RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
+install(TARGETS nanovdb_validate RUNTIME DESTINATION ${NANOVDB_INSTALL_BINDIR})
@@ -1,5 +1,5 @@
 // Copyright Contributors to the OpenVDB Project
-// SPDX-License-Identifier: MPL-2.0
+// SPDX-License-Identifier: Apache-2.0

 /*!
    \file   nanovdb_convert.cc
@@ -35,6 +35,8 @@ void usage [[noreturn]] (const std::string& progName, int exitStatus = EXIT_FAIL
              << "--fp8\tQuantize float grids to 8 bits\n"
              << "--fp16\tQuantize float grids to 16 bits\n"
              << "--fpN\tQuantize float grids to variable bit depth (use -a or -r to specify a tolerance)\n"
+              << "--index\tProduce an IndexGrid where all float values are encoded as sidecar data\n"
+              << "--onIndex\tProduce an IndexGrid where active float values are encoded as sidecar data\n"
              << "-g,--grid name\tConvert all grids matching the specified string name\n"
              << "-h,--help\tPrints this message\n"
              << "-r,--rel-error float\t Relative error tolerance used for variable bit depth quantization\n"
@@ -57,14 +59,14 @@ int main(int argc, char* argv[])
 {
    int exitStatus = EXIT_SUCCESS;

-    nanovdb::io::Codec       codec = nanovdb::io::Codec::NONE;// compression codec for the file
-    nanovdb::tools::StatsMode       sMode = nanovdb::tools::StatsMode::Default;
-    nanovdb::CheckMode    cMode = nanovdb::CheckMode::Default;
-    nanovdb::GridType        qMode = nanovdb::GridType::Unknown;//specify the quantization mode
-    bool                     verbose = false, overwrite = false, dither = false, absolute = true;
-    float                    tolerance = -1.0f;
-    std::string              gridName;
-    std::vector<std::string> fileNames;
+    nanovdb::io::Codec        codec = nanovdb::io::Codec::NONE;// compression codec for the file
+    nanovdb::tools::StatsMode sMode = nanovdb::tools::StatsMode::Default;
+    nanovdb::CheckMode        cMode = nanovdb::CheckMode::Default;
+    nanovdb::GridType         qMode = nanovdb::GridType::Unknown;//specify the quantization mode
+    bool                      verbose = false, overwrite = false, dither = false, absolute = true;
+    float                     tolerance = -1.0f;
+    std::string               gridName;
+    std::vector<std::string>  fileNames;
    auto toLowerCase = [](std::string &str) {
        std::transform(str.begin(), str.end(), str.begin(),[](unsigned char c){return std::tolower(c);});
    };
@@ -85,6 +87,10 @@ int main(int argc, char* argv[])
                qMode = nanovdb::GridType::Fp16;
            } else if (arg == "--fpN") {
                qMode = nanovdb::GridType::FpN;
+            } else if (arg == "--index") {
+                qMode = nanovdb::GridType::Index;
+            } else if (arg == "--onIndex") {
+                qMode = nanovdb::GridType::OnIndex;
            } else if (arg == "-h" || arg == "--help") {
                usage(argv[0], EXIT_SUCCESS);
            } else if (arg == "-b" || arg == "--blosc") {
@@ -169,17 +175,22 @@ int main(int argc, char* argv[])
        std::cerr << "Expected at least one input file followed by exactly one output file\n" << std::endl;
        usage(argv[0]);
    }
-    const std::string outputFile = fileNames.back();
+    const std::string outputFile = fileNames.back();// last file is always the output file
    const std::string ext = outputFile.substr(outputFile.find_last_of(".") + 1);
    bool              toNanoVDB = false;
    if (ext == "nvdb") {
        toNanoVDB = true;
-    } else if (ext != "vdb") {
+    } else if (ext == "vdb") {
+        if (qMode != nanovdb::GridType::Unknown) {
+            std::cerr << "The options are incompatible an output of type OpenVDB" << std::endl;
+            usage(argv[0]);
+        }
+    } else {
        std::cerr << "Unrecognized file extension: \"" << ext << "\"\n" << std::endl;
        usage(argv[0]);
    }

-    fileNames.pop_back();
+    fileNames.pop_back();// remove the output file name

    if (!overwrite) {
        std::ifstream is(outputFile, std::ios::in | std::ios::binary);
@@ -203,9 +214,16 @@ int main(int argc, char* argv[])

    auto openToNano = [&](const openvdb::GridBase::Ptr& base)
    {
-        using SrcGridT = openvdb::FloatGrid;
-        if (auto floatGrid = openvdb::GridBase::grid<SrcGridT>(base)) {
-            nanovdb::tools::CreateNanoGrid<SrcGridT> s(*floatGrid);
+        const bool includeStats = false, includeTiles = false;
+        const int verb = verbose ? 1 : 0;
+        if (qMode == nanovdb::GridType::OnIndex) {
+            return nanovdb::tools::openToIndexVDB<nanovdb::ValueOnIndex>(base, 1u, includeStats, includeTiles, verb);
+        } else if (qMode == nanovdb::GridType::Index) {
+            return nanovdb::tools::openToIndexVDB<nanovdb::ValueIndex>(base, 1u, includeStats, includeTiles, verb);
+        }
+
+        if (auto floatGrid = openvdb::GridBase::grid<openvdb::FloatGrid>(base)) {
+            nanovdb::tools::CreateNanoGrid<openvdb::FloatGrid> s(*floatGrid);
            s.setStats(sMode);
            s.setChecksum(cMode);
            s.enableDithering(dither);
@@ -227,7 +245,7 @@ int main(int argc, char* argv[])
                break;
            }// end of switch
        }
-        return nanovdb::tools::openToNanoVDB(base, sMode, cMode, verbose ? 1 : 0);
+        return nanovdb::tools::openToNanoVDB(base, sMode, cMode, verb);
    };
    try {
        if (toNanoVDB) { // OpenVDB -> NanoVDB
@@ -237,26 +255,21 @@ int main(int argc, char* argv[])
                    std::cerr << "Since the last file has extension .nvdb the remaining input files were expected to have extensions .vdb\n" << std::endl;
                    usage(argv[0]);
                }
-                if (verbose)
-                    std::cout << "Opening OpenVDB file named \"" << inputFile << "\"" << std::endl;
+                if (verbose) std::cout << "Opening OpenVDB file named \"" << inputFile << "\"" << std::endl;
                openvdb::io::File file(inputFile);
                file.open(false); //disable delayed loading
                if (gridName.empty()) {// convert all grid in the file
                    auto grids = file.getGrids();
                    std::vector<nanovdb::GridHandle<nanovdb::HostBuffer> > handles;
                    for (auto& grid : *grids) {
-                        if (verbose) {
-                            std::cout << "Converting OpenVDB grid named \"" << grid->getName() << "\" to NanoVDB" << std::endl;
-                        }
+                        if (verbose) std::cout << "Converting OpenVDB grid named \"" << grid->getName() << "\" to NanoVDB" << std::endl;
                        handles.push_back(openToNano(grid));
                    } // loop over OpenVDB grids in file
                    auto handle = nanovdb::mergeGrids<nanovdb::HostBuffer, std::vector>(handles);
                    nanovdb::io::writeGrid(os, handle, codec);
                } else {// convert only grid with matching name
                    auto grid = file.readGrid(gridName);
-                    if (verbose) {
-                        std::cout << "Converting OpenVDB grid named \"" << grid->getName() << "\" to NanoVDB" << std::endl;
-                    }
+                    if (verbose) std::cout << "Converting OpenVDB grid named \"" << grid->getName() << "\" to NanoVDB" << std::endl;
                    auto handle = openToNano(grid);
                    nanovdb::io::writeGrid(os, handle, codec);
                }
@@ -269,15 +282,13 @@ int main(int argc, char* argv[])
                    std::cerr << "Since the last file has extension .vdb the remaining input files were expected to have extensions .nvdb\n" << std::endl;
                    usage(argv[0]);
                }
-                if (verbose)
-                    std::cout << "Opening NanoVDB file named \"" << inputFile << "\"" << std::endl;
+                if (verbose) std::cout << "Opening NanoVDB file named \"" << inputFile << "\"" << std::endl;
                if (gridName.empty()) {
                    auto handles = nanovdb::io::readGrids(inputFile, verbose);
                    for (auto &h : handles) {
                        for (uint32_t i = 0; i < h.gridCount(); ++i) {
-                            if (verbose)
-                                std::cout << "Converting NanoVDB grid named \"" << h.gridMetaData(i)->shortGridName() << "\" to OpenVDB" << std::endl;
-                            grids->push_back(nanovdb::tools::nanoToOpenVDB(h, 0, i));
+                            if (verbose) std::cout << "Converting NanoVDB grid named \"" << h.gridMetaData(i)->shortGridName() << "\" to OpenVDB" << std::endl;
+                            grids->push_back(nanovdb::tools::nanoToOpenVDB(h, i));
                        }
                    }
                } else {
@@ -286,8 +297,7 @@ int main(int argc, char* argv[])
                        std::cerr << "File did not contain a NanoVDB grid named \"" << gridName << "\"\n" << std::endl;
                        usage(argv[0]);
                    }
-                    if (verbose)
-                        std::cout << "Converting NanoVDB grid named \"" << handle.gridMetaData()->shortGridName() << "\" to OpenVDB" << std::endl;
+                    if (verbose) std::cout << "Converting NanoVDB grid named \"" << handle.gridMetaData()->shortGridName() << "\" to OpenVDB" << std::endl;
                    grids->push_back(nanovdb::tools::nanoToOpenVDB(handle));
                }
            } // loop over input files
@@ -1,5 +1,5 @@
 // Copyright Contributors to the OpenVDB Project
-// SPDX-License-Identifier: MPL-2.0
+// SPDX-License-Identifier: Apache-2.0

 /*!
    \file   nanovdb_print.cc
@@ -166,7 +166,7 @@ int main(int argc, char* argv[])
            auto       classWidth = std::string("Class").length() + padding;
            auto       codecWidth = std::string("Codec").length() + padding;
            auto       ibboxWidth = std::string("Index Bounding Box").length() + padding;
-            auto       wbboxWidth = std::string("World BBox").length() + padding;
+            auto       wbboxWidth = std::string("World Bounding Box").length() + padding;
            auto       sizeWidth = std::string("Size").length() + padding;
            auto       fileWidth = std::string("File").length() + padding;
            auto       voxelsWidth = std::string("# Voxels").length() + padding;
@@ -175,6 +175,7 @@ int main(int argc, char* argv[])
            auto       configWidth = std::string("32^3->16^3->8^3").length() + padding;
            auto       tileWidth = std::string("# Active tiles").length() + padding;
            auto       resWidth = std::string("Resolution").length() + padding;
+            auto       blindWidth = std::string("Blind data count").length() + padding;
            for (auto& m : list) {
                width(nameWidth, m.gridName);
                width(typeWidth, nanovdb::toStr(str, m.gridType));
@@ -190,6 +191,7 @@ int main(int argc, char* argv[])
                width(tileWidth, nodesToStr(m.tileCount));
                width(voxelsWidth, std::to_string(m.voxelCount));
                width(voxelSizeWidth, Vec3dToStr(m.voxelSize));
+                width(blindWidth, std::to_string(m.blindDataCount));
            }
            std::cout << "\nThe file \"" << file << "\" contains the following ";
            if (list.size()>1) {
@@ -214,7 +216,8 @@ int main(int argc, char* argv[])
                std::cout << std::left << std::setw(configWidth) << "32^3->16^3->8^3"
                          << std::left << std::setw(tileWidth)   << "# Active tiles"
                          << std::left << std::setw(ibboxWidth)  << "Index Bounding Box"
-                          << std::left << std::setw(wbboxWidth)  << "World Bounding Box";
+                          << std::left << std::setw(wbboxWidth)  << "World Bounding Box"
+                          << std::left << std::setw(blindWidth)  << "Blind data count";
            }
            std::cout << std::endl;
            int n = 0;
@@ -238,11 +241,12 @@ int main(int argc, char* argv[])
                    std::cout << std::left << std::setw(configWidth) << nodesToStr(m.nodeCount)
                              << std::left << std::setw(tileWidth)   << nodesToStr(m.tileCount)
                              << std::left << std::setw(ibboxWidth)  << ibboxToStr(m.indexBBox)
-                              << std::left << std::setw(wbboxWidth)  << wbboxToStr(m.worldBBox);
+                              << std::left << std::setw(wbboxWidth)  << wbboxToStr(m.worldBBox)
+                              << std::left << std::setw(blindWidth)  << m.blindDataCount;
                }
                std::cout << std::endl;
-            }
-        }
+            }// loop over FileGridMetaData in list from a single file
+        }// loop over files
        if (verbose) {
            size_t w = 0;
            switch (mode) {
@@ -1,3 +1,5 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
 import argparse
 import os
 from pathlib import Path
@@ -1,4 +1,6 @@
 #!/bin/bash
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
 #Usage process all files in this directory or optionally specify a target directory

 # Define directory in which to find files
@@ -1,5 +1,5 @@
 // Copyright Contributors to the OpenVDB Project
-// SPDX-License-Identifier: MPL-2.0
+// SPDX-License-Identifier: Apache-2.0

 /*!
    \file   nanovdb_validate.cc
@@ -1,5 +1,5 @@
 // Copyright Contributors to the OpenVDB Project
-// SPDX-License-Identifier: MPL-2.0
+// SPDX-License-Identifier: Apache-2.0

 /*!
    \file DeviceBuffer.h
@@ -8,7 +8,7 @@

    \date January 8, 2020

-    \brief Implements a simple dual (host/device) CUDA buffer.
+    \brief DeviceBuffer has one pinned host buffer and multiple device CUDA buffers

    \note This file has no device-only kernel functions,
          which explains why it's a .h and not .cuh file.
@@ -17,6 +17,8 @@
 #ifndef NANOVDB_CUDA_DEVICEBUFFER_H_HAS_BEEN_INCLUDED
 #define NANOVDB_CUDA_DEVICEBUFFER_H_HAS_BEEN_INCLUDED

+#include <cuda.h>
+#include <memory>// for std::shared_ptr
 #include <nanovdb/HostBuffer.h>// for BufferTraits
 #include <nanovdb/util/cuda/Util.h>// for cudaMalloc/cudaMallocManaged/cudaFree

@@ -34,44 +36,80 @@ namespace cuda {// =============================================================
 class DeviceBuffer
 {
    uint64_t mSize; // total number of bytes managed by this buffer (assumed to be identical for host and device)
-    void *mCpuData, *mGpuData; // raw pointers to the host and device buffers
-    bool mManaged;
+    void *mCpuData, **mGpuData; // raw pointers to the host and device buffers
+    int   mDeviceCount, mManaged;// if mManaged is non-zero this class is responsible for allocating and freeing memory buffers. Otherwise this is assumed to be handled externally
+
+    /// @brief Initialize buffer
+    /// @param size byte size of buffer to be initialized
+    /// @param device id of the device on which to initialize the buffer
+    /// @note All existing buffers are first cleared
+    /// @warning size is expected to be non-zero. Use clear() clear buffer!
+    void init(uint64_t size, int device, cudaStream_t stream);

 public:
-    /// @brief Static factory method that return an instance of this buffer
-    /// @param size byte size of buffer to be initialized
-    /// @param dummy this argument is currently ignored but required to match the API of the HostBuffer
-    /// @param host If true buffer is initialized only on the host/CPU, else on the device/GPU
-    /// @param stream optional stream argument (defaults to stream NULL)
-    /// @return An instance of this class using move semantics
-    static DeviceBuffer create(uint64_t size, const DeviceBuffer* dummy = nullptr, bool host = true, void* stream = nullptr);

-    /// @brief Static factory method that return an instance of this buffer that wraps externally managed memory
-    /// @param size byte size of buffer specified by external memory
-    /// @param cpuData pointer to externally managed host memory
-    /// @param gpuData pointer to externally managed device memory
-    /// @return An instance of this class using move semantics
-    static DeviceBuffer create(uint64_t size, void* cpuData, void* gpuData);
+    using PtrT = std::shared_ptr<DeviceBuffer>;
+
+    /// @brief Default constructor of an empty buffer
+    DeviceBuffer() : mSize(0), mCpuData(nullptr), mGpuData(nullptr), mDeviceCount(0), mManaged(0){}
+
+    /// @brief Constructor with a specified device and size
+    /// @param size byte size of buffer to be initialized
+    /// @param device id of the device on which to initialize the buffer
+    /// @param stream cuda stream
+    DeviceBuffer(uint64_t size, int device = cudaCpuDeviceId, cudaStream_t stream = 0) : DeviceBuffer()
+    {
+        this->init(size, device, stream);
+    }

    /// @brief Constructor
    /// @param size byte size of buffer to be initialized
-    /// @param host If true buffer is initialized only on the host/CPU, else on the device/GPU
+    /// @param host If true buffer is initialized only on the host/CPU, else on the current device/GPU
    /// @param stream optional stream argument (defaults to stream NULL)
-    DeviceBuffer(uint64_t size = 0, bool host = true, void* stream = nullptr)
-        : mSize(0)
-        , mCpuData(nullptr)
-        , mGpuData(nullptr)
-        , mManaged(false)
+    DeviceBuffer(uint64_t size, bool host, void* stream) : DeviceBuffer()
    {
-        if (size > 0) this->init(size, host, stream);
+        int device = cudaCpuDeviceId;
+        if (!host) cudaCheck(cudaGetDevice(&device));
+        this->init(size, device, reinterpret_cast<cudaStream_t>(stream));
    }

+    /// @brief Constructor for externally managed host and device buffers
+    /// @param size byte size of the two external buffers
+    /// @param cpuData host buffer, assumed to NOT be NULL
+    /// @param gpuData device buffer, assumed to NOT be NULL;
+    /// @note The device buffer, @c gpuData, will be associated
+    ///       with the current device ID given by cudaGetDevice
    DeviceBuffer(uint64_t size, void* cpuData, void* gpuData)
        : mSize(size)
        , mCpuData(cpuData)
-        , mGpuData(gpuData)
-        , mManaged(false)
+        , mManaged(0)
    {
+        cudaCheck(cudaGetDeviceCount(&mDeviceCount));
+        mGpuData = new void*[mDeviceCount]();// NULL initialization
+        NANOVDB_ASSERT(cpuData);
+        NANOVDB_ASSERT(gpuData);
+        int device = 0;
+        cudaCheck(cudaGetDevice(&device));
+        mGpuData[device] = gpuData;
+    }
+
+    /// @brief Constructor for externally managed host and multiple device buffers
+    /// @param size byte size of the two external buffers
+    /// @param cpuData host buffer, assumed to NOT be NULL
+    /// @param list list of device IDs and external device buffers, all assumed to not be NULL
+    DeviceBuffer(uint64_t size, void* cpuData, std::initializer_list<std::pair<int,void*>> list)
+        : mSize(size)
+        , mCpuData(cpuData)
+        , mManaged(0)
+    {
+        NANOVDB_ASSERT(cpuData);
+        cudaCheck(cudaGetDeviceCount(&mDeviceCount));
+        mGpuData = new void*[mDeviceCount]();// NULL initialization
+        for (auto &p : list) {
+            NANOVDB_ASSERT(p.first>=0 && p.first<mDeviceCount);
+            NANOVDB_ASSERT(p.second);
+            mGpuData[p.first] = p.second;
+        }
    }

    /// @brief Disallow copy-construction
@@ -82,138 +120,283 @@ public:
        : mSize(other.mSize)
        , mCpuData(other.mCpuData)
        , mGpuData(other.mGpuData)
+        , mDeviceCount(other.mDeviceCount)
        , mManaged(other.mManaged)
    {
-        other.mSize = 0;
-        other.mCpuData = nullptr;
-        other.mGpuData = nullptr;
-        other.mManaged = false;
+        other.mCpuData = other.mGpuData = nullptr;
+        other.mSize = other.mDeviceCount = other.mManaged = 0;
    }

+    /// @brief Copy-constructor from a HostBuffer
+    /// @param buffer host buffer from which to copy data
+    /// @param device id of the device on which to initialize the buffer
+    /// @param stream cuda stream
+    DeviceBuffer(const HostBuffer& buffer, int device = cudaCpuDeviceId, cudaStream_t stream = 0)
+        : DeviceBuffer(buffer.size(), device, stream)
+    {
+        if (mCpuData) {
+            cudaCheck(cudaMemcpy(mCpuData, buffer.data(), mSize, cudaMemcpyHostToHost));
+        } else if (mGpuData[device]) {
+            cudaCheck(cudaMemcpyAsync(mGpuData[device], buffer.data(), mSize, cudaMemcpyHostToDevice, stream));
+        }
+    }
+
+     /// @brief Destructor frees memory on both the host and device
+    ~DeviceBuffer() { this->clear(); };
+
+    /// @brief Static factory method that return an instance of this buffer
+    /// @param size byte size of buffer to be initialized
+    /// @param dummy this argument is currently ignored but required to match the API of the HostBuffer
+    /// @param host If true buffer is initialized only on the host/CPU, else only on the device/GPU
+    /// @param stream optional stream argument (defaults to stream NULL)
+    /// @return An instance of this class using move semantics
+    static DeviceBuffer create(uint64_t size, const DeviceBuffer* dummy, bool host, void* stream){return DeviceBuffer(size, host, stream);}
+
+    /// @brief Static factory method that returns an instance of this buffer
+    /// @param size byte size of buffer to be initialized
+    /// @param dummy this argument is currently ignored but required to match the API of the HostBuffer
+    /// @param device id of the device on which to initialize the buffer
+    /// @param stream cuda stream
+    static DeviceBuffer create(uint64_t size, const DeviceBuffer* dummy = nullptr, int device = cudaCpuDeviceId, cudaStream_t stream = 0){return DeviceBuffer(size, device, stream);}
+
+    /// @brief Static factory method that returns an instance of this buffer that wraps externally managed memory
+    /// @param size byte size of buffer specified by external memory
+    /// @param cpuData pointer to externally managed host memory
+    /// @param gpuData pointer to externally managed device memory
+    /// @return An instance of this class using move semantics
+    static DeviceBuffer create(uint64_t size, void* cpuData, void* gpuData) {return DeviceBuffer(size, cpuData, gpuData);}
+
+    /// @brief  Static factory method that returns an instance of this buffer that wraps externally managed host and device memory
+    /// @param size byte size of buffer to be initialized
+    /// @param cpuData  pointer to externally managed host memory
+    /// @param list list of device IDs and device memory pointers
+    static DeviceBuffer create(uint64_t size, void* cpuData, std::initializer_list<std::pair<int,void*>> list) {return DeviceBuffer(size, cpuData, list);}
+
+    /// @brief Static factory method that returns an instance of this buffer constructed from a HostBuffer
+    /// @param buffer host buffer from which to copy data
+    /// @param device id of the device on which to initialize the buffer
+    /// @param stream cuda stream
+    static DeviceBuffer create(const HostBuffer& buffer, int device = cudaCpuDeviceId, cudaStream_t stream = 0) {return DeviceBuffer(buffer, device, stream);}
+
+    ///////////////////////////////////////////////////////////////////////
+
+    /// @{
+    /// @brief Factory methods that create a shared pointer to an DeviceBuffer instance
+    static PtrT createPtr(uint64_t size, const DeviceBuffer* = nullptr, int device = cudaCpuDeviceId, cudaStream_t stream = 0) {return std::make_shared<DeviceBuffer>(size, device, stream);}
+    static PtrT createPtr(uint64_t size, void* cpuData, void* gpuData) {return std::make_shared<DeviceBuffer>(size, cpuData, gpuData);}
+    static PtrT createPtr(uint64_t size, void* cpuData, std::initializer_list<std::pair<int,void*>> list) {return std::make_shared<DeviceBuffer>(size, cpuData, list);}
+    static PtrT createPtr(const HostBuffer& buffer, int device = cudaCpuDeviceId, cudaStream_t stream = 0) {return std::make_shared<DeviceBuffer>(buffer, device, stream);}
+    /// @}
+
+    ///////////////////////////////////////////////////////////////////////
+
    /// @brief Disallow copy assignment operation
    DeviceBuffer& operator=(const DeviceBuffer&) = delete;

    /// @brief Move copy assignment operation
-    DeviceBuffer& operator=(DeviceBuffer&& other) noexcept
-    {
-        this->clear();
-        mSize = other.mSize;
-        mCpuData = other.mCpuData;
-        mGpuData = other.mGpuData;
-        mManaged = other.mManaged;
-        other.mSize = 0;
-        other.mCpuData = nullptr;
-        other.mGpuData = nullptr;
-        other.mManaged = false;
-        return *this;
-    }
+    DeviceBuffer& operator=(DeviceBuffer&& other) noexcept;

-    /// @brief Destructor frees memory on both the host and device
-    ~DeviceBuffer() { this->clear(); };
+    ///////////////////////////////////////////////////////////////////////

-    /// @brief Initialize buffer
-    /// @param size byte size of buffer to be initialized
-    /// @param host If true buffer is initialized only on the host/CPU, else on the device/GPU
-    /// @note All existing buffers are first cleared
-    /// @warning size is expected to be non-zero. Use clear() clear buffer!
-    void init(uint64_t size, bool host = true, void* stream = nullptr);
-
-    /// @brief Retuns a raw pointer to the host/CPU buffer managed by this allocator.
+    /// @brief Retuns a raw void pointer to the host/CPU buffer managed by this allocator.
    /// @warning Note that the pointer can be NULL!
    void* data() const { return mCpuData; }

-    /// @brief Retuns a raw pointer to the device/GPU buffer managed by this allocator.
+    /// @brief Returns an offset pointer of a specific type from the allocated host memory
+    /// @tparam T Type of the pointer returned
+    /// @param count Numbers of elements of @c parameter type T to skip
+    /// @warning might return NULL
+    template <typename T>
+    T* data(ptrdiff_t count = 0, int device = cudaCpuDeviceId) const
+    {
+        NANOVDB_ASSERT(device >= cudaCpuDeviceId && device < mDeviceCount);
+        void *ptr = device == cudaCpuDeviceId ? mCpuData : mGpuData[device];
+        return ptr ? reinterpret_cast<T*>(ptr) + count : nullptr;
+    }
+
+    /// @brief Returns a byte offset void pointer from the allocated host memory
+    /// @param byteOffset offset of return pointer in units of bytes
+    /// @warning assumes that this instance is not empty!
+    void* data(ptrdiff_t byteOffset, int device = cudaCpuDeviceId) const
+    {
+        NANOVDB_ASSERT(device >= cudaCpuDeviceId && device < mDeviceCount);
+        void *ptr = device == cudaCpuDeviceId ? mCpuData : mGpuData[device];
+        return ptr ? reinterpret_cast<char*>(ptr) + byteOffset : nullptr;
+    }
+
+    ///////////////////////////////////////////////////////////////////////
+
+    /// @brief Retuns a raw pointer to the specified device/GPU buffer managed by this allocator.
    /// @warning Note that the pointer can be NULL!
-    void* deviceData() const { return mGpuData; }
+    void* deviceData(int device) const {
+        NANOVDB_ASSERT(device >= 0 && device < mDeviceCount);
+        return mGpuData[device];
+    }

-    /// @brief  Upload this buffer from the host to the device, i.e. CPU -> GPU.
-    /// @param stream optional CUDA stream (defaults to CUDA stream 0)
-    /// @param sync if false the memory copy is asynchronous
-    /// @note If the device/GPU buffer does not exist it is first allocated
-    /// @warning Assumes that the host/CPU buffer already exists
-    void deviceUpload(void* stream = nullptr, bool sync = true) const;
+    /// @brief Retuns a raw pointer to the current device/GPU buffer managed by this allocator.
+    /// @warning Note that the pointer can be NULL!
+    void* deviceData() const {
+        int device = cudaCpuDeviceId;
+        cudaCheck(cudaGetDevice(&device));
+        return this->deviceData(device);
+    }

-    /// @brief Upload this buffer from the device to the host, i.e. GPU -> CPU.
-    /// @param stream optional CUDA stream (defaults to CUDA stream 0)
+    ///////////////////////////////////////////////////////////////////////
+
+    /// @brief Uploads buffer on the host to a specific device. If it doesn't exist it's created first.
+    /// @param device Device ID that the data is copied to
+    /// @param stream cuda stream
+    /// @param sync if false the memory copy is asynchronous.
+    /// @warning Assumes that the host buffer already exists!
+    /// @note determine the current device with cudaGetDevice
+    void deviceUpload(int device = 0, cudaStream_t stream = 0, bool sync = true);
+    void deviceUpload(int device, void* stream, bool sync){this->deviceUpload(device, cudaStream_t(stream), sync);}
+
+    /// @brief Upload buffer from the host to ALL the existing devices, i.e. CPU -> GPU.
+    ///        If no device buffers exist one is created for the current device (typically 0)
+    ///        and subsequently populated with the host data.
+    /// @param stream CUDA stream.
+    /// @param sync if false the memory copy is asynchronous.
+    /// @warning Assumes that the host buffer already exists!
+    void deviceUpload(cudaStream_t stream, bool sync);
+    void deviceUpload(void* stream, bool sync) {this->deviceUpload(cudaStream_t(stream), sync);}
+
+    ///////////////////////////////////////////////////////////////////////
+
+    /// @brief Download data from a specified device to the host. If the host buffer des not exist it will first be allocated
+    /// @param device device ID to download source data from
+    /// @param stream cuda stream
+    /// @param sync if false the memory copy is asynchronous.
+    /// @warning Assumes that the specifed device buffer already exists!
+    void deviceDownload(int device = 0, cudaStream_t stream = 0, bool sync = true);
+    void deviceDownload(int device, void* stream , bool sync) {this->deviceDownload(device, cudaStream_t(stream), sync);}
+
+    /// @brief Download the buffer from the current device to the host, i.e. GPU -> CPU.
+    ///        If the host buffer des not exist it will first be allocated
+    /// @param stream CUDA stream
    /// @param sync if false the memory copy is asynchronous
    /// @note If the host/CPU buffer does not exist it is first allocated
    /// @warning Assumes that the device/GPU buffer already exists
-    void deviceDownload(void* stream = nullptr, bool sync = true) const;
+    void deviceDownload(void* stream, bool sync);
+
+    ///////////////////////////////////////////////////////////////////////

    /// @brief Returns the size in bytes of the raw memory buffer managed by this allocator.
    uint64_t size() const { return mSize; }
+    uint64_t capacity() const {return this->size();}

-    //@{
+    /// @brief Returns the number of buffers that are not NULL
+    int bufferCount() const {
+        int count = mCpuData ? 1 : 0;
+        for (int i=0; i<mDeviceCount; ++i) if (mGpuData[i]) ++count;
+        return count;
+    }
+
+    int deviceCount() const {return mDeviceCount;}
+
+    /// @{
    /// @brief Returns true if this allocator is empty, i.e. has no allocated memory
    bool empty() const { return mSize == 0; }
-    bool isEmpty() const { return mSize == 0; }
-    //@}
+    bool isEmpty() const { return this->empty(); }
+    /// @}

    /// @brief De-allocate all memory managed by this allocator and set all pointers to NULL
-    void clear(void* stream = nullptr);
+    void clear(cudaStream_t stream = 0);
+    void clear(void* stream){this->clear(cudaStream_t(stream));}

 }; // DeviceBuffer class

 // --------------------------> Implementations below <------------------------------------

-inline DeviceBuffer DeviceBuffer::create(uint64_t size, const DeviceBuffer*, bool host, void* stream)
+inline DeviceBuffer& DeviceBuffer::operator=(DeviceBuffer&& other) noexcept
 {
-    return DeviceBuffer(size, host, stream);
+    if (mManaged) {// first free all the managed data buffers
+        cudaCheck(cudaFreeHost(mCpuData));
+        for (int i=0; i<mDeviceCount; ++i) cudaCheck(util::cuda::freeAsync(mGpuData[i], 0));
+    }
+    delete [] mGpuData;
+    mSize    = other.mSize;
+    mCpuData = other.mCpuData;
+    mGpuData = other.mGpuData;
+    mDeviceCount = other.mDeviceCount;
+    mManaged = other.mManaged;
+    other.mCpuData = nullptr;
+    other.mGpuData = nullptr;
+    other.mSize = 0;
+    other.mDeviceCount = 0;
+    other.mManaged = 0;
+    return *this;
 }

-inline DeviceBuffer DeviceBuffer::create(uint64_t size, void* cpuData, void* gpuData)
+inline void DeviceBuffer::init(uint64_t size, int device, cudaStream_t stream)
 {
-    return DeviceBuffer(size, cpuData, gpuData);
-}
-
-inline void DeviceBuffer::init(uint64_t size, bool host, void* stream)
-{
-    if (mSize>0) this->clear(stream);
-    NANOVDB_ASSERT(size > 0);
-    if (host) {
+    if (size==0) return;
+    cudaCheck(cudaGetDeviceCount(&mDeviceCount));
+    mGpuData = new void*[mDeviceCount]();// NULL initialization
+    NANOVDB_ASSERT(device >= cudaCpuDeviceId && device < mDeviceCount);
+    if (device == cudaCpuDeviceId) {
        cudaCheck(cudaMallocHost((void**)&mCpuData, size)); // un-managed pinned memory on the host (can be slow to access!). Always 32B aligned
        checkPtr(mCpuData, "cuda::DeviceBuffer::init: failed to allocate host buffer");
    } else {
-        cudaCheck(util::cuda::mallocAsync((void**)&mGpuData, size, reinterpret_cast<cudaStream_t>(stream))); // un-managed memory on the device, always 32B aligned!
-        checkPtr(mGpuData, "cuda::DeviceBuffer::init: failed to allocate device buffer");
+        cudaCheck(util::cuda::mallocAsync(mGpuData+device, size, stream)); // un-managed memory on the device, always 32B aligned!
+        checkPtr(mGpuData[device], "cuda::DeviceBuffer::init: failed to allocate device buffer");
    }
    mSize = size;
-    mManaged = true;
+    mManaged = 1;// i.e. this instance is responsible for allocating and delete memory
 } // DeviceBuffer::init

-inline void DeviceBuffer::deviceUpload(void* stream, bool sync) const
+inline void DeviceBuffer::deviceUpload(int device, cudaStream_t stream, bool sync)
 {
-    if (!mManaged) throw std::runtime_error("DeviceBuffer::deviceUpload called on externally managed memory. Replace deviceUpload call with the appropriate external copy operation.");
-
-    checkPtr(mCpuData, "uninitialized cpu data");
-    if (mGpuData == nullptr) {
-        cudaCheck(util::cuda::mallocAsync((void**)&mGpuData, mSize, reinterpret_cast<cudaStream_t>(stream))); // un-managed memory on the device, always 32B aligned!
+    NANOVDB_ASSERT(device >= 0 && device < mDeviceCount);// should be device and not the host
+    checkPtr(mCpuData, "uninitialized cpu source data");
+    if (mGpuData[device] == nullptr) {
+        if (mManaged==0) throw std::runtime_error("DeviceBuffer::deviceUpload called on externally managed memory that wasn\'t allocated.");
+        cudaCheck(util::cuda::mallocAsync(mGpuData+device, mSize, stream)); // un-managed memory on the device, always 32B aligned!
    }
-    checkPtr(mGpuData, "uninitialized gpu data");
-    cudaCheck(cudaMemcpyAsync(mGpuData, mCpuData, mSize, cudaMemcpyHostToDevice, reinterpret_cast<cudaStream_t>(stream)));
-    if (sync) cudaCheck(cudaStreamSynchronize(reinterpret_cast<cudaStream_t>(stream)));
-} // DeviceBuffer::gpuUpload
+    checkPtr(mGpuData[device], "uninitialized gpu destination data");
+    cudaCheck(cudaMemcpyAsync(mGpuData[device], mCpuData, mSize, cudaMemcpyHostToDevice, stream));
+    if (sync) cudaCheck(cudaStreamSynchronize(stream));
+} // DeviceBuffer::deviceUpload

-inline void DeviceBuffer::deviceDownload(void* stream, bool sync) const
+inline void DeviceBuffer::deviceUpload(cudaStream_t stream, bool sync)
 {
-    if (!mManaged) throw std::runtime_error("DeviceBuffer::deviceDownload called on externally managed memory. Replace deviceDownload call with the appropriate external copy operation.");
+    int device = 0;
+    cudaGetDevice(&device);
+    this->deviceUpload(device, stream, sync);
+} // DeviceBuffer::deviceUpload

-    checkPtr(mGpuData, "uninitialized gpu data");
+inline void DeviceBuffer::deviceDownload(int device, cudaStream_t stream, bool sync)
+{
+    NANOVDB_ASSERT(device >= 0 && device < mDeviceCount);
+    checkPtr(mGpuData[device], "uninitialized gpu source data");// no source data on the specified device
    if (mCpuData == nullptr) {
+        if (mManaged==0) throw std::runtime_error("DeviceBuffer::deviceDownload called on uninitialized cpu destination memory that is externally managed.");
        cudaCheck(cudaMallocHost((void**)&mCpuData, mSize)); // un-managed pinned memory on the host (can be slow to access!). Always 32B aligned
    }
-    checkPtr(mCpuData, "uninitialized cpu data");
-    cudaCheck(cudaMemcpyAsync(mCpuData, mGpuData, mSize, cudaMemcpyDeviceToHost, reinterpret_cast<cudaStream_t>(stream)));
-    if (sync) cudaCheck(cudaStreamSynchronize(reinterpret_cast<cudaStream_t>(stream)));
-} // DeviceBuffer::gpuDownload
+    checkPtr(mCpuData, "uninitialized cpu destination data");
+    cudaCheck(cudaMemcpyAsync(mCpuData, mGpuData[device], mSize, cudaMemcpyDeviceToHost, stream));
+    if (sync) cudaCheck(cudaStreamSynchronize(stream));
+} // DeviceBuffer::deviceDownload

-inline void DeviceBuffer::clear(void *stream)
+inline void DeviceBuffer::deviceDownload(void* stream, bool sync)
 {
-    if (mManaged && mGpuData) cudaCheck(util::cuda::freeAsync(mGpuData, reinterpret_cast<cudaStream_t>(stream)));
-    if (mManaged && mCpuData) cudaCheck(cudaFreeHost(mCpuData));
-    mCpuData = mGpuData = nullptr;
+    int device = 0;
+    cudaCheck(cudaGetDevice(&device));
+    this->deviceDownload(device, cudaStream_t(stream), sync);
+} // DeviceBuffer::deviceDownload
+
+inline void DeviceBuffer::clear(cudaStream_t stream)
+{
+    if (mManaged) {// free all the managed data buffers
+        cudaCheck(cudaFreeHost(mCpuData));
+        for (int i=0; i<mDeviceCount; ++i) cudaCheck(util::cuda::freeAsync(mGpuData[i], stream));
+    }
+    delete [] mGpuData;
+    mCpuData = nullptr;
+    mGpuData = nullptr;
    mSize = 0;
-    mManaged = false;
+    mDeviceCount = 0;
+    mManaged = 0;
 } // DeviceBuffer::clear

 }// namespace cuda
@@ -0,0 +1,229 @@
+// Copyright Contributors to the OpenVDB Project
+// SPDX-License-Identifier: Apache-2.0
+
+/*!
+    \file DeviceMesh.h
+
+    \brief nanovdb::cuda::DeviceMesh encapsulates device IDs, CUDA streams,
+           and NCCL communicators in order to facilitate multi-GPU applications.
+*/
+
+#ifndef NANOVDB_CUDA_DEVICEMESH_H_HAS_BEEN_INCLUDED
+#define NANOVDB_CUDA_DEVICEMESH_H_HAS_BEEN_INCLUDED
+
+#include <vector>
+#include <algorithm>
+#include <thread>
+
+#include <nanovdb/util/cuda/Util.h>
+#ifdef NANOVDB_USE_NCCL
+#include <nccl.h>
+#endif
+
+namespace nanovdb {
+
+namespace cuda {
+
+namespace detail {
+
+/// @brief RAII class that caches/restores the current device at construction/destruction
+class DeviceGuard {
+    public:
+        DeviceGuard() { cudaGetDevice(&deviceId); }
+        ~DeviceGuard() { cudaSetDevice(deviceId); }
+
+        /// @{
+        /// @brief DeviceGuard is not copyable nor movable
+        DeviceGuard(const DeviceGuard&) = delete;
+        DeviceGuard& operator=(const DeviceGuard&) = delete;
+        DeviceGuard(DeviceGuard&& other) = delete;
+        DeviceGuard& operator=(DeviceGuard&& other) = delete;
+        /// @}
+    private:
+        int deviceId = -1;
+};
+
+}
+
+/// @brief POD struct representing a device id and a stream on that device
+struct DeviceNode
+{
+    int id = -1;
+    cudaStream_t stream = cudaStream_t(0);
+};
+
+/// @brief This class wraps a vector of per-device IDs and CUDA streams and holds inter-device connectivity information and NCCL comms.
+class DeviceMesh
+{
+    using DeviceVecT = std::vector<DeviceNode>;
+    using const_iterator = DeviceVecT::const_iterator;
+    using size_type = DeviceVecT::size_type;
+public:
+    /// @brief Constructs a device mesh for all devices on the host and initializes a CUDA stream (and NCCL communicator) for each device
+    DeviceMesh();
+    /// @brief Destroys the per-device CUDA stream and finalizes and destroys the per-device NCCL communicators
+    ~DeviceMesh();
+    /// @brief Disallow copy-construction
+    DeviceMesh(const DeviceMesh&) = delete;
+    /// @brief Move constructor.  Underlying CUDA streams and NCCL communicators are not reinitialized.
+    /// @param DeviceMesh instance that will be moved into this DeviceMesh.
+    DeviceMesh(DeviceMesh&&) noexcept;
+    /// @brief Disallow copy-assignment
+    DeviceMesh& operator=(const DeviceMesh&) = delete;
+    /// @brief Move assignment. Underlying CUDA streams and NCCL communicators are not reinitialized.
+    /// @param DeviceMesh instance that will be moved into this DeviceMesh.
+    DeviceMesh& operator=(DeviceMesh&&) noexcept;
+
+    /// @brief Returns the number of devices
+    size_type deviceCount() const { return mDeviceNodes.size(); };
+
+    /// @brief Returns a const reference to the DeviceNode for a particular device
+    const DeviceNode& operator [](int deviceId) const { return mDeviceNodes[deviceId]; }
+
+    /// @brief Returns an iterator to the first device node
+    const_iterator begin() const { return mDeviceNodes.begin(); };
+    /// @brief Returns an iterator past the last device node
+    const_iterator end() const { return mDeviceNodes.end(); };
+
+#ifdef NANOVDB_USE_NCCL
+    /// @brief Returns the NCCL communicator for a particular device
+    ncclComm_t comm(int deviceId) const { return mComms[deviceId]; }
+#endif
+
+    /// @brief Returns whether or not peer to peer access is supported between two devices.
+    bool canAccessPeer(int deviceId, int peerId) const { return mConnectivity[deviceId][peerId] & (1 << cudaDevP2PAttrAccessSupported); }
+
+private:
+    /// @brief Returns whether or not a device supports managed memory
+    bool hasManagedMemory(int deviceId) const {
+        int managedMemoryValue = 0;
+        cudaDeviceGetAttribute(&managedMemoryValue, cudaDevAttrManagedMemory, deviceId);
+        return static_cast<bool>(managedMemoryValue);
+    }
+
+    DeviceVecT mDeviceNodes;
+
+#ifdef NANOVDB_USE_NCCL
+    std::vector<ncclComm_t> mComms;
+#endif
+    std::vector<std::vector<uint32_t>> mConnectivity;
+};
+
+inline DeviceMesh::DeviceMesh()
+{
+    detail::DeviceGuard deviceGuard;
+
+    int deviceCount = -1;
+    cudaGetDeviceCount(&deviceCount);
+
+    for (int deviceId = 0; deviceId < deviceCount; ++deviceId) {
+        NANOVDB_ASSERT(hasManagedMemory(deviceId));
+        cudaSetDevice(deviceId);
+        cudaStream_t stream;
+        cudaStreamCreate(&stream);
+        mDeviceNodes.push_back({deviceId, stream});
+    }
+
+#ifdef NANOVDB_USE_NCCL
+    mComms.resize(deviceCount);
+    ncclCommInitAll(mComms.data(), deviceCount, nullptr);
+#endif
+
+    mConnectivity.resize(deviceCount);
+    for (const auto& deviceNode : mDeviceNodes) {
+        mConnectivity[deviceNode.id].resize(deviceCount);
+        for (const auto& peerNode : mDeviceNodes) {
+            mConnectivity[deviceNode.id][peerNode.id] = 0;
+            if (deviceNode.id != peerNode.id) {
+                int peerAccessSupportedValue = 0;
+                cudaDeviceGetP2PAttribute(&peerAccessSupportedValue, cudaDevP2PAttrAccessSupported, deviceNode.id, peerNode.id);
+                if (peerAccessSupportedValue) {
+                    mConnectivity[deviceNode.id][peerNode.id] |= (1 << cudaDevP2PAttrAccessSupported);
+                }
+            }
+        }
+    }
+}
+
+inline DeviceMesh::DeviceMesh(DeviceMesh&& other) noexcept
+    : mDeviceNodes(std::move(other.mDeviceNodes)),
+#ifdef NANOVDB_USE_NCCL
+      mComms(std::move(other.mComms)),
+#endif
+      mConnectivity(std::move(other.mConnectivity))
+{
+}
+
+inline DeviceMesh::~DeviceMesh()
+{
+    detail::DeviceGuard deviceGuard;
+
+#ifdef NANOVDB_USE_NCCL
+    std::for_each(mComms.begin(), mComms.end(), [](ncclComm_t comm) {
+        ncclCommFinalize(comm);
+        ncclCommDestroy(comm);
+    });
+#endif
+
+    std::for_each(mDeviceNodes.begin(), mDeviceNodes.end(), [](DeviceNode& deviceNode) {
+        cudaSetDevice(deviceNode.id);
+        cudaStreamDestroy(deviceNode.stream);
+    });
+}
+
+inline DeviceMesh& DeviceMesh::operator=(DeviceMesh&& other) noexcept
+{
+    mDeviceNodes = std::move(other.mDeviceNodes);
+#ifdef NANOVDB_USE_NCCL
+    mComms = std::move(other.mComms);
+#endif
+    mConnectivity = std::move(other.mConnectivity);
+    return *this;
+}
+
+namespace detail {
+
+inline void* queryAllocationGranularityEntryPoint()
+{
+    void* entryPoint = nullptr;
+#if CUDART_VERSION >= 12500 // cudaGetDriverEntryPointByVersion was added in CUDA 12.5 with cudaGetDriverEntryPoint being potentially deprecated
+    cudaDriverEntryPointQueryResult queryResult = cudaDriverEntryPointSymbolNotFound;
+    cudaCheck(cudaGetDriverEntryPointByVersion("cuMemGetAllocationGranularity", &entryPoint, 12000, cudaEnableDefault, &queryResult));
+    NANOVDB_ASSERT(queryResult == cudaDriverEntryPointSuccess);
+#elif CUDART_VERSION >= 12000 // queryResult argument was added in CUDA 12
+    cudaDriverEntryPointQueryResult queryResult = cudaDriverEntryPointSymbolNotFound;
+    cudaCheck(cudaGetDriverEntryPoint("cuMemGetAllocationGranularity", &entryPoint, cudaEnableDefault, &queryResult));
+    NANOVDB_ASSERT(queryResult == cudaDriverEntryPointSuccess);
+#else
+    cudaCheck(cudaGetDriverEntryPoint("cuMemGetAllocationGranularity", &entryPoint, cudaEnableDefault));
+#endif
+    return entryPoint;
+}// queryAllocationGranularityEntryPoint
+
+}
+
+/// @brief Returns the minimum page size (in bytes) across all devices on the system
+inline size_t minDevicePageSize(const DeviceMesh& mesh)
+{
+    using FuncT = CUresult(size_t*, const CUmemAllocationProp*, CUmemAllocationGranularity_flags);
+    static FuncT* functPtr = reinterpret_cast<FuncT*>(detail::queryAllocationGranularityEntryPoint());
+
+    NANOVDB_ASSERT(functPtr);
+    CUmemAllocationProp prop = {};
+    prop.type = CU_MEM_ALLOCATION_TYPE_PINNED;
+    prop.location.type = CU_MEM_LOCATION_TYPE_DEVICE;
+    size_t minGranularity = 0;
+    for (const auto& node : mesh) {
+        prop.location.id = node.id;
+        size_t granularity = 0;
+        (*functPtr)(&granularity, &prop, CU_MEM_ALLOC_GRANULARITY_MINIMUM);
+        if (minGranularity < granularity) minGranularity = granularity;
+    }
+    return minGranularity;
+}// minDevicePageSize
+
+} // namespace cuda
+
+} // namespace nanovdb
+
+#endif // end of NANOVDB_CUDA_DEVICEMESH_H_HAS_BEEN_INCLUDED
@@ -0,0 +1,35 @@
+// Copyright Contributors to the OpenVDB Project
+// SPDX-License-Identifier: Apache-2.0
+//
+#ifndef NANOVDB_CUDA_DEVICERESOURCE_H_HAS_BEEN_INCLUDED
+#define NANOVDB_CUDA_DEVICERESOURCE_H_HAS_BEEN_INCLUDED
+
+#include <cuda_runtime_api.h>
+#include <nanovdb/util/cuda/Util.h>
+
+namespace nanovdb {
+
+namespace cuda {
+
+class DeviceResource
+{
+public:
+    // cudaMalloc aligns memory to 256 bytes by default
+    static constexpr size_t DEFAULT_ALIGNMENT = 256;
+
+    static void* allocateAsync(size_t bytes, size_t, cudaStream_t stream) {
+        void* p = nullptr;
+        cudaCheck(util::cuda::mallocAsync(&p, bytes, stream));
+        return p;
+    }
+
+    static void deallocateAsync(void *p, size_t, size_t, cudaStream_t stream) {
+        cudaCheck(util::cuda::freeAsync(p, stream));
+    }
+};
+
+}
+
+} // namespace nanovdb::cuda
+
+#endif // end of NANOVDB_CUDA_DEVICERESOURCE_H_HAS_BEEN_INCLUDED
@@ -0,0 +1,124 @@
+// Copyright Contributors to the OpenVDB Project
+// SPDX-License-Identifier: Apache-2.0
+
+/*!
+    \file DeviceStreamMap.h
+
+    \author Ken Museth
+
+    \date October 15, 2024
+
+    \brief nanovdb::cuda::DeviceStreamMap maps device IDs to CUDA streams,
+           which is useful for multi-GPU applications.
+*/
+
+#ifndef NANOVDB_CUDA_DEVICESTREAMMAP_H_HAS_BEEN_INCLUDED
+#define NANOVDB_CUDA_DEVICESTREAMMAP_H_HAS_BEEN_INCLUDED
+
+#include <cuda.h>
+#include <map>
+#include <vector>
+#include <nanovdb/util/cuda/Util.h>// for cudaCheck, deviceCount etc
+
+namespace nanovdb {// ================================================================
+
+namespace cuda {// ===================================================================
+
+/// @brief map from a device ID to an associated cuda stream. Useful for multi-GPU applications.
+class DeviceStreamMap : public std::map<int, cudaStream_t>
+{
+     using FuncT = CUresult(size_t*, const CUmemAllocationProp*, CUmemAllocationGranularity_flags);
+     FuncT *mFunctPtr;
+
+public:
+
+    enum DeviceType { Any = 0, PeerToPeer = 1, Unified = 3 };
+
+    /// @brief Initiates a map between CUDA device IDs and corresponding streams that satisfy certain constraints.
+    ///        All devices should be able to access memory on all the other devices!
+    /// @param t Type of device to include in map. Any means all available devices, PeerToPeer means only devices that
+    ///          access all another devices are included, and Unified means all devices support unified memory, concurrent access,
+    ///          and can be access by all other devices.
+    /// @param exclude optional list of device IDs to exclude from the map
+    /// @param verbose  0 means quiet, 1 means print if a device is ignores and 2 means print is a device is included
+    DeviceStreamMap(DeviceType t = DeviceType::Unified, std::vector<int> exclude = {}, int verbose = 0);
+
+    /// @brief Destructor
+    ~DeviceStreamMap();
+
+    /// @brief returns the minimum page size of all the devices in this map
+    size_t getMinPageSize() const;
+
+    /// @brief Print information about all the devices included in this map
+    void printDevInfo(std::FILE* file = stdout) const {for (auto &p : *this) util::cuda::printDevInfo(p.first, nullptr, file);}
+
+    /// @brief Returns the number of device associated with this map
+    int deviceCount() const {return this->size();}
+
+};// DeviceStreamMap
+
+DeviceStreamMap::DeviceStreamMap(DeviceType t, std::vector<int> exclude, int verbose)
+{
+    std::initializer_list<cudaDeviceAttr> filter = {cudaDevAttrUnifiedAddressing, cudaDevAttrConcurrentManagedAccess};
+    const int devCount = util::cuda::deviceCount(), current = util::cuda::currentDevice();
+    for (int dev = 0; dev < devCount; ++dev) {
+        int check = 1;
+        for (auto it=exclude.begin();         check && it!=exclude.end(); ++it) if (dev == *it) check = 0;
+        for (auto it=filter.begin(); (t&2) && check && it!=filter.end(); ++it) cudaCheck(cudaDeviceGetAttribute( &check, *it, dev));
+        for (auto it= this->begin(); (t&1) && check && it!= this->end(); ++it) cudaCheck(cudaDeviceCanAccessPeer(&check, dev, it->first));
+        if (check) {
+            cudaCheck(cudaSetDevice(dev));
+            cudaStream_t stream;
+            cudaCheck(cudaStreamCreate(&stream));
+            if (verbose>1) util::cuda::printDevInfo(dev, "Using");
+            (*this)[dev] = stream;
+        } else if (verbose) util::cuda::printDevInfo(dev, "Ignoring");
+    }
+    cudaCheck(cudaSetDevice(current));// reset to the previous device
+
+    void* entryPoint = nullptr;
+#if CUDART_VERSION >= 13000
+    cudaDriverEntryPointQueryResult queryResult;
+    cudaCheck(cudaGetDriverEntryPointByVersion("cuMemGetAllocationGranularity", &entryPoint, 13000, cudaEnableDefault, &queryResult));
+    NANOVDB_ASSERT(queryResult == cudaDriverEntryPointSuccess);
+#elif CUDART_VERSION >= 12000// queryResult argument was added in CUDA 12
+    cudaDriverEntryPointQueryResult queryResult;
+    cudaCheck(cudaGetDriverEntryPoint("cuMemGetAllocationGranularity", &entryPoint, cudaEnableDefault, &queryResult));
+    NANOVDB_ASSERT(queryResult == cudaDriverEntryPointSuccess);
+#else
+    cudaCheck(cudaGetDriverEntryPoint("cuMemGetAllocationGranularity", &entryPoint, cudaEnableDefault));
+#endif
+    mFunctPtr = reinterpret_cast<FuncT*>(entryPoint);
+}// DeviceStreamMap::DeviceStreamMap
+
+DeviceStreamMap::~DeviceStreamMap()
+{
+    const int current = util::cuda::currentDevice();
+    for (auto& [device, stream] : *this) {
+        cudaCheck(cudaSetDevice(device));
+        cudaCheck(cudaStreamDestroy(stream));
+    }
+    cudaCheck(cudaSetDevice(current));// reset to the previous device
+}
+
+inline size_t DeviceStreamMap::getMinPageSize() const
+{
+    NANOVDB_ASSERT(mFunctPtr);
+    CUmemAllocationProp prop = {};
+    prop.type = CU_MEM_ALLOCATION_TYPE_PINNED;
+    prop.location.type = CU_MEM_LOCATION_TYPE_DEVICE;
+    size_t minGranularity = 0;
+    for (auto it = this->begin(); it!=this->end(); ++it) {
+        prop.location.id = it->first;
+        size_t granularity = 0;
+        (*mFunctPtr)(&granularity, &prop, CU_MEM_ALLOC_GRANULARITY_MINIMUM);
+        if (minGranularity < granularity) minGranularity = granularity;
+    }
+    return minGranularity;
+}// DeviceStreamMap::getMinPageSize
+
+}// namespace cuda
+
+}// namespace nanovdb
+
+#endif // end of NANOVDB_CUDA_DEVICESTREAMMAP_H_HAS_BEEN_INCLUDED
@@ -1,5 +1,5 @@
 // Copyright Contributors to the OpenVDB Project
-// SPDX-License-Identifier: MPL-2.0
+// SPDX-License-Identifier: Apache-2.0

 /*!
    \file nanovdb/cuda/GridHandle.cuh
@@ -25,22 +25,25 @@ namespace nanovdb {

 namespace cuda {

-namespace {// anonymous namespace
-__global__ void cpyGridHandleMeta(const GridData *d_data, GridHandleMetaData *d_meta)
+namespace detail {
+
+static __global__ void cpyGridHandleMeta(const GridData *d_data, GridHandleMetaData *d_meta)
 {
    nanovdb::cpyGridHandleMeta(d_data, d_meta);
 }

-__global__ void updateGridCount(GridData *d_data, uint32_t gridIndex, uint32_t gridCount, bool *d_dirty)
+static __global__ void updateGridCount(GridData *d_data, uint32_t gridIndex, uint32_t gridCount, bool *d_dirty)
 {
    NANOVDB_ASSERT(gridIndex < gridCount);
-    if (*d_dirty = d_data->mGridIndex != gridIndex || d_data->mGridCount != gridCount) {
+    *d_dirty = (d_data->mGridIndex != gridIndex) || (d_data->mGridCount != gridCount);
+    if (*d_dirty) {
        d_data->mGridIndex = gridIndex;
        d_data->mGridCount = gridCount;
        if (d_data->mChecksum.isEmpty()) *d_dirty = false;// no need to update checksum if it didn't already exist
    }
 }
-}// anonymous namespace
+
+}// namespace detail

 template<typename BufferT, template <class, class...> class VectorT = std::vector>
 inline typename util::enable_if<BufferTraits<BufferT>::hasDeviceDual, VectorT<GridHandle<BufferT>>>::type
@@ -51,20 +54,22 @@ splitGridHandles(const GridHandle<BufferT> &handle, const BufferT* other = nullp
    VectorT<GridHandle<BufferT>> handles(handle.gridCount());
    bool dirty, *d_dirty;// use this to check if the checksum needs to be recomputed
    cudaCheck(util::cuda::mallocAsync((void**)&d_dirty, sizeof(bool), stream));
+    int device = util::cuda::currentDevice();
    for (uint32_t n=0; n<handle.gridCount(); ++n) {
-        auto buffer = BufferT::create(handle.gridSize(n), other, false, stream);
+        auto buffer = BufferT::create(handle.gridSize(n), other, device, stream);
        GridData *dst = reinterpret_cast<GridData*>(buffer.deviceData());
        const GridData *src = reinterpret_cast<const GridData*>(ptr);
        cudaCheck(cudaMemcpyAsync(dst, src, handle.gridSize(n), cudaMemcpyDeviceToDevice, stream));
-        updateGridCount<<<1, 1, 0, stream>>>(dst, 0u, 1u, d_dirty);
+        detail::updateGridCount<<<1, 1, 0, stream>>>(dst, 0u, 1u, d_dirty);
        cudaCheckError();
        cudaCheck(cudaMemcpyAsync(&dirty, d_dirty, sizeof(bool), cudaMemcpyDeviceToHost, stream));
+        cudaCheck(cudaStreamSynchronize(stream));
        if (dirty) tools::cuda::updateChecksum(dst, CheckMode::Partial, stream);
        handles[n] = nanovdb::GridHandle<BufferT>(std::move(buffer));
        ptr = util::PtrAdd(ptr, handle.gridSize(n));
    }
    cudaCheck(util::cuda::freeAsync(d_dirty, stream));
-    return std::move(handles);
+    return handles;
 }// cuda::splitGridHandles

 template<typename BufferT, template <class, class...> class VectorT>
@@ -77,7 +82,8 @@ mergeGridHandles(const VectorT<GridHandle<BufferT>> &handles, const BufferT* oth
        gridCount += h.gridCount();
        for (uint32_t n=0; n<h.gridCount(); ++n) size += h.gridSize(n);
    }
-    auto buffer = BufferT::create(size, other, false, stream);
+    int device = util::cuda::currentDevice();
+    auto buffer = BufferT::create(size, other, device, stream);
    void *dst = buffer.deviceData();
    bool dirty, *d_dirty;// use this to check if the checksum needs to be recomputed
    cudaCheck(util::cuda::mallocAsync((void**)&d_dirty, sizeof(bool), stream));
@@ -86,9 +92,10 @@ mergeGridHandles(const VectorT<GridHandle<BufferT>> &handles, const BufferT* oth
        for (uint32_t n=0; n<h.gridCount(); ++n) {
            cudaCheck(cudaMemcpyAsync(dst, src, h.gridSize(n), cudaMemcpyDeviceToDevice, stream));
            GridData *data = reinterpret_cast<GridData*>(dst);
-            updateGridCount<<<1, 1, 0, stream>>>(data, counter++, gridCount, d_dirty);
+            detail::updateGridCount<<<1, 1, 0, stream>>>(data, counter++, gridCount, d_dirty);
            cudaCheckError();
            cudaCheck(cudaMemcpyAsync(&dirty, d_dirty, sizeof(bool), cudaMemcpyDeviceToHost, stream));
+            cudaCheck(cudaStreamSynchronize(stream));
            if (dirty) tools::cuda::updateChecksum(data, CheckMode::Partial, stream);
            dst = util::PtrAdd(dst, h.gridSize(n));
            src = util::PtrAdd(src, h.gridSize(n));
@@ -129,7 +136,7 @@ GridHandle<BufferT>::GridHandle(T&& buffer)
            if (!tmp.isValid()) throw std::runtime_error("GridHandle was constructed with an invalid device buffer");
            GridHandleMetaData *d_metaData;
            cudaMalloc((void**)&d_metaData, tmp.mGridCount*sizeof(GridHandleMetaData));
-            cuda::cpyGridHandleMeta<<<1,1>>>(d_data, d_metaData);
+            cuda::detail::cpyGridHandleMeta<<<1,1>>>(d_data, d_metaData);
            mMetaData.resize(tmp.mGridCount);
            cudaCheck(cudaMemcpy(mMetaData.data(), d_metaData,tmp.mGridCount*sizeof(GridHandleMetaData), cudaMemcpyDeviceToHost));
            cudaCheck(cudaFree(d_metaData));
@@ -1,5 +1,5 @@
 // Copyright Contributors to the OpenVDB Project
-// SPDX-License-Identifier: MPL-2.0
+// SPDX-License-Identifier: Apache-2.0

 /*!
    \file nanovdb/cuda/NodeManager.cuh
@@ -37,16 +37,14 @@ createNodeManager(const NanoGrid<BuildT> *d_grid,
                  const BufferT& pool = BufferT(),
                  cudaStream_t stream = 0)
 {
-    auto buffer = BufferT::create(sizeof(NodeManagerData), &pool, false, stream);
+    int device = 0;
+    cudaCheck(cudaGetDevice(&device));
+    auto buffer = BufferT::create(sizeof(NodeManagerData), &pool, device, stream);
    auto *d_data = (NodeManagerData*)buffer.deviceData();
    size_t size = 0u, *d_size;
    cudaCheck(util::cuda::mallocAsync((void**)&d_size, sizeof(size_t), stream));
    util::cuda::lambdaKernel<<<1, 1, 0, stream>>>(1, [=] __device__(size_t) {
-#ifdef NANOVDB_USE_NEW_MAGIC_NUMBERS
-        *d_data = NodeManagerData{NANOVDB_MAGIC_NODE,   0u, (void*)d_grid, {0u,0u,0u}};
-#else
-        *d_data = NodeManagerData{NANOVDB_MAGIC_NUMB, 0u, (void*)d_grid, {0u,0u,0u}};
-#endif
+        *d_data = NodeManagerData((void*)d_grid);
        *d_size = sizeof(NodeManagerData);
        auto &tree = d_grid->tree();
        if (NodeManager<BuildT>::FIXED_SIZE && d_grid->isBreadthFirst()) {
@@ -62,7 +60,7 @@ createNodeManager(const NanoGrid<BuildT> *d_grid,
    cudaCheck(cudaMemcpyAsync(&size, d_size, sizeof(size_t), cudaMemcpyDeviceToHost, stream));
    cudaCheck(util::cuda::freeAsync(d_size, stream));
    if (size > sizeof(NodeManagerData)) {
-        auto tmp = BufferT::create(size, &pool, false, stream);// only allocate buffer on the device
+        auto tmp = BufferT::create(size, &pool, device, stream);// only allocate buffer on the device
        cudaCheck(cudaMemcpyAsync(tmp.deviceData(), buffer.deviceData(), sizeof(NodeManagerData), cudaMemcpyDeviceToDevice, stream));
        buffer = std::move(tmp);
        d_data = reinterpret_cast<NodeManagerData*>(buffer.deviceData());
@@ -0,0 +1,66 @@
+// Copyright Contributors to the OpenVDB Project
+// SPDX-License-Identifier: Apache-2.0
+//
+/// @brief Defines a simple memory pool used to call cub functions that use dynamic temporary storage
+///
+/// @details See nanovdb/tools/cuda/PointToGrid.cuh and nanovdb/tools/cuda/DistributedPointToGrid.cuh
+///          for examples. Also note that this explains the somewhat unusual API with direct access to
+///          protected member data.
+//
+#ifndef NANOVDB_CUDA_TEMPPOOL_H_HAS_BEEN_INCLUDED
+#define NANOVDB_CUDA_TEMPPOOL_H_HAS_BEEN_INCLUDED
+
+#include <nanovdb/cuda/DeviceResource.h>
+
+namespace nanovdb {
+
+namespace cuda {
+
+template <class Resource>
+class TempPool {
+public:
+
+    /// @brief Default c-tor of an empty memory pool.
+    TempPool() : mData(nullptr), mSize(0), mRequestedSize(0) {}
+
+    /// @brief Destructor
+    ~TempPool() {
+        mRequestedSize = 0;
+        Resource::deallocateAsync(mData, mSize, Resource::DEFAULT_ALIGNMENT, nullptr);
+        mData = nullptr;
+        mSize = 0;
+    }
+
+    /// @brief Returns a non-const void pointer to the data managed by this instance.
+    void* data() {return mData;}
+
+    /// @brief Returns a non-const reference to the actual size of the data managed by this instance.
+    size_t& size() {return mSize;}
+
+    /// @brief Returns a non-const reference to the requested size of the data managed by this instance.
+    /// @note This requested size should always be less than or smaller than the actual size().
+    size_t& requestedSize() {return mRequestedSize;}
+
+    /// @brief Re-allocation of the data managed by this instance. Only has affect if the pool in empty or
+    ///        the requested memory is larger than the existing size.
+    /// @param stream cuda stream used for asynchronous de-allocation and allocation.
+    void reallocate(cudaStream_t stream) {
+        if (!mData || mRequestedSize > mSize) {
+            Resource::deallocateAsync(mData, mSize, Resource::DEFAULT_ALIGNMENT, stream);
+            mData = Resource::allocateAsync(mRequestedSize, Resource::DEFAULT_ALIGNMENT, stream);
+            mSize = mRequestedSize;
+        }
+    }
+private:
+    void  *mData;
+    size_t mSize;
+    size_t mRequestedSize;
+};// TempPool<Resource> class
+
+using TempDevicePool = TempPool<DeviceResource>;
+
+} // namespace cuda
+
+} // namespace nanovdb
+
+#endif // end of NANOVDB_CUDA_TEMPPOOL_H_HAS_BEEN_INCLUDED
@@ -0,0 +1,326 @@
+// Copyright Contributors to the OpenVDB Project
+// SPDX-License-Identifier: Apache-2.0
+
+/*!
+    \file UnifiedBuffer.h
+
+    \author Ken Museth
+
+    \date October 15, 2024
+
+    \brief nanovdb::cuda::UnifiedBuffer that uses unified memory management
+
+    \note This file has no device-only kernel functions,
+          which explains why it's a .h and not .cuh file.
+*/
+
+#ifndef NANOVDB_CUDA_UNIFIEDBUFFER_H_HAS_BEEN_INCLUDED
+#define NANOVDB_CUDA_UNIFIEDBUFFER_H_HAS_BEEN_INCLUDED
+
+#include <cuda.h>
+#include <memory>// for std::shared_ptr
+#include <nanovdb/HostBuffer.h>// for BufferTraits
+#include <nanovdb/util/cuda/Util.h>// for cudaCheck
+
+namespace nanovdb {// ================================================================
+
+namespace cuda {// ===================================================================
+
+/// @brief  buffer, used for instance by the GridHandle, to allocate unified memory that
+///         can be resized and shared between multiple devices and the host.
+class UnifiedBuffer
+{
+    void *mPtr;
+    size_t mSize, mCapacity;
+public:
+
+    using PtrT = std::shared_ptr<UnifiedBuffer>;
+
+    /// @brief Default constructor of an empty buffer
+    UnifiedBuffer() : mPtr(nullptr), mSize(0), mCapacity(0){}
+
+    /// @brief Constructor that specifies both the size and capacity
+    /// @param size size of the buffer in bytes, indication what is actually used
+    /// @param capacity number of bytes in the virtual page table, i.e max size for growing
+    /// @note Capacity can be over-estimated to allow for future growth. Memory is not allocated
+    ///       with this constructor, only a page table. Allocation happens on usage or when calling prefetch
+    UnifiedBuffer(size_t size, size_t capacity) : mPtr(nullptr), mSize(size), mCapacity(capacity)
+    {
+        assert(mSize <= mCapacity);
+        cudaCheck(cudaMallocManaged(&mPtr, mCapacity, cudaMemAttachGlobal));
+    }
+
+     /// @brief Similar to the constructor above except the size and capacity are equal, so no future growth is supported
+    UnifiedBuffer(size_t size) : UnifiedBuffer(size, size){}
+
+    /// @brief Constructor that specifies the size, capacity, and device (for prefetching)
+    /// @param size
+    /// @param capacity
+    /// @param device
+    /// @param stream
+    UnifiedBuffer(uint64_t size, uint64_t capacity, int device, cudaStream_t stream = 0) : mPtr(nullptr), mSize(size), mCapacity(size)
+    {
+        assert(mSize <= mCapacity);
+        cudaCheck(cudaMallocManaged(&mPtr, mCapacity, cudaMemAttachGlobal));
+        cudaCheck(util::cuda::memAdvise(mPtr, size, cudaMemAdviseSetPreferredLocation, device));
+        cudaCheck(util::cuda::memPrefetchAsync(mPtr, size, device, stream));
+    }
+
+    /// @brief Constructor with a specified device
+    /// @param size
+    /// @param device
+    /// @param stream
+    UnifiedBuffer(uint64_t size, int device, cudaStream_t stream = 0) : UnifiedBuffer(size, size, device, stream){}
+
+     /// @brief Disallow copy-construction
+    UnifiedBuffer(const UnifiedBuffer&) = delete;
+
+    /// @brief Move copy-constructor
+    UnifiedBuffer(UnifiedBuffer&& other) noexcept
+        : mPtr(other.mPtr)
+        , mSize(other.mSize)
+        , mCapacity(other.mCapacity)
+    {
+        other.mPtr = nullptr;
+        other.mSize = other.mCapacity = 0;
+    }
+
+    /// @brief Destructor
+    ~UnifiedBuffer(){cudaCheck(cudaFree(mPtr));}
+
+    ///////////////////////////////////////////////////////////////////////
+
+    //@{
+    /// @brief Factory methods that create an UnifiedBuffer instance and returns it with move semantics
+    static UnifiedBuffer create(size_t size, size_t capacity) {return UnifiedBuffer(size, capacity);}
+    static UnifiedBuffer create(size_t size) {return UnifiedBuffer(size);}
+    ///@}
+
+    //@{
+    /// @brief Factory methods that create a shared pointer to an UnifiedBuffer instance
+    static PtrT createPtr(size_t size, size_t capacity) {return std::make_shared<UnifiedBuffer>(size, capacity);}
+    static PtrT createPtr(size_t size) {return std::make_shared<UnifiedBuffer>(size);}
+    ///@}
+
+    /// @brief Legacy factory method that mirrors DeviceBuffer. It creates a UnifiedBuffer from a size and a reference buffer.
+    ///        If a reference buffer is provided and its non-empty, it is used to defined the capacity of the new buffer
+    /// @param size Size on bytes of the new buffer
+    /// @param reference reference buffer optionally used to define the capacity
+    /// @param host Ignored for now
+    /// @param stream cuda stream
+    /// @return An instance of a new UnifiedBuffer using move semantics
+    static UnifiedBuffer create(size_t size, const UnifiedBuffer* reference, int device, cudaStream_t stream)
+    {
+        const size_t capacity = (reference && reference->capacity()) ? reference->capacity() : size;
+        UnifiedBuffer buffer(size, capacity);
+        cudaCheck(util::cuda::memAdvise(buffer.mPtr, size, cudaMemAdviseSetPreferredLocation, device));
+        cudaCheck(util::cuda::memPrefetchAsync(buffer.mPtr, size, device, stream));
+        return buffer;
+    }
+
+    /// @brief Factory method that created a buffer on the host of the specified size. If the
+    ///        reference buffer has a capacity it is used. Also the buffer is prefetched to the host
+    /// @param size byte size of buffer initiated on the host
+    /// @param reference optional reference buffer from which the capacity is derived
+    static UnifiedBuffer create(size_t size, const UnifiedBuffer* reference){return create(size, reference, cudaCpuDeviceId, (cudaStream_t)0);}
+
+    /// @brief Factory method that created a buffer on the host or device of the specified size. If the
+    ///        reference buffer has a capacity it is used. Also the buffer is prefetched to the host or (current) device
+    /// @param size byte size of buffer initiated on the device or host
+    /// @param reference optional reference buffer from which the capacity is derived
+    /// @param host If true the buffer will be prefetched to the host, else to the current device
+    /// @param stream optional cuda stream
+    static UnifiedBuffer create(size_t size, const UnifiedBuffer* reference, bool host, void* stream = nullptr)
+    {
+        int device = cudaCpuDeviceId;
+        if (!host) cudaGetDevice(&device);
+        return create(size, reference, device, (cudaStream_t)stream);
+    }
+
+    /// @brief Free all memory and reset this instance to empty
+    void clear()
+    {
+        cudaCheck(cudaFree(mPtr));
+        mPtr = nullptr;
+        mSize = mCapacity = 0;
+    }
+
+    /// @brief Disallow copy assignment operation
+    UnifiedBuffer& operator=(const UnifiedBuffer&) = delete;
+
+    /// @brief Allow move assignment operation
+    UnifiedBuffer& operator=(UnifiedBuffer&& other)
+    {
+        cudaCheck(cudaFree(mPtr));
+        mPtr = other.mPtr;
+        mSize = other.mSize;
+        mCapacity = other.mCapacity;
+        other.mPtr = nullptr;
+        other.mSize = other.mCapacity = 0;
+        return *this;
+    }
+
+    /// @brief initialize buffer as a new with the specified size and capacity
+    /// @param size size of memory block to be used in bytes
+    /// @param capacity size of page table in bytes
+    void init(size_t size, size_t capacity)
+    {
+        NANOVDB_ASSERT(size <= capacity);
+        cudaCheck(cudaFree(mPtr));
+        mSize = size;
+        mCapacity = capacity;
+        cudaCheck(cudaMallocManaged(&mPtr, capacity, cudaMemAttachGlobal));
+    }
+
+    /// @brief Resize the memory block managed by this buffer. If the current capacity is larger than the new size this method
+    ///        simply redefines size. Otherwise a new page-table is defined, with the specified advice, and the old block is copied to the new block.
+    /// @param size size of the new memory block
+    /// @param dev the device ID on which to apply each advice provided in list, cudaCpuDeviceId = -1, 0, 1, ...
+    /// @param list advices to be applied to the resized range
+    void resize(size_t size, int dev = cudaCpuDeviceId, std::initializer_list<cudaMemoryAdvise> list = {cudaMemAdviseSetPreferredLocation})
+    {
+        if (size <= mCapacity) {
+            mSize = size;
+        } else {
+            void *ptr = 0;
+            cudaCheck(cudaMallocManaged(&ptr, size, cudaMemAttachGlobal));
+            if (dev > -2) for (auto a : list) cudaCheck(util::cuda::memAdvise(ptr, size, a, dev));
+            if (mSize > 0) {// copy over data from the old memory block
+                cudaCheck(cudaMemcpy(ptr, mPtr, std::min(mSize, size), cudaMemcpyDefault));
+                cudaCheck(cudaFree(mPtr));
+            }
+            mPtr = ptr;
+            mSize = mCapacity = size;
+        }
+    }
+
+    /// @brief Apply a single advise to a memory block
+    /// @param byteOffset offset in bytes marking the beginning of the memory block to be advised
+    /// @param size size in bytes of the memory block to be advised.
+    /// @param dev the device ID on which to apply the advice provided in adv, cudaCpuDeviceId = -1, 0, 1, ...
+    /// @param adv advice to be applied to the resized range
+    void advise(ptrdiff_t byteOffset, size_t size, int dev, cudaMemoryAdvise adv) const
+    {
+        cudaCheck(util::cuda::memAdvise(util::PtrAdd(mPtr, byteOffset), size, adv, dev));
+    }
+
+    /// @brief Apply a list of advices to a memory block
+    /// @param byteOffset offset in bytes marking the beginning of the memory block to be advised
+    /// @param size size in bytes of the memory block to be advised.
+    /// @param dev the device ID to prefetch to, cudaCpuDeviceId = -1, 0, 1, ...
+    /// @param list list of cuda advises
+    void advise(ptrdiff_t byteOffset, size_t size, int dev, std::initializer_list<cudaMemoryAdvise> list) const
+    {
+        void *ptr = util::PtrAdd(mPtr, byteOffset);
+        for (auto a : list)  cudaCheck(util::cuda::memAdvise(ptr, size, a, dev));
+    }
+
+    /// @brief Prefetches data to the specified device, i.e. ensure the device has an up-to-date copy of the memory specified
+    /// @param byteOffset offset in bytes marking the beginning of the memory block to be prefetched
+    /// @param size size in bytes of the memory block to be prefetched. The default value of zero means copy all @c this->size() bytes.
+    /// @param dev the device ID to prefetch to, cudaCpuDeviceId = -1, 0, 1, ...
+    /// @param stream  cuda stream
+    void prefetch(ptrdiff_t byteOffset = 0, size_t size = 0, int dev = cudaCpuDeviceId, cudaStream_t stream = cudaStreamPerThread) const
+    {
+        cudaCheck(util::cuda::memPrefetchAsync(util::PtrAdd(mPtr, byteOffset), size ? size : mSize, dev, stream));
+    }
+
+    ///////////////////////////////////////////////////////////////////////
+
+    /// @brief Prefetches all data to the specified device
+    /// @param device device ID, cudaCpuDeviceId = -1, 0, 1, ...
+    /// @param stream cuda stream
+    /// @param sync if false the memory copy is asynchronous
+    /// @note Legacy method included for compatibility with DeviceBuffer
+    void deviceUpload(int device = 0, cudaStream_t stream = cudaStreamPerThread, bool sync = false) const
+    {
+        cudaCheck(util::cuda::memPrefetchAsync(mPtr, mSize, device, stream));
+        if (sync) cudaCheck(cudaStreamSynchronize(stream));
+    }
+    void deviceUpload(int device, void* stream, bool sync) const{this->deviceUpload(device, cudaStream_t(stream));}
+
+    /// @brief Prefetches all data to the current device, as given by cudaGetDevice
+    /// @param stream cuda stream
+    /// @param sync if false the memory copy is asynchronous
+    /// @note Legacy method included for compatibility with DeviceBuffer
+    void deviceUpload(void* stream, bool sync) const{
+        int device = 0;
+        cudaCheck(cudaGetDevice(&device));
+        this->deviceUpload(device, cudaStream_t(stream), sync);
+    }
+
+    ///////////////////////////////////////////////////////////////////////
+
+    /// @brief Prefetches all data to the host
+    /// @param stream cuda stream
+    /// @param sync if false the memory copy is asynchronous
+    void deviceDownload(cudaStream_t stream = 0, bool sync = false) const
+    {
+        cudaCheck(util::cuda::memPrefetchAsync(mPtr, mSize, cudaCpuDeviceId, stream));
+        if (sync) cudaCheck(cudaStreamSynchronize(stream));
+    }
+
+    /// @brief Legacy
+    /// @param stream
+    /// @param sync
+    void deviceDownload(void* stream, bool sync) const{this->deviceDownload(cudaStream_t(stream), sync);}
+
+    // used by GridHandle
+    void deviceDownload(int dummmy, void* stream, bool sync) const{this->deviceDownload(cudaStream_t(stream), sync);}
+
+    ///////////////////////////////////////////////////////////////////////
+
+    /// @brief Returns a raw pointer to the unified memory managed by this instance.
+    /// @warning Note that the pointer can be NULL!
+    void* data() const {return mPtr;}
+
+    /// @brief Returns an offset pointer of a specific type from the allocated unified memory
+    /// @tparam T Type of the pointer returned
+    /// @param count Numbers of elements of @c parameter type T to skip (or offset) the return pointer
+    /// @warning assumes that this instance is not empty!
+    template <typename T>
+    T* data(ptrdiff_t count = 0) const {
+        NANOVDB_ASSERT(mPtr != nullptr || count == 0);
+        return reinterpret_cast<T*>(mPtr) + count;
+    }
+
+    /// @brief Returns a byte offset void pointer from the unified memory
+    /// @param byteOffset Number of bytes to skip (or offset) the return pointer
+    /// @warning assumes that this instance is not empty!
+    void* data(ptrdiff_t byteOffset) const {
+        NANOVDB_ASSERT(mPtr != nullptr || byteOffset == 0);
+        return util::PtrAdd(mPtr, byteOffset);
+    }
+
+    /// @brief Legacy
+    /// @return
+    void* deviceData()    const {return mPtr;}
+    void* deviceData(int) const {return mPtr;}
+
+    /// @brief Size of the allocated pages in this instance
+    /// @return number bytes allocated by this instance
+    size_t size() const {return mSize;}
+
+    /// @brief Capacity of this instance, i.e. room in page table
+    /// @return number of bytes reserved, but not necessarily allocated, by this instance
+    size_t capacity() const {return mCapacity;}
+
+    //@{
+    /// @brief Returns true if this allocator is empty, i.e. has no allocated memory
+    inline bool empty() const { return mPtr == nullptr; }
+    inline bool isEmpty() const { return this->empty(); }
+    //@}
+
+};// UnifiedBuffer
+
+}// namespace cuda
+
+template<>
+struct BufferTraits<cuda::UnifiedBuffer>
+{
+    static constexpr bool hasDeviceDual = true;
+};
+
+}// namespace nanovdb
+
+#endif // end of NANOVDB_CUDA_UNIFIEDBUFFER_H_HAS_BEEN_INCLUDED
@@ -1,3 +1,5 @@
+# Copyright Contributors to the OpenVDB Project
+# SPDX-License-Identifier: Apache-2.0
 # find_package(doxygen REQUIRED dot  )
 if(WIN32)
  set(DOXYGEN_EXECUTABLE "C:/Program Files/doxygen/bin/doxygen.exe")
@@ -1,5 +1,5 @@
 # Copyright Contributors to the OpenVDB Project
-# SPDX-License-Identifier: MPL-2.0
+# SPDX-License-Identifier: Apache-2.0
 #
 #[=======================================================================[

@@ -7,7 +7,7 @@

 #]=======================================================================]

-cmake_minimum_required(VERSION 3.18)
+cmake_minimum_required(VERSION 3.24)
 project(NanoVDBExamples LANGUAGES CXX)

 include(GNUInstallDirs)
@@ -78,7 +78,7 @@ function(NANOVDB_EXAMPLE)
  target_include_directories(${EXAMPLE_NAME} PUBLIC ex_util)
  target_link_libraries(${EXAMPLE_NAME} PRIVATE nanovdb)

-  install(TARGETS ${EXAMPLE_NAME} DESTINATION ${CMAKE_INSTALL_DOCDIR}/examples)
+  install(TARGETS ${EXAMPLE_NAME} DESTINATION ${NANOVDB_INSTALL_DOCDIR}/examples)
 endfunction()

 # -----------------------------------------------------------------------
@@ -100,14 +100,24 @@ nanovdb_example(NAME "ex_read_nanovdb_sphere")
 nanovdb_example(NAME "ex_read_nanovdb_sphere_accessor")
 nanovdb_example(NAME "ex_read_nanovdb_sphere_accessor_cuda")
 nanovdb_example(NAME "ex_index_grid_cuda")
-nanovdb_example(NAME "ex_nodemanager_cuda")
+nanovdb_example(NAME "ex_nodemanager_cuda" OPENVDB)
 nanovdb_example(NAME "ex_voxels_to_grid_cuda")
+nanovdb_example(NAME "ex_voxels_to_grid_mgpu")
 nanovdb_example(NAME "ex_modify_nanovdb_thrust")
 nanovdb_example(NAME "ex_map_pool_buffer")
 nanovdb_example(NAME "ex_bump_pool_buffer")
 nanovdb_example(NAME "ex_collide_level_set")
 nanovdb_example(NAME "ex_raytrace_fog_volume")
 nanovdb_example(NAME "ex_raytrace_level_set")
+nanovdb_example(NAME "ex_dilate_nanovdb_cuda" OPENVDB)
+nanovdb_example(NAME "ex_merge_nanovdb_cuda" OPENVDB)
+nanovdb_example(NAME "ex_refine_nanovdb_cuda" OPENVDB)
+nanovdb_example(NAME "ex_coarsen_nanovdb_cuda" OPENVDB)
+
+if(CUDAToolkit_FOUND)
+  nanovdb_example(NAME "ex_make_mgpu_nanovdb") # requires cuRAND
+  target_link_libraries(ex_make_mgpu_nanovdb PRIVATE CUDA::curand)
+endif()

 if(NANOVDB_USE_MAGICAVOXEL)
  nanovdb_example(NAME "ex_vox_to_nanovdb")
@@ -1,5 +1,5 @@
 // Copyright Contributors to the OpenVDB Project
-// SPDX-License-Identifier: MPL-2.0
+// SPDX-License-Identifier: Apache-2.0

 #include <nanovdb/tools/CreatePrimitives.h>
 #include <nanovdb/io/IO.h>
@@ -0,0 +1,150 @@
+// Copyright Contributors to the OpenVDB Project
+// SPDX-License-Identifier: Apache-2.0
+
+// the following files are from OpenVDB
+#include <openvdb/tools/Composite.h>
+#include <openvdb/util/CpuTimer.h>
+
+// the following files are from NanoVDB
+#include <nanovdb/NanoVDB.h>
+#include <nanovdb/cuda/DeviceBuffer.h>
+#include <nanovdb/tools/CreateNanoGrid.h>
+
+template<class CoordT>
+inline CoordT
+coarsenCoord(const CoordT& coord)
+{
+    auto coarsenComponent = [](const typename CoordT::ValueType n) {return (n>=0) ? (n>>1) : -((-n+1)>>1);};
+    CoordT result;
+    result[0] = coarsenComponent(coord[0]);
+    result[1] = coarsenComponent(coord[1]);
+    result[2] = coarsenComponent(coord[2]);
+    return result;
+}
+
+template<typename BuildT>
+void mainCoarsenGrid(
+    nanovdb::NanoGrid<BuildT> *deviceGridOriginal,
+    nanovdb::NanoGrid<BuildT> *deviceGridCoarsened,
+    nanovdb::NanoGrid<BuildT> *indexGridOriginal,
+    nanovdb::NanoGrid<BuildT> *indexGridCoarsened,
+    uint32_t benchmark_iters
+);
+
+/// @brief This example depends on OpenVDB, NanoVDB, and CUDA
+int main(int argc, char *argv[])
+{
+    using GridT = openvdb::FloatGrid;
+    using BuildT = nanovdb::ValueOnIndex;
+
+    openvdb::util::CpuTimer cpuTimer;
+    const bool printGridDiagnostics = true;
+
+    try {
+        if (argc<2) OPENVDB_THROW(openvdb::ValueError, "usage: "+std::string(argv[0])+" input.vdb [<iterations>]\n");
+        int benchmark_iters = 10;
+        if (argc > 2) sscanf(argv[2], "%d", &benchmark_iters);
+
+        // Read the initial level set from file
+
+        cpuTimer.start("Read input VDB file");
+        openvdb::initialize();
+        openvdb::io::File inFile(argv[1]);
+        inFile.open(false); // disable delayed loading
+        auto baseGrids = inFile.getGrids();
+        inFile.close();
+        auto grid = openvdb::gridPtrCast<GridT>(baseGrids->at(0));
+        if (!grid) OPENVDB_THROW(openvdb::ValueError, "First grid is not a FloatGrid\n");
+        cpuTimer.stop();
+
+        // Convert to indexGrid (original, un-coarsened)
+        cpuTimer.start("Converting openVDB input to indexGrid (original version)");
+        auto handleOriginal = nanovdb::tools::openToIndexVDB<BuildT, nanovdb::cuda::DeviceBuffer>(
+            grid,
+            0u,    // Don't copy data channel
+            false, // No stats
+            false, // No tiles
+            1      // Verbose mode
+        );
+        auto *indexGridOriginal = handleOriginal.grid<BuildT>();
+        cpuTimer.stop();
+
+        if (printGridDiagnostics) {
+            std::cout << "============ Original Grid ===========" << std::endl;
+            std::cout << "Allocated values [valueCount()]       : " << indexGridOriginal->valueCount() << std::endl;
+            std::cout << "Active voxels    [activeVoxelCount()] : " << indexGridOriginal->activeVoxelCount() << std::endl;
+            auto minCorner = indexGridOriginal->indexBBox().min(), maxCorner = indexGridOriginal->indexBBox().max();
+            std::cout << "Index-space bounding box              : [" << minCorner.x() << "," << minCorner.y() << "," << minCorner.z()
+                      << "] -> [" << maxCorner.x() << "," << maxCorner.y() << "," << maxCorner.z() << "]" << std::endl;
+            std::cout << "Leaf nodes                            : " << indexGridOriginal->tree().nodeCount(0) << std::endl;
+            std::cout << "Lower internal nodes                  : " << indexGridOriginal->tree().nodeCount(1) << std::endl;
+            std::cout << "Upper internal nodes                  : " << indexGridOriginal->tree().nodeCount(2) << std::endl;
+            std::cout << "Leaf-level occupancy                  : "
+                      << 100.f * (float)(indexGridOriginal->activeVoxelCount())/(float)(indexGridOriginal->tree().nodeCount(0) * 512)
+                      << "%" << std::endl;
+            std::cout << "Memory usage                          : " << indexGridOriginal->gridSize() << " bytes" << std::endl;
+        }
+        // Coarsening (CPU/OpenVDB version)
+        cpuTimer.start("Coarsening OpenVDB (on CPU)");
+        using TreeT = GridT::TreeType;
+        using LeafManagerT = openvdb::tree::LeafManager<const TreeT>;
+        LeafManagerT leafMgr(grid->tree());
+        auto coarsenedGrid = openvdb::FloatGrid::create(grid->background());
+        coarsenedGrid->setTransform(grid->transform().copy());
+        coarsenedGrid->setName(grid->getName());
+        auto dstAcc = coarsenedGrid->getAccessor();
+        for (std::size_t leafID = 0; leafID < leafMgr.leafCount(); ++leafID) {
+            const auto& srcLeaf = leafMgr.leaf(leafID);
+            for (auto iter = srcLeaf.cbeginValueOn(); iter; ++iter) {
+                const auto dstCoord = coarsenCoord(iter.getCoord());
+                if (!dstAcc.isValueOn(dstCoord))
+                    dstAcc.setValue(dstCoord, iter.getValue());
+            }
+        }
+        cpuTimer.stop();
+
+        // Convert to indexGrid (coarsened)
+        cpuTimer.start("Converting openVDB input to indexGrid (coarsened version)");
+        auto handleCoarsened = nanovdb::tools::openToIndexVDB<BuildT, nanovdb::cuda::DeviceBuffer>(
+            coarsenedGrid,
+            0u,    // Don't copy data channel
+            false, // No stats
+            false, // No tiles
+            1      // Verbose mode
+        );
+        cpuTimer.stop();
+
+        auto *indexGridCoarsened = handleCoarsened.grid<BuildT>();
+
+        if (printGridDiagnostics) {
+            std::cout << "=========== Coarsened Grid ===========" << std::endl;
+            std::cout << "Allocated values [valueCount()]       : " << indexGridCoarsened->valueCount() << std::endl;
+            std::cout << "Active voxels    [activeVoxelCount()] : " << indexGridCoarsened->activeVoxelCount() << std::endl;
+            auto minCorner = indexGridCoarsened->indexBBox().min(), maxCorner = indexGridCoarsened->indexBBox().max();
+            std::cout << "Index-space bounding box              : [" << minCorner.x() << "," << minCorner.y() << "," << minCorner.z()
+                      << "] -> [" << maxCorner.x() << "," << maxCorner.y() << "," << maxCorner.z() << "]" << std::endl;
+            std::cout << "Leaf nodes                            : " << indexGridCoarsened->tree().nodeCount(0) << std::endl;
+            std::cout << "Lower internal nodes                  : " << indexGridCoarsened->tree().nodeCount(1) << std::endl;
+            std::cout << "Upper internal nodes                  : " << indexGridCoarsened->tree().nodeCount(2) << std::endl;
+            std::cout << "Leaf-level occupancy                  : "
+                      << 100.f * (float)(indexGridCoarsened->activeVoxelCount())/(float)(indexGridCoarsened->tree().nodeCount(0) * 512)
+                      << "%" << std::endl;
+            std::cout << "Memory usage                          : " << indexGridCoarsened->gridSize() << " bytes" << std::endl;
+        }
+
+        // Copy both NanoVDB grids to GPU
+        handleOriginal.deviceUpload();
+        handleCoarsened.deviceUpload();
+        auto* deviceGridOriginal = handleOriginal.deviceGrid<BuildT>();
+        auto* deviceGridCoarsened = handleCoarsened.deviceGrid<BuildT>();
+        if (!deviceGridOriginal || !deviceGridCoarsened)
+            OPENVDB_THROW(openvdb::RuntimeError, "Failure while uploading indexGrids to GPU");
+
+        // Launch benchmark
+        mainCoarsenGrid( deviceGridOriginal, deviceGridCoarsened, indexGridOriginal, indexGridCoarsened, benchmark_iters );
+    }
+    catch (const std::exception& e) {
+        std::cerr << "An exception occurred: \"" << e.what() << "\"" << std::endl;
+    }
+    return 0;
+}
@@ -0,0 +1,56 @@
+// Copyright Contributors to the OpenVDB Project
+// SPDX-License-Identifier: Apache-2.0
+
+#include <nanovdb/tools/cuda/CoarsenGrid.cuh>
+
+template<typename T>
+bool bufferCheck(const T* deviceBuffer, const T* hostBuffer, size_t elem_count) {
+    T* tmpBuffer = new T[elem_count];
+    cudaCheck(cudaMemcpy(tmpBuffer, deviceBuffer, elem_count * sizeof(T), cudaMemcpyDeviceToHost));
+    bool same = true;
+    for (int i=0; same && i< elem_count; ++i) { same = (tmpBuffer[i] == hostBuffer[i]); }
+    delete [] tmpBuffer;
+    return same;
+}
+
+template<typename BuildT>
+void mainCoarsenGrid(
+    nanovdb::NanoGrid<BuildT> *deviceGridOriginal,
+    nanovdb::NanoGrid<BuildT> *deviceGridCoarsened,
+    nanovdb::NanoGrid<BuildT> *indexGridOriginal,
+    nanovdb::NanoGrid<BuildT> *indexGridCoarsened,
+    uint32_t benchmark_iters)
+{
+    nanovdb::util::cuda::Timer gpuTimer;
+
+    // Initialize coarsener
+    nanovdb::tools::cuda::CoarsenGrid<BuildT> coarsener( deviceGridOriginal );
+    coarsener.setChecksum(nanovdb::CheckMode::Default);
+    coarsener.setVerbose(1);
+
+    auto handle = coarsener.getHandle();
+    auto dstGrid = handle.template deviceGrid<BuildT>();
+
+    // Check for correctness
+    if (bufferCheck((char*)dstGrid, (char*)indexGridCoarsened->data(), indexGridCoarsened->gridSize()))
+        std::cout << "Result of CoarsenGrid check out CORRECT against reference" << std::endl;
+    else
+        std::cout << "Result of CoarsenGrid compares INCORRECT against reference" << std::endl;
+
+    // Re-run warm-started iterations
+    coarsener.setVerbose(0);
+    for (int i = 0; i < benchmark_iters; i++) {
+        gpuTimer.start("Re-running entire coarsening after warmstart");
+        auto dummyHandle = coarsener.getHandle();
+        gpuTimer.stop();
+    }
+}
+
+template
+void mainCoarsenGrid(
+    nanovdb::NanoGrid<nanovdb::ValueOnIndex> *deviceGridOriginal,
+    nanovdb::NanoGrid<nanovdb::ValueOnIndex> *deviceGridCoarsened,
+    nanovdb::NanoGrid<nanovdb::ValueOnIndex> *indexGridOriginal,
+    nanovdb::NanoGrid<nanovdb::ValueOnIndex> *indexGridCoarsened,
+    uint32_t benchmark_iters
+);
@@ -1,5 +1,5 @@
 // Copyright Contributors to the OpenVDB Project
-// SPDX-License-Identifier: MPL-2.0
+// SPDX-License-Identifier: Apache-2.0

 #pragma once

@@ -1,5 +1,5 @@
 // Copyright Contributors to the OpenVDB Project
-// SPDX-License-Identifier: MPL-2.0
+// SPDX-License-Identifier: Apache-2.0

 #include <algorithm>
 #include <iostream>
@@ -37,10 +37,8 @@ int main(int ac, char** av)

        const int numPoints = 10000000;

-        BufferT positionBuffer;
-        positionBuffer.init(numPoints * sizeof(float) * 3);
-        BufferT velocityBuffer;
-        velocityBuffer.init(numPoints * sizeof(float) * 3);
+        BufferT positionBuffer(numPoints * sizeof(float) * 3);
+        BufferT velocityBuffer(numPoints * sizeof(float) * 3);

        runNanoVDB(handle, numIterations, numPoints, positionBuffer, velocityBuffer);
 #if defined(NANOVDB_USE_OPENVDB)
@@ -1,5 +1,5 @@
 // Copyright Contributors to the OpenVDB Project
-// SPDX-License-Identifier: MPL-2.0
+// SPDX-License-Identifier: Apache-2.0

 #define _USE_MATH_DEFINES
 #include <cmath>
@@ -1,5 +1,5 @@
 // Copyright Contributors to the OpenVDB Project
-// SPDX-License-Identifier: MPL-2.0
+// SPDX-License-Identifier: Apache-2.0

 #if defined(NANOVDB_USE_OPENVDB)

@@ -0,0 +1,133 @@
+// Copyright Contributors to the OpenVDB Project
+// SPDX-License-Identifier: Apache-2.0
+
+// the following files are from OpenVDB
+#include <openvdb/tools/Morphology.h>
+#include <openvdb/util/CpuTimer.h>
+
+// the following files are from NanoVDB
+#include <nanovdb/NanoVDB.h>
+#include <nanovdb/cuda/DeviceBuffer.h>
+#include <nanovdb/tools/CreateNanoGrid.h>
+
+template<typename BuildT>
+void mainDilateGrid(
+    nanovdb::NanoGrid<BuildT> *deviceGridOriginal,
+    nanovdb::NanoGrid<BuildT> *deviceGridDilated,
+    nanovdb::NanoGrid<BuildT> *indexGridOriginal,
+    nanovdb::NanoGrid<BuildT> *indexGridDilated,
+    uint32_t nnType,
+    uint32_t benchmark_iters
+);
+
+/// @brief This example depends on OpenVDB, NanoVDB, and CUDA
+int main(int argc, char *argv[])
+{
+    using GridT = openvdb::FloatGrid;
+    using BuildT = nanovdb::ValueOnIndex;
+
+    // Select the type of dilation here. The NN_EDGE case supports leaf dilation too (currently)
+    // openvdb::tools::NearestNeighbors nnType = openvdb::tools::NN_FACE_EDGE_VERTEX;
+    openvdb::tools::NearestNeighbors nnType = openvdb::tools::NN_FACE;
+
+    openvdb::util::CpuTimer cpuTimer;
+    const bool printGridDiagnostics = true;
+
+    try {
+
+        if (argc<2) OPENVDB_THROW(openvdb::ValueError, "usage: "+std::string(argv[0])+" input.vdb [<iterations>]\n");
+        int benchmark_iters = 10;
+        if (argc > 2) sscanf(argv[2], "%d", &benchmark_iters);
+
+        // Read the initial level set from file
+
+        cpuTimer.start("Read input VDB file");
+        openvdb::initialize();
+        openvdb::io::File inFile(argv[1]);
+        inFile.open(false); // disable delayed loading
+        auto baseGrids = inFile.getGrids();
+        inFile.close();
+        auto grid = openvdb::gridPtrCast<GridT>(baseGrids->at(0));
+        openvdb::FloatGrid* ptr = grid.get(); // raw pointer
+        if (!grid) OPENVDB_THROW(openvdb::ValueError, "First grid is not a FloatGrid\n");
+        cpuTimer.stop();
+
+        // Convert to indexGrid (original, un-dilated)
+        cpuTimer.start("Converting openVDB input to indexGrid (original version)");
+        auto handleOriginal = nanovdb::tools::openToIndexVDB<BuildT, nanovdb::cuda::DeviceBuffer>(
+            grid,
+            0u,    // Don't copy data channel
+            false, // No stats
+            false, // No tiles
+            1      // Verbose mode
+        );
+        auto *indexGridOriginal = handleOriginal.grid<BuildT>();
+        cpuTimer.stop();
+
+        if (printGridDiagnostics) {
+            std::cout << "============ Original Grid ===========" << std::endl;
+            std::cout << "Allocated values [valueCount()]       : " << indexGridOriginal->valueCount() << std::endl;
+            std::cout << "Active voxels    [activeVoxelCount()] : " << indexGridOriginal->activeVoxelCount() << std::endl;
+            auto minCorner = indexGridOriginal->indexBBox().min(), maxCorner = indexGridOriginal->indexBBox().max();
+            std::cout << "Index-space bounding box              : [" << minCorner.x() << "," << minCorner.y() << "," << minCorner.z()
+                      << "] -> [" << maxCorner.x() << "," << maxCorner.y() << "," << maxCorner.z() << "]" << std::endl;
+            std::cout << "Leaf nodes                            : " << indexGridOriginal->tree().nodeCount(0) << std::endl;
+            std::cout << "Lower internal nodes                  : " << indexGridOriginal->tree().nodeCount(1) << std::endl;
+            std::cout << "Upper internal nodes                  : " << indexGridOriginal->tree().nodeCount(2) << std::endl;
+            std::cout << "Leaf-level occupancy                  : "
+                      << 100.f * (float)(indexGridOriginal->activeVoxelCount())/(float)(indexGridOriginal->tree().nodeCount(0) * 512)
+                      << "%" << std::endl;
+            std::cout << "Memory usage                          : " << indexGridOriginal->gridSize() << " bytes" << std::endl;
+        }
+
+        // Dilation (CPU/OpenVDB version)
+        cpuTimer.start("Dilating openVDB (on CPU)");
+        openvdb::tools::dilateActiveValues(grid->tree(), 1, nnType);
+        cpuTimer.stop();
+
+        // Convert to indexGrid (dilated)
+        cpuTimer.start("Converting openVDB input to indexGrid (dilated version)");
+        auto handleDilated = nanovdb::tools::openToIndexVDB<BuildT, nanovdb::cuda::DeviceBuffer>(
+            grid,
+            0u,    // Don't copy data channel
+            false, // No stats
+            false, // No tiles
+            1      // Verbose mode
+        );
+        cpuTimer.stop();
+
+        auto *indexGridDilated = handleDilated.grid<BuildT>();
+
+        if (printGridDiagnostics) {
+            std::cout << "============ Dilated Grid ============" << std::endl;
+            std::cout << "Allocated values [valueCount()]       : " << indexGridDilated->valueCount() << std::endl;
+            std::cout << "Active voxels    [activeVoxelCount()] : " << indexGridDilated->activeVoxelCount() << std::endl;
+            auto minCorner = indexGridDilated->indexBBox().min(), maxCorner = indexGridDilated->indexBBox().max();
+            std::cout << "Index-space bounding box              : [" << minCorner.x() << "," << minCorner.y() << "," << minCorner.z()
+                      << "] -> [" << maxCorner.x() << "," << maxCorner.y() << "," << maxCorner.z() << "]" << std::endl;
+            std::cout << "Leaf nodes                            : " << indexGridDilated->tree().nodeCount(0) << std::endl;
+            std::cout << "Lower internal nodes                  : " << indexGridDilated->tree().nodeCount(1) << std::endl;
+            std::cout << "Upper internal nodes                  : " << indexGridDilated->tree().nodeCount(2) << std::endl;
+            std::cout << "Leaf-level occupancy                  : "
+                      << 100.f * (float)(indexGridDilated->activeVoxelCount())/(float)(indexGridDilated->tree().nodeCount(0) * 512)
+                      << "%" << std::endl;
+            std::cout << "Memory usage                          : " << indexGridDilated->gridSize() << " bytes" << std::endl;
+        }
+
+        // Copy both NanoVDB grids to GPU
+        handleOriginal.deviceUpload();
+        handleDilated.deviceUpload();
+        auto* deviceGridOriginal = handleOriginal.deviceGrid<BuildT>();
+        auto* deviceGridDilated = handleDilated.deviceGrid<BuildT>();
+        if (!deviceGridOriginal || !deviceGridDilated)
+            OPENVDB_THROW(openvdb::RuntimeError, "Failure while uploading indexGrids to GPU");
+
+        // Launch benchmark
+        mainDilateGrid( deviceGridOriginal, deviceGridDilated, indexGridOriginal, indexGridDilated, nnType, benchmark_iters );
+
+    }
+    catch (const std::exception& e) {
+        std::cerr << "An exception occurred: \"" << e.what() << "\"" << std::endl;
+    }
+    return 0;
+}
@@ -0,0 +1,102 @@
+// Copyright Contributors to the OpenVDB Project
+// SPDX-License-Identifier: Apache-2.0
+
+#include <nanovdb/tools/cuda/DilateGrid.cuh>
+#include <nanovdb/tools/cuda/PruneGrid.cuh>
+#include <nanovdb/util/cuda/Injection.cuh>
+
+template<typename T>
+bool bufferCheck(const T* deviceBuffer, const T* hostBuffer, size_t elem_count) {
+    T* tmpBuffer = new T[elem_count];
+    cudaCheck(cudaMemcpy(tmpBuffer, deviceBuffer, elem_count * sizeof(T), cudaMemcpyDeviceToHost));
+    bool same = true;
+    for (int i=0; same && i< elem_count; ++i) { same = (tmpBuffer[i] == hostBuffer[i]); }
+    delete [] tmpBuffer;
+    return same;
+}
+
+template<typename BuildT>
+void mainDilateGrid(
+    nanovdb::NanoGrid<BuildT> *deviceGridOriginal,
+    nanovdb::NanoGrid<BuildT> *deviceGridDilated,
+    nanovdb::NanoGrid<BuildT> *indexGridOriginal,
+    nanovdb::NanoGrid<BuildT> *indexGridDilated,
+    uint32_t nnType,
+    uint32_t benchmark_iters)
+{
+    nanovdb::util::cuda::Timer gpuTimer;
+
+    // Initialize dilator
+    nanovdb::tools::cuda::DilateGrid<BuildT> dilator( deviceGridOriginal );
+    dilator.setOperation(nanovdb::tools::morphology::NearestNeighbors(nnType));
+    dilator.setChecksum(nanovdb::CheckMode::Default);
+    dilator.setVerbose(1);
+
+    auto handle = dilator.getHandle();
+    auto dstGrid = handle.template deviceGrid<BuildT>();
+
+    // Check for correctness
+    if (bufferCheck((char*)dstGrid, (char*)indexGridDilated->data(), indexGridDilated->gridSize()))
+        std::cout << "Result of DilateGrid check out CORRECT against reference" << std::endl;
+    else
+        std::cout << "Result of DilateGrid compares INCORRECT against reference" << std::endl;
+
+    // Re-run warm-started iterations
+    dilator.setVerbose(0);
+    for (int i = 0; i < benchmark_iters; i++) {
+        gpuTimer.start("Re-running entire dilation after warmstart");
+        auto dummyHandle = dilator.getHandle();
+        gpuTimer.stop();
+    }
+
+    uint32_t dstLeafCount = nanovdb::util::cuda::DeviceGridTraits<BuildT>::getTreeData(dstGrid).mNodeCount[0];
+    nanovdb::cuda::DeviceBuffer dstLeafMaskBuffer;
+    nanovdb::Mask<3>* dstLeafMasks = nullptr;
+    if (dstLeafCount) {
+        dstLeafMaskBuffer = nanovdb::cuda::DeviceBuffer::create( std::size_t(dstLeafCount) * sizeof(nanovdb::Mask<3>), nullptr, false );
+        dstLeafMasks = static_cast<nanovdb::Mask<3>*>(dstLeafMaskBuffer.deviceData());
+        if (!dstLeafMasks) throw std::runtime_error("No GPU buffer for dstLeafMask");
+    }
+
+    const unsigned int numThreads = 128;
+    auto numBlocks = [numThreads] (unsigned int n) {return (n + numThreads - 1) / numThreads;};
+    gpuTimer.start("Injecting un-dilated topology as a pruning mask");
+    if (dstLeafCount)
+        nanovdb::util::cuda::lambdaKernel<<<numBlocks(dstLeafCount), numThreads>>>(dstLeafCount,
+            nanovdb::util::cuda::InjectGridMaskFunctor<BuildT>(),
+            deviceGridOriginal, dstGrid, dstLeafMasks );
+    gpuTimer.stop();
+
+    // Initialize pruner
+    nanovdb::tools::cuda::PruneGrid<BuildT> pruner( dstGrid, dstLeafMasks );
+    pruner.setChecksum(nanovdb::CheckMode::Default);
+    pruner.setVerbose(1);
+
+    auto prunedHandle = pruner.getHandle();
+    auto prunedGrid = prunedHandle.template deviceGrid<BuildT>();
+
+    // Check for correctness
+    if (bufferCheck((char*)prunedGrid, (char*)indexGridOriginal->data(), indexGridOriginal->gridSize()))
+        std::cout << "Result of PruneGrid check out CORRECT against reference" << std::endl;
+    else
+        std::cout << "Result of PruneGrid compares INCORRECT against reference" << std::endl;
+
+    // Re-run warm-started iterations
+    pruner.setVerbose(0);
+    for (int i = 0; i < benchmark_iters; i++) {
+        gpuTimer.start("Re-running entire pruning after warmstart");
+        auto dummyHandle = pruner.getHandle();
+        gpuTimer.stop();
+    }
+
+}
+
+template
+void mainDilateGrid(
+    nanovdb::NanoGrid<nanovdb::ValueOnIndex> *deviceGridOriginal,
+    nanovdb::NanoGrid<nanovdb::ValueOnIndex> *deviceGridDilated,
+    nanovdb::NanoGrid<nanovdb::ValueOnIndex> *indexGridOriginal,
+    nanovdb::NanoGrid<nanovdb::ValueOnIndex> *indexGridDilated,
+    uint32_t nnType,
+    uint32_t benchmark_iters
+);
@@ -1,5 +1,5 @@
 // Copyright Contributors to the OpenVDB Project
-// SPDX-License-Identifier: MPL-2.0
+// SPDX-License-Identifier: Apache-2.0

 #include <nanovdb/tools/CreateNanoGrid.h>
 #include <nanovdb/tools/CreatePrimitives.h>// for nanovdb::tools::createLevelSetSphere
@@ -1,18 +1,19 @@
 // Copyright Contributors to the OpenVDB Project
-// SPDX-License-Identifier: MPL-2.0
+// SPDX-License-Identifier: Apache-2.0

 #include <nanovdb/NanoVDB.h> // this defined the core tree data structure of NanoVDB accessable on both the host and device
 #include <nanovdb/cuda/GridHandle.cuh>// required since GridHandle<DeviceBuffer> has device code
-#include <stdio.h> // for printf
+#include <cstdio> // for printf
+#include <cinttypes>

 // This is called by the host only
 void cpu_kernel(const nanovdb::NanoGrid<nanovdb::ValueOnIndex>* cpuGrid)
 {
    nanovdb::ChannelAccessor<float, nanovdb::ValueOnIndex> acc(*cpuGrid);
    //printf("\nNanoVDB CPU: channels=%u values=%lu\n", acc.grid().blindDataCount(), acc.root().maximum());
-    printf("NanoVDB CPU; %lu\n", acc.idx(  0, 0, 0));
-    printf("NanoVDB CPU; %lu\n", acc.idx( 99, 0, 0));
-    printf("NanoVDB CPU; %lu\n", acc.idx(100, 0, 0));
+    printf("NanoVDB CPU; %" PRIu64 "\n", acc.idx(  0, 0, 0));
+    printf("NanoVDB CPU; %" PRIu64 "\n", acc.idx( 99, 0, 0));
+    printf("NanoVDB CPU; %" PRIu64 "\n", acc.idx(100, 0, 0));
    printf("NanoVDB CPU; %4.2f\n",   acc(  0, 0, 0));
    printf("NanoVDB CPU; %4.2f\n",   acc( 99, 0, 0));
    printf("NanoVDB CPU; %4.2f\n",   acc(100, 0, 0));
@@ -23,9 +24,9 @@ __global__ void gpu_kernel(const nanovdb::NanoGrid<nanovdb::ValueOnIndex>* gpuGr
 {
    nanovdb::ChannelAccessor<float, nanovdb::ValueOnIndex> acc(*gpuGrid);
    //printf("\nNanoVDB GPU: channels=%u values=%lu\n", gpuGrid->blindDataCount(), acc.root().maximum());
-    printf("NanoVDB GPU; %lu\n", acc.idx(  0, 0, 0));
-    printf("NanoVDB GPU; %lu\n", acc.idx( 99, 0, 0));
-    printf("NanoVDB GPU; %lu\n", acc.idx(100, 0, 0));
+    printf("NanoVDB GPU; %" PRIu64 "\n", acc.idx(  0, 0, 0));
+    printf("NanoVDB GPU; %" PRIu64 "\n", acc.idx( 99, 0, 0));
+    printf("NanoVDB GPU; %" PRIu64 "\n", acc.idx(100, 0, 0));
    printf("NanoVDB GPU; %4.2f\n",   acc(  0, 0, 0));
    printf("NanoVDB GPU; %4.2f\n",   acc( 99, 0, 0));
    printf("NanoVDB GPU; %4.2f\n",   acc(100, 0, 0));
@@ -39,4 +40,4 @@ extern "C" void launch_kernels(const nanovdb::NanoGrid<nanovdb::ValueOnIndex>* g
    gpu_kernel<<<1, 1, 0, stream>>>(gpuGrid); // Launch the device kernel asynchronously

    cpu_kernel(cpuGrid); // Launch the host "kernel" (synchronously)
-}
+}
@@ -1,5 +1,5 @@
 // Copyright Contributors to the OpenVDB Project
-// SPDX-License-Identifier: MPL-2.0
+// SPDX-License-Identifier: Apache-2.0

 #include <nanovdb/tools/GridBuilder.h>
 #include <nanovdb/tools/CreateNanoGrid.h>
@@ -1,5 +1,5 @@
 // Copyright Contributors to the OpenVDB Project
-// SPDX-License-Identifier: MPL-2.0
+// SPDX-License-Identifier: Apache-2.0

 #undef NANOVDB_USE_OPENVDB // Prevents include/openvdb/points/AttributeArray.h:1841:25: error: ‘stride’ cannot be used as a function

@@ -34,6 +34,7 @@ int main()

        cudaStream_t stream; // Create a CUDA stream to allow for asynchronous copy of pinned CUDA memory.
        cudaStreamCreate(&stream);
+
        handle.deviceUpload(stream, false); // Copy the NanoVDB grid to the GPU asynchronously
        auto* gpuGrid = handle.deviceGrid<float>(); // get a (raw) pointer to a NanoVDB grid of value type float on the GPU

@@ -1,5 +1,5 @@
 // Copyright Contributors to the OpenVDB Project
-// SPDX-License-Identifier: MPL-2.0
+// SPDX-License-Identifier: Apache-2.0

 #include <nanovdb/NanoVDB.h> // this defined the core tree data structure of NanoVDB accessable on both the host and device
 #include <nanovdb/cuda/GridHandle.cuh>// required since GridHandle<DeviceBuffer> has device code
@@ -1,5 +1,5 @@
 // Copyright Contributors to the OpenVDB Project
-// SPDX-License-Identifier: MPL-2.0
+// SPDX-License-Identifier: Apache-2.0

 #include <nanovdb/tools/GridBuilder.h>
 #include <nanovdb/tools/CreateNanoGrid.h>
@@ -0,0 +1,302 @@
+// Copyright Contributors to the OpenVDB Project
+// SPDX-License-Identifier: Apache-2.0
+#include <nanovdb/tools/CreatePrimitives.h>
+#include <nanovdb/cuda/UnifiedBuffer.h>
+#include <nanovdb/cuda/DeviceMesh.h>
+#include <nanovdb/util/cuda/Timer.h>
+
+#include <cassert>
+#include <cinttypes>
+#include <cuda.h>
+#include <curand.h>
+#include <mma.h>
+#include <device_launch_parameters.h>
+#include <iostream>
+#include <random>
+#include <thread>
+#include <type_traits>
+#include <map>
+
+namespace {// anonymous namespace
+
+template <typename frag_t>
+inline __device__
+void to_TF32(frag_t& frag) {
+#pragma unroll
+    for (int t = 0; t < frag.num_elements; t++)
+        frag.x[t] =  nvcuda::wmma::__float_to_tf32(frag.x[t]);
+}
+
+__global__
+__launch_bounds__(256)
+void stencilConvolve_v7(nanovdb::NanoGrid<nanovdb::ValueOnIndex> *deviceGrid, int leafOffset,
+    float *inputBuffer, uint32_t *haloIndices, float *haloBuffer,
+    float *stencil, float *outputBuffer, float *denseOutputBuffer)
+{
+    static constexpr int Di = 64;
+    static constexpr int Do = 128;
+    static constexpr int M = 16;
+    static constexpr int N = 16;
+    static constexpr int K = 8;
+
+    using InputBufferType = float (&)[][Di];
+    using HaloBufferType = float (&)[4][4][6][Di];
+    using StencilType = float (&)[3][3][3][Di][Do];
+    using OutputBufferType = float (&)[][Do];
+    using DenseOutputBufferType = float (&)[2][2][4][Do];
+    using SpokeStencilType = float (&)[2][2][4][Di];
+
+    InputBufferType mInputBuffer = reinterpret_cast<InputBufferType>(*inputBuffer);
+    OutputBufferType mOutputBuffer = reinterpret_cast<OutputBufferType>(*outputBuffer);
+    StencilType mStencil = reinterpret_cast<StencilType>(*stencil);
+
+    const int Bk = (blockIdx.x & 1) * 4;
+    const int Bj = ((blockIdx.x >> 1) & 3) * 2;
+    const int Bi = ((blockIdx.x >> 3) & 3) * 2;
+    const int leafId = (blockIdx.x >> 5) + leafOffset;
+    const int tid = threadIdx.x;
+
+    using LeafNodeType = nanovdb::NanoGrid<nanovdb::ValueOnIndex>::TreeType::LeafNodeType;
+    const auto& tree = deviceGrid->tree();
+    auto acc = tree.getAccessor();
+    const LeafNodeType& leaf = tree.template getFirstNode<LeafNodeType>()[leafId];
+    const nanovdb::Coord leafOrigin = leaf.origin();
+    const auto& valueMask = leaf.valueMask();
+
+    uint64_t activeMask = valueMask.words()[Bi] | valueMask.words()[Bi + 1];
+    activeMask &= (0xffffUL << (Bj << 3));
+    activeMask &= (0xf0f0f0f0f0f0f0fUL << Bk);
+    if (!activeMask) return;
+
+    int II = (tid >> 6) & 0x3;
+    int E  =  tid       & 0x3f;
+
+    __shared__ float sBufferRaw[6144]; // 4x4x6 array of elements of size Di=64
+    HaloBufferType sHaloBuffer = reinterpret_cast<HaloBufferType>(sBufferRaw[0]);
+    __shared__ float sSpokeStencil[2][2][4][Di];
+    __shared__ float sOutputBuffer[2][2][4][Do];
+
+    auto origin = leafOrigin.offsetBy(Bi, Bj, Bk);
+
+    // -----------------------------
+    for (int jj = 0; jj < 4; jj++)
+        for (int kk = 0; kk < 6; kk++) {
+            const auto& offset = acc.getValue(origin.offsetBy(II - 1, jj - 1, kk - 1));
+            sHaloBuffer[II][jj][kk][E] = mInputBuffer[offset][E];
+        }
+    __syncthreads();
+
+// -----------------------------
+
+    using a_frag_t = nvcuda::wmma::fragment<nvcuda::wmma::matrix_a, M, N, K, nvcuda::wmma::precision::tf32, nvcuda::wmma::row_major>;
+    nvcuda::wmma::fragment<nvcuda::wmma::matrix_b, M, N, K, nvcuda::wmma::precision::tf32, nvcuda::wmma::row_major> b_frag;
+    nvcuda::wmma::fragment<nvcuda::wmma::accumulator, M, N, K, float> c_frag;
+    __shared__ float sFragBuffer[8][32][4];
+    const int warpID = tid >> 5;
+    const int laneID = tid & 0x1f;
+    nvcuda::wmma::fill_fragment(c_frag, 0.f);
+
+    for (int di = 0; di <= 2; di++)
+    for (int dj = 0; dj <= 2; dj++)
+    for (int dk = 0; dk <= 2; dk++) {
+        // Create a contiguous copy of the 2x2x4 Spoke-Stencil
+        // No sychthreads here, because we take care of that at the end of the loop
+        for (int b = 0; b < 2 * 2 * 4 * Di; b += blockDim.x) {
+            auto eid = tid + b;
+            int ii = ( eid >> 9 ) & 0x1;  // 1-wide in X
+            int jj = ( eid >> 8 ) & 0x1;  // 2-wide in Y
+            int kk = ( eid >> 6 ) & 0x3;  // 4-wide in Z
+            int ee = ( eid >> 0 ) & 0x3f; // 64 entries
+            sSpokeStencil[ii][jj][kk][ee] = sHaloBuffer[ii + di][jj + dj][kk + dk][ee];
+        }
+        __syncthreads();
+
+        {
+            int inBase = warpID << 3;
+
+            a_frag_t& a_frag = *reinterpret_cast<a_frag_t*>(sFragBuffer[warpID][laneID]);
+            nvcuda::wmma::load_matrix_sync(a_frag, &sSpokeStencil[0][0][0][inBase], 64);
+            to_TF32(a_frag);
+
+            __syncthreads();
+        }
+
+#pragma unroll
+        for (int sweep = 0; sweep < 8; ++sweep) {
+            const int inBlock = sweep;
+            const int outBlock = warpID;
+            const int inBase = inBlock << 3;
+            const int outBase = outBlock << 4;
+
+            a_frag_t& a_frag = *reinterpret_cast<a_frag_t*>(sFragBuffer[inBlock][laneID]);
+
+            nvcuda::wmma::load_matrix_sync(b_frag, &mStencil[di][dj][dk][inBase][outBase], 128);
+            to_TF32(b_frag);
+            nvcuda::wmma::mma_sync(c_frag, a_frag, b_frag, c_frag);
+
+        }
+    }
+    nvcuda::wmma::store_matrix_sync(&sOutputBuffer[0][0][0][warpID << 4], c_frag, 128, nvcuda::wmma::mem_row_major);
+
+    // Sparse commit phase
+    __syncthreads();
+    const int warpI     = (tid >> 8) & 0x1;
+    const int warpJ     = (tid >> 7) & 0x1;
+    const int warpK     = (tid >> 5) & 0x3;
+    const int elementID = laneID | ((tid & 0x200) >> 4);
+
+    // NB: this if fixed for 256 threads/block
+    const int elementsPerSM = 32;
+    const int xSpanPerSM = 1;
+    const int ySpanPerSM = 2;
+
+#pragma unroll
+    for (int xOffset = 0; xOffset < 2; xOffset += xSpanPerSM)
+    for (int yOffset = 0; yOffset < 2; yOffset += ySpanPerSM) {
+        const auto coord = origin.offsetBy(warpI + xOffset, warpJ + yOffset, warpK);
+        const auto& offset = acc.getValue(coord);
+#pragma unroll
+        for (int elementOffset = 0; elementOffset < Do; elementOffset += elementsPerSM)
+            mOutputBuffer[offset][elementID + elementOffset] = sOutputBuffer[warpI + xOffset][warpJ + yOffset][warpK][elementID + elementOffset];
+    }
+}// stencilConvolve_v7
+
+}// anonymous namespace
+
+void testConvolution()
+{
+    static constexpr int Di = 64, Do = 128;
+    using InputBufferType  = std::array<float, Di>;
+    using OutputBufferType = std::array<float, Do>;
+    using StencilType = float (&)[3][3][3][Di][Do];
+
+    float* stencilHostPtr = new float[Do * 27 * Di];
+
+    std::mt19937 gen(42u);
+    std::uniform_real_distribution<float> uniform_dist(-1., 1.);
+
+    for (int i = 0; i < Do * 27 * Di; i++) stencilHostPtr[i] = uniform_dist(gen);
+
+    nanovdb::cuda::DeviceMesh deviceMesh;
+    const size_t deviceCount = deviceMesh.deviceCount();
+    std::cout << "Number of devices that supports unified memory: " << deviceCount << std::endl;
+
+    // Calculate minimum page size which corresponds to minimum physical allocation granularity
+    size_t minGranularity = nanovdb::cuda::minDevicePageSize(deviceMesh);
+
+    // Ensure that we don't split an input and/or output feature across page boundaries
+    minGranularity = std::lcm(std::lcm(minGranularity, sizeof(InputBufferType)), sizeof(OutputBufferType));
+    size_t valueCountGranularity = minGranularity / min(sizeof(InputBufferType), sizeof(OutputBufferType));
+
+    // Initialize and replicate an IndexGrid on each device
+    auto floatHandle = nanovdb::tools::createLevelSetSphere<float>(100, nanovdb::Vec3d(0), 1, 3, nanovdb::Vec3d(0), "test");
+    nanovdb::FloatGrid* floatGrid = floatHandle.grid<float>();
+
+    using BufferT = nanovdb::cuda::DeviceBuffer;
+    auto indexHandle = nanovdb::tools::createNanoGrid<nanovdb::FloatGrid, nanovdb::ValueOnIndex, BufferT>(*floatGrid, 0u, false, false, 1);
+    std::for_each(deviceMesh.begin(), deviceMesh.end(), [&](const nanovdb::cuda::DeviceNode& node) {// copy host buffer to all the device buffers
+        cudaCheck(cudaSetDevice(node.id));
+        indexHandle.deviceUpload(node.id, node.stream, true);
+    });
+    auto* indexGrid = indexHandle.grid<nanovdb::ValueOnIndex>();
+
+    auto ceil = [](size_t x, size_t y)->size_t{return ((x + y - 1) / y) * y;};
+    const size_t valueCount = ceil(indexGrid->valueCount(), deviceCount * valueCountGranularity);
+
+    const size_t inputAllocationSize  = valueCount * sizeof(InputBufferType);
+    const size_t outputAllocationSize = valueCount * sizeof(OutputBufferType);
+
+    nanovdb::cuda::UnifiedBuffer inputBuffer(inputAllocationSize, 2*inputAllocationSize);// over-allocate
+    nanovdb::cuda::UnifiedBuffer outputBuffer(outputAllocationSize);
+
+    const size_t deviceValueCount = valueCount / deviceCount;
+
+    // Randomly initialize input features
+    std::vector<std::thread> threads;
+    std::for_each(deviceMesh.begin(), deviceMesh.end(), [&](const nanovdb::cuda::DeviceNode& node) {
+        threads.emplace_back([&](int device, cudaStream_t stream) {
+            cudaCheck(cudaSetDevice(device));
+            float* inputStripePtr  =  inputBuffer.data<float>(deviceValueCount * Di * device);
+            float* outputStripePtr = outputBuffer.data<float>(deviceValueCount * Do * device);
+
+            unsigned long long seed = 42u;
+            curandGenerator_t rng;
+            curandStatus_t curandStatus = curandCreateGenerator(&rng, CURAND_RNG_PSEUDO_DEFAULT);
+            curandStatus = curandSetStream(rng, stream);
+            curandStatus = curandSetPseudoRandomGeneratorSeed(rng, seed);
+            curandStatus = curandGenerateUniform(rng, inputStripePtr,  deviceValueCount * Di);
+            curandStatus = curandGenerateUniform(rng, outputStripePtr, deviceValueCount * Do);
+            curandStatus = curandDestroyGenerator(rng);
+            NANOVDB_ASSERT(curandStatus == CURAND_STATUS_SUCCESS);
+
+            cudaCheck(cudaStreamSynchronize(stream));
+
+            // If we use managed memory, we need to advise about the usage of the memory range in order to obtain an
+            // equivalently optimal paging strategy. cudaMemAdviseSetReadMostly instructs the "paging policy" that data
+            // is far more likely to be read than written, cudaMemAdviseSetPreferredLocation suggests the preferred device to place the data on, and cudaMemAdviseSetAccessedBy is a hint about the which devices are accessing the data.
+            const size_t inPageSize  = deviceValueCount * Di * sizeof(float), inPageOffset  =  inPageSize * device;// in bytes
+            const size_t outPageSize = deviceValueCount * Do * sizeof(float), outPageOffset = outPageSize * device;// in bytes
+            inputBuffer.advise(inPageOffset, inPageSize, device, {cudaMemAdviseSetReadMostly, cudaMemAdviseSetPreferredLocation});
+            inputBuffer.prefetch(inPageOffset, inPageSize, device, stream);
+            outputBuffer.advise(outPageOffset, outPageSize, device, cudaMemAdviseSetPreferredLocation);
+            std::for_each(deviceMesh.begin(), deviceMesh.end(), [&](const nanovdb::cuda::DeviceNode& otherNode) {
+                inputBuffer.advise(  inPageOffset,  inPageSize, otherNode.id, cudaMemAdviseSetAccessedBy);
+                outputBuffer.advise(outPageOffset, outPageSize, otherNode.id, cudaMemAdviseSetAccessedBy);
+            });
+        }, node.id, node.stream);
+    });
+    std::for_each(threads.begin(), threads.end(), [](std::thread& t) { t.join(); });
+    threads.clear();
+
+    // Spawn a convolution kernel on each device that operates on a disjoint subset of the leaves. The leaves operated on by
+    // each device corresponds approximately to the features in the virtual address range that are physically allocated on
+    // the same device.
+    auto **timers = new nanovdb::util::cuda::Timer*[deviceCount];
+    const size_t leafNodeCount = indexGrid->tree().nodeCount(0);
+    std::for_each(deviceMesh.begin(), deviceMesh.end(), [&](const nanovdb::cuda::DeviceNode& node) {
+        threads.emplace_back([&](int device, cudaStream_t stream) {
+            cudaCheck(cudaSetDevice(device));
+            timers[device] = new nanovdb::util::cuda::Timer(stream);
+
+            void* stencilDevicePtr;
+            cudaCheck(nanovdb::util::cuda::mallocAsync(&stencilDevicePtr, sizeof(float) * Do * 27 * Di, stream));
+            cudaCheck(cudaMemcpyAsync(stencilDevicePtr, stencilHostPtr, sizeof(float) * Do * 27 * Di, cudaMemcpyHostToDevice, stream));
+
+            size_t deviceLeafNodeCount = (leafNodeCount + deviceCount - 1) / deviceCount;
+            const size_t deviceLeafNodeOffset = deviceLeafNodeCount * device;
+            deviceLeafNodeCount = std::min(deviceLeafNodeCount, leafNodeCount - deviceLeafNodeOffset);
+            auto deviceIndexGrid =  reinterpret_cast<nanovdb::OnIndexGrid*>(indexHandle.deviceData(device));
+
+            // Run 10 warmup iterations
+            float* stencilPtr = (float*)stencilDevicePtr;
+            dim3 blockDim(256);
+            for (int k = 0; k < 10; ++k) {
+                stencilConvolve_v7<<<deviceLeafNodeCount * 2 * 4 * 4, blockDim, 0, stream>>>(
+                    deviceIndexGrid, deviceLeafNodeOffset, inputBuffer.data<float>(), nullptr, nullptr, stencilPtr, outputBuffer.data<float>(), nullptr);
+            }
+
+            timers[device]->start();
+            stencilConvolve_v7<<<deviceLeafNodeCount * 2 * 4 * 4, blockDim, 0, stream>>>(
+                deviceIndexGrid, deviceLeafNodeOffset, inputBuffer.data<float>(), nullptr, nullptr, stencilPtr, outputBuffer.data<float>(), nullptr);
+            timers[device]->record();
+            cudaCheck(nanovdb::util::cuda::freeAsync(stencilDevicePtr, stream));
+            cudaCheck(cudaStreamSynchronize(stream));
+        }, node.id, node.stream);
+    });
+    std::for_each(threads.begin(), threads.end(), [](std::thread& t) { t.join(); });
+    threads.clear();
+    delete[] stencilHostPtr;
+
+    std::for_each(deviceMesh.begin(), deviceMesh.end(), [&](const nanovdb::cuda::DeviceNode& node) {
+        timers[node.id]->print("Device " + std::to_string(node.id) + " GPU convolution ", std::cout);
+        delete timers[node.id];
+    });
+    delete[] timers;
+    std::cout << "Multi-GPU sparse convolution test complete!" << std::endl;
+}// testConvolution
+
+int main(int argc, char** argv)
+{
+    testConvolution();
+    return 0;
+}
@@ -1,5 +1,5 @@
 // Copyright Contributors to the OpenVDB Project
-// SPDX-License-Identifier: MPL-2.0
+// SPDX-License-Identifier: Apache-2.0

 #include <nanovdb/tools/CreatePrimitives.h>

@@ -1,5 +1,5 @@
 // Copyright Contributors to the OpenVDB Project
-// SPDX-License-Identifier: MPL-2.0
+// SPDX-License-Identifier: Apache-2.0

 #include <nanovdb/tools/GridBuilder.h>
 #include <nanovdb/tools/CreateNanoGrid.h>
@@ -1,5 +1,5 @@
 // Copyright Contributors to the OpenVDB Project
-// SPDX-License-Identifier: MPL-2.0
+// SPDX-License-Identifier: Apache-2.0

 #include <nanovdb/tools/CreatePrimitives.h>
 #include <nanovdb/io/IO.h>
@@ -0,0 +1,166 @@
+// Copyright Contributors to the OpenVDB Project
+// SPDX-License-Identifier: Apache-2.0
+
+// the following files are from OpenVDB
+#include <openvdb/tools/Composite.h>
+#include <openvdb/util/CpuTimer.h>
+
+// the following files are from NanoVDB
+#include <nanovdb/NanoVDB.h>
+#include <nanovdb/cuda/DeviceBuffer.h>
+#include <nanovdb/tools/CreateNanoGrid.h>
+
+template<typename BuildT>
+void mainMergeGrids(
+    nanovdb::NanoGrid<BuildT> *deviceSrcGrid1,
+    nanovdb::NanoGrid<BuildT> *deviceSrcGrid2,
+    nanovdb::NanoGrid<BuildT> *deviceDstReferenceGrid,
+    nanovdb::NanoGrid<BuildT> *hostSrcGrid1,
+    nanovdb::NanoGrid<BuildT> *hostSrcGrid2,
+    nanovdb::NanoGrid<BuildT> *hostDstReferenceGrid,
+    uint32_t benchmark_iters
+);
+
+/// @brief This example depends on OpenVDB, NanoVDB, and CUDA
+int main(int argc, char *argv[])
+{
+    using GridT = openvdb::FloatGrid;
+    using BuildT = nanovdb::ValueOnIndex;
+
+    openvdb::util::CpuTimer cpuTimer;
+    const bool printGridDiagnostics = true;
+
+    try {
+
+        if (argc < 3) OPENVDB_THROW(openvdb::ValueError, "usage: "+std::string(argv[0])+" input1.vdb input2.vdb [<iterations>]\n");
+        int benchmark_iters = 10;
+        if (argc > 3) sscanf(argv[3], "%d", &benchmark_iters);
+
+        // Read the openvdb inputs from files
+
+        cpuTimer.start("Read first input VDB file");
+        openvdb::initialize();
+        openvdb::io::File inFile1(argv[1]);
+        inFile1.open(false); // disable delayed loading
+        auto baseGrids1 = inFile1.getGrids();
+        inFile1.close();
+        auto grid1 = openvdb::gridPtrCast<GridT>(baseGrids1->at(0));
+        if (!grid1) OPENVDB_THROW(openvdb::ValueError, "First grid is not a FloatGrid\n");
+        cpuTimer.stop();
+
+        cpuTimer.start("Read second input VDB file");
+        openvdb::initialize();
+        openvdb::io::File inFile2(argv[2]);
+        inFile2.open(false); // disable delayed loading
+        auto baseGrids2 = inFile2.getGrids();
+        inFile2.close();
+        auto grid2 = openvdb::gridPtrCast<GridT>(baseGrids2->at(0));
+        if (!grid2) OPENVDB_THROW(openvdb::ValueError, "Second grid is not a FloatGrid\n");
+        cpuTimer.stop();
+
+        // Convert to indexGrid
+        cpuTimer.start("Converting openVDB input to indexGrid (first component)");
+        auto srcHandle1 = nanovdb::tools::openToIndexVDB<BuildT, nanovdb::cuda::DeviceBuffer>(
+            grid1,
+            0u,    // Don't copy data channel
+            false, // No stats
+            false, // No tiles
+            1      // Verbose mode
+        );
+        auto *srcIndexGrid1 = srcHandle1.grid<BuildT>();
+        cpuTimer.stop();
+
+        cpuTimer.start("Converting openVDB input to indexGrid (second component)");
+        auto srcHandle2 = nanovdb::tools::openToIndexVDB<BuildT, nanovdb::cuda::DeviceBuffer>(
+            grid2,
+            0u,    // Don't copy data channel
+            false, // No stats
+            false, // No tiles
+            1      // Verbose mode
+        );
+        auto *srcIndexGrid2 = srcHandle2.grid<BuildT>();
+        cpuTimer.stop();
+
+        if (printGridDiagnostics) {
+            std::cout << "============ Component Grid #1 =======  " << std::endl;
+            std::cout << "Allocated values [valueCount()]       : " << srcIndexGrid1->valueCount() << std::endl;
+            std::cout << "Active voxels    [activeVoxelCount()] : " << srcIndexGrid1->activeVoxelCount() << std::endl;
+            auto minCorner = srcIndexGrid1->indexBBox().min(), maxCorner = srcIndexGrid1->indexBBox().max();
+            std::cout << "Index-space bounding box              : [" << minCorner.x() << "," << minCorner.y() << "," << minCorner.z()
+                      << "] -> [" << maxCorner.x() << "," << maxCorner.y() << "," << maxCorner.z() << "]" << std::endl;
+            std::cout << "Leaf nodes                            : " << srcIndexGrid1->tree().nodeCount(0) << std::endl;
+            std::cout << "Lower internal nodes                  : " << srcIndexGrid1->tree().nodeCount(1) << std::endl;
+            std::cout << "Upper internal nodes                  : " << srcIndexGrid1->tree().nodeCount(2) << std::endl;
+            std::cout << "Leaf-level occupancy                  : "
+                      << 100.f * (float)(srcIndexGrid1->activeVoxelCount())/(float)(srcIndexGrid1->tree().nodeCount(0) * 512)
+                      << "%" << std::endl;
+            std::cout << "Memory usage                          : " << srcIndexGrid1->gridSize() << " bytes" << std::endl;
+            std::cout << "============ Component Grid #2 =======  " << std::endl;
+            std::cout << "Allocated values [valueCount()]       : " << srcIndexGrid2->valueCount() << std::endl;
+            std::cout << "Active voxels    [activeVoxelCount()] : " << srcIndexGrid2->activeVoxelCount() << std::endl;
+            minCorner = srcIndexGrid2->indexBBox().min(), maxCorner = srcIndexGrid2->indexBBox().max();
+            std::cout << "Index-space bounding box              : [" << minCorner.x() << "," << minCorner.y() << "," << minCorner.z()
+                      << "] -> [" << maxCorner.x() << "," << maxCorner.y() << "," << maxCorner.z() << "]" << std::endl;
+            std::cout << "Leaf nodes                            : " << srcIndexGrid2->tree().nodeCount(0) << std::endl;
+            std::cout << "Lower internal nodes                  : " << srcIndexGrid2->tree().nodeCount(1) << std::endl;
+            std::cout << "Upper internal nodes                  : " << srcIndexGrid2->tree().nodeCount(2) << std::endl;
+            std::cout << "Leaf-level occupancy                  : "
+                      << 100.f * (float)(srcIndexGrid2->activeVoxelCount())/(float)(srcIndexGrid2->tree().nodeCount(0) * 512)
+                      << "%" << std::endl;
+            std::cout << "Memory usage                          : " << srcIndexGrid2->gridSize() << " bytes" << std::endl;
+        }
+
+        // Merge (CPU/OpenVDB version)
+        cpuTimer.start("Merging openVDBs (on CPU)");
+        auto mergedGrid = grid1->deepCopy();
+        openvdb::tools::compSum(*mergedGrid, *grid2);
+        cpuTimer.stop();
+
+        // Convert to indexGrid
+        cpuTimer.start("Converting merged openVDB output to indexGrid");
+        auto dstReferenceHandle = nanovdb::tools::openToIndexVDB<BuildT, nanovdb::cuda::DeviceBuffer>(
+            mergedGrid,
+            0u,    // Don't copy data channel
+            false, // No stats
+            false, // No tiles
+            1      // Verbose mode
+        );
+        auto *dstReferenceGrid = dstReferenceHandle.grid<BuildT>();
+        cpuTimer.stop();
+
+        if (printGridDiagnostics) {
+            std::cout << "============ Merged Grid (CPU) =======  " << std::endl;
+            std::cout << "Allocated values [valueCount()]       : " << dstReferenceGrid->valueCount() << std::endl;
+            std::cout << "Active voxels    [activeVoxelCount()] : " << dstReferenceGrid->activeVoxelCount() << std::endl;
+            auto minCorner = dstReferenceGrid->indexBBox().min(), maxCorner = dstReferenceGrid->indexBBox().max();
+            std::cout << "Index-space bounding box              : [" << minCorner.x() << "," << minCorner.y() << "," << minCorner.z()
+                      << "] -> [" << maxCorner.x() << "," << maxCorner.y() << "," << maxCorner.z() << "]" << std::endl;
+            std::cout << "Leaf nodes                            : " << dstReferenceGrid->tree().nodeCount(0) << std::endl;
+            std::cout << "Lower internal nodes                  : " << dstReferenceGrid->tree().nodeCount(1) << std::endl;
+            std::cout << "Upper internal nodes                  : " << dstReferenceGrid->tree().nodeCount(2) << std::endl;
+            std::cout << "Leaf-level occupancy                  : "
+                      << 100.f * (float)(dstReferenceGrid->activeVoxelCount())/(float)(dstReferenceGrid->tree().nodeCount(0) * 512)
+                      << "%" << std::endl;
+            std::cout << "Memory usage                          : " << dstReferenceGrid->gridSize() << " bytes" << std::endl;
+        }
+
+        // Copy both NanoVDB grids to GPU
+        srcHandle1.deviceUpload();
+        srcHandle2.deviceUpload();
+        dstReferenceHandle.deviceUpload();
+        auto* deviceSrcGrid1 = srcHandle1.deviceGrid<BuildT>();
+        auto* deviceSrcGrid2 = srcHandle2.deviceGrid<BuildT>();
+        auto* deviceDstReferenceGrid = dstReferenceHandle.deviceGrid<BuildT>();
+        if (!deviceSrcGrid1 || !deviceSrcGrid2 || !deviceDstReferenceGrid)
+            OPENVDB_THROW(openvdb::RuntimeError, "Failure while uploading indexGrids to GPU");
+
+        // Launch benchmark
+        mainMergeGrids(
+            deviceSrcGrid1, deviceSrcGrid2, deviceDstReferenceGrid,
+            srcIndexGrid1, srcIndexGrid2, dstReferenceGrid, benchmark_iters);
+    }
+    catch (const std::exception& e) {
+        std::cerr << "An exception occurred: \"" << e.what() << "\"" << std::endl;
+    }
+    return 0;
+}
@@ -0,0 +1,59 @@
+// Copyright Contributors to the OpenVDB Project
+// SPDX-License-Identifier: Apache-2.0
+
+#include <nanovdb/tools/cuda/MergeGrids.cuh>
+
+template<typename T>
+bool bufferCheck(const T* deviceBuffer, const T* hostBuffer, size_t elem_count) {
+    T* tmpBuffer = new T[elem_count];
+    cudaCheck(cudaMemcpy(tmpBuffer, deviceBuffer, elem_count * sizeof(T), cudaMemcpyDeviceToHost));
+    bool same = true;
+    for (int i=0; same && i< elem_count; ++i) { same = (tmpBuffer[i] == hostBuffer[i]); }
+    delete [] tmpBuffer;
+    return same;
+}
+
+template<typename BuildT>
+void mainMergeGrids(
+    nanovdb::NanoGrid<BuildT> *deviceSrcGrid1,
+    nanovdb::NanoGrid<BuildT> *deviceSrcGrid2,
+    nanovdb::NanoGrid<BuildT> *deviceDstReferenceGrid,
+    nanovdb::NanoGrid<BuildT> *hostSrcGrid1,
+    nanovdb::NanoGrid<BuildT> *hostSrcGrid2,
+    nanovdb::NanoGrid<BuildT> *hostDstReferenceGrid,
+    uint32_t benchmark_iters)
+{
+    nanovdb::util::cuda::Timer gpuTimer;
+
+    // Initialize converter
+    nanovdb::tools::cuda::MergeGrids<BuildT> converter( deviceSrcGrid1, deviceSrcGrid2 );
+    converter.setChecksum(nanovdb::CheckMode::Default);
+    converter.setVerbose(1);
+
+    auto handle = converter.getHandle();
+    auto dstGrid = handle.template deviceGrid<BuildT>();
+
+    // Check for correctness
+    if (bufferCheck((char*)dstGrid, (char*)hostDstReferenceGrid->data(), hostDstReferenceGrid->gridSize()))
+        std::cout << "Result of MergeGrids check out CORRECT against reference" << std::endl;
+    else
+        std::cout << "Result of MergeGrids compares INCORRECT against reference" << std::endl;
+
+    // Re-run warm-started iterations
+    converter.setVerbose(0);
+    for (int i = 0; i < benchmark_iters; i++) {
+        gpuTimer.start("Re-running entire dilation after warmstart");
+        auto handle2 = converter.getHandle();
+        gpuTimer.stop();
+    }
+}
+
+template
+void mainMergeGrids(
+    nanovdb::NanoGrid<nanovdb::ValueOnIndex> *deviceSrcGrid1,
+    nanovdb::NanoGrid<nanovdb::ValueOnIndex> *deviceSrcGrid2,
+    nanovdb::NanoGrid<nanovdb::ValueOnIndex> *deviceDstReferenceGrid,
+    nanovdb::NanoGrid<nanovdb::ValueOnIndex> *hostSrcGrid1,
+    nanovdb::NanoGrid<nanovdb::ValueOnIndex> *hostSrcGrid2,
+    nanovdb::NanoGrid<nanovdb::ValueOnIndex> *hostDstReferenceGrid,
+    uint32_t benchmark_iters);
--- a/Show More
+++ b/Show More