New doc

Contribution guidelines (#506 )
* Signed-off-by:Alexandra Epanchinzeva alexandra.epanchinzeva@intel.com * Minor fixes * Update CONTRIBUTING.md * Update CONTRIBUTING.md * Update CONTRIBUTING.md
2021-08-31 11:13:53 +05:00 · 2021-08-26 17:55:23 +03:00 · 2021-08-25 13:12:07 +03:00 · 2021-08-24 14:23:05 +03:00 · 2021-08-24 10:40:49 +03:00 · 2021-08-24 10:29:57 +03:00
525 changed files with 27186 additions and 6172 deletions
@@ -0,0 +1,19 @@
+# Copyright (c) 2021 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# DISCLAIMER: Bazel support is community-based. The maintainers do not
+# use Bazel internally. The Bazel build can have security risks or 
+# optimization gaps.
+
+build --symlink_prefix=/ # Out of source build
@@ -0,0 +1 @@
+4.1.0
@@ -0,0 +1,60 @@
+#!/bin/bash
+#
+# Copyright (c) 2021 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+SCAN_TARGET=$1
+
+SKIP_PATTERN='*/.github/*'
+
+# Ignored cases
+IGNORE_COMMAND="sed -e /.*\\sOd\\s*=.*/d \
+-e /.*\\sOt\\s*=.*/d \
+-e /.*\\siff\\s*=.*/d \
+-e /.*\\sith\\s*=.*/d \
+-e /.*\\scas\\s*=.*/d \
+-e /.*\\sCAS\\s*=.*/d \
+-e /.*\\ssom\\s*=.*/d \
+-e /.*\\sSOM\\s*=.*/d \
+-e /.*\\suint\\s*=.*/d \
+-e /.*\\sUINT\\s*=.*/d \
+-e /.*\\scopyable\\s*=.*/d \
+-e /.*\\sCopyable\\s*=.*/d \
+-e /.*\\sFo\\s*=.*/d \
+-e /.*pipeline_filters.h.*nd\\s*=.*/d \
+-e /.*ittnotify.h.*unx\\s*=.*/d \
+-e /.*bzlib.cpp.*MSDOS\\s*=.*/d \
+-e /.*test_task.cpp.*tE\\s*=.*/d \
+-e /.*backend.cpp.*resSize\\s*=.*/d \
+-e /.*test_join_node.h.*Ned\\s*=.*/d \
+-e /.*test_indexer_node.cpp.*OT\\s*=.*/d \
+-e /.*allocator_stl_test_common.h.*Aci*\\s*=.*/d \
+-e /.*seismic_video.cpp.*DialogBox\\s*=.*/d \
+-e /.*test_composite_node.cpp.*que\\s*=.*/d \
+-e /.*blocksort.cpp.*hiSt\\s*=.*/d \
+-e /.*compress.cpp.*fave\\s*=.*/d \
+-e /.*count_strings.cpp.*ue\\s*=.*/d \
+-e /.*count_strings.cpp.*nd\\s*=.*/d \
+-e /.*count_strings.cpp.*ths\\s*=.*/d \
+-e /.*polygon_overlay\/README.md.*ist\\s*=.*/d \
+-e /.*_pipeline_filters.h.*nd\\s*=.*/d \
+-e /.*sub_string_finder\/README.md.*ba\\s*=.*/d"
+
+SCAN_RESULT=`codespell --quiet-level=2 --skip "${SKIP_PATTERN}" ${SCAN_TARGET}`
+SCAN_RESULT=`echo -e "${SCAN_RESULT}" | ${IGNORE_COMMAND}`
+echo "${SCAN_RESULT}"
+
+if [[ ! -z ${SCAN_RESULT} ]]; then
+    exit 1
+fi
@@ -0,0 +1,316 @@
+# Copyright (c) 2021 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: oneTBB CI
+
+on:
+  push:
+    branches: [master]
+
+  pull_request:
+    branches: [master]
+    types:
+      - opened
+      - synchronize
+      - reopened
+
+env:
+  BUILD_CONCURRENCY: 2
+  MACOS_BUILD_CONCURRENCY: 3
+  TEST_TIMEOUT: 180
+
+jobs:
+  codespell:
+    runs-on: [ubuntu-20.04]
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v2
+      - name: Run scan
+        run: |
+          sudo apt update && sudo apt install -y codespell
+          ${GITHUB_WORKSPACE}/.github/scripts/codespell.sh `pwd`
+
+  examples_clang-format:
+    runs-on: [ubuntu-20.04]
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v2
+      - name: Run scan
+        run: |
+          cp -r examples examples_formatted
+          find examples_formatted -regex '.*\.\(cpp\|hpp\)' -exec clang-format-9 -style=file -i {} \;
+          diff -r examples examples_formatted
+
+  documentation:
+    needs: [codespell]
+    env:
+      BUILD_TYPE: oss
+    runs-on: [ubuntu-20.04]
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install prerequisites
+        run: |
+          pip3 install git+https://github.com/executablebooks/sphinx-book-theme.git
+          echo GITHUB_SHA_SHORT=${GITHUB_SHA::8} >> $GITHUB_ENV
+          mkdir html
+      - name: Build documentation
+        run: |
+          export BUILD_TYPE=${BUILD_TYPE} && sphinx-build doc/main html
+          tar -czvf html.tar.gz html/
+      - name: Save docs
+        uses: actions/upload-artifact@v2.2.1
+        with:
+          name: oneTBB-html-docs-${{ env.GITHUB_SHA_SHORT }}
+          path: html.tar.gz
+
+  pages:
+    if: ${{ github.ref == 'refs/heads/master' }}
+    runs-on: ubuntu-latest
+    needs: [documentation]
+    steps:
+      - name: Checkout gh-pages
+        uses: actions/checkout@v2
+        with:
+          ref: gh-pages
+          path: gh-pages
+      - name: Set env
+        run: echo GITHUB_SHA_SHORT=${GITHUB_SHA::8} >> $GITHUB_ENV
+      - name: Download documetation
+        uses: actions/download-artifact@v2
+        with:
+          name: oneTBB-html-docs-${{ env.GITHUB_SHA_SHORT }}
+      - name: Publish to github pages
+        run: |
+          tar -xvf html.tar.gz
+          cd gh-pages
+          rm -rf *
+          touch .nojekyll # https://github.blog/2009-12-29-bypassing-jekyll-on-github-pages/
+          cp -r ../html/* .
+          git config user.name github-actions
+          git config user.email github-actions@github.com
+          git add .
+          git commit --reset-author --amend -m "Update from GitHub Actions"
+          git push --force origin gh-pages
+
+  python_module_test_ubuntu18-04:
+    runs-on: [ubuntu-18.04]
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v2
+      - name: Run testing
+        run: |
+          mkdir build && cd build
+          cmake -DTBB4PY_BUILD=ON -DCMAKE_CXX_COMPILER=g++ -DCMAKE_C_COMPILER=gcc ..
+          make VERBOSE=1 -j${BUILD_CONCURRENCY} python_build
+          ctest -R python_test --output-on-failure --timeout ${TEST_TIMEOUT}
+
+  linux-testing:
+    name: ${{ matrix.os }}_${{ matrix.cxx_compiler }}_cxx${{ matrix.std }}_${{ matrix.build_type }}_preview=${{ matrix.preview }}
+    runs-on: ['${{ matrix.os }}']
+    timeout-minutes: 45
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: ubuntu-18.04
+            c_compiler: gcc
+            cxx_compiler: g++
+            std: 14
+            build_type: relwithdebinfo
+            preview: 'OFF'
+          - os: ubuntu-20.04
+            c_compiler: gcc
+            cxx_compiler: g++
+            std: 17
+            build_type: release
+            preview: 'ON'
+          - os: ubuntu-20.04
+            c_compiler: gcc-10
+            cxx_compiler: g++-10
+            std: 20
+            build_type: debug
+            preview: 'ON'
+    steps:
+      - uses: actions/checkout@v2
+      - name: Run testing
+        shell: bash
+        run: |
+          set -x
+          mkdir build && cd build
+          cmake -DCMAKE_CXX_STANDARD=${{ matrix.std }} -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
+            -DCMAKE_CXX_COMPILER=${{ matrix.cxx_compiler }} -DCMAKE_C_COMPILER=${{ matrix.c_compiler }} -DTBB_CPF=${{ matrix.preview }} ..
+          make VERBOSE=1 -j${BUILD_CONCURRENCY}
+          ctest --timeout ${TEST_TIMEOUT} --output-on-failure
+
+  macos-testing:
+    name: ${{ matrix.os }}_${{ matrix.cxx_compiler }}_cxx${{ matrix.std }}_${{ matrix.build_type }}_preview=${{ matrix.preview }}
+    runs-on: ['${{ matrix.os }}']
+    timeout-minutes: 45
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: macos-10.15
+            c_compiler: clang
+            cxx_compiler: clang++
+            std: 14
+            build_type: relwithdebinfo
+            preview: 'ON'
+    steps:
+      - uses: actions/checkout@v2
+      - name: Run testing
+        shell: bash
+        run: |
+          set -x
+          mkdir build && cd build
+          cmake -DCMAKE_CXX_STANDARD=${{ matrix.std }} -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
+            -DCMAKE_CXX_COMPILER=${{ matrix.cxx_compiler }} -DCMAKE_C_COMPILER=${{ matrix.c_compiler }} -DTBB_CPF=${{ matrix.preview }} ..
+          make VERBOSE=1 -j${MACOS_BUILD_CONCURRENCY}
+          ctest --timeout ${TEST_TIMEOUT} --output-on-failure
+
+  windows-testing:
+    name: ${{ matrix.job_name }}
+    runs-on: ['${{ matrix.os }}']
+    timeout-minutes: 45
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: windows-2016
+            generator: Visual Studio 15 2017
+            c_compiler: cl
+            cxx_compiler: cl
+            std: 14
+            build_type: relwithdebinfo
+            preview: 'ON'
+            job_name: windows_cl2017_cxx14_relwithdebinfo_preview=ON
+          - os: windows-2019
+            generator: Visual Studio 16 2019
+            c_compiler: cl
+            cxx_compiler: cl
+            std: 17
+            build_type: relwithdebinfo
+            preview: 'OFF'
+            job_name: windows_cl2019_cxx17_relwithdebinfo_preview=OFF
+    steps:
+      - uses: actions/checkout@v2
+      - name: Run testing
+        run: |
+          mkdir build
+          cd build
+          cmake -G "${{ matrix.generator }}" -A x64 -DCMAKE_CXX_STANDARD=${{ matrix.std }} `
+            -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DCMAKE_CXX_COMPILER=${{ matrix.cxx_compiler }} `
+            -DCMAKE_C_COMPILER=${{ matrix.c_compiler }} -DTBB_CPF=${{ matrix.preview }} ..
+          cmake --build . --config ${{ matrix.build_type }} -j -v
+          ctest -C ${{ matrix.build_type }} --timeout ${env:TEST_TIMEOUT} --output-on-failure
+
+  linux-examples-testing:
+    name: examples_${{ matrix.os }}_${{ matrix.cxx_compiler }}_cxx${{ matrix.std }}_${{ matrix.build_type }}_preview=${{ matrix.preview }}
+    runs-on: ['${{ matrix.os }}']
+    timeout-minutes: 20
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: ubuntu-18.04
+            c_compiler: gcc
+            cxx_compiler: g++
+            std: 14
+            build_type: relwithdebinfo
+            preview: 'OFF'
+          - os: ubuntu-20.04
+            c_compiler: gcc
+            cxx_compiler: g++
+            std: 17
+            build_type: release
+            preview: 'ON'
+          - os: ubuntu-20.04
+            c_compiler: gcc-10
+            cxx_compiler: g++-10
+            std: 20
+            build_type: debug
+            preview: 'ON'
+    steps:
+      - uses: actions/checkout@v2
+      - name: Run testing
+        shell: bash
+        run: |
+          set -x
+          mkdir build && cd build
+          cmake -DCMAKE_CXX_STANDARD=${{ matrix.std }} -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
+            -DCMAKE_CXX_COMPILER=${{ matrix.cxx_compiler }} -DCMAKE_C_COMPILER=${{ matrix.c_compiler }} \
+            -DTBB_CPF=${{ matrix.preview }} -DTBB_TEST=OFF -DTBB_EXAMPLES=ON ..
+          cmake --build . -v --target light_test_examples
+
+  macos-examples-testing:
+    name: examples_${{ matrix.os }}_${{ matrix.cxx_compiler }}_cxx${{ matrix.std }}_${{ matrix.build_type }}_preview=${{ matrix.preview }}
+    runs-on: ['${{ matrix.os }}']
+    timeout-minutes: 20
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: macos-10.15
+            c_compiler: clang
+            cxx_compiler: clang++
+            std: 14
+            build_type: relwithdebinfo
+            preview: 'ON'
+    steps:
+      - uses: actions/checkout@v2
+      - name: Run testing
+        shell: bash
+        run: |
+          set -x
+          mkdir build && cd build
+          cmake -DCMAKE_CXX_STANDARD=${{ matrix.std }} -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
+            -DCMAKE_CXX_COMPILER=${{ matrix.cxx_compiler }} -DCMAKE_C_COMPILER=${{ matrix.c_compiler }} \
+            -DTBB_CPF=${{ matrix.preview }} -DTBB_TEST=OFF -DTBB_EXAMPLES=ON ..
+          cmake --build . -v --target light_test_examples
+
+  windows-examples-testing:
+    name: ${{ matrix.job_name }}
+    runs-on: ['${{ matrix.os }}']
+    timeout-minutes: 20
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: windows-2016
+            generator: Visual Studio 15 2017
+            c_compiler: cl
+            cxx_compiler: cl
+            std: 14
+            build_type: relwithdebinfo
+            preview: 'ON'
+            job_name: examples_windows_cl2017_cxx14_relwithdebinfo_preview=ON
+          - os: windows-2019
+            generator: Visual Studio 16 2019
+            c_compiler: cl
+            cxx_compiler: cl
+            std: 17
+            build_type: relwithdebinfo
+            preview: 'OFF'
+            job_name: examples_windows_cl2019_cxx17_relwithdebinfo_preview=OFF
+    steps:
+      - uses: actions/checkout@v2
+      - name: Run testing
+        run: |
+          mkdir build
+          cd build
+          cmake -G "${{ matrix.generator }}" -A x64 -DCMAKE_CXX_STANDARD=${{ matrix.std }} `
+            -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} -DCMAKE_CXX_COMPILER=${{ matrix.cxx_compiler }} `
+            -DCMAKE_C_COMPILER=${{ matrix.c_compiler }} -DTBB_CPF=${{ matrix.preview }} -DTBB_TEST=OFF -DTBB_EXAMPLES=ON ..
+          cmake --build . -v --target light_test_examples
@@ -0,0 +1,112 @@
+# Copyright (c) 2021 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# DISCLAIMER: Bazel support is community-based. The maintainers do not
+# use Bazel internally. The Bazel build can have security risks or
+# optimization gaps.
+
+package(
+    default_visibility = ["//visibility:public"],
+)
+
+cc_library(
+    name = "tbb",
+    srcs = glob([
+        "src/tbb/*.cpp",
+        "src/tbb/*.h",
+    ]) + select({
+        "@platforms//cpu:x86_64": glob(["src/tbb/tools_api/**/*.h"]),
+        "//conditions:default": [],
+    }),
+    hdrs = glob([
+        "include/tbb/*.h",
+        "include/oneapi/*.h",
+        "include/oneapi/tbb/*.h",
+        "include/oneapi/tbb/detail/*.h",
+    ]),
+    copts = [
+        "-w",
+    ],
+    defines =
+        select({
+            "@platforms//cpu:x86_64": ["__TBB_NO_IMPLICIT_LINKAGE"],
+            "//conditions:default": [
+                "USE_PTHREAD",
+            ],
+        }),
+    includes = [
+        "include",
+    ],
+    linkopts =
+        select({
+            "@bazel_tools//platforms:windows": [],
+            "//conditions:default": [
+                "-ldl",
+                "-pthread",
+                "-lrt",
+            ],
+        }),
+    local_defines = select({
+        "@platforms//cpu:x86_64": [
+            "__TBB_USE_ITT_NOTIFY",
+        ],
+        "//conditions:default": [],
+    }) + [
+        "__TBB_BUILD",
+    ],
+    textual_hdrs = select({
+        "@platforms//cpu:x86_64": [
+            "src/tbb/tools_api/ittnotify_static.c",
+        ],
+        "//conditions:default": [],
+    }),
+)
+
+cc_library(
+    name = "tbbmalloc",
+    srcs =
+        glob([
+            "src/tbbmalloc/*.h",
+            "src/tbb/*.h",
+            "src/tbbmalloc_proxy/*.h",
+        ]) + [
+            "src/tbbmalloc/backend.cpp",
+            "src/tbbmalloc/backref.cpp",
+            "src/tbbmalloc/frontend.cpp",
+            "src/tbbmalloc/large_objects.cpp",
+            "src/tbbmalloc/tbbmalloc.cpp",
+        ],
+    hdrs = glob([
+        "include/tbb/*.h",
+        "include/oneapi/tbb/detail/*.h",
+        "include/oneapi/tbb/*.h",
+    ]),
+    includes = [
+        "include",
+    ],
+    local_defines = [
+        "__TBBMALLOC_BUILD",
+    ],
+)
+
+cc_library(
+    name = "tbbmalloc_proxy",
+    srcs = [
+        "src/tbbmalloc_proxy/function_replacement.cpp",
+        "src/tbbmalloc_proxy/proxy.cpp",
+    ],
+    deps = [
+        ":tbbmalloc",
+    ],
+)
@@ -0,0 +1,60 @@
+# Bazel build support
+
+The main build system of oneTBB is CMake.
+Bazel support is community-based.
+The maintainers do not use Bazel internally.
+The Bazel configuration may not include recommended compiler and linker flags used in official CMake configuration.
+
+The Bazel build of oneTBB currently only aims for a subset of oneTBB that suffices restricted use cases of the usage of oneTBB.
+Pull requests to improve the Bazel build experience are welcomed.
+
+The standard approach of how Bazel handles third-party libraries is static linking. 
+Even this is not recommended by the oneTBB maintainers this is chosen since this is considered as the best practice in the Bazel ecosystem.
+
+## Example usage
+
+1. [Install Bazel](https://docs.bazel.build/versions/main/install.html).
+
+2. Create the following files in one folder (this folder will be considered as the workspace folder):
+
+_WORKSPACE.bazel_:
+```
+load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository")
+
+git_repository(
+    name = "oneTBB",
+    branch = "master",
+    remote = "https://github.com/oneapi-src/oneTBB/",
+)
+```
+
+_BUILD_:
+```
+cc_binary(
+    name = "Demo",
+    srcs = ["main.cpp"],
+    deps = ["@oneTBB//:tbb"],
+)
+```
+
+_main.cpp_:
+```
+#include "oneapi/tbb/version.h"
+
+#include <iostream>
+
+int main() {
+    std::cout << "Hello from oneTBB "
+              << TBB_VERSION_MAJOR << "."
+              << TBB_VERSION_MINOR << "."
+              << TBB_VERSION_PATCH
+              << "!" << std::endl;
+
+    return 0;
+}
+```
+
+3. Switch to the folder where you create the files.
+
+4. Execute the command `bazel run //:Demo`.
+As an expected output, you should see the oneTBB version.
@@ -98,14 +98,18 @@ option(TBB_STRICT "Treat compiler warnings as errors" ON)
 option(TBB_WINDOWS_DRIVER "Build as Universal Windows Driver (UWD)" OFF)
 option(TBB_NO_APPCONTAINER "Apply /APPCONTAINER:NO (for testing binaries for Windows Store)" OFF)
 option(TBB4PY_BUILD "Enable tbb4py build" OFF)
+option(TBB_BUILD "Enable tbb build" ON)
+option(TBBMALLOC_BUILD "Enable tbbmalloc build" ON)
 option(TBB_CPF "Enable preview features of the library" OFF)
 option(TBB_FIND_PACKAGE "Enable search for external oneTBB using find_package instead of build from sources" OFF)
+option(TBB_DISABLE_HWLOC_AUTOMATIC_SEARCH "Disable HWLOC automatic search by pkg-config tool" OFF)

 if (NOT DEFINED BUILD_SHARED_LIBS)
    set(BUILD_SHARED_LIBS ON)
 endif()

 if (NOT BUILD_SHARED_LIBS)
+    set(CMAKE_POSITION_INDEPENDENT_CODE ON)
    message(WARNING "You are building oneTBB as a static library. This is highly discouraged and such configuration is not supported. Consider building a dynamic library to avoid unforeseen issues.")
 endif()

@@ -190,12 +194,16 @@ if (TBB_FIND_PACKAGE OR TBB_DIR)
    message(STATUS "Using external TBB for testing")
    find_package(TBB REQUIRED)
 else()
-    add_subdirectory(src/tbb)
+    if (TBB_BUILD)
+        add_subdirectory(src/tbb)
+    endif()
    if (NOT "${CMAKE_SYSTEM_PROCESSOR}" MATCHES "mips")
-        add_subdirectory(src/tbbmalloc)
-        add_subdirectory(src/tbbmalloc_proxy)
+        if (TBBMALLOC_BUILD)
+            add_subdirectory(src/tbbmalloc)
+            add_subdirectory(src/tbbmalloc_proxy)
+        endif()
        if (APPLE)
-            message(STATUS "TBBBind build target is disabled due to unsupported environment")
+            message(STATUS "TBBBind build targets are disabled due to unsupported environment")
        else()
            add_subdirectory(src/tbbbind)
        endif()
@@ -0,0 +1,55 @@
+<!--
+******************************************************************************
+* 
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*******************************************************************************/-->
+
+# How to Contribute
+As an open source project, we welcome community contributions to oneAPI Threading Building Blocks (oneTBB).  This document explains how to participate in project conversations, log bugs and enhancement requests, and submit code patches to the project. 
+
+## Licensing 
+
+Licensing is very important to open source projects. It helps ensure the software continues to be available under the terms that the author desired. The oneTBB project uses the [Apache 2.0 License](https://github.com/oneapi-src/oneTBB/blob/master/LICENSE.txt), a permissive open source license that allows you to freely use, modify, and distribute your own products that include Apache 2.0 licensed software. By contributing to the oneTBB project, you agree to the license and copyright terms therein and release your own contributions under these terms. 
+
+Some imported or reused components within oneTBB use other licenses, as described in [third-party-programs.txt](https://github.com/oneapi-src/oneTBB/blob/master/third-party-programs.txt). By carefully reviewing potential contributions and enforcing a [Developer Certification of Origin (DCO)](https://developercertificate.org/) for contributed code, we can ensure that the community can develop products with oneTBB without concerns over patent or copyright issues. 
+
+The DCO is an attestation attached to every contribution made by every developer. In the commit message of the contribution, (described later), the developer simply adds a Signed-off-by statement and thereby agrees to the DCO. 
+
+## Prerequisites 
+
+As a contributor, you’ll want to be familiar with the oneTBB project and the repository layout. You should also know how to use it as explained in the [oneTBB documentation](https://oneapi-src.github.io/oneTBB/) and how to set up your build development environment to configure, build, and test oneTBB as explained in the [oneTBB Build System Description](cmake/README.md). 
+
+## Issues 
+If you face a problem, first check out open [oneTBB GitHub issues](https://github.com/oneapi-src/oneTBB/issues) to see if the issue you’d like to address is already reported. You may find users that have encountered the bug you’re finding or have similar ideas for changes or additions.
+
+You can use issues to report a problem, make a feature request, or add comments on an existing issue. 
+
+## Pull Requests 
+
+You can find all [open oneTBB pull requests](https://github.com/oneapi-src/oneTBB/pulls) on GitHub. 
+
+No anonymous contributions are accepted. The name in the commit message Signed-of-by line and your email must match the change authorship information.  Make sure your .gitconfig is set up correctly so you can use `git commit -s` for signing your patches: 
+
+`git config --global user.name "Taylor Developer"`
+
+`git config --global user.email taylor.developer@company.com`
+ 
+
+Before contributing changes directly to the oneTBB repository: 
+
+* Make sure you can build the product and run all the tests with your patch. 
+* For a larger feature, provide a relevant test. 
+* Document your code. The oneTBB project uses reStructuredText for documentation.  
+* Submit a pull request into the master branch. You can submit changes with a pull request (preferred) or by sending an email patch.  
+
+Continuous Integration (CI) testing is enabled for the repository. Your pull request must pass all checks before it can be merged. We will review your contribution and may provide feedback to guide you if any additional fixes or modifications are necessary. When reviewed and accepted, your pull request will be merged into our GitHub repository. 
@@ -0,0 +1,34 @@
+# Installation from Sources
+
+Required Software:
+* DPC++/C++ Compiler
+* Microsoft* Visual Studio* (Windows* OS only)
+* GNU Compilers (gcc) 4.8.5–10
+* GNU C Library (glibc) version 2.17–2.32
+* Clang* 4.2–11.0
+
+The project uses CMake build configuration. Make sure that you have CMake version 3.1 or higher. 
+
+Before installing the library, learn how to [configure, build, and test](cmake\README.md) the project. 
+
+NOTE: Be careful about installation. Avoid commands like `make install` unless you fully understand the consequences.
+
+The following installation components are supported:
+
+`runtime` - oneTBB runtime package (core shared libraries and .dll files on Windows).
+`devel` - oneTBB development package (header files, CMake integration files, library symbolic links, and .lib files on Windows).
+`tbb4py` - oneTBB Module for Python.
+
+To install the specific components after configuration and build, run:
+
+```bash
+cmake -DCOMPONENT=<component> [-DBUILD_TYPE=<build-type>] -P cmake_install.cmake
+```
+
+Simple packaging using CPack is supported.
+The following commands allow to create a simple portable package that includes header files, libraries, and integration files for CMake:
+
+```bash
+cmake <options> ..
+cpack
+```
@@ -1,49 +1,34 @@
 # oneAPI Threading Building Blocks
-[![Apache License Version 2.0](https://img.shields.io/badge/license-Apache_2.0-green.svg)](LICENSE.txt)
+[![Apache License Version 2.0](https://img.shields.io/badge/license-Apache_2.0-green.svg)](LICENSE.txt) [![oneTBB CI](https://github.com/oneapi-src/oneTBB/actions/workflows/ci.yml/badge.svg)](https://github.com/oneapi-src/oneTBB/actions/workflows/ci.yml?query=branch%3Amaster)

 oneAPI Threading Building Blocks (oneTBB) lets you easily write parallel C++ programs that take
-full advantage of multicore performance, that are portable, composable and have future-proof scalability.
+full advantage of multicore performance that are portable, composable, and have future-proof scalability.

-## Release Information
-Here are [Release Notes]( https://software.intel.com/en-us/articles/intel-oneapi-threading-building-blocks-release-notes) and
-[System Requirements](https://software.intel.com/en-us/articles/intel-oneapi-threading-building-blocks-system-requirements).
+## Installation
+You can install oneTBB from [GitHub*](https://github.com/oneapi-src/oneTBB/releases).
+See [Installation from Sources](INSTALL.md) for details.

 ## Documentation
-* [oneTBB documentation](https://software.intel.com/en-us/oneapi-tbb-documentation)
-* README for build system: [cmake/README.md](cmake/README.md)
+* [oneTBB Specification](https://spec.oneapi.com/versions/latest/elements/oneTBB/source/nested-index.html)
+* [oneTBB Developer Guide and Reference](https://oneapi-src.github.io/oneTBB/)
+* [Migrating from TBB to oneTBB](https://oneapi-src.github.io/oneTBB/tbb_userguide/Migration_Guide.html)
+* [README for build system](cmake/README.md)
+* [Basic support for the Bazel build system](Bazel.md)

 ## Support
 Please report issues and suggestions via
-[GitHub issues](https://github.com/oneapi-src/oneTBB/issues) or start a topic on the
-[oneTBB forum](https://community.intel.com/t5/Intel-oneAPI-Threading-Building/bd-p/oneapi-threading-building-blocks).
+[GitHub issues](https://github.com/oneapi-src/oneTBB/issues).

 ## How to Contribute
-To contribute to oneTBB, please open a GitHub pull request (preferred) or send us a patch by e-mail.
-
-Please use the sign-off line at the end of the patch.
-Your signature certifies that you wrote the patch or
-otherwise have the right to pass it on as an open-source patch.
-The rules are pretty simple:
-if you can certify the https://developercertificate.org then you just add a line to every git commit message:
-```
-Signed-off-by: Name Surname <user@email.com>
-```
-Use your real name (sorry, no pseudonyms or anonymous contributions).
-
-If you set your `user.name` and `user.email` git configs, you can sign your
-commit automatically with `git commit -s`.
+We welcome community contributions, so check our [contributing guidelines](CONTRIBUTING.md)
+to learn more.

 ## License
 oneAPI Threading Building Blocks is licensed under [Apache License, Version 2.0](LICENSE.txt).
 By its terms, contributions submitted to the project are also done under that license.

-## Security
-To report security issues please go to https://intel.com/security.
-
 ## Engineering team contacts
 * [E-mail us.](mailto:inteltbbdevelopers@intel.com)

 ------------------------------------------------------------------------
-Intel and the Intel logo are trademarks of Intel Corporation or its subsidiaries in the U.S. and/or other countries.
-
-\* Other names and brands may be claimed as the property of others.
+\* All names and brands may be claimed as the property of others.
@@ -0,0 +1,19 @@
+# Copyright (c) 2021 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# DISCLAIMER: Bazel support is community-based. The maintainers do not
+# use Bazel internally. The Bazel build can have security risks or 
+# optimization gaps.
+
+workspace(name = "oneTBB")
@@ -9,10 +9,13 @@ TBB_STRICT:BOOL - Treat compiler warnings as errors (ON by default)
 TBB_SANITIZE:STRING - Sanitizer parameter, passed to compiler/linker
 TBB_SIGNTOOL:FILEPATH - Tool for digital signing, used in post install step for libraries if provided.
 TBB_SIGNTOOL_ARGS:STRING - Additional arguments for TBB_SIGNTOOL, used if TBB_SIGNTOOL is set.
-TBB4PY_BUILD:BOOL - Enable  Intel(R) oneAPI Threading Building Blocks (oneTBB) Python module build (OFF by default)
+TBB_BUILD:BOOL - Enable Intel(R) oneAPI Threading Building Blocks (oneTBB) build (ON by default)
+TBBMALLOC_BUILD:BOOL - Enable Intel(R) oneAPI Threading Building Blocks (oneTBB) memory allocator build (ON by default)
+TBB4PY_BUILD:BOOL - Enable Intel(R) oneAPI Threading Building Blocks (oneTBB) Python module build (OFF by default)
 TBB_CPF:BOOL - Enable preview features of the library (OFF by default)
 TBB_INSTALL_VARS:BOOL - Enable auto-generated vars installation(packages generated by `cpack` and `make install` will also include the vars script)(OFF by default)
 TBB_VALGRIND_MEMCHECK:BOOL - Enable scan for memory leaks using Valgrind (OFF by default)
+TBB_DISABLE_HWLOC_AUTOMATIC_SEARCH - Disable HWLOC automatic search by pkg-config tool (OFF by default)
 ```

 ## Configure, build and test
@@ -38,21 +41,27 @@ cmake <options> <repo_root>

 Some useful options:
 - `-G <generator>` - specify particular project generator, see `cmake --help` for details.
- `-DCMAKE_BUILD_TYPE=Debug` - specify for Debug build, it doesn't applicable for multi-config generators, e.g. for Visual Studio generator).
+- `-DCMAKE_BUILD_TYPE=Debug` - specify for Debug build, it isn't applicable for multi-config generators, e.g. for Visual Studio generator).

 #### TBBBind library configuration

-The TBBbind library has two versions: `tbbbind` and `tbbbind_2_0`. Each of these versions is linked with corresponding HWLOC library version: `tbbbind` links with HWLOC 1.11.x , `tbbbind_2_0` links with HWLOC 2.x.
+The TBBbind library has three versions: `tbbbind`, `tbbbind_2_0` and `tbbbind_2_5`. Each of these versions is linked with corresponding HWLOC library version: `tbbbind` links with HWLOC 1.11.x , `tbbbind_2_0` links with HWLOC 2.1-2.4, `tbbbind_2_5` links with HWLOC 2.5 and later.

-To enable the TBBBind library build or special tests that performed inside the HWLOC environment, please specify the following CMake variables:
+The search for a suitable version of the HWLOC library is enabled by default, but if you want to use a specific version of the library, you can specify the path to it manually using the following CMake variables:

 - `CMAKE_HWLOC_<HWLOC_VER>_LIBRARY_PATH` - path to the corresponding HWLOC version shared library on Linux or path to `.lib` file on Windows.
 - `CMAKE_HWLOC_<HWLOC_VER>_INCLUDE_PATH` - path to the corresponding HWLOC version includes directory.

+Automatic HWLOC searching requires CMake >= 3.6.
+
 Windows requires one additional variable for correct TBBBind library building:
 - `CMAKE_HWLOC_<HWLOC_VER>_DLL_PATH` - path to the corresponding HWLOC version `.dll` file.

-`HWLOC_VER` variable used above can be replaced with one of the two values: `1_11` for the `tbbbind` library configuration and `2` for the `tbbbind_2_0` library configuration.
+`HWLOC_VER` substring used above can be replaced with one of the three values: `1_11` for the `tbbbind` library configuration, `2` for the `tbbbind_2_0`, and `2_5` for the `tbbbind_2_5` library configuration.
+
+If you specify variables for several TBBBind versions, then building process for all of these versions will be performed during single build session.
+
+Specify the `TBB_DISABLE_HWLOC_AUTOMATIC_SEARCH` to disable HWLOC libraries automatic search.

 ### Build

@@ -274,6 +283,7 @@ Variable | Description
 `TBB_LIB_LINK_LIBS `       | Applied to libraries as link libraries
 `TBB_TEST_COMPILE_FLAGS`   | Applied to tests as compile options

+Please specify the `LINK` options prefixed with dash(-) for MSVC(Visual Studio) compiler with CMake < 3.13 to avoid issues caused by `target_link_libraries` CMake command usage.

 #### Specific options

@@ -31,7 +31,8 @@ if (NOT TBB_STRICT AND COMMAND tbb_remove_compile_flag)
    tbb_remove_compile_flag(-Werror)
 endif()

-if (NOT "${CMAKE_OSX_ARCHITECTURES}" MATCHES "^arm64$" AND CMAKE_SYSTEM_PROCESSOR STREQUAL x86_64)
+# Enable Intel(R) Transactional Synchronization Extensions (-mrtm) on relevant processors
+if (NOT "${CMAKE_OSX_ARCHITECTURES}" MATCHES "^arm64$" AND CMAKE_SYSTEM_PROCESSOR MATCHES "(x86_64|AMD64)") # OSX systems are 64-bit only
    set(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} -mrtm)
 endif()

@@ -12,12 +12,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

-if (APPLE)
+if (MINGW)
+    set(TBB_LINK_DEF_FILE_FLAG "")
+    set(TBB_DEF_FILE_PREFIX "")
+elseif (APPLE)
    set(TBB_LINK_DEF_FILE_FLAG -Wl,-exported_symbols_list,)
    set(TBB_DEF_FILE_PREFIX mac${TBB_ARCH})

    # For correct ucontext.h structures layout
    set(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} -D_XOPEN_SOURCE)
+elseif (MSVC)
+    include(${CMAKE_CURRENT_LIST_DIR}/MSVC.cmake)
+    return()
 else()
    set(TBB_LINK_DEF_FILE_FLAG -Wl,--version-script=)
    set(TBB_DEF_FILE_PREFIX lin${TBB_ARCH})
@@ -37,15 +43,27 @@ if (NOT TBB_STRICT AND COMMAND tbb_remove_compile_flag)
    tbb_remove_compile_flag(-Werror)
 endif()

-if (CMAKE_SYSTEM_PROCESSOR STREQUAL x86_64)
-    set(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} -mrtm)
+# Enable Intel(R) Transactional Synchronization Extensions (-mrtm) and WAITPKG instructions support (-mwaitpkg) on relevant processors
+if (CMAKE_SYSTEM_PROCESSOR MATCHES "(x86|AMD64)")
+    set(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} -mrtm $<$<NOT:$<VERSION_LESS:${CMAKE_CXX_COMPILER_VERSION},12.0>>:-mwaitpkg>)
 endif()

-set(TBB_COMMON_LINK_LIBS dl)
+set(TBB_COMMON_LINK_LIBS ${CMAKE_DL_LIBS})

 if (ANDROID_PLATFORM)
    set(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} $<$<NOT:$<CONFIG:Debug>>:-D_FORTIFY_SOURCE=2>)
 endif()

+if (MINGW)
+    list(APPEND TBB_COMMON_COMPILE_FLAGS -U__STRICT_ANSI__)
+endif()
+
+# Enabling LTO on Android causes the NDK bug.
+# NDK throws the warning: "argument unused during compilation: '-Wa,--noexecstack'"
+if (NOT ANDROID_PLATFORM AND BUILD_SHARED_LIBS)
+    set(TBB_IPO_COMPILE_FLAGS $<$<NOT:$<CONFIG:Debug>>:-flto>)
+    set(TBB_IPO_LINK_FLAGS $<$<NOT:$<CONFIG:Debug>>:-flto>)
+endif()
+
 # TBB malloc settings
 set(TBBMALLOC_LIB_COMPILE_FLAGS -fno-rtti -fno-exceptions)
@@ -12,7 +12,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

-if (APPLE)
+if (MINGW)
+    set(TBB_LINK_DEF_FILE_FLAG "")
+    set(TBB_DEF_FILE_PREFIX "")
+elseif (APPLE)
    set(TBB_LINK_DEF_FILE_FLAG -Wl,-exported_symbols_list,)
    set(TBB_DEF_FILE_PREFIX mac${TBB_ARCH})

@@ -32,11 +35,14 @@ if (NOT CMAKE_GENERATOR MATCHES "Ninja" AND NOT CMAKE_CXX_DEPENDS_USE_COMPILER)
    set(TBB_MMD_FLAG -MMD)
 endif()

-if (CMAKE_SYSTEM_PROCESSOR STREQUAL x86_64)
-    set(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} -mrtm)
+# Enable Intel(R) Transactional Synchronization Extensions (-mrtm) and WAITPKG instructions support (-mwaitpkg) on relevant processors
+if (CMAKE_SYSTEM_PROCESSOR MATCHES "(x86|AMD64)")
+    set(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} -mrtm $<$<AND:$<NOT:$<CXX_COMPILER_ID:Intel>>,$<NOT:$<VERSION_LESS:${CMAKE_CXX_COMPILER_VERSION},11.0>>>:-mwaitpkg>)
 endif()

-set(TBB_COMMON_LINK_LIBS dl)
+if (NOT MINGW)
+    set(TBB_COMMON_LINK_LIBS dl)
+endif()

 # Ignore -Werror set through add_compile_options() or added to CMAKE_CXX_FLAGS if TBB_STRICT is disabled.
 if (NOT TBB_STRICT AND COMMAND tbb_remove_compile_flag)
@@ -48,9 +54,19 @@ if (NOT ${CMAKE_CXX_COMPILER_ID} STREQUAL Intel)
    set(TBB_DSE_FLAG $<$<NOT:$<VERSION_LESS:${CMAKE_CXX_COMPILER_VERSION},6.0>>:-flifetime-dse=1>)
 endif()

-# Workaround for heavy tests
+# Workaround for heavy tests and too many symbols in debug (rellocation truncated to fit: R_MIPS_CALL16)
 if ("${CMAKE_SYSTEM_PROCESSOR}" MATCHES "mips")
-    set(TBB_TEST_COMPILE_FLAGS ${TBB_TEST_COMPILE_FLAGS} -DTBB_TEST_LOW_WORKLOAD)
+    set(TBB_TEST_COMPILE_FLAGS ${TBB_TEST_COMPILE_FLAGS} -DTBB_TEST_LOW_WORKLOAD $<$<CONFIG:DEBUG>:-mxgot>)
+endif()
+
+if (MINGW)
+    list(APPEND TBB_COMMON_COMPILE_FLAGS -U__STRICT_ANSI__)
+endif()
+
+# For some reason GCC does not instrument code with Thread Sanitizer when lto is enabled and C linker is used.
+if (NOT TBB_SANITIZE MATCHES "thread")
+    set(TBB_IPO_COMPILE_FLAGS $<$<NOT:$<CONFIG:Debug>>:-flto>)
+    set(TBB_IPO_LINK_FLAGS $<$<NOT:$<CONFIG:Debug>>:-flto>)
 endif()

 # TBB malloc settings
@@ -16,11 +16,14 @@ if (MSVC)
    include(${CMAKE_CURRENT_LIST_DIR}/MSVC.cmake)
    set(TBB_WARNING_LEVEL ${TBB_WARNING_LEVEL} /W3)
    set(TBB_OPENMP_FLAG /Qopenmp)
+    set(TBB_IPO_COMPILE_FLAGS $<$<NOT:$<CONFIG:Debug>>:/Qipo>)
+    set(TBB_IPO_LINK_FLAGS $<$<NOT:$<CONFIG:Debug>>:/INCREMENTAL:NO>)
 elseif (APPLE)
    include(${CMAKE_CURRENT_LIST_DIR}/AppleClang.cmake)
    set(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} -fstack-protector -Wformat -Wformat-security
                                 $<$<NOT:$<CONFIG:Debug>>:-fno-omit-frame-pointer -qno-opt-report-embed -D_FORTIFY_SOURCE=2>)
    set(TBB_OPENMP_FLAG -qopenmp)
+    set(TBB_IPO_COMPILE_FLAGS $<$<NOT:$<CONFIG:Debug>>:-ipo>)
 else()
    include(${CMAKE_CURRENT_LIST_DIR}/GNU.cmake)
    set(TBB_LIB_LINK_FLAGS ${TBB_LIB_LINK_FLAGS} -static-intel -Wl,-z,relro,-z,now,)
@@ -28,4 +31,6 @@ else()
                                 $<$<NOT:$<CONFIG:Debug>>:-qno-opt-report-embed -D_FORTIFY_SOURCE=2>
                                 $<$<EQUAL:${TBB_ARCH},32>:-falign-stack=maintain-16-byte>)
    set(TBB_OPENMP_FLAG -qopenmp)
+    set(TBB_IPO_COMPILE_FLAGS $<$<NOT:$<CONFIG:Debug>>:-ipo>)
 endif()
+set(TBB_IPO_LINK_FLAGS ${TBB_IPO_LINK_FLAGS} ${TBB_IPO_COMPILE_FLAGS})
@@ -0,0 +1,23 @@
+# Copyright (c) 2020-2021 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if (WIN32)
+    include(${CMAKE_CURRENT_LIST_DIR}/MSVC.cmake)
+    set(TBB_IPO_COMPILE_FLAGS $<$<NOT:$<CONFIG:Debug>>:/Qipo>)
+    set(TBB_IPO_LINK_FLAGS $<$<NOT:$<CONFIG:Debug>>:/INCREMENTAL:NO>)
+else()
+    include(${CMAKE_CURRENT_LIST_DIR}/Clang.cmake)
+    set(TBB_IPO_COMPILE_FLAGS $<$<NOT:$<CONFIG:Debug>>:-ipo>)
+endif()
+set(TBB_IPO_LINK_FLAGS ${TBB_IPO_LINK_FLAGS} ${TBB_IPO_COMPILE_FLAGS})
@@ -19,9 +19,11 @@ set(TBB_DEF_FILE_PREFIX win${TBB_ARCH})
 # TODO: consider use of CMP0092 CMake policy.
 string(REGEX REPLACE "/W[0-4]" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")

-# Warning suppression C4324: structure was padded due to alignment specifier
 set(TBB_WARNING_LEVEL $<$<NOT:$<CXX_COMPILER_ID:Intel>>:/W4> $<$<BOOL:${TBB_STRICT}>:/WX>)
+
+# Warning suppression C4324: structure was padded due to alignment specifier
 set(TBB_WARNING_SUPPRESS /wd4324)
+
 set(TBB_TEST_COMPILE_FLAGS /bigobj)
 if (MSVC_VERSION LESS_EQUAL 1900)
    # Warning suppression C4503 for VS2015 and earlier:
@@ -40,7 +42,7 @@ endif()

 if (WINDOWS_STORE OR TBB_WINDOWS_DRIVER)
    set(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} /D_WIN32_WINNT=0x0A00)
-    set(TBB_COMMON_LINK_FLAGS /NODEFAULTLIB:kernel32.lib /INCREMENTAL:NO)
+    set(TBB_COMMON_LINK_FLAGS -NODEFAULTLIB:kernel32.lib -INCREMENTAL:NO)
    set(TBB_COMMON_LINK_LIBS OneCore.lib)
 endif()

@@ -55,7 +57,7 @@ if (WINDOWS_STORE)
    string(REGEX REPLACE "WindowsApp.lib" "" CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES}")

    if (TBB_NO_APPCONTAINER)
-        set(TBB_LIB_LINK_FLAGS ${TBB_LIB_LINK_FLAGS} /APPCONTAINER:NO)
+        set(TBB_LIB_LINK_FLAGS ${TBB_LIB_LINK_FLAGS} -APPCONTAINER:NO)
    endif()
 endif()

@@ -69,4 +71,15 @@ if (TBB_WINDOWS_DRIVER)
    set(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} /D _UNICODE /DUNICODE /DWINAPI_FAMILY=WINAPI_FAMILY_APP /D__WRL_NO_DEFAULT_LIB__)
 endif()

+if (CMAKE_CXX_COMPILER_ID MATCHES "(Clang|IntelLLVM)")
+    if (CMAKE_SYSTEM_PROCESSOR MATCHES "(x86|AMD64)")
+        set(TBB_COMMON_COMPILE_FLAGS ${TBB_COMMON_COMPILE_FLAGS} -mrtm -mwaitpkg)
+    endif()
+    set(TBB_OPENMP_NO_LINK_FLAG TRUE)
+    set(TBB_IPO_COMPILE_FLAGS $<$<NOT:$<CONFIG:Debug>>:-flto>)
+else()
+    set(TBB_IPO_COMPILE_FLAGS $<$<NOT:$<CONFIG:Debug>>:/GL>)
+    set(TBB_IPO_LINK_FLAGS $<$<NOT:$<CONFIG:Debug>>:-LTCG> $<$<NOT:$<CONFIG:Debug>>:-INCREMENTAL:NO>)
+endif()
+
 set(TBB_OPENMP_FLAG /openmp)
@@ -0,0 +1,18 @@
+# Copyright (c) 2021 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/compilers/GNU.cmake)
+
+# Remove dl library not present in QNX systems
+unset(TBB_COMMON_LINK_LIBS)
@@ -40,6 +40,10 @@ function(tbb_generate_config)

    set(TBB_VERSION ${tbb_gen_cfg_VERSION})

+    set(_tbb_pc_lib_name tbb)
+    set(_prefix_for_pc_file "\${pcfiledir}/../../")
+    set(_includedir_for_pc_file "\${prefix}/include")
+
    set(TBB_COMPONENTS_BIN_VERSION "
 set(_tbb_bin_version ${tbb_gen_cfg_TBB_BINARY_VERSION})
 set(_tbbmalloc_bin_version ${tbb_gen_cfg_TBBMALLOC_BINARY_VERSION})
@@ -60,12 +64,20 @@ else ()
    set(_tbb_subdir ia32/gcc4.8)
 endif()
 ")
+
+            set(_libdir_for_pc_file "\${prefix}/lib/intel64/gcc4.8")
+            configure_file(${_tbb_gen_cfg_path}/../integration/pkg-config/tbb.pc.in ${config_install_dir}/tbb.pc @ONLY)
+
+            set(_libdir_for_pc_file "\${prefix}/lib/ia32/gcc4.8")
+            configure_file(${_tbb_gen_cfg_path}/../integration/pkg-config/tbb.pc.in ${config_install_dir}/tbb32.pc @ONLY)
        endif()
    elseif (tbb_gen_cfg_SYSTEM_NAME STREQUAL "Darwin")
        set(TBB_LIB_PREFIX "lib")
        set(TBB_LIB_EXT "\${_\${_tbb_component}_bin_version}.dylib")
        set(TBB_IMPLIB_RELEASE "")
        set(TBB_IMPLIB_DEBUG "")
+        set(_libdir_for_pc_file "\${prefix}/lib")
+        configure_file(${_tbb_gen_cfg_path}/../integration/pkg-config/tbb.pc.in ${config_install_dir}/tbb.pc @ONLY)
    elseif (tbb_gen_cfg_SYSTEM_NAME STREQUAL "Windows")
        set(TBB_LIB_PREFIX "")
        set(TBB_LIB_EXT "dll")
@@ -95,6 +107,13 @@ else ()
    set(_tbb_subdir ia32/\${_tbb_subdir})
 endif()
 ")
+            set(_tbb_pc_lib_name ${_tbb_pc_lib_name}${TBB_BINARY_VERSION})
+
+            set(_libdir_for_pc_file "\${prefix}/lib/intel64/vc14")
+            configure_file(${_tbb_gen_cfg_path}/../integration/pkg-config/tbb.pc.in ${config_install_dir}/tbb.pc @ONLY)
+
+            set(_libdir_for_pc_file "\${prefix}/lib/ia32/vc14")
+            configure_file(${_tbb_gen_cfg_path}/../integration/pkg-config/tbb.pc.in ${config_install_dir}/tbb32.pc @ONLY)
        endif()

        set(TBB_HANDLE_BIN_VERSION "
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

-list(APPEND HWLOC_REQUIRED_VERSIONS 1_11 2 2_4)
+list(APPEND HWLOC_REQUIRED_VERSIONS 1_11 2 2_5)

 foreach(hwloc_version ${HWLOC_REQUIRED_VERSIONS})
    if (NOT WIN32)
@@ -45,19 +45,22 @@ endforeach()

 unset(HWLOC_TARGET_NAME)

-if (NOT HWLOC_TARGET_EXPLICITLY_DEFINED)
+if (NOT HWLOC_TARGET_EXPLICITLY_DEFINED AND
+    # No hwloc auto detection for cross compilation
+    NOT CMAKE_CROSSCOMPILING AND
+    NOT TBB_DISABLE_HWLOC_AUTOMATIC_SEARCH
+)
    find_package(PkgConfig QUIET)
-    if (PKG_CONFIG_FOUND)
-        pkg_search_module(HWLOC hwloc)
-        if (HWLOC_FOUND)
+    if (PKG_CONFIG_FOUND AND NOT CMAKE_VERSION VERSION_LESS 3.6)
+        pkg_search_module(HWLOC hwloc IMPORTED_TARGET)
+        if (TARGET PkgConfig::HWLOC)
            if (HWLOC_VERSION VERSION_LESS 2)
                set(TBBBIND_LIBRARY_NAME tbbbind)
-            elseif(HWLOC_VERSION VERSION_LESS 2.4)
+            elseif(HWLOC_VERSION VERSION_LESS 2.5)
                set(TBBBIND_LIBRARY_NAME tbbbind_2_0)
            else()
-                set(TBBBIND_LIBRARY_NAME tbbbind_2_4)
+                set(TBBBIND_LIBRARY_NAME tbbbind_2_5)
            endif()
        endif()
    endif()
 endif()
-
@@ -34,5 +34,7 @@ if (NOT ${FLAG_DISPLAY_NAME})
    "please try another compiler or omit TBB_SANITIZE variable")
 endif()

+set(TBB_TESTS_ENVIRONMENT ${TBB_TESTS_ENVIRONMENT} "LSAN_OPTIONS=suppressions=${CMAKE_CURRENT_SOURCE_DIR}/cmake/suppressions/lsan.suppressions")
+
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${TBB_SANITIZE_OPTION}")
 set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${TBB_SANITIZE_OPTION}")
@@ -26,22 +26,18 @@ set(DLL_REL_PATH "redist")
 file(READ ${CMAKE_CURRENT_LIST_DIR}/../../include/oneapi/tbb/version.h _tbb_version_info)
 string(REGEX REPLACE ".*#define TBB_VERSION_MAJOR ([0-9]+).*" "\\1" _tbb_ver_major "${_tbb_version_info}")
 string(REGEX REPLACE ".*#define TBB_VERSION_MINOR ([0-9]+).*" "\\1" _tbb_ver_minor "${_tbb_version_info}")
-string(REGEX REPLACE ".*#define TBB_INTERFACE_VERSION ([0-9]+).*" "\\1" _tbb_interface_ver "${_tbb_version_info}")
+string(REGEX REPLACE ".*#define TBB_VERSION_PATCH ([0-9]+).*" "\\1" _tbb_ver_patch "${_tbb_version_info}")
 string(REGEX REPLACE ".*#define __TBB_BINARY_VERSION ([0-9]+).*" "\\1" TBB_BINARY_VERSION "${_tbb_version_info}")
 file(READ ${CMAKE_CURRENT_LIST_DIR}/../../CMakeLists.txt _tbb_cmakelist)
 string(REGEX REPLACE ".*TBBMALLOC_BINARY_VERSION ([0-9]+).*" "\\1" TBBMALLOC_BINARY_VERSION "${_tbb_cmakelist}")
 set(TBBMALLOC_PROXY_BINARY_VERSION ${TBBMALLOC_BINARY_VERSION})
 string(REGEX REPLACE ".*TBBBIND_BINARY_VERSION ([0-9]+).*" "\\1" TBBBIND_BINARY_VERSION "${_tbb_cmakelist}")

-# Parse patch and tweak versions from interface version: e.g. 12014 --> 01 - patch version, 4 - tweak version.
-math(EXPR _tbb_ver_patch "${_tbb_interface_ver} % 1000 / 10")
-math(EXPR _tbb_ver_tweak "${_tbb_interface_ver} % 10")
-
 set(COMMON_ARGS
    TBB_ROOT_REL_PATH ${TBB_ROOT_REL_PATH}
    INC_REL_PATH ${INC_REL_PATH}
    LIB_REL_PATH ${LIB_REL_PATH}
-    VERSION ${_tbb_ver_major}.${_tbb_ver_minor}.${_tbb_ver_patch}.${_tbb_ver_tweak}
+    VERSION ${_tbb_ver_major}.${_tbb_ver_minor}.${_tbb_ver_patch}
    TBB_BINARY_VERSION ${TBB_BINARY_VERSION}
    TBBMALLOC_BINARY_VERSION ${TBBMALLOC_BINARY_VERSION}
    TBBMALLOC_PROXY_BINARY_VERSION ${TBBMALLOC_PROXY_BINARY_VERSION}
@@ -0,0 +1,2 @@
+# LSAN suppression for ltdl library known issue.
+leak:libltdl.so
@@ -36,9 +36,11 @@ macro(tbb_install_target target)
            DESTINATION ${CMAKE_INSTALL_LIBDIR}
            COMPONENT devel)

-    install(TARGETS ${target}
-        LIBRARY
-            DESTINATION ${CMAKE_INSTALL_LIBDIR}
-            NAMELINK_ONLY
-            COMPONENT devel)
+    if (BUILD_SHARED_LIBS)
+        install(TARGETS ${target}
+            LIBRARY
+                DESTINATION ${CMAKE_INSTALL_LIBDIR}
+                NAMELINK_ONLY
+                COMPONENT devel)
+    endif()
 endmacro()
@@ -0,0 +1,262 @@
+# -*- coding: utf-8 -*-
+#
+# Configuration file for the Sphinx documentation builder.
+#
+# This file does only contain a selection of the most common options. For a
+# full list see the documentation:
+# http://www.sphinx-doc.org/en/master/config
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+SOURCE_DIR = os.path.dirname(__file__)
+LATEX_DIR = os.path.join(SOURCE_DIR, '_latex')
+PREAMBLE_FILE = os.path.join(LATEX_DIR, 'preamble.tex')
+TITLE_PAGE_FILE = os.path.join(LATEX_DIR, 'title_page.tex')
+
+BUILD_TYPE = os.getenv("BUILD_TYPE")
+
+# -- Project information -----------------------------------------------------
+
+
+project = u'Intel® oneAPI Threading Building Blocks (oneTBB)'
+copyright = u'2021, Intel Corporation'
+author = u'Intel'
+
+# The short X.Y version
+version = u''
+# The full version, including alpha/beta/rc tags
+release = u''
+
+
+# -- General configuration ---------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'sphinx.ext.autodoc',
+    'sphinx.ext.doctest',
+    'sphinx.ext.intersphinx',
+    'sphinx.ext.todo',
+    'sphinx.ext.coverage',
+    'sphinx.ext.imgmath',
+    'sphinx.ext.ifconfig',
+    'sphinx.ext.viewcode',
+    'sphinx.ext.githubpages',
+]
+
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The master toctree document.
+#master_doc = 'main/title_main'
+master_doc = 'index'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = []
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = None
+
+# Syntax highlighting for the :: directive
+highlight_language = 'cpp' 
+
+
+rst_prolog = """
+.. |full_name| replace:: Intel\ |reg|\  oneAPI Threading Building Blocks (oneTBB)
+.. |short_name| replace:: oneTBB
+.. |product| replace:: oneTBB
+.. |reg| unicode:: U+000AE
+.. |copy| unicode:: U+000A9
+.. |base_tk| replace:: Intel\ |reg|\  oneAPI Base Toolkit
+.. |dpcpp| replace:: Intel\ |reg|\  oneAPI DPC++/C++ Compiler
+    """
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+html_theme = 'sphinx_book_theme'
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+
+
+# Custom sidebar templates, must be a dictionary that maps document names
+# to template names.
+#
+# The default sidebars (for documents that don't match any pattern) are
+# defined by theme itself.  Builtin themes are using these templates by
+# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
+# 'searchbox.html']``.
+#
+# html_sidebars = {}
+
+
+# -- Options for HTMLHelp output ---------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'sphinx-infodevdoc'
+
+
+# -- Options for LaTeX output ------------------------------------------------
+
+#latex_engine = 'xelatex'
+#PDF_TITLE = 'Information Development Template'
+#
+#with open(PREAMBLE_FILE, 'r', encoding='utf-8') as f:
+#    PREAMBLE = f.read()
+#
+#with open(TITLE_PAGE_FILE, 'r', encoding='utf-8') as f:
+#    TITLE_PAGE = f.read().replace('<PDF_TITLE>', PDF_TITLE)
+#
+#
+#latex_elements = {
+#    # The paper size ('letterpaper' or 'a4paper').
+#    #
+#    'extraclassoptions': 'openany,oneside',
+#    'babel' : '\\usepackage[english]{babel}',
+#    'papersize': 'a4paper',
+#    'releasename':" ",
+#    # Sonny, Lenny, Glenn, Conny, Rejne, Bjarne and Bjornstrup
+#    # 'fncychap': '\\usepackage[Lenny]{fncychap}',
+#    'fncychap':  '',
+#    #'fontpkg': '\\usepackage{amsmath,amsfonts,amssymb,amsthm}',
+#
+#    'figure_align':'htbp',
+#    # The font size ('10pt', '11pt' or '12pt').
+#    #
+#    'pointsize': '12pt',
+#
+#    # Additional stuff for the LaTeX preamble.
+#    #
+#    'preamble': PREAMBLE,
+#
+#    'maketitle': TITLE_PAGE,
+#    # Latex figure (float) alignment
+#    #
+#    # 'figure_align': 'htbp',
+#    'sphinxsetup': \
+#        'hmargin={0.7in,0.7in}, vmargin={1in,1in}, \
+#        verbatimwithframe=true, \
+#        TitleColor={rgb}{0,0.686,0.941}, \
+#        HeaderFamily=\\rmfamily\\bfseries, \
+#        InnerLinkColor={rgb}{0,0.686,0.941}, \
+#        OuterLinkColor={rgb}{0,0.686,0.941}',
+#
+#    'tableofcontents':' '
+#}
+#
+#latex_logo = '_latex/intel_logo.png'
+## Grouping the document tree into LaTeX files. List of tuples
+## (source start file, target name, title,
+##  author, documentclass [howto, manual, or own class]).
+#latex_documents = [
+#    (master_doc, 'sphinx-infodev.tex', u'sphinx-infodev Documentation',
+#     u'Intel', 'manual'),
+#]
+
+#breathe_projects = {                                       #todd-mod
+#    project: "../doxygen/xml"
+#}
+#breathe_default_project = project
+
+# Setup the exhale extension
+#exhale_args = {                                            #todd-mod
+#    # These arguments are required
+#    "containmentFolder":     "./api",
+#    "rootFileName":          "library_root.rst",
+#    "rootFileTitle":         "Library API",
+#    "doxygenStripFromPath":  "..",
+#    "fullApiSubSectionTitle": 'Full API'
+#}
+
+
+# -- Options for manual page output ------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (master_doc, 'sphinx-infodev', u'sphinx-infodev Documentation',
+     [author], 1)
+]
+
+
+# -- Options for Texinfo output ----------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, 'sphinx-infodev', u'sphinx-infodev Documentation',
+     author, 'sphinx-infodev', 'One line description of project.',
+     'Miscellaneous'),
+]
+
+
+# -- Options for Epub output -------------------------------------------------
+
+# Bibliographic Dublin Core info.
+epub_title = project
+
+# The unique identifier of the text. This can be a ISBN number
+# or the project homepage.
+#
+# epub_identifier = ''
+
+# A unique identification for the text.
+#
+# epub_uid = ''
+
+# A list of files that should not be packed into the epub file.
+epub_exclude_files = ['search.html']
+
+
+# -- Extension configuration -------------------------------------------------
+
+# -- Options for intersphinx extension ---------------------------------------
+
+# Example configuration for intersphinx: refer to the Python standard library.
+intersphinx_mapping = {'https://docs.python.org/': None}
+
+# -- Options for todo extension ----------------------------------------------
+
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+todo_include_todos = True
@@ -0,0 +1,97 @@
+.. _Get_Started_Guide
+
+Get Started with |full_name|
+============================
+
+
+|full_name| enables you to simplify parallel programming by breaking 
+computation into parallel running tasks. oneTBB is available as a stand-alone
+product and as part of the |base_tk|.
+
+|short_name| is a runtime-based parallel programming model for C++ code that uses threads.
+It consists of a template-based runtime library to help you harness the latent performance
+of multi-core processors. Use |short_name| to write scalable applications that:
+
+- Specify logical parallel structure instead of threads
+- Emphasize data parallel programming
+- Take advantage of concurrent collections and parallel algorithms
+
+Before You Begin
+****************
+
+After installing |short_name|, you need to set the environment variables:
+  
+#. Go to the oneTBB installation directory (``<install_dir>``). By default, ``<install_dir>`` is the following:
+     
+   * On Linux* OS:
+	 
+     * For superusers (root): ``/opt/intel/oneapi``
+     * For ordinary users (non-root): ``$HOME/intel/oneapi``
+     
+   * On Windows* OS:
+
+     * ``<Program Files>\Intel\oneAPI``
+
+#. Set the environment variables, using the script in <install_dir>, by running
+     
+   * On Linux* OS:
+	 
+     ``vars.{sh|csh} in <install_dir>/tbb/latest/env``
+	   
+   * On Windows* OS:
+	 
+     ``vars.bat in <install_dir>/tbb/latest/env``
+
+
+System Requirements
+*******************
+
+Refer to the `oneTBB System Requirements <https://software.intel.com/content/www/us/en/develop/articles/intel-oneapi-threading-building-blocks-system-requirements.html>`_.
+
+Find more
+*********
+
+.. list-table:: 
+   :widths: 40 60
+   :header-rows: 0
+
+
+   * - 
+	   - `oneTBB Community Forum <https://community.intel.com/>`_
+	   - `Product FAQs <https://software.intel.com/content/www/us/en/develop/support/faq-product.html>`_
+	   - `Support requests <https://software.intel.com/content/www/us/en/develop/articles/how-to-create-a-support-request-at-online-service-center.html>`_
+     - Use these resources if you need support with oneTBB.
+   
+   * - `Release Notes <https://software.intel.com/content/www/us/en/develop/articles/intel-oneapi-threading-building-blocks-release-notes.html>`_
+     - Find up-to-date information about the product, including detailed notes, known issues, and changes.
+   
+   * - `Documentation <https://software.intel.com/content/www/us/en/develop/documentation/onetbb-documentation/top.html>`_: `Developer Guide <https://software.intel.com/content/www/us/en/develop/documentation/onetbb-documentation/top/onetbb-developer-guide.html>`_ and `API Reference <https://software.intel.com/content/www/us/en/develop/documentation/onetbb-documentation/top/onetbb-api-reference.html>`_
+     - Learn to use oneTBB.   
+   * - `GitHub* <https://github.com/oneapi-src/oneTBB>`_
+     - Find oneTBB implementation in open source.
+   
+
+Notices and Disclaimers
+***********************
+
+Intel technologies may require enabled hardware, software or service activation.
+
+No product or component can be absolutely secure.
+
+Your costs and results may vary.
+
+© Intel Corporation. Intel, the Intel logo, and other Intel marks are trademarks
+of Intel Corporation or its subsidiaries. Other names and brands may be claimed
+as the property of others.
+
+No license (express or implied, by estoppel or otherwise) to any intellectual
+property rights is granted by this document.
+
+The products described may contain design defects or errors known as errata which
+ay cause the product to deviate from published specifications. Current
+characterized errata are available on request.
+
+Intel disclaims all express and implied warranties, including without limitation,
+the implied warranties of merchantability, fitness for a particular purpose,
+and non-infringement, as well as any warranty arising from course of performance,
+course of dealing, or usage in trade.
@@ -0,0 +1,53 @@
+//configure tms    
+var wapLocalCode = 'us-en'; 
+//dynamically set per localized site, see mapping table for values    
+var wapSection = "oneapi-docs";     
+//load tms    
+(function() {                 
+    var host = (window.document.location.protocol == 'http:') ? "http://www.intel.com" : "https://www.intel.com";
+    var url = host+"/content/dam/www/global/wap/tms-loader.js"; //wap file url
+    var po = document.createElement('script'); 
+    po.type = 'text/javascript'; 
+    po.async = true;  
+    po.src = url;                 
+    var s = document.getElementsByTagName('script')[0]; 
+    s.parentNode.insertBefore(po, s);     
+})(); 
+
+window.MathJax = {
+    TeX: {
+    Macros: {
+        src: '\\operatorname{src}',
+        srclayer: '\\operatorname{src\\_layer}',
+        srciter: '\\operatorname{src\\_iter}',
+        srciterc: '\\operatorname{src\\_iter\\_c}',
+        weights: '\\operatorname{weights}',
+        weightslayer: '\\operatorname{weights\\_layer}',
+        weightsiter: '\\operatorname{weights\\_iter}',
+        weightspeephole: '\\operatorname{weights\\_peephole}',
+        weightsprojection: '\\operatorname{weights\\_projection}',
+        bias: '\\operatorname{bias}',
+        dst: '\\operatorname{dst}',
+        dstlayer: '\\operatorname{dst\\_layer}',
+        dstiter: '\\operatorname{dst\\_iter}',
+        dstiterc: '\\operatorname{dst\\_iter\\_c}',
+        diffsrc: '\\operatorname{diff\\_src}',
+        diffsrclayer: '\\operatorname{diff\\_src\\_layer}',
+        diffsrciter: '\\operatorname{diff\\_src\\_iter}',
+        diffsrciterc: '\\operatorname{diff\\_src\\_iter\\_c}',
+        diffweights: '\\operatorname{diff\\_weights}',
+        diffweightslayer: '\\operatorname{diff\\_weights\\_layer}',
+        diffweightsiter: '\\operatorname{diff\\_weights\\_iter}',
+        diffweightspeephole: '\\operatorname{diff\\_weights\\_peephole}',
+        diffweightsprojection: '\\operatorname{diff\\_weights\\_projection}',
+        diffbias: '\\operatorname{diff\\_bias}',
+        diffdst: '\\operatorname{diff\\_dst}',
+        diffdstlayer: '\\operatorname{diff\\_dst\\_layer}',
+        diffdstiter: '\\operatorname{diff\\_dst\\_iter}',
+        diffdstiterc: '\\operatorname{diff\\_dst\\_iter\\_c}',
+        diffgamma: '\\operatorname{diff\\_\\gamma}',
+        diffbeta: '\\operatorname{diff\\_\\beta}',
+        workspace: '\\operatorname{workspace}'
+    }
+    }
+}
@@ -0,0 +1,13 @@
+/* override table width restrictions */
+@media screen and (min-width: 767px) {
+
+   .wy-table-responsive table td {
+      /* !important prevents the common CSS stylesheets from overriding
+         this as on RTD they are loaded after this stylesheet */
+      white-space: normal !important;
+   }
+
+   .wy-table-responsive {
+      overflow: visible !important;
+   }
+}
@@ -0,0 +1,294 @@
+# -*- coding: utf-8 -*-
+#
+# Configuration file for the Sphinx documentation builder.
+#
+# This file does only contain a selection of the most common options. For a
+# full list see the documentation:
+# http://www.sphinx-doc.org/en/master/config
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+SOURCE_DIR = os.path.dirname(__file__)
+LATEX_DIR = os.path.join(SOURCE_DIR, '_latex')
+PREAMBLE_FILE = os.path.join(LATEX_DIR, 'preamble.tex')
+TITLE_PAGE_FILE = os.path.join(LATEX_DIR, 'title_page.tex')
+
+BUILD_TYPE = os.getenv("BUILD_TYPE")
+
+# -- Project information -----------------------------------------------------
+
+if BUILD_TYPE == 'oneapi':
+    project = u'Intel® oneAPI Threading Building Blocks (oneTBB)'
+else:
+    project = u'oneAPI Threading Building Blocks (oneTBB)'
+copyright = u'2021, Intel Corporation'
+author = u'Intel'
+
+# The short X.Y version
+version = u''
+# The full version, including alpha/beta/rc tags
+release = u''
+
+
+# -- General configuration ---------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+    'sphinx.ext.autodoc',
+    'sphinx.ext.doctest',
+    'sphinx.ext.intersphinx',
+    'sphinx.ext.todo',
+    'sphinx.ext.coverage',
+    'sphinx.ext.imgmath',
+    'sphinx.ext.ifconfig',
+    'sphinx.ext.viewcode',
+    'sphinx.ext.githubpages'
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The master toctree document.
+#master_doc = 'main/title_main'
+master_doc = 'index'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = []
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = None
+
+# Syntax highlighting for the :: directive
+highlight_language = 'cpp' 
+
+if BUILD_TYPE == 'oneapi':
+    rst_prolog = """
+.. |full_name| replace:: Intel\ |reg|\  oneAPI Threading Building Blocks (oneTBB)
+.. |short_name| replace:: oneTBB
+.. |product| replace:: oneTBB
+.. |reg| unicode:: U+000AE
+.. |copy| unicode:: U+000A9
+.. |base_tk| replace:: Intel\ |reg|\  oneAPI Base Toolkit
+.. |dpcpp| replace:: Intel\ |reg|\  oneAPI DPC++/C++ Compiler
+    """
+else:
+    rst_prolog = """
+.. |full_name| replace:: oneAPI Threading Building Blocks (oneTBB)
+.. |short_name| replace:: oneTBB
+.. |product| replace:: oneTBB
+.. |reg| unicode:: U+000AE
+.. |copy| unicode:: U+000A9
+.. |base_tk| replace:: Intel\ |reg|\  oneAPI Base Toolkit
+.. |dpcpp| replace:: Intel\ |reg|\  oneAPI DPC++/C++ Compiler
+    """
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+if BUILD_TYPE == 'oneapi':
+    html_theme = 'sphinx_rtd_theme'
+else:
+    html_theme = 'sphinx_book_theme'
+    html_theme_options = {
+    'repository_url': 'https://github.com/oneapi-src/oneTBB',
+    'path_to_docs': 'doc/main',
+    'use_issues_button': True,
+    'use_edit_page_button': True,
+    'repository_branch': 'master',
+    'extra_footer': '<p align="right"><a href="https://www.intel.com/content/www/us/en/privacy/intel-cookie-notice.html">Cookies</a></p>'
+    }
+
+
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+if BUILD_TYPE == 'oneapi':
+    html_context = {
+        'css_files': [
+            '_static/theme_overrides.css',  # override wide tables in RTD theme
+        ],
+    }
+else:
+    html_js_files = ['custom.js']
+    html_logo = '_static/oneAPI-rgb-rev-100.png'
+    html_favicon = '_static/favicons.png'
+
+# Custom sidebar templates, must be a dictionary that maps document names
+# to template names.
+#
+# The default sidebars (for documents that don't match any pattern) are
+# defined by theme itself.  Builtin themes are using these templates by
+# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
+# 'searchbox.html']``.
+#
+# html_sidebars = {}
+
+
+# -- Options for HTMLHelp output ---------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'sphinx-infodevdoc'
+
+
+# -- Options for LaTeX output ------------------------------------------------
+
+#latex_engine = 'xelatex'
+#PDF_TITLE = 'Information Development Template'
+#
+#with open(PREAMBLE_FILE, 'r', encoding='utf-8') as f:
+#    PREAMBLE = f.read()
+#
+#with open(TITLE_PAGE_FILE, 'r', encoding='utf-8') as f:
+#    TITLE_PAGE = f.read().replace('<PDF_TITLE>', PDF_TITLE)
+#
+#
+#latex_elements = {
+#    # The paper size ('letterpaper' or 'a4paper').
+#    #
+#    'extraclassoptions': 'openany,oneside',
+#    'babel' : '\\usepackage[english]{babel}',
+#    'papersize': 'a4paper',
+#    'releasename':" ",
+#    # Sonny, Lenny, Glenn, Conny, Rejne, Bjarne and Bjornstrup
+#    # 'fncychap': '\\usepackage[Lenny]{fncychap}',
+#    'fncychap':  '',
+#    #'fontpkg': '\\usepackage{amsmath,amsfonts,amssymb,amsthm}',
+#
+#    'figure_align':'htbp',
+#    # The font size ('10pt', '11pt' or '12pt').
+#    #
+#    'pointsize': '12pt',
+#
+#    # Additional stuff for the LaTeX preamble.
+#    #
+#    'preamble': PREAMBLE,
+#
+#    'maketitle': TITLE_PAGE,
+#    # Latex figure (float) alignment
+#    #
+#    # 'figure_align': 'htbp',
+#    'sphinxsetup': \
+#        'hmargin={0.7in,0.7in}, vmargin={1in,1in}, \
+#        verbatimwithframe=true, \
+#        TitleColor={rgb}{0,0.686,0.941}, \
+#        HeaderFamily=\\rmfamily\\bfseries, \
+#        InnerLinkColor={rgb}{0,0.686,0.941}, \
+#        OuterLinkColor={rgb}{0,0.686,0.941}',
+#
+#    'tableofcontents':' '
+#}
+#
+#latex_logo = '_latex/intel_logo.png'
+## Grouping the document tree into LaTeX files. List of tuples
+## (source start file, target name, title,
+##  author, documentclass [howto, manual, or own class]).
+#latex_documents = [
+#    (master_doc, 'sphinx-infodev.tex', u'sphinx-infodev Documentation',
+#     u'Intel', 'manual'),
+#]
+
+#breathe_projects = {                                       #todd-mod
+#    project: "../doxygen/xml"
+#}
+#breathe_default_project = project
+
+# Setup the exhale extension
+#exhale_args = {                                            #todd-mod
+#    # These arguments are required
+#    "containmentFolder":     "./api",
+#    "rootFileName":          "library_root.rst",
+#    "rootFileTitle":         "Library API",
+#    "doxygenStripFromPath":  "..",
+#    "fullApiSubSectionTitle": 'Full API'
+#}
+
+
+# -- Options for manual page output ------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (master_doc, 'sphinx-infodev', u'sphinx-infodev Documentation',
+     [author], 1)
+]
+
+
+# -- Options for Texinfo output ----------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, 'sphinx-infodev', u'sphinx-infodev Documentation',
+     author, 'sphinx-infodev', 'One line description of project.',
+     'Miscellaneous'),
+]
+
+
+# -- Options for Epub output -------------------------------------------------
+
+# Bibliographic Dublin Core info.
+epub_title = project
+
+# The unique identifier of the text. This can be a ISBN number
+# or the project homepage.
+#
+# epub_identifier = ''
+
+# A unique identification for the text.
+#
+# epub_uid = ''
+
+# A list of files that should not be packed into the epub file.
+epub_exclude_files = ['search.html']
+
+
+# -- Extension configuration -------------------------------------------------
+
+# -- Options for intersphinx extension ---------------------------------------
+
+# Example configuration for intersphinx: refer to the Python standard library.
+intersphinx_mapping = {'https://docs.python.org/': None}
+
+# -- Options for todo extension ----------------------------------------------
+
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+todo_include_todos = True
@@ -0,0 +1,13 @@
+|full_name|
+===========
+
+.. toctree::
+   :maxdepth: 2
+
+   intro/help_support
+   intro/notation
+   intro/intro
+   intro/Benefits
+   tbb_userguide/title
+   reference/reference
+   intro/notices_and_disclaimers
@@ -0,0 +1,88 @@
+.. _Benefits:
+
+|short_name| Benefits
+=====================
+
+
+|full_name| is a library that helps you leverage multi-core performance
+without having to be a threading expert. Typically you can improve
+performance for multi-core processors by implementing the key points
+explained in the early sections of the Developer Guide. As your
+expertise grows, you may want to dive into more complex subjects that
+are covered in advanced sections.
+
+
+There are a variety of approaches to parallel programming, ranging from
+using platform-dependent threading primitives to exotic new languages.
+The advantage of oneTBB is that it works at a higher level than raw
+threads, yet does not require exotic languages or compilers. You can use
+it with any compiler supporting ISO C++. The library differs from
+typical threading packages in the following ways:
+
+
+-  **oneTBB enables you to specify logical paralleism instead of
+   threads**. Most threading packages require you to specify threads.
+   Programming directly in terms of threads can be tedious and lead to
+   inefficient programs, because threads are low-level, heavy constructs
+   that are close to the hardware. Direct programming with threads
+   forces you to efficiently map logical tasks onto threads. In
+   contrast, the oneTBB run-time library automatically maps logical
+   parallelism onto threads in a way that makes efficient use of
+   processor resources.
+
+
+-  **oneTBB targets threading for performance**. Most general-purpose
+   threading packages support many different kinds of threading, such as
+   threading for asynchronous events in graphical user interfaces. As a
+   result, general-purpose packages tend to be low-level tools that
+   provide a foundation, not a solution. Instead, oneTBB focuses on the
+   particular goal of parallelizing computationally intensive work,
+   delivering higher-level, simpler solutions.
+
+
+-  **oneTBB is compatible with other threading packages.** Because the
+   library is not designed to address all threading problems, it can
+   coexist seamlessly with other threading packages.
+
+
+-  **oneTBB emphasizes scalable, data parallel programming**. Breaking a
+   program up into separate functional blocks, and assigning a separate
+   thread to each block is a solution that typically does not scale well
+   since typically the number of functional blocks is fixed. In
+   contrast, oneTBB emphasizes *data-parallel* programming, enabling
+   multiple threads to work on different parts of a collection.
+   Data-parallel programming scales well to larger numbers of processors
+   by dividing the collection into smaller pieces. With data-parallel
+   programming, program performance increases as you add processors.
+
+
+-  **oneTBB relies on generic programming**. Traditional libraries
+   specify interfaces in terms of specific types or base classes.
+   Instead, oneAPI Threading Building Blocks uses generic programming.
+   The essence of generic programming is writing the best possible
+   algorithms with the fewest constraints. The C++ Standard Template
+   Library (STL) is a good example of generic programming in which the
+   interfaces are specified by *requirements* on types. For example, C++
+   STL has a template function ``sort`` that sorts a sequence abstractly
+   defined in terms of iterators on the sequence. The requirements on
+   the iterators are:
+
+
+   -  Provide random access
+
+
+   -  The expression ``*i<*j`` is true if the item pointed to by
+      iterator ``i`` should precede the item pointed to by iterator
+      ``j``, and false otherwise.
+
+
+   -  The expression ``swap(*i,*j)`` swaps two elements.
+
+
+Specification in terms of requirements on types enables the template to
+sort many different representations of sequences, such as vectors and
+deques. Similarly, the oneTBB templates specify requirements on types,
+not particular types, and thus adapt to different data representations.
+Generic programming enables oneTBB to deliver high performance
+algorithms with broad applicability.
+
@@ -0,0 +1,26 @@
+.. _help_support:
+
+Getting Help and Support
+========================
+
+
+
+
+
+.. container:: section
+
+
+   .. rubric:: Getting Technical Support
+      :class: sectiontitle
+
+   If you did not register your Intel® software product during
+   installation, please do so now at the Intel® Software Development
+   Products Registration Center. Registration entitles you to free
+   technical support, product updates and upgrades for the duration of
+   the support term.
+
+
+   For general information about Intel technical support, product
+   updates, user forums, FAQs, tips and tricks and other support
+   questions, go to: http://www.intel.com/software/products/support/.
+
@@ -0,0 +1,39 @@
+.. _intro:
+
+Introduction
+============
+
+
+|full_name| is a library that supports scalable parallel programming using
+standard ISO C++ code. It does not require special languages or
+compilers. It is designed to promote scalable data parallel programming.
+Additionally, it fully supports nested parallelism, so you can build
+larger parallel components from smaller parallel components. To use the
+library, you specify tasks, not threads, and let the library map tasks
+onto threads in an efficient manner.
+
+
+Many of the library interfaces employ generic programming, in which
+interfaces are defined by requirements on types and not specific types.
+The C++ Standard Template Library (STL) is an example of generic
+programming. Generic programming enables oneTBB to be flexible yet
+efficient. The generic interfaces enable you to customize components to
+your specific needs.
+
+
+.. note:: 
+   |full_name| requires C++11 standard compiler support.
+
+
+The net result is that oneTBB enables you to specify parallelism far
+more conveniently than using raw threads, and at the same time can
+improve performance.
+
+
+.. admonition:: Product and Performance Information 
+
+   Performance varies by use, configuration and other factors. Learn more at `www.intel.com/PerformanceIndex <https://www.intel.com/PerformanceIndex>`_.
+   Notice revision #20201201 
+
+
+
@@ -0,0 +1,39 @@
+.. _introducing_main:
+
+Introduction
+============
+
+
+|full_name| is a library that supports scalable parallel programming using
+standard ISO C++ code. It does not require special languages or
+compilers. It is designed to promote scalable data parallel programming.
+Additionally, it fully supports nested parallelism, so you can build
+larger parallel components from smaller parallel components. To use the
+library, you specify tasks, not threads, and let the library map tasks
+onto threads in an efficient manner.
+
+
+Many of the library interfaces employ generic programming, in which
+interfaces are defined by requirements on types and not specific types.
+The C++ Standard Template Library (STL) is an example of generic
+programming. Generic programming enables oneTBB to be flexible yet
+efficient. The generic interfaces enable you to customize components to
+your specific needs.
+
+
+.. note:: 
+   |full_name| requires C++11 standard compiler support.
+
+
+The net result is that oneTBB enables you to specify parallelism far
+more conveniently than using raw threads, and at the same time can
+improve performance.
+
+
+.. admonition:: Product and Performance Information 
+
+   Performance varies by use, configuration and other factors. Learn more at `www.intel.com/PerformanceIndex <https://www.intel.com/PerformanceIndex>`_.
+   Notice revision #20201201 
+
+
+
@@ -0,0 +1,106 @@
+.. _notation:
+
+Notational Conventions
+======================
+
+
+The following conventions may be used in this document.
+
+
+.. container:: tablenoborder
+
+
+   .. list-table:: 
+      :header-rows: 1
+
+      * -     Convention     
+        -     Explanation     
+        -     Example     
+      * -     \ *Italic*     
+        -     Used for introducing new terms, denotation of terms,    placeholders, or titles of manuals.    
+        -     The filename consists of the *basename* and the *extension*.     
+      * -     \ ``Monospace``     
+        -     Indicates directory paths and filenames, commands and    command line        options, function names, methods,   classes, data structures in body text, source code.    
+        -     \ ``ippsapi.h``        \ ``\alt\include``           Use the okCreateObjs() function to...          \ ``printf("hello, world\n");``    
+      * -     [ ]     
+        -     Items enclosed in brackets are optional.     
+        -     Fa[c]        Indicates Fa or Fac.     
+      * -     { \| }     
+        -     Braces and vertical bars indicate the choice of one item    from a selection of two or more items.    
+        -     X{K \| W \| P}        Indicates XK, XW, or XP.       
+      * -     "[" "]" "{"    | " }" "|"    
+        -     Writing a metacharacter in quotation marks negates the      syntactical meaning stated above;   | the character is taken as a literal.    
+        -     "[" X "]" [ Y ]        Denotes the letter X    enclosed in brackets, optionally followed by the letter Y.    
+      * -     ...     
+        -     The ellipsis indicates that the previous item can be    repeated several times.    
+        -     \ ``filename`` ...        Indicates that one or    more filenames can be specified.    
+      * -     ,...     
+        -     The ellipsis preceded by a comma indicates that the      previous item can be repeated several times,   | separated by commas.    
+        -     \ ``word`` ,...        Indicates that one or    more words can be specified. If more than one word is specified, the   words are comma-separated.    
+
+
+
+
+.. container:: section
+
+
+   Class members are summarized by informal class declarations that
+   describe the class as it seems to clients, not how it is actually
+   implemented. For example, here is an informal declaration of class
+   ``Foo``:
+
+
+   ::
+
+
+      class Foo {
+      public:
+          int x();
+          int y;
+          ~Foo();
+      };
+
+
+   The actual implementation might look like:
+
+
+   ::
+
+
+      namespace internal {
+          class FooBase  {
+          protected:
+              int x();
+          };
+
+
+          class Foo_v3: protected FooBase {
+          private:
+              int internal_stuff;
+          public:
+              using FooBase::x;
+              int y;
+          };
+      }
+
+
+      typedef internal::Foo_v3 Foo;
+
+
+   The example shows two cases where the actual implementation departs
+   from the informal declaration:
+
+
+   -  ``Foo`` is actually a typedef to ``Foo_v3``.
+
+
+   -  Method ``x()`` is inherited from a protected base class.
+
+
+   -  The destructor is an implicit method generated by the compiler.
+
+
+   The informal declarations are intended to show you what you need to
+   know to use the class without the distraction of irrelevant clutter
+   particular to the implementation.
+
@@ -0,0 +1,49 @@
+.. _notices_and_disclaimers:
+
+Notices and Disclaimers
+=======================
+
+
+Intel technologies may require enabled hardware, software or service
+activation.
+
+
+No product or component can be absolutely secure.
+
+
+Your costs and results may vary.
+
+
+© Intel Corporation. Intel, the Intel logo, and other Intel marks are
+trademarks of Intel Corporation or its subsidiaries. Other names and
+brands may be claimed as the property of others.
+
+
+Intel's compilers may or may not optimize to the same degree for
+non-Intel microprocessors for optimizations that are not unique to Intel
+microprocessors. These optimizations include SSE2, SSE3, and SSSE3
+instruction sets and other optimizations. Intel does not guarantee the
+availability, functionality, or effectiveness of any optimization on
+microprocessors not manufactured by Intel. Microprocessor-dependent
+optimizations in this product are intended for use with Intel
+microprocessors. Certain optimizations not specific to Intel
+microarchitecture are reserved for Intel microprocessors. Please refer
+to the applicable product User and Reference Guides for more information
+regarding the specific instruction sets covered by this notice.
+
+
+No license (express or implied, by estoppel or otherwise) to any
+intellectual property rights is granted by this document.
+
+
+The products described may contain design defects or errors known as
+errata which may cause the product to deviate from published
+specifications. Current characterized errata are available on request.
+
+
+Intel disclaims all express and implied warranties, including without
+limitation, the implied warranties of merchantability, fitness for a
+particular purpose, and non-infringement, as well as any warranty
+arising from course of performance, course of dealing, or usage in
+trade.
+
@@ -0,0 +1,165 @@
+.. _task_scheduler_handle_reference:
+
+task_scheduler_handle Class
+===========================
+
+.. note::
+    To enable this feature, set the ``TBB_PREVIEW_WAITING_FOR_WORKERS`` macro to 1.
+
+.. contents::
+    :local:
+    :depth: 1
+
+Description
+***********
+
+The ``oneapi::tbb::task_scheduler_handle`` class and the ``oneapi::tbb::finalize`` function allow to wait for completion of worker threads.
+
+When the ``oneapi::tbb::finalize`` function is called with an ``oneapi::tbb::task_scheduler_handle`` instance, it blocks the calling
+thread until the completion of all worker threads that were implicitly created by the library.
+
+API
+***
+
+Header
+------
+
+.. code:: cpp
+
+    #include <oneapi/tbb/global_control.h>
+
+Synopsis
+--------
+
+.. code:: cpp
+
+    namespace oneapi {
+        namespace tbb {
+
+            class task_scheduler_handle {
+            public:
+                task_scheduler_handle() = default;
+                ~task_scheduler_handle();
+
+                task_scheduler_handle(const task_scheduler_handle& other) = delete;
+                task_scheduler_handle(task_scheduler_handle&& other) noexcept;
+                task_scheduler_handle& operator=(const task_scheduler_handle& other) = delete;
+                task_scheduler_handle& operator=(task_scheduler_handle&& other) noexcept;
+
+                explicit operator bool() const noexcept;
+
+                static task_scheduler_handle get();
+
+                static void release(task_scheduler_handle& handle);
+            };
+
+            void finalize(task_scheduler_handle& handle);
+            bool finalize(task_scheduler_handle& handle, const std::nothrow_t&) noexcept;
+
+        } // namespace tbb
+    } // namespace oneapi
+
+Member Functions
+----------------
+
+.. cpp:function:: task_scheduler_handle()
+
+    **Effects**: Creates an instance of the ``task_scheduler_handle`` class that does not contain any reference to the task scheduler.
+
+-------------------------------------------------------
+
+.. cpp:function:: ~task_scheduler_handle()
+
+    **Effects**: Destroys an instance of the ``task_scheduler_handle`` class.
+    Releases a reference to the task scheduler and deactivates an instance of the ``task_scheduler_handle`` class.
+
+-------------------------------------------------------
+
+.. cpp:function:: task_scheduler_handle(task_scheduler_handle&& other) noexcept
+
+    **Effects**: Creates an instance of the ``task_scheduler_handle`` class that references the task scheduler referenced by ``other``. In turn, ``other`` releases its reference to the task scheduler.
+
+-------------------------------------------------------
+
+.. cpp:function:: task_scheduler_handle& operator=(task_scheduler_handle&& other) noexcept
+
+    **Effects**: Releases a reference to the task scheduler referenced by ``this``. Adds a reference to the task scheduler referenced by ``other``.
+In turn, ``other`` releases its reference to the task scheduler.
+
+-------------------------------------------------------
+
+.. cpp:function:: explicit operator bool() const noexcept
+
+    **Returns**: ``true`` if ``this`` references any task scheduler; ``false`` otherwise.
+
+-------------------------------------------------------
+
+.. cpp:function:: task_scheduler_handle get()
+
+    **Returns**: An instance of the ``task_scheduler_handle`` class that holds a reference to the task scheduler preventing
+    its premature destruction.
+
+-------------------------------------------------------
+
+.. cpp:function:: void release(task_scheduler_handle& handle)
+
+    **Effects**: Releases a reference to the task scheduler and deactivates an instance of the ``task_scheduler_handle``
+    class. Non-blocking method.
+
+Non-member Functions
+--------------------
+
+.. cpp:function:: void finalize(task_scheduler_handle& handle)
+
+    **Effects**: Blocks the program execution until all worker threads have been completed. Throws the ``oneapi::tbb::unsafe_wait``
+    exception if it is not safe to wait for the completion of the worker threads.
+
+The following conditions should be met for finalization to succeed:
+
+- No active (not yet terminated) instances of class ``task_arena`` exist in the whole program;
+- ``task_scheduler_handle::release`` is called for each other active instance of class ``task_scheduler_handle``, possibly by different application threads.
+
+Under these conditions, it is guaranteed that at least one ``finalize`` call succeeds,
+at which point all worker threads have been completed.
+If calls are performed simultaneously, more than one call might succeed.
+
+.. note::
+
+    If you know how many active ``task_scheduler_handle`` instances exist in the program,
+    it is necessary to ``release`` all but the last one, then call ``finalize`` for
+    the last instance.
+
+.. caution::
+
+  The method always fails if called within a task, a parallel algorithm, or a flow graph node.
+
+-------------------------------------------------------
+
+.. cpp:function:: bool finalize(task_scheduler_handle& handle, const std::nothrow_t&) noexcept
+
+    **Effects**: Blocks the program execution until all worker threads have been completed. Same as above, but returns ``true`` if all worker
+    threads have been completed successfully, or ``false`` if it is not safe to wait for the completion of the worker threads.
+
+Examples
+********
+
+.. code:: cpp
+
+    #define TBB_PREVIEW_WAITING_FOR_WORKERS 1
+    #include <oneapi/tbb/global_control.h>
+    #include <oneapi/tbb/parallel_for.h>
+
+    #include <iostream>
+
+    int main() {
+        oneapi::tbb::task_scheduler_handle handle = oneapi::tbb::task_scheduler_handle::get();
+        // Do some parallel work here, e.g.
+        oneapi::tbb::parallel_for(0, 10000, [](int){});
+        try {
+            oneapi::tbb::finalize(handle);
+            // oneTBB worker threads are terminated at this point.
+        } catch (const oneapi::tbb::unsafe_wait&) {
+            std::cerr << "Failed to terminate the worker threads." << std::endl;
+        }
+        return 0;
+    }
@@ -0,0 +1,163 @@
+.. _concurrent_lru_cache:
+
+concurrent_lru_cache
+====================
+
+.. note::
+   To enable this feature, define the ``TBB_PREVIEW_CONCURRENT_LRU_CACHE`` macro to 1.
+
+A Class Template for Least Recently Used cache with concurrent operations.
+
+.. contents::
+    :local:
+    :depth: 1
+
+Description
+***********
+
+A ``concurrent_lru_cache`` container maps keys to values with the ability
+to limit the number of stored unused values. For each key, there is at most one item
+stored in the container.
+
+The container permits multiple threads to concurrently retrieve items from it.
+
+The container tracks which items are in use by returning a proxy
+``concurrent_lru_cache::handle`` object that refers to an item instead of its value.
+Once there are no ``handle`` objects holding reference to an item, it is considered unused.
+
+The container stores all the items that are currently in use plus a limited
+number of unused items. Excessive unused items are erased according to
+least recently used policy.
+
+When no item is found for a given key, the container calls the user-specified
+``value_function_type`` object to construct a value for the key, and stores that value.
+The ``value_function_type`` object must be thread-safe.
+
+API
+***
+
+Header
+------
+
+.. code:: cpp
+
+    #include "oneapi/tbb/concurrent_lru_cache.h"
+
+Synopsis
+--------
+
+.. code:: cpp
+
+    namespace oneapi {
+        namespace tbb {
+            template <typename Key, typename Value, typename ValueFunctionType = Value (*)(Key)>
+            class concurrent_lru_cache {
+            public:
+                using key_type = Key;
+                using value_type = Value;
+                using pointer = value_type*;
+                using const_pointer = const value_type*;
+                using reference = value_type&;
+                using const_reference = const value_type&;
+
+                using value_function_type = ValueFunctionType;
+
+                class handle {
+                public:
+                    handle();
+                    handle( handle&& other );
+
+                    ~handle();
+
+                    handle& operator=( handle&& other );
+
+                    operator bool() const;
+                    value_type& value();
+                }; // class handle
+
+                concurrent_lru_cache( value_function_type f, std::size_t number_of_lru_history_items );
+                ~concurrent_lru_cache();
+
+                handle operator[]( key_type key );
+            }; // class concurrent_lru_cache
+        } // namespace tbb
+    } // namespace oneapi
+
+Member Functions
+----------------
+
+.. cpp:function:: concurrent_lru_cache( value_function_type f, std::size_t number_of_lru_history_items );
+
+    **Effects**: Constructs an empty cache that can keep up to ``number_of_lru_history_items``
+    unused values, with a function object ``f`` for constructing new values.
+
+-------------------------------------------------------
+
+.. cpp:function:: ~concurrent_lru_cache();
+
+    **Effects**: Destroys the ``concurrent_lru_cache``. Calls the destructors of the stored elements and
+    deallocates the used storage.
+
+The behavior is undefined in case of concurrent operations with ``*this``.
+
+-------------------------------------------------------
+
+.. cpp:function:: handle operator[]( key_type k );
+
+    **Effects**: Searches the container for an item that corresponds to the given key.
+    If such an item is not found, the user-specified function object is called to
+    construct a value that is inserted into the container.
+
+    **Returns**: a ``handle`` object holding reference to the matching value.
+
+Member Objects
+--------------
+
+``handle`` class
+^^^^^^^^^^^^^^^^
+
+**Member Functions**
+
+.. cpp:function:: handle();
+
+    **Effects**: Constructs a ``handle`` object that does not refer to any value.
+
+--------------------------------------------------
+
+.. cpp:function:: handle( handle&& other );
+
+    **Effects**: Transfers the reference to the value stored in ``concurrent_lru_cache``
+    from ``other`` to the newly constructed object. Upon completion,
+    ``other`` no longer refers to any value.
+
+---------------------------------------------------
+
+.. cpp:function:: ~handle();
+
+    **Effects**: Releases the reference (if it exists) to a value stored in ``concurrent_lru_cache``.
+
+The behavior is undefined for concurrent operations with ``*this``.
+
+---------------------------------------------------
+
+.. cpp:function:: handle& operator=( handle&& other );
+
+    **Effects**: Transfers the reference to a value stored in ``concurrent_lru_cache`` from ``other``
+    to ``*this``. If existed, the previous reference held by ``*this`` is released. Upon
+    completion ``other`` no longer refers to any value.
+
+    **Returns**: a reference to ``*this``.
+
+---------------------------------------------------
+
+.. cpp:function:: operator bool() const;
+
+    **Returns**: ``true`` if ``*this`` holds reference to a value, ``false`` otherwise.
+
+---------------------------------------------------
+
+.. cpp:function:: value_type& value();
+
+    **Returns**: a reference to a ``value_type`` object stored in ``concurrent_lru_cache``.
+
+The behavior is undefined if ``*this`` does not refer to any value.
@@ -0,0 +1,118 @@
+.. _constraints_extensions:
+
+task_arena::constraints extensions
+======================================
+
+.. note::
+    To enable this feature, set the ``TBB_PREVIEW_TASK_ARENA_CONSTRAINTS_EXTENSION`` macro to 1.
+
+.. contents::
+    :local:
+    :depth: 1
+
+Description
+***********
+
+These extensions allow to customize ``tbb::task_arena::constraints`` with the following properties:
+
+* On machines with Intel® Hybrid Technology set the preferred core type for threads working within the task arena.
+* Limit the maximum number of threads that can be scheduled to one core simultaneously.
+
+API
+***
+
+Header
+------
+
+.. code:: cpp
+
+    #include <oneapi/tbb/task_arena.h>
+
+Synopsis
+--------
+
+.. code:: cpp
+
+    namespace oneapi {
+        namespace tbb {
+
+            class task_arena {
+            public:
+                struct constraints {
+                    constraints& set_numa_id(numa_node_id id);
+                    constraints& set_max_concurrency(int maximal_concurrency);
+                    constraints& set_core_type(core_type_id id);
+                    constraints& set_max_threads_per_core(int threads_number);
+
+                    numa_node_id numa_id = task_arena::automatic;
+                    int max_concurrency = task_arena::automatic;
+                    core_type_id core_type = task_arena::automatic;
+                    int max_threads_per_core = task_arena::automatic;
+                }; // struct constraints
+            }; // class task_arena
+
+        } // namespace tbb
+    } // namespace oneapi
+
+Member Functions
+----------------
+
+.. cpp:function:: constraints& set_numa_id(numa_node_id id)
+
+    Sets the ``numa_id`` field to the ``id``.
+
+    **Returns:** Reference to ``*this``.
+
+.. cpp:function:: constraints& set_max_concurrency(int maximal_concurrency)
+
+    Sets the ``max_concurrency`` field to the ``maximal_concurrency``.
+
+    **Returns:** Reference to ``*this``.
+
+.. cpp:function:: constraints& set_core_type(core_type_id id)
+
+    Sets the ``core_type`` field to the ``id``.
+
+    **Returns:** Reference to ``*this``.
+
+.. cpp:function:: constraints& set_max_threads_per_core(int threads_number)
+
+    Sets the ``max_threads_per_core`` field to the ``threads_number``.
+
+    **Returns:** Reference to ``*this``.
+
+Member Objects
+--------------
+
+.. cpp:member:: numa_node_id numa_id
+
+    An integral logical index uniquely identifying a NUMA node. All threads joining the
+    ``task_arena`` are bound to this NUMA node.
+
+    .. note::
+
+        To obtain a valid NUMA node ID, call ``oneapi::tbb::info::numa_nodes()``.
+
+.. cpp:member:: int max_concurrency
+
+    The maximum number of threads that can participate in work processing within the
+    ``task_arena`` at the same time.
+
+
+.. cpp:member:: core_type_id core_type
+
+    An integral logical index uniquely identifying a core type. All threads joining the
+    ``task_arena`` are bound to this core type.
+
+    .. note::
+
+        To obtain a valid core type node ID, call ``oneapi::tbb::info::core_types()``.
+
+.. cpp:member:: int max_threads_per_core
+
+    The maximum number of threads that can be scheduled to one core simultaneously.
+
+See also:
+
+* :doc:`oneapi::tbb::info namespace preview extensions <info_namespace_extensions>`
+* `oneapi::tbb::task_arena specification <https://spec.oneapi.com/versions/latest/elements/oneTBB/source/task_scheduler/task_arena/task_arena_cls.html>`_
@@ -0,0 +1,104 @@
+.. _constructors_for_fg_nodes:
+
+Constructors for Flow Graph nodes
+=================================
+
+.. note::
+   To enable this feature, define the ``TBB_PREVIEW_FLOW_GRAPH_FEATURES`` macro to 1.
+
+.. contents::
+    :local:
+    :depth: 1
+
+Description
+***********
+
+The "Helper Functions for Expressing Graphs" feature adds a set of new constructors
+that can be used to construct a node that ``follows`` or ``precedes`` a set of nodes.
+
+Where possible, the constructors support Class Template Argument Deduction (since C++17).
+
+API
+***
+
+Header
+------
+
+.. code:: cpp
+
+    #include <oneapi/tbb/flow_graph.h>
+
+Syntax
+------
+
+.. code:: cpp
+
+    // continue_node
+    continue_node(follows(...), Body body, Policy = Policy());
+    continue_node(precedes(...), Body body, Policy = Policy());
+
+    continue_node(follows(...), int number_of_predecessors, Body body, Policy = Policy());
+    continue_node(precedes(...), int number_of_predecessors, Body body, Policy = Policy());
+
+    // function_node
+    function_node(follows(...), std::size_t concurrency, Policy = Policy());
+    function_node(precedes(...), std::size_t concurrency, Policy = Policy());
+
+    // input_node
+    input_node(precedes(...), body);
+
+    // multifunction_node
+    multifunction_node(follows(...), std::size_t concurrency, Body body);
+    multifunction_node(precedes(...), std::size_t concurrency, Body body);
+
+    // async_node
+    async_node(follows(...), std::size_t concurrency, Body body);
+    async_node(precedes(...), std::size_t concurrency, Body body);
+
+    // overwrite_node
+    explicit overwrite_node(follows(...));
+    explicit overwrite_node(precedes(...));
+
+    // write_once_node
+    explicit write_once_node(follows(...));
+    explicit write_once_node(precedes(...));
+
+    // buffer_node
+    explicit buffer_node(follows(...));
+    explicit buffer_node(precedes(...));
+
+    // queue_node
+    explicit queue_node(follows(...));
+    explicit queue_node(precedes(...));
+
+    // priority_queue_node
+    explicit priority_queue_node(follows(...), const Compare& comp = Compare());
+    explicit priority_queue_node(precedes(...), const Compare& compare = Compare());
+
+    // sequencer_node
+    sequencer_node(follows(...), const Sequencer& s);
+    sequencer_node(precedes(...), const Sequencer& s);
+
+    // limiter_node
+    limiter_node(follows(...), std::size_t threshold);
+    limiter_node(precedes(...), std::size_t threshold);
+
+    // broadcast_node
+    explicit broadcast_node(follows(...));
+    explicit broadcast_node(precedes(...));
+
+    // join_node
+    explicit join_node(follows(...), Policy = Policy());
+    explicit join_node(precedes(...), Policy = Policy());
+
+    // split_node
+    explicit split_node(follows(...));
+    explicit split_node(precedes(...));
+
+    // indexer_node
+    indexer_node(follows(...));
+    indexer_node(precedes(...));
+
+See Also
+********
+:ref:`follows_precedes`
@@ -0,0 +1,158 @@
+.. _custom_mutex_chmap:
+
+The customizing mutex type for ``concurrent_hash_map``
+======================================================
+
+.. note::
+    To enable this feature, define the ``TBB_PREVIEW_CONCURRENT_HASH_MAP_EXTENSIONS`` macro to 1.
+
+.. contents::
+    :local:
+    :depth: 1
+
+Description
+***********
+
+oneTBB ``concurrnent_hash_map`` class uses reader-writer mutex
+to provide thread safety and avoid data races for insert, lookup, and erasure operations. This feature adds an extra template parameter
+for ``concurrent_hash_map`` that allows to customize the type of the reader-writer mutex.
+
+API
+***
+
+Header
+------
+
+.. code:: cpp
+
+    #include <oneapi/tbb/concurrent_hash_map.h>
+
+Synopsis
+--------
+
+.. code:: cpp
+
+    namespace oneapi {
+    namespace tbb {
+
+        template <typename Key, typename T,
+                typename HashCompare = tbb_hash_compare<Key>,
+                typename Allocator = tbb_allocator<std::pair<const Key, T>>,
+                typename Mutex = spin_rw_mutex>
+        class concurrent_hash_map {
+            using mutex_type = Mutex;
+        };
+
+    } // namespace tbb
+    } // namespace oneapi
+
+Type requirements
+-----------------
+
+The type of the mutex passed as a template argument for ``concurrent_hash_map`` should
+meet the requirements of `ReaderWriterMutex <https://spec.oneapi.com/versions/latest/elements/oneTBB/source/named_requirements/mutexes/rw_mutex.html>`_.
+It should also provide the following API:
+
+.. cpp:function:: bool ReaderWriterMutex::scoped_lock::is_writer() const;
+
+**Returns**: ``true`` if the ``scoped_lock`` object acquires the mutex as a writer, ``false`` otherwise.
+
+The behavior is undefined if the ``scoped_lock`` object does not acquire the mutex.
+
+``oneapi::tbb::spin_rw_mutex``, ``oneapi::tbb::speculative_spin_rw_mutex``, ``oneapi::tbb::queuing_rw_mutex``, ``oneapi::tbb::null_rw_mutex``,
+and ``oneapi::tbb::rw_mutex`` meet the requirements above.
+
+.. rubric:: Example
+
+The example below demonstrates how to wrap ``std::shared_mutex`` (C++17) to meet the requirements
+of `ReaderWriterMutex` and how to customize ``concurrent_hash_map`` to use this mutex.
+
+.. code:: cpp
+
+    #define TBB_PREVIEW_CONCURRENT_HASH_MAP_EXTENSIONS 1
+    #include "oneapi/tbb/concurrent_hash_map.h"
+    #include <shared_mutex>
+
+    class SharedMutexWrapper {
+    public:
+        // ReaderWriterMutex requirements
+
+        static constexpr bool is_rw_mutex = true;
+        static constexpr bool is_recursive_mutex = false;
+        static constexpr bool is_fair_mutex = false;
+
+        class scoped_lock {
+        public:
+            scoped_lock() : my_mutex_ptr(nullptr), my_writer_flag(false) {}
+            scoped_lock(SharedMutexWrapper& mutex, bool write = true)
+                : my_mutex_ptr(&mutex), my_writer_flag(write)
+            {
+                if (my_writer_flag) {
+                    my_mutex_ptr->my_mutex.lock();
+                } else {
+                    my_mutex_ptr->my_mutex.lock_shared();
+                }
+            }
+
+            ~scoped_lock() {
+                if (my_mutex_ptr) release();
+            }
+
+            void acquire(SharedMutexWrapper& mutex, bool write = true) {
+                if (my_mutex_ptr) release();
+
+                my_mutex_ptr = &mutex;
+                my_writer_flag = write;
+
+                if (my_writer_flag) {
+                    my_mutex_ptr->my_mutex.lock();
+                } else {
+                    my_mutex_ptr->my_mutex.lock_shared();
+                }
+            }
+
+            void release() {
+                if (my_writer_flag) {
+                    my_mutex_ptr->my_mutex.unlock();
+                } else {
+                    my_mutex_ptr->my_mutex.unlock_shared();
+                }
+            }
+
+            bool upgrade_to_writer() {
+                // std::shared_mutex does not have the upgrade/downgrade parallel_for_each_semantics
+                if (my_writer_flag) return true; // Already a writer
+
+                my_mutex_ptr->my_mutex.unlock_shared();
+                my_mutex_ptr->my_mutex.lock();
+                return false; // The lock was reacquired
+            }
+
+            bool downgrade_to_reader() {
+                if (!my_writer_flag) return true; // Already a reader
+
+                my_mutex_ptr->my_mutex.unlock();
+                my_mutex_ptr->my_mutex.lock_shared();
+                return false;
+            }
+
+            bool is_writer() const {
+                return my_writer_flag;
+            }
+
+        private:
+            SharedMutexWrapper* my_mutex_ptr;
+            bool                my_writer_flag;
+        };
+    private:
+        std::shared_mutex my_mutex;
+    }; // struct SharedMutexWrapper
+
+    int main() {
+        using map_type = oneapi::tbb::concurrent_hash_map<int, int,
+                                                          oneapi::tbb::tbb_hash_compare<int>,
+                                                          oneapi::tbb::tbb_allocator<std::pair<const int, int>>,
+                                                          SharedMutexWrapper>;
+
+        map_type map; // This object will use SharedMutexWrapper for thread safety of insert/find/erase operations
+    }
@@ -0,0 +1,78 @@
+.. _follows_precedes:
+
+``follows`` and ``precedes`` function templates
+===============================================
+
+.. note::
+   To enable this feature, define the ``TBB_PREVIEW_FLOW_GRAPH_FEATURES`` macro to 1.
+
+The ``follows`` and ``precedes`` helper functions aid in expressing
+dependencies between nodes when building oneTBB flow graphs. These helper functions can
+only be used while constructing the node.
+
+.. contents::
+    :local:
+    :depth: 1
+
+Description
+***********
+
+The ``follows`` helper function specifies that the node being constructed is
+the successor of the set of nodes passed as an argument.
+
+The ``precedes`` helper function specifies that the node being constructed is
+the predecessor of the set of nodes passed as an argument.
+
+Functions ``follows`` and ``precedes`` are meant to replace the graph argument, which is
+passed as the first argument to the constructor of the node. The graph argument for the
+node being constructed is obtained either from the specified node set or the sequence of nodes passed
+to ``follows`` or ``precedes``.
+
+If the nodes passed to ``follows`` or ``precedes`` belong to
+different graphs, the behavior is undefined.
+
+API
+***
+
+Header
+------
+
+.. code:: cpp
+
+    #include <oneapi/tbb/flow_graph.h>
+
+Syntax
+------
+
+.. code:: cpp
+
+    // node_set is an exposition-only name for the type returned from make_node_set function
+
+    template <typename NodeType, typename... NodeTypes>
+    /*unspecified*/ follows( node_set<NodeType, NodeTypes...>& set );
+
+    template <typename NodeType, typename... NodeTypes>
+    /*unspecified*/ follows( NodeType& node, NodeTypes&... nodes );
+
+    template <typename NodeType, typename... NodeTypes>
+    /*unspecified*/ precedes( node_set<NodeType, NodeTypes...>& set );
+
+    template <typename NodeType, typename... NodeTypes>
+    /*unspecified*/ precedes( NodeType& node, NodeTypes&... nodes );
+
+Input Parameters
+----------------
+
+Either a set or a sequence of nodes can be used as arguments for ``follows`` and
+``precedes``. The following expressions are equivalent:
+
+.. code-block:: cpp
+    :caption: A set of nodes as an input
+
+    auto handlers = make_node_set(n1, n2, n3);
+    broadcast_node<int> input(precedes(handlers));
+
+.. code-block:: cpp
+    :caption: A sequence of nodes as an input
+ 
+    broadcast_node<int> input(precedes(n1, n2, n3));
@@ -0,0 +1,118 @@
+.. _helpers_for_expressing_graphs:
+
+Helper Functions for Expressing Graphs
+======================================
+
+.. note::
+   To enable this feature, define the ``TBB_PREVIEW_FLOW_GRAPH_FEATURES`` macro to 1.
+
+Helper functions are intended to make creation of the flow graphs less verbose.
+
+.. contents::
+    :local:
+    :depth: 1
+
+Description
+***********
+
+This feature adds ``make_edges``, ``make_node_set``,
+``follows`` and ``precedes`` functions to ``oneapi::tbb::flow`` namespace.
+These functions simplify the process of building flow graphs by allowing to gather nodes
+into sets and connect them to other nodes in the graph.
+
+API
+***
+
+.. toctree::
+    :titlesonly:
+
+    constructors_for_nodes
+    follows_and_precedes_functions
+    make_node_set_function
+    make_edges_function
+
+Example
+*******
+
+Consider the graph depicted below.
+
+.. figure:: ./Resources/fg_api_graph_structure.png
+    :align: center
+
+In the examples below, C++17 Class Template Argument Deduction is used
+to avoid template parameter specification where possible.
+
+**Regular API**
+
+.. code:: cpp
+
+    #include <oneapi/tbb/flow_graph.h>
+
+    int main() {
+        using namespace oneapi::tbb::flow;
+
+        graph g;
+
+        broadcast_node<int> input(g);
+
+        function_node doubler(g, unlimited, [](const int& v) { return 2 * v; });
+        function_node squarer(g, unlimited, [](const int&) { return v * v; });
+        function_node cuber(g, unlimited, [](const int& v) { return v * v * v; });
+
+        join_node<std::tuple<int, int, int>> join(g);
+
+        int sum = 0;
+        function_node summer(g, serial, [&](const std::tuple<int, int, int>& v) {
+            int sub_sum = std::get<0>(v) + std::get<1>(v) + std::get<2>(v);
+            sum += sub_sum;
+            return sub_sum;
+        });
+
+        make_edge(input, doubler);
+        make_edge(input, squarer);
+        make_edge(input, cuber);
+        make_edge(doubler, std::get<0>(join.input_ports()));
+        make_edge(squarer, std::get<1>(join.input_ports()));
+        make_edge(cuber, std::get<2>(join.input_ports()));
+        make_edge(join, summer);
+
+        for (int i = 1; i <= 10; ++i) {
+            input.try_put(i);
+        }
+        g.wait_for_all();
+    }
+
+**Preview API**
+
+.. code:: cpp
+
+    #define TBB_PREVIEW_FLOW_GRAPH_FEATURES 1
+    #include <oneapi/tbb/flow_graph.h>
+
+    int main() {
+        using namespace oneapi::tbb::flow;
+
+        graph g;
+
+        function_node doubler(g, unlimited, [](const int& v) { return 2 * v; });
+        function_node squarer(g, unlimited, [](const int&) { return v * v; });
+        function_node cuber(g, unlimited, [](const int& v) { return v * v * v; });
+
+        auto handlers = make_node_set(doubler, squarer, cuber);
+
+        broadcast_node input(precedes(handlers));
+        join_node join(follows(handlers));
+
+        int sum = 0;
+        function_node summer(follows(join), serial,
+                             [&](const std::tuple<int, int, int>& v) {
+                                 int sub_sum = std::get<0>(v) + std::get<1>(v) + std::get<2>(v);
+                                 sum += sub_sum;
+                                 return sub_sum;
+                             });
+
+        for (int i = 1; i <= 10; ++i) {
+            input.try_put(i);
+        }
+        g.wait_for_all();
+    }
@@ -0,0 +1,241 @@
+.. _heterogeneous_extensions_chmap:
+
+Heterogeneous overloads for ``concurrent_hash_map`` member functions
+====================================================================
+
+.. note::
+    To enable this feature, define the ``TBB_PREVIEW_CONCURRENT_HASH_MAP_EXTENSIONS`` macro to 1.
+
+A set of overloads for ``concurrent_hash_map`` member functions that allow to search, erase, and insert
+elements into the container without creating a temporary ``key_type`` object.
+
+.. contents::
+    :local:
+    :depth: 1
+
+Description
+***********
+
+Heterogeneous overloads allow you to perform insert, lookup, and erasure operations on ``concurrent_hash_map`` object
+using an object of the type that is different from ``key_type`` but comparable with it.
+
+All member functions described below only participate in overload resolution if ``HashCompareType::is_transparent``
+is valid and denotes a type.``HashCompareType`` is a type of the ``HashCompare`` passed as a template argument
+for ``concurrent_hash_map``. It means that the ``HashCompare`` object calculates a hash and compares keys for
+equality without creating a temporary ``key_type`` object.
+
+API
+***
+
+Header
+------
+
+.. code:: cpp
+
+    #include "oneapi/tbb/concurrent_hash_map.h"
+
+Synopsis
+--------
+
+.. code:: cpp
+
+    namespace oneapi {
+        namespace tbb {
+            template <typename Key, typename Mapped,
+                      typename HashCompare = tbb_hash_compare<Key>,
+                      typename Allocator = tbb_allocator<std::pair<const Key, Mapped>>>
+            class concurrent_hash_map {
+            public:
+                // Insertion
+                template <typename K>
+                bool insert( accessor& result, const K& k );
+
+                template <typename K>
+                bool insert( const_accessor& result, const K& k );
+
+                // Lookup
+                template <typename K>
+                bool find( accessor& result, const K& k );
+
+                template <typename K>
+                bool find( const_accessor& result, const K& k ) const;
+
+                template <typename K>
+                size_type count( const K& k ) const;
+
+                template <typename K>
+                std::pair<iterator, iterator> equal_range( const K& k );
+
+                template <typename K>
+                std::pair<const_iterator, const_iterator> equal_range( const K& k ) const;
+
+                // Erasure
+                template <typename K>
+                bool erase( const K& k );
+            };
+        } // namespace tbb
+    } // namespace oneapi
+
+Member functions
+----------------
+
+Insertion
+^^^^^^^^^
+
+.. code:: cpp
+
+    template <typename K>
+    bool insert( accessor& result, const K& k );
+
+    template <typename K>
+    bool insert( const_accessor& result, const K& k );
+
+If the accessor ``result`` is not empty, releases the ``result`` and tries to
+insert the value constructed from ``{k, mapped_type()}`` into the container.
+
+Sets the ``result`` to provide access to the inserted element or to the element with the key that
+compares `equivalent` to the value ``k``.
+
+This overload only participates in overload resolution if ``std::is_constructible<key_type, const K&>`` is ``true``.
+
+**Returns**: ``true`` if the insertion was applied, ``false`` otherwise.
+
+Lookup
+^^^^^^
+
+.. code:: cpp
+
+    template <typename K>
+    bool find( accessor& result, const K& k );
+
+    template <typename K>
+    bool find( const_accessor& result, const K& k ) const;
+
+If the accessor ``result`` is not empty, releases the ``result``.
+
+If an element with the key that compares `equivalent` to the value ``k`` exists,
+sets the ``result`` to provide access to this element.
+
+**Returns**: ``true`` if an element with the key that compares `equivalent` to the value ``k`` is found,
+``false`` otherwise.
+
+------------------------------------------------
+
+.. code:: cpp
+
+    template <typename K>
+    size_type count( const K& k ) const;
+
+**Returns**: ``1`` if an element with the key that compares `equivalent` to the value ``k`` exists, ``0`` otherwise.
+
+------------------------------------------------
+
+.. code:: cpp
+
+    template <typename K>
+    std::pair<iterator, iterator> equal_range( const K& k );
+
+    template <typename K>
+    std::pair<const_iterator, const_iterator> equal_range( const K& k ) const;
+
+**Returns**:
+
+- A pair of iterators ``{f, l}`` if an element with the key that compares `equivalent` to the value ``k`` exists in the container.
+  Here ``f`` is an iterator to this element, ``l`` is ``std::next(f)``.
+- ``{end(), end()}`` otherwise.
+
+.. rubric:: Example
+
+The example below demonstrates how to use heterogeneous lookup feature to find an object with the key of type ``std::string``
+using an object of type ``const char*`` without conversions.
+
+.. code:: cpp
+
+    #define TBB_PREVIEW_CONCURRENT_HASH_MAP_EXTENSIONS 1
+    #include "oneapi/tbb/concurrent_hash_map.h"
+    #include <string>
+    #include <cstring>
+
+    // HashCompare an object that can calculate the hash code for
+    // std::string only and compare strings for equality
+    class RegularHashCompare {
+    private:
+        std::hash<std::string> my_hasher;
+    public:
+        std::size_t hash( const std::string& key ) const {
+            return my_hasher(key);
+        }
+
+        bool equal( const std::string& key1, const std::string& key2 ) const {
+            return key1 == key2;
+        }
+    };
+
+    // HashCompare an object that can calculate the hash code for
+    // std::string and const char*, and compare them for equality
+    class TransparentHashCompare {
+    private:
+        std::hash<char> my_hasher;
+
+        // Calculates a hash for the array of chars
+        std::size_t calculate_hash( const char* ptr ) const {
+            std::size_t h = 0;
+            for (auto c = ptr; *c; ++c) {
+                h = h ^ my_hasher(*c);
+            }
+            return h;
+        }
+    public:
+        using is_transparent = void;
+
+        std::size_t hash( const char* key ) const {
+            return calculate_hash(key);
+        }
+
+        std::size_t hash( const std::string& key ) const {
+            return calculate_hash(key.c_str());
+        }
+
+        bool equal( const char* key1, const char* key2 ) const {
+            return std::strcmp(key1, key2) == 0;
+        }
+
+        bool equal( const char* key1, const std::string& key2 ) const {
+            return std::strcmp(key1, key2.c_str()) == 0;
+        }
+
+        bool equal( const std::string& key1, const char* key2 ) const {
+            return std::strcmp(key1.c_str(), key2) == 0;
+        }
+
+        bool equal( const std::string& key1, const std::string& key2 ) const {
+            return std::strcmp(key1.c_str(), key2.c_str()) == 0;
+        }
+    };
+
+    int main() {
+        using regular_hash_map =
+            oneapi::tbb::concurrent_hash_map<std::string, int, RegularHashCompare>;
+        using transparent_hash_map =
+            oneapi::tbb::concurrent_hash_map<std::string, int, TransparentHashCompare>;
+
+        using regular_accessor = typename regular_hash_map::accessor;
+        using transparent_accessor = typename transparent_hash_map::accessor;
+
+        // Accessors
+        regular_accessor reg_accessor;
+        transparent_accessor tran_accessor;
+
+        // Maps
+        regular_hash_map regular_map;
+        transparent_hash_map tran_map;
+
+        // Heterogeneous overloads do not participate in overload resolution
+        // Such a call matches on the find overload, which accepts key_type (std::string)
+        // Creates a temporary key_type (std::string) object because of implicit conversion
+        bool result = regular_map.find(reg_accessor, "abc");
+
+        // Heterogeneous overloads participate in overload resolution
+        // No implicit conversion from const char* to std::string takes place
+        result = tran_map.find(tran_accessor, "abc");
+    }
@@ -0,0 +1,17 @@
+.. _info_namespace:
+
+oneapi::tbb::info namespace
+===========================
+
+The ``oneapi::tbb::info`` namespace satisfies `the corresponding oneTBB specification section
+<https://spec.oneapi.com/versions/latest/elements/oneTBB/source/info_namespace.html>`_.
+
+The |full_name| implementation requires `the hwloc library <https://www-lb.open-mpi.org/projects/hwloc>`_
+to query NUMA(version >= 1.11) and Hybrid CPUs(version >= 2.4) topology information.
+
+See also:
+
+* `info namespace specification <https://spec.oneapi.com/versions/latest/elements/oneTBB/source/info_namespace.html>`_
+* `oneapi::tbb::task_arena specification <https://spec.oneapi.com/versions/latest/elements/oneTBB/source/task_scheduler/task_arena/task_arena_cls.html>`_
+* :doc:`oneapi::tbb::info namespace preview extensions <info_namespace_extensions>`
+* :doc:`task_arena::constraints class preview extensions <constraints_extensions>`
@@ -0,0 +1,71 @@
+.. _info_namespace_extensions:
+
+oneapi::tbb::info namespace extensions
+======================================
+
+.. note::
+    To enable this feature, set the ``TBB_PREVIEW_TASK_ARENA_CONSTRAINTS_EXTENSION`` macro to 1.
+
+.. contents::
+    :local:
+    :depth: 1
+
+Description
+***********
+
+These extensions allow to query information about execution environment.
+
+.. contents::
+    :local:
+    :depth: 1
+
+API
+***
+
+Header
+------
+
+.. code:: cpp
+
+    #include <oneapi/tbb/info.h>
+
+Syntax
+------
+
+.. code:: cpp
+
+    namespace oneapi {
+        namespace tbb {
+            using core_type_id = /*implementation-defined*/;
+            namespace info {
+                std::vector<core_type_id> core_types();
+                int default_concurrency(task_arena::constraints c);
+            }
+        }
+    }
+
+Types
+-----
+
+``core_type_id`` - Represents core type identifier.
+
+Functions
+---------
+
+.. cpp:function:: std::vector<core_type_id> core_types()
+
+    Returns the vector of integral indexes that indicate available core types.
+    The indexes are sorted from the least performant to the most performant core type.
+
+    .. note::
+        If error occurs during system topology parsing, returns vector containing single element
+        that equals to ``task_arena::automatic``.
+
+.. cpp:function:: int default_concurrency(task_arena::constraints c)
+
+    Returns concurrency level for the given constraints.
+
+See also:
+
+* :doc:`task_arena::constraints class preview extensions <constraints_extensions>`
+* `info namespace specification <https://spec.oneapi.com/versions/latest/elements/oneTBB/source/info_namespace.html>`_
@@ -0,0 +1,81 @@
+.. _make_edges:
+
+``make_edges`` function template
+================================
+
+.. note::
+   To enable this feature, define the ``TBB_PREVIEW_FLOW_GRAPH_FEATURES`` macro to 1.
+
+.. contents::
+    :local:
+    :depth: 1
+
+Description
+***********
+
+The ``make_edges`` function template creates edges between a single node
+and each node in a set of nodes.
+
+There are two ways to connect nodes in a set and a single node using
+``make_edges``:
+
+.. figure:: ./Resources/make_edges_usage.png
+   :align: center
+
+API
+***
+
+Header
+------
+
+.. code:: cpp
+
+    #include <oneapi/tbb/flow_graph.h>
+
+Syntax
+------
+
+.. code:: cpp
+
+    // node_set is an exposition-only name for the type returned from make_node_set function
+
+    template <typename NodeType, typename Node, typename... Nodes>
+    void make_edges(node_set<Node, Nodes...>& set, NodeType& node);
+
+    template <typename NodeType, typename Node, typename... Nodes>
+    void make_edges(NodeType& node, node_set<Node, Nodes...>& set);
+
+Example
+-------
+
+The example implements the graph structure in the picture below.
+
+.. figure:: ./Resources/make_edges_example.png
+    :align: center
+
+.. code:: cpp
+
+    #define TBB_PREVIEW_FLOW_GRAPH_FEATURES 1
+    #include <oneapi/tbb/flow_graph.h>
+
+    int main() {
+        using namespace oneapi::tbb::flow;
+
+        graph g;
+        broadcast_node<int> input(g);
+
+        function_node doubler(g, unlimited, [](const int& i) { return 2 * i; });
+        function_node squarer(g, unlimited, [](const int& i) { return i * i; });
+        function_node cuber(g, unlimited, [](const int& i) { return i * i * i; });
+
+        buffer_node<int> buffer(g);
+
+        auto handlers = make_node_set(doubler, squarer, cuber);
+        make_edges(input, handlers);
+        make_edges(handlers, buffer);
+
+        for (int i = 1; i <= 10; ++i) {
+            input.try_put(i);
+        }
+        g.wait_for_all();
+    }
@@ -0,0 +1,42 @@
+.. _make_node_set:
+
+``make_node_set`` function template
+===================================
+
+.. note::
+   To enable this feature, define the ``TBB_PREVIEW_FLOW_GRAPH_FEATURES`` macro to 1.
+
+.. contents::
+    :local:
+    :depth: 1
+
+Description
+***********
+
+The ``make_node_set`` function template creates a set of nodes that
+can be passed as arguments to ``make_edges``, ``follows`` and ``precedes`` functions.
+
+API
+***
+
+Header
+------
+
+.. code:: cpp
+
+    #include <oneapi/tbb/flow_graph.h>
+
+Syntax
+------
+
+.. code:: cpp
+
+    template <typename Node, typename... Nodes>
+    /*unspecified*/ make_node_set( Node& node, Nodes&... nodes );
+
+See Also
+********
+
+:ref:`make_edges`
+
+:ref:`follows_precedes`
@@ -0,0 +1,93 @@
+.. _mutex:
+
+mutex
+==========
+
+.. note::
+   To enable this feature, define the ``TBB_PREVIEW_MUTEXES`` macro to 1.
+
+Description
+***********
+
+A ``mutex`` is a class that models the `Mutex requirement <https://spec.oneapi.com/versions/latest/elements/oneTBB/source/named_requirements/mutexes/mutex.html>`_,
+using adaptive approach: the combination of spinlock and waiting on system primitives.
+The ``mutex`` class satisfies all of the mutex requirements described in the [thread.mutex.requirements] section of the ISO C++ standard.
+The ``mutex`` class is not fair or recursive.
+
+API
+***
+
+Header
+------
+
+.. code:: cpp
+
+    #include <oneapi/tbb/mutex.h>
+
+Synopsis
+--------
+
+.. code:: cpp
+
+    namespace oneapi {
+        namespace tbb {
+            class mutex {
+            public:
+                mutex() noexcept;
+                ~mutex();
+
+                mutex(const mutex&) = delete;
+                mutex& operator=(const mutex&) = delete;
+
+                class scoped_lock;
+
+                void lock();
+                bool try_lock();
+                void unlock();
+
+                static constexpr bool is_rw_mutex = false;
+                static constexpr bool is_recursive_mutex = false;
+                static constexpr bool is_fair_mutex = false;
+            };
+        }
+    }
+
+Member classes
+--------------
+
+.. namespace:: tbb::mutex
+	       
+.. cpp:class:: scoped_lock
+
+    The corresponding ``scoped_lock`` class. See the `Mutex requirement <https://spec.oneapi.com/versions/latest/elements/oneTBB/source/named_requirements/mutexes/mutex.html>`_.
+
+Member functions
+----------------
+
+.. cpp:function:: mutex()
+
+    Constructs ``mutex`` with unlocked state.
+
+--------------------------------------------------
+
+.. cpp:function:: ~mutex()
+
+    Destroys an unlocked ``mutex``.
+
+--------------------------------------------------
+
+.. cpp:function:: void lock()
+
+    Acquires a lock. It uses adaptive logic for waiting: it blocks after particular time period of busy wait.
+
+--------------------------------------------------
+
+.. cpp:function:: bool try_lock()
+
+    Tries to acquire a lock (non-blocking). Returns **true** if succeeded; **false** otherwise.
+
+--------------------------------------------------
+
+.. cpp:function:: void unlock()
+
+    Releases the lock held by a current thread.
@@ -0,0 +1,66 @@
+.. _parallel_for_each_semantics:
+
+parallel_for_each Body semantics and requirements
+=================================================
+
+.. contents::
+    :local:
+    :depth: 1
+
+Description
+***********
+
+This page clarifies `ParallelForEachBody <https://spec.oneapi.com/versions/latest/elements/oneTBB/source/named_requirements/algorithms/par_for_each_body.html>`_
+named requirements for ``tbb::parallel_for_each`` algorithm specification.
+
+.. code:: cpp
+
+    namespace oneapi {
+        namespace tbb {
+
+            template <typaname InputIterator, typename Body>
+            void parallel_for_each( InputIterator first, InputIterator last, Body body ); // overload (1)
+            template <typename InputIterator, typename Body>
+            void parallel_for_each( InputIterator first, InputIterator last, Body body, task_group_context& group ); // overload (2)
+
+            template <typename Container, typename Body>
+            void parallel_for_each( Container& c, Body body ); // overload (3)
+            template <typename Container, typename Body>
+            void parallel_for_each( Container& c, Body body, task_group_context& group ); // overload (4)
+
+            template <typename Container, typename Body>
+            void parallel_for_each( const Container& c, Body body ); // overload (5)
+            template <typename Container, typename Body>
+            void parallel_for_each( const Container& c, Body body, task_group_context& group ); // overload (6)
+
+        } // namespace tbb
+    } // namespace oneapi
+
+Terms
+-----
+
+* ``iterator`` determines the type of the iterator passed into ``parallel_for_each`` algorithm (which is ``InputIterator`` for overloads `(1)` and `(2)`
+  and ``decltype(std::begin(c))`` for overloads `(3) - (6)`)
+* ``value_type`` - the type ``typename std::iterator_traits<iterator>::value_type``
+* ``reference`` -  the type ``typename std::iterator_traits<iterator>::reference``.
+
+Requirements for different iterator types
+-----------------------------------------
+
+If the ``iterator`` satisfies `Input iterator` named requirements from [input.iterators] ISO C++ Standard section and do not satisfies
+`Forward iterator` named requirements from [forward.iterators] ISO C++ Standard section, ``tbb::parallel_for_each`` requires the execution
+of the ``body`` with an object of type ``const value_type&`` or ``value_type&&`` to be well-formed. If both forms are well-formed, an overload with
+rvalue reference will be preferred.
+
+.. caution::
+
+  If the ``Body`` only takes non-const lvalue reference to ``value_type``, named requirements above are violated and the program can be ill-formed.
+
+If the ``iterator`` satisfies `Forward iterator` named requirements from [forward.iterators] ISO C++ Standard section, ``tbb::parallel_for_each`` requires the execution of the ``body``
+with an object of type ``reference`` to be well-formed.
+
+Requirements for ``Body`` with ``feeder`` argument
+--------------------------------------------------
+
+Additional elements submitted into ``tbb::parallel_for_each`` through the ``feeder::add`` passes to the ``Body`` as rvalues and therefore the corresponding
+execution of the ``Body`` is required to be well-formed.
@@ -0,0 +1,58 @@
+.. _reference:
+
+|short_name| API Reference
+==========================
+
+For oneTBB API Reference, refer to `oneAPI Specification <https://spec.oneapi.com/>`_. The current supported
+version of oneAPI Specification is 1.0.
+
+Specification extensions
+************************
+
+|full_name| implements the `oneTBB specification <https://spec.oneapi.com/versions/latest/elements/oneTBB/source/nested-index.html>`_.
+This document provides additional details or restrictions where necessary.
+It also describes features that are not included in the oneTBB specification.
+
+.. toctree::
+    :titlesonly:
+
+    info_namespace
+    parallel_for_each_semantics
+
+Preview features
+****************
+
+A preview feature is a component of oneTBB introduced to receive early feedback from
+users.
+
+The key properties of a preview feature are:
+
+- It is off by default and must be explicitly enabled.
+- It is intended to have a high quality implementation.
+- There is no guarantee of future existence or compatibility.
+- It may have limited or no support in tools such as correctness analyzers, profilers and debuggers.
+
+
+.. caution::
+    A preview feature is subject to change in future. It might be removed or significantly
+    altered in future releases. Changes to a preview feature do NOT require
+    usual deprecation and removal process. Therefore, using preview features in production code
+    is strongly discouraged.
+
+.. toctree::
+    :titlesonly:
+
+    type_specified_message_keys
+    blocking_terminate
+    scalable_memory_pools
+    helpers_for_expressing_graphs
+    concurrent_lru_cache_cls
+    constraints_extensions
+    info_namespace_extensions
+    task_group_extensions
+    task_arena_extensions 
+    this_task_arena_extensions
+    mutex_cls
+    rw_mutex_cls
+    heterogeneous_extensions_chmap
+    custom_mutex_chmap
@@ -0,0 +1,116 @@
+.. _rw_mutex:
+
+rw_mutex
+=============
+
+.. note::
+   To enable this feature, define the ``TBB_PREVIEW_MUTEXES`` macro to 1.
+
+Description
+***********
+
+A ``rw_mutex`` is a class that models the `ReaderWriterMutex requirement <https://spec.oneapi.com/versions/latest/elements/oneTBB/source/named_requirements/mutexes/rw_mutex.html>`_,
+using adaptive approach: the combination of spinlock and waiting on system primitives.
+The ``rw_mutex`` class satisfies all of the shared mutex requirements described in the [thread.sharedmutex.requirements] section of the ISO C++ standard.
+The ``rw_mutex`` class is unfair reader-writer lock with writer-preference.
+
+API
+***
+
+Header
+------
+
+.. code:: cpp
+
+    #include <oneapi/tbb/rw_mutex.h>
+
+Synopsis
+--------
+.. code:: cpp
+
+    namespace oneapi {
+        namespace tbb {
+            class rw_mutex {
+            public:
+                rw_mutex() noexcept;
+                ~rw_mutex();
+
+                rw_mutex(const rw_mutex&) = delete;
+                rw_mutex& operator=(const rw_mutex&) = delete;
+
+                class scoped_lock;
+
+                // exclusive ownership
+                void lock();
+                bool try_lock();
+                void unlock();
+
+                // shared ownership
+                void lock_shared();
+                bool try_lock_shared();
+                void unlock_shared();
+
+                static constexpr bool is_rw_mutex = true;
+                static constexpr bool is_recursive_mutex = false;
+                static constexpr bool is_fair_mutex = false;
+            };
+        }
+    }
+
+Member classes
+--------------
+
+.. namespace:: tbb::rw_mutex
+	       
+.. cpp:class:: scoped_lock
+
+    The corresponding scoped-lock class. See the `ReaderWriterMutex requirement <https://spec.oneapi.com/versions/latest/elements/oneTBB/source/named_requirements/mutexes/rw_mutex.html>`_.
+
+Member functions
+----------------
+
+.. cpp:function:: rw_mutex()
+
+    Constructs unlocked ``rw_mutex``.
+
+--------------------------------------------------
+
+.. cpp:function:: ~rw_mutex()
+
+    Destroys unlocked ``rw_mutex``.
+
+--------------------------------------------------
+
+.. cpp:function:: void lock()
+
+    Acquires a lock. It uses adaptive logic for waiting: it blocks after particular time period of busy wait.
+
+--------------------------------------------------
+
+.. cpp:function:: bool try_lock()
+
+    Tries to acquire a lock (non-blocking) on write. Returns **true** if succeeded; **false** otherwise.
+
+--------------------------------------------------
+
+.. cpp:function:: void unlock()
+
+    Releases the write lock held by the current thread.
+
+--------------------------------------------------
+
+.. cpp:function:: void lock_shared()
+
+    Acquires a lock on read. It uses adaptive logic for waiting: it blocks after particular time period of busy wait.
+
+--------------------------------------------------
+
+.. cpp:function:: bool try_lock_shared()
+
+    Tries to acquire the lock (non-blocking) on read. Returns **true** if succeeded; **false** otherwise.
+
+--------------------------------------------------
+
+.. cpp:function:: void unlock_shared()
+
+    Releases the read lock held by the current thread.
@@ -0,0 +1,43 @@
+.. _scalable_memory_pools_reference:
+
+Scalable Memory Pools
+=====================
+
+.. note::
+   To enable this feature, set the ``TBB_PREVIEW_MEMORY_POOL`` macro to 1.
+
+Memory pools allocate and free memory from a specified region or an underlying allocator using
+thread-safe, scalable operations. The  following table summarizes the Memory Pool named requirement.
+Here, ``P`` represents an instance of the memory pool class.
+
+.. container:: tablenoborder
+
+   .. list-table:: 
+      :header-rows: 1
+
+      * -    Pseudo-Signature
+        -    Semantics
+      * -    \ ``~P() throw();``
+        -    Destructor. Frees all the allocated memory.
+      * -    \ ``void P::recycle();``
+        -    Frees all the allocated memory.
+      * -    \ ``void* P::malloc(size_t n);``
+        -    Returns a pointer to ``n`` bytes allocated from the memory pool.
+      * -    \ ``void P::free(void* ptr);``
+        -    Frees the memory object specified via ``ptr`` pointer.
+      * -    \ ``void* P::realloc(void* ptr, size_t n);``
+        -    Reallocates the memory object pointed by ``ptr`` to ``n`` bytes.
+
+.. container:: section
+
+    .. rubric:: Model Types
+        :class: sectiontitle
+
+    The ``memory_pool`` template class and the ``fixed_pool`` class meet the Memory Pool named requirement.
+
+.. toctree::
+    :titlesonly:
+
+    scalable_memory_pools/memory_pool_cls
+    scalable_memory_pools/fixed_pool_cls
+    scalable_memory_pools/memory_pool_allocator_cls
@@ -0,0 +1,76 @@
+.. _fixed_pool_cls:
+
+fixed_pool
+==========
+
+.. note::
+   To enable this feature, set the ``TBB_PREVIEW_MEMORY_POOL`` macro to 1.
+
+A class for scalable memory allocation from a buffer of fixed size.
+
+.. contents::
+    :local:
+    :depth: 1
+
+Description
+***********
+
+``fixed_pool`` allocates and frees memory in a way that scales with the number of processors.
+All the memory available for the allocation is initially passed through arguments of the constructor.
+``fixed_pool`` meet the :doc:`Memory Pool named requirement<../scalable_memory_pools>`.
+
+API
+***
+
+Header
+------
+
+.. code:: cpp
+
+    #include "oneapi/tbb/memory_pool.h"
+
+Synopsis
+--------
+
+.. code:: cpp
+
+    namespace oneapi {
+        namespace tbb {
+            class fixed_pool {
+            public:
+                fixed_pool(void *buffer, size_t size);
+                fixed_pool(const fixed_pool& other) = delete;
+                fixed_pool& operator=(const fixed_pool& other) = delete;
+                ~fixed_pool();
+
+                void recycle();
+                void* malloc(size_t size);
+                void free(void* ptr);
+                void* realloc(void* ptr, size_t size);
+            };
+        } // namespace tbb
+    } // namespace oneapi
+
+Member Functions
+----------------
+
+.. cpp:function:: fixed_pool(void *buffer, size_t size)
+
+    **Effects**: Constructs a memory pool to manage the memory of size ``size`` pointed to by ``buffer``.
+    Throws the ``bad_alloc`` exception if the library fails to construct an instance of the class.
+
+Examples
+********
+
+The code below provides a simple example of allocation from a fixed pool.
+
+.. code:: cpp
+
+    #define TBB_PREVIEW_MEMORY_POOL 1
+    #include "oneapi/tbb/memory_pool.h"
+    ...
+    char buf[1024*1024];
+    oneapi::tbb::fixed_pool my_pool(buf, 1024*1024);
+    void* my_ptr = my_pool.malloc(10);
+    my_pool.free(my_ptr);}
+
@@ -0,0 +1,117 @@
+.. _memory_pool_allocator_cls:
+
+memory_pool_allocator
+=====================
+
+.. note::
+   To enable this feature, set the ``TBB_PREVIEW_MEMORY_POOL`` macro to 1.
+
+A class template that provides a memory pool with a C++ allocator interface.
+
+.. contents::
+    :local:
+    :depth: 1
+
+Description
+***********
+
+``memory_pool_allocator`` meets the allocator requirements from the [allocator.requirements] ISO C++ Standard section
+It also provides a constructor to allocate and deallocate memory.
+This constructor is linked with an instance of either the ``memory_pool`` or the ``fixed_pool`` class.
+The class is mainly intended for enabling memory pools within STL containers.
+
+API
+***
+
+Header
+------
+
+.. code:: cpp
+
+    #include "oneapi/tbb/memory_pool.h"
+
+Synopsis
+--------
+
+.. code:: cpp
+
+    namespace oneapi {
+        namespace tbb {
+            template<typename T>
+            class memory_pool_allocator {
+            public:
+                using value_type = T;
+                using pointer = value_type*;
+                using const_pointer = const value_type*;
+                using reference = value_type&;
+                using const_reference = const value_type&;
+                using size_type = size_t;
+                using difference_type = ptrdiff_t;
+                template<typename U> struct rebind {
+                    using other = memory_pool_allocator<U>;
+                };
+                explicit memory_pool_allocator(memory_pool &pool) throw();
+                explicit memory_pool_allocator(fixed_pool &pool) throw();
+                memory_pool_allocator(const memory_pool_allocator& src) throw();
+                template<typename U>
+                memory_pool_allocator(const memory_pool_allocator<U,P>& src) throw();
+                pointer address(reference x) const;
+                const_pointer address(const_reference x) const;
+                pointer allocate(size_type n, const void* hint=0);
+                void deallocate(pointer p, size_type);
+                size_type max_size() const throw();
+                void construct(pointer p, const T& value);
+                void destroy(pointer p);
+            };
+
+            template<>
+            class memory_pool_allocator<void> {
+            public:
+                using pointer = void*;
+                using const_pointer = const void*;
+                using value_type = void;
+                template<typename U> struct rebind {
+                    using other = memory_pool_allocator<U>;
+                };
+                memory_pool_allocator(memory_pool &pool) throw();
+                memory_pool_allocator(fixed_pool &pool) throw();
+                memory_pool_allocator(const memory_pool_allocator& src) throw();
+                template<typename U>
+                memory_pool_allocator(const memory_pool_allocator<U>& src) throw();
+            };
+        } // namespace tbb
+    } // namespace oneapi
+
+    template<typename T, typename U>
+    inline bool operator==( const memory_pool_allocator<T>& a,
+                           const memory_pool_allocator<U>& b);
+    template<typename T, typename U>
+    inline bool operator!=( const memory_pool_allocator<T>& a,
+                            const memory_pool_allocator<U>& b);
+
+Member Functions
+----------------
+
+.. cpp:function:: explicit memory_pool_allocator(memory_pool &pool)
+
+    **Effects**: Constructs a memory pool allocator serviced by ``memory_pool`` instance pool.
+
+-------------------------------------------------------
+
+.. cpp:function:: explicit memory_pool_allocator(fixed_pool &pool)
+
+    **Effects**: Constructs a memory pool allocator serviced by ``fixed_pool`` instance pool.
+
+Examples
+********
+
+The code below provides a simple example of container construction with the use of a memory pool.
+
+.. code:: cpp
+
+    #define TBB_PREVIEW_MEMORY_POOL 1
+    #include "oneapi/tbb/memory_pool.h"
+    ...
+    typedef oneapi::tbb::memory_pool_allocator<int>
+    pool_allocator_t;
+    std::list<int, pool_allocator_t> my_list(pool_allocator_t( my_pool ));
@@ -0,0 +1,80 @@
+.. _memory_pool_cls:
+
+memory_pool
+===========
+
+.. note::
+   To enable this feature, set the ``TBB_PREVIEW_MEMORY_POOL`` macro to 1.
+
+A class template for scalable memory allocation from memory blocks provided by an underlying allocator.
+
+.. contents::
+    :local:
+    :depth: 1
+
+Description
+***********
+
+A ``memory_pool`` allocates and frees memory in a way that scales with the number of processors.
+The memory is obtained as big chunks from an underlying allocator specified by the template
+argument. The latter must satisfy the subset of the allocator requirements from the [allocator.requirements]
+ISO C++ Standard section. A ``memory_pool`` meet the :doc:`Memory Pool named requirement<../scalable_memory_pools>`.
+
+.. caution::
+
+    If the underlying allocator refers to another scalable memory pool, the inner pool (or pools)
+    must be destroyed before the outer pool is destroyed or recycled.
+
+API
+***
+
+Header
+------
+
+.. code:: cpp
+
+    #include "oneapi/tbb/memory_pool.h"
+
+Synopsis
+--------
+
+.. code:: cpp
+
+    namespace oneapi {
+        namespace tbb {
+            template <typename Alloc>
+            class memory_pool {
+            public:
+                explicit memory_pool(const Alloc &src = Alloc());
+                memory_pool(const memory_pool& other) = delete;
+                memory_pool& operator=(const memory_pool& other) = delete;
+                ~memory_pool();
+                void recycle();
+                void *malloc(size_t size);
+                void free(void* ptr);
+               void *realloc(void* ptr, size_t size);
+            };
+        }
+    }
+
+Member Functions
+----------------
+
+.. cpp:function:: explicit memory_pool(const Alloc &src = Alloc())
+
+    **Effects**: Constructs a memory pool with an instance of underlying memory allocator of type ``Alloc`` copied from ``src``.
+    Throws the ``bad_alloc`` exception if runtime fails to construct an instance of the class.
+
+Examples
+********
+
+The code below provides a simple example of allocation from an extensible memory pool.
+
+.. code:: cpp
+
+    #define TBB_PREVIEW_MEMORY_POOL 1
+    #include "oneapi/tbb/memory_pool.h"
+    ...
+    oneapi::tbb::memory_pool<std::allocator<char> > my_pool;
+    void* my_ptr = my_pool.malloc(10);
+    my_pool.free(my_ptr);
@@ -0,0 +1,61 @@
+.. _task_arena_extensions:
+
+task_arena extensions
+=====================
+
+.. note::
+    To enable these extensions, set the ``TBB_PREVIEW_TASK_GROUP_EXTENSIONS`` macro to 1.
+
+.. contents::
+    :local:
+    :depth: 1
+
+Description
+***********
+
+|full_name| implementation extends the `tbb::task_arena specification <https://spec.oneapi.com/versions/latest/elements/oneTBB/source/task_scheduler/task_arena/task_arena_cls.html>`_ 
+with an overload of ``enqueue`` method accepting ``task_handle``. 
+   
+
+API
+***
+
+Header
+------
+
+.. code:: cpp
+
+    #include <oneapi/tbb/task_arena.h>
+
+Synopsis
+--------
+
+.. code:: cpp
+
+    namespace oneapi {
+        namespace tbb {
+   
+           class task_arena {
+           public:
+               void enqueue(task_handle&& h);        
+           }; 
+
+        } // namespace tbb
+    } // namespace oneapi
+
+Member Functions
+----------------
+
+.. cpp:function:: void enqueue(task_handle&& h)   
+     
+Enqueues a task owned by ``h`` into the ``task_arena`` for procession.
+ 
+Behavior of this function is identical to generic version (``template<typename F> void task_arena::enqueue(F&& f)``) except parameter type. 
+
+.. note:: 
+   ``h`` should not be empty to avoid undefined behavior.
+             
+.. rubric:: See also
+
+* `oneapi::tbb::task_arena specification <https://spec.oneapi.com/versions/latest/elements/oneTBB/source/task_scheduler/task_arena/task_arena_cls.html>`_
+* :ref:`task_handle` 
@@ -0,0 +1,102 @@
+.. _task_group_extensions:
+
+task_group extensions
+=====================
+
+.. note::
+    To enable these extensions, set the ``TBB_PREVIEW_TASK_GROUP_EXTENSIONS`` macro to 1.
+
+.. contents::
+    :local:
+    :depth: 1
+
+Description
+***********
+
+|full_name| implementation extends the `tbb::task_group specification <https://spec.oneapi.com/versions/latest/elements/oneTBB/source/task_scheduler/task_group/task_group_cls.html>`_ with the following members:
+
+  - Constructor that takes a custom ``tbb::task_group_context`` object as an argument.
+  - Methods to create and run deferred tasks with ``task_handle``. 
+  - Requirements for a user-provided function object.
+   
+
+API
+***
+
+Header
+------
+
+.. code:: cpp
+
+    #include <oneapi/tbb/task_group.h>
+
+Synopsis
+--------
+
+.. code:: cpp
+
+    namespace oneapi {
+        namespace tbb {
+   
+           class task_group {
+           public:
+               task_group(task_group_context& context);
+               
+               template<typename F>
+               task_handle defer(F&& f);
+                   
+               void run(task_handle&& h);
+
+               //only F return type requirements have changed              
+               template<typename F>
+               void run(F&& f);
+           }; 
+
+        } // namespace tbb
+    } // namespace oneapi
+
+Member Functions
+----------------
+
+.. cpp:function:: task_group(task_group_context& context)
+
+Constructs an empty ``task_group``, which tasks are associated with the ``context``.
+
+.. cpp:function:: template<typename F> task_handle  defer(F&& f)
+
+Creates a deferred task to compute ``f()`` and returns ``task_handle`` pointing to it.
+
+The task is not scheduled for execution until explicitly requested. For example, with the ``task_group::run`` method.
+However, the task is still added into the ``task_group``, thus the ``task_group::wait`` method waits until the ``task_handle`` is either scheduled or destroyed.
+
+The ``F`` type must meet the `Function Objects` requirements described in the [function.objects] section of the ISO C++ Standard.
+
+As an optimization hint, ``F`` might return a ``task_handle``, which task object can be executed next.
+
+.. note::
+   The ``task_handle`` returned by the function must be created with ``*this`` ``task_group``. It means, with the one for which run method is called, otherwise it is an undefined behavior. 
+
+**Returns:** ``task_handle`` object pointing to task to compute ``f()``.
+
+.. cpp:function:: void run(task_handle&& h)
+   
+Schedules the task object pointed by the ``h`` for execution.
+
+.. note::
+   The failure to satisfy the following conditions leads to undefined behavior:
+      * ``h`` is not empty.
+      * ``*this`` is the same ``task_group`` that ``h`` is created with.    
+
+.. cpp:function:: template<typename F> void  run(F&& f)
+
+As an optimization hint, ``F`` might return a ``task_handle``, which task object can be executed next.
+
+.. note::
+   The ``task_handle`` returned by the function must be created with ``*this`` ``task_group``. It means, with the one for which run method is called, otherwise it is an undefined behavior. 
+    
+               
+.. rubric:: See also
+
+* `oneapi::tbb::task_group specification <https://spec.oneapi.com/versions/latest/elements/oneTBB/source/task_scheduler/task_group/task_group_cls.html>`_
+* `oneapi::tbb::task_group_context specification <https://spec.oneapi.com/versions/latest/elements/oneTBB/source/task_scheduler/scheduling_controls/task_group_context_cls.html>`_
+* :doc:`oneapi::tbb::task_handle class <task_group_extensions/task_handle>`
@@ -0,0 +1,101 @@
+.. _task_handle:
+
+task_handle
+===========
+
+.. note::    To enable these extensions, set the ``TBB_PREVIEW_TASK_GROUP_EXTENSIONS`` macro to 1.
+
+
+.. contents::
+    :local:
+    :depth: 1
+
+Description
+***********
+
+This class owns a deferred task object. 
+
+API
+***
+
+Header
+------
+
+.. code:: cpp
+
+    #include <oneapi/tbb/task_group.h>
+
+Synopsis
+--------
+
+.. code:: cpp
+
+    namespace oneapi {
+        namespace tbb {
+      
+            class task_handle {
+            public:
+                task_handle();
+                task_handle(task_handle&& src);
+                
+                ~task_handle();
+                
+                task_handle& operator=(task_handle&& src);
+            
+                explicit operator bool() const noexcept;
+            }; 
+            
+            bool operator==(task_handle const& h, std::nullptr_t) noexcept;
+            bool operator==(std::nullptr_t, task_handle const& h) noexcept;
+            
+            bool operator!=(task_handle const& h, std::nullptr_t) noexcept;
+            bool operator!=(std::nullptr_t, task_handle const& h) noexcept;
+                      
+        } // namespace tbb
+    } // namespace oneapi
+
+Member Functions
+----------------
+
+.. cpp:function:: task_handle()
+
+Creates an empty ``task_handle`` object.
+
+.. cpp:function:: task_handle(task_handle&& src)
+
+Constructs ``task_handle`` object with the content of ``src`` using move semantics. ``src`` is left in an empty state.
+
+.. cpp:function:: ~task_handle()
+
+Destroys the ``task_handle`` object and associated task if it exists. 
+
+.. cpp:function:: task_handle& operator=(task_handle&& src)
+
+Replaces the content of ``task_handle`` object with the content of ``src`` using move semantics. ``src`` is left in an empty state.
+The previously associated task object, if any, is destroyed before the assignment. 
+
+**Returns:** Reference to ``*this``.
+
+.. cpp:function:: explicit operator bool() const noexcept
+
+Checks if ``*this`` has an associated task object.
+
+**Returns:** ``true`` if ``*this`` is not empty, ``false`` otherwise.
+
+Non-Member Functions
+--------------------
+
+.. code:: cpp
+
+    bool operator==(task_handle const& h, std::nullptr_t) noexcept
+    bool operator==(std::nullptr_t, task_handle const& h) noexcept
+
+**Returns**: ``true`` if ``h`` is empty, ``false`` otherwise.
+
+.. code:: cpp
+
+    bool operator!=(task_handle const& h, std::nullptr_t) noexcept
+    bool operator!=(std::nullptr_t, task_handle const& h) noexcept
+
+**Returns**: ``true`` if ``h`` is not empty, ``false`` otherwise.
+   
@@ -0,0 +1,71 @@
+.. _this_task_arena_extensions:
+
+this_task_arena extensions
+==========================
+
+.. note::
+    To enable these extensions, set the ``TBB_PREVIEW_TASK_GROUP_EXTENSIONS`` macro to 1.
+
+.. contents::
+    :local:
+    :depth: 1
+
+Description
+***********
+
+|full_name| implementation extends the `tbb::this_task_arena specification <https://spec.oneapi.com/versions/latest/elements/oneTBB/source/task_scheduler/task_arena/this_task_arena_ns.html>`_
+with an overloaded ``enqueue`` function. 
+   
+
+API
+***
+
+Header
+------
+
+.. code:: cpp
+
+    #include <oneapi/tbb/task_arena.h>
+
+Synopsis
+--------
+
+.. code:: cpp
+
+    namespace oneapi {
+        namespace tbb {
+            namespace this_task_arena {
+                void enqueue(task_handle&& h);
+      
+                template<typename F>
+                void enqueue(F&& f) ;
+            } // namespace this_task_arena
+        } // namespace tbb
+    } // namespace oneapi
+
+Member Functions
+----------------
+
+   
+.. cpp:function:: template<typename F> void enqueue(F&& f)
+  
+Enqueues a task into the ``task_arena`` currently used by the calling thread to process the specified functor, then immediately returns.
+The ``F`` type must meet the `Function Objects` requirements described in the [function.objects] section of the ISO C++ Standard.
+
+Behavior of this function is identical to ``template<typename F> void task_arena::enqueue(F&& f)`` applied to ``task_arena`` object constructed with ``attach`` parameter.     
+
+.. cpp:function:: void enqueue(task_handle&& h)   
+     
+Enqueues a task owned by ``h`` into the ``task_arena`` that is currently used by the calling thread.
+
+Behavior of this function is identical to generic version (``template<typename F> void enqueue(F&& f)``) except the parameter type. 
+
+.. note:: 
+   ``h`` should not be empty to avoid undefined behavior.
+ 
+        
+.. rubric:: See also:
+
+* `oneapi::tbb::this_task_arena specification <https://spec.oneapi.com/versions/latest/elements/oneTBB/source/task_scheduler/task_arena/this_task_arena_ns.html>`_
+* `oneapi::tbb::task_arena specification <https://spec.oneapi.com/versions/latest/elements/oneTBB/source/task_scheduler/task_arena/task_arena_cls.html>`_
+* :ref:`task_handle` 
@@ -0,0 +1,69 @@
+.. _class_join_node_extension:
+
+Type-specified message keys for join_node
+=========================================
+
+.. note::
+    To enable this feature, define the ``TBB_PREVIEW_FLOW_GRAPH_FEATURES`` macro to 1.
+
+.. contents::
+    :local:
+    :depth: 1
+
+Description
+***********
+
+The extension allows a key matching ``join_node`` to obtain keys via functions associated with
+its input types. The extension simplifies the existing approach by removing the need to
+provide a function object for each input port of ``join_node``.
+
+API
+***
+
+Header
+------
+
+.. code:: cpp
+
+   #include "oneapi/tbb/flow_graph.h"
+
+Syntax
+------
+
+The extension adds a special constructor to the ``join_node`` interface when the
+``key_matching<typename K, class KHash=tbb_hash_compare>`` policy is
+used. The constructor has the following signature:
+
+.. code:: cpp
+
+   join_node( graph &g )
+
+When constructed this way, a ``join_node`` calls the
+``key_from_message`` function for each incoming message to obtain the key associated
+with it. The default implementation of ``key_from_message`` is the following
+
+.. code:: cpp
+
+   namespace oneapi {
+       namespace tbb {
+           namespace flow {
+               template <typename K, typename T>
+               K key_from_message( const T &t ) {
+                   return t.key();
+               }
+           }
+       }
+   }
+
+``T`` is one of the user-provided types in ``OutputTuple`` and is
+used to construct the ``join_node``, and ``K`` is the key type
+of the node.
+By default, the ``key()`` method defined in the message class will be called.
+Alternatively, the user can define its own ``key_from_message`` function in the
+same namespace with the message type. This function will be found via C++ argument-dependent
+lookup and used in place of the default implementation.
+
+See Also
+********
+
+`join_node Specification <https://spec.oneapi.com/versions/latest/elements/oneTBB/source/flow_graph/join_node_cls.html>`_
@@ -0,0 +1,96 @@
+.. _Advanced_Example:
+
+Advanced Example
+================
+
+
+An example of a more advanced associative operation is to find the index
+where ``Foo(i)`` is minimized. A serial version might look like this:
+
+
+::
+
+
+   long SerialMinIndexFoo( const float a[], size_t n ) {
+       float value_of_min = FLT_MAX;        // FLT_MAX from <climits>
+       long index_of_min = -1;
+       for( size_t i=0; i<n; ++i ) {
+           float value = Foo(a[i]);
+           if( value<value_of_min ) {
+               value_of_min = value;
+               index_of_min = i;
+           }
+       }  
+       return index_of_min;
+   }
+
+
+The loop works by keeping track of the minimum value found so far, and
+the index of this value. This is the only information carried between
+loop iterations. To convert the loop to use ``parallel_reduce``, the
+function object must keep track of the carried information, and how to
+merge this information when iterations are spread across multiple
+threads. Also, the function object must record a pointer to ``a`` to
+provide context.
+
+
+The following code shows the complete function object.
+
+
+::
+
+
+   class MinIndexFoo {
+       const float *const my_a;
+   public:
+       float value_of_min;
+       long index_of_min; 
+       void operator()( const blocked_range<size_t>& r ) {
+           const float *a = my_a;
+           for( size_t i=r.begin(); i!=r.end(); ++i ) {
+              float value = Foo(a[i]);    
+              if( value<value_of_min ) {
+                  value_of_min = value;
+                  index_of_min = i;
+              }
+           }
+       }
+    
+
+       MinIndexFoo( MinIndexFoo& x, split ) : 
+           my_a(x.my_a), 
+           value_of_min(FLT_MAX),    // FLT_MAX from <climits>
+           index_of_min(-1) 
+      {}
+    
+
+       void join( const SumFoo& y ) {
+           if( y.value_of_min<value_of_min ) {
+               value_of_min = y.value_of_min;
+               index_of_min = y.index_of_min;
+           }
+       }
+                
+
+       MinIndexFoo( const float a[] ) :
+           my_a(a), 
+           value_of_min(FLT_MAX),    // FLT_MAX from <climits>
+           index_of_min(-1),
+       {}
+   };
+
+
+Now ``SerialMinIndex`` can be rewritten using ``parallel_reduce`` as
+shown below:
+
+
+::
+
+
+   long ParallelMinIndexFoo( float a[], size_t n ) {
+       MinIndexFoo mif(a);
+       parallel_reduce(blocked_range<size_t>(0,n), mif );
+      
+
+    return mif.index_of_min;
+   }
@@ -0,0 +1,81 @@
+.. _Advanced_Topic_Other_Kinds_of_Iteration_Spaces:
+
+Advanced Topic: Other Kinds of Iteration Spaces
+===============================================
+
+
+The examples so far have used the class ``blocked_range<T>`` to specify
+ranges. This class is useful in many situations, but it does not fit
+every situation. You can use |full_name|
+to define your own iteration space objects. The object must specify how
+it can be split into subspaces by providing a basic splitting
+constructor, an optional proportional splitting constructor (accompanied
+with a trait value that enables its usage), and two predicate methods.
+If your class is called ``R``, the methods and constructors should be as
+follows:
+
+
+::
+
+
+   class R {
+       // True if range is empty
+       bool empty() const;
+       // True if range can be split into non-empty subranges
+       bool is_divisible() const;
+       // Splits r into subranges r and *this
+       R( R& r, split );
+       // Splits r into subranges r and *this in proportion p
+       R( R& r, proportional_split p );
+       // Allows usage of proportional splitting constructor
+       static const bool is_splittable_in_proportion = true;
+       ...
+   };
+       
+
+
+
+The method ``empty`` should return true if the range is empty. The
+method ``is_divisible`` should return true if the range can be split
+into two non-empty subspaces, and such a split is worth the overhead.
+The basic splitting constructor should take two arguments:
+
+
+-  The first of type ``R``
+
+
+-  The second of type oneapi::tbb::split
+
+
+The second argument is not used; it serves only to distinguish the
+constructor from an ordinary copy constructor. The basic splitting
+constructor should attempt to split ``r`` roughly into two halves, and
+update ``r`` to be the first half, and set the constructed object as the
+second half.
+
+
+Unlike the basic splitting constructor, the proportional splitting
+constructor is optional and takes the second argument of type
+``oneapi::tbb::proportional_split``. The type has methods ``left`` and ``right``
+that return the values of the proportion. These values should be used to
+split ``r`` accordingly, so that the updated ``r`` corresponds to the
+left part of the proportion, and the constructed object corresponds to
+the right part. The proportional splitting constructor will be used only
+if the static constant ``is_splittable_in_proportion`` is defined in the
+class and assigned the value of ``true``.
+
+
+Both splitting constructors should guarantee that the updated ``r`` part
+and the constructed object are not empty. The parallel algorithm
+templates call the splitting constructors on ``r`` only if
+``r.is_divisible`` is true.
+
+
+The iteration space does not have to be linear. Look at
+``oneapi/tbb/blocked_range2d.h`` for an example of a range that is
+two-dimensional. Its splitting constructor attempts to split the range
+along its longest axis. When used with ``parallel_for``, it causes the
+loop to be "recursively blocked" in a way that improves cache usage.
+This nice cache behavior means that using ``parallel_for`` over a
+``blocked_range2d<T>`` can make a loop run faster than the sequential
+equivalent, even on a single processor.
@@ -0,0 +1,42 @@
+.. _Allocator_Configuration:
+
+Configuring the Memory Allocator
+================================
+
+
+The oneTBB memory allocator provides the following API functions and
+environment variables to configure its behavior:
+
+
+-  the ``scalable_allocation_command`` function instructs the allocator
+   to perform a certain action, such as cleaning up its internal memory
+   buffers.
+
+
+-  the ``scalable_allocation_mode`` function allows an application to
+   set certain parameters for the memory allocator, such as an option to
+   map memory in huge pages or define a recommended heap size. These
+   settings take effect until modified by another call to
+   ``scalable_allocation_mode``.
+
+
+Some of the memory allocator parameters can also be set via system
+environment variables. It can be useful to adjust the behavior without
+modifying application source code, to ensure that a setting takes effect
+as early as possible, or to avoid explicit dependency on the oneTBB
+allocator binaries. The following environment variables are recognized:
+
+
+-  ``TBB_MALLOC_USE_HUGE_PAGES`` controls usage of huge pages for memory
+   mapping.
+
+
+-  ``TBB_MALLOC_SET_HUGE_OBJECT_THRESHOLD`` defines the lower bound for
+   the size (bytes), that is interpreted as huge and not released during
+   regular cleanup operations.
+
+
+These variables only take effect at the time the memory manager is
+initialized; later environment changes are ignored. A call to
+``scalable_allocation_mode`` overrides the effect of the corresponding
+environment variable.
@@ -0,0 +1,23 @@
+.. _Automatic_Chunking:
+
+Automatic Chunking
+==================
+
+
+A parallel loop construct incurs overhead cost for every chunk of work
+that it schedules. |full_name|
+chooses chunk sizes automatically, depending upon load balancing
+needs. The heuristic attempts to limit overheads while
+still providing ample opportunities for load balancing.
+
+
+.. CAUTION::
+   Typically a loop needs to take at least a million clock cycles to
+   make it worth using ``parallel_for``. For example, a loop that takes
+   at least 500 microseconds on a 2 GHz processor might benefit from
+   ``parallel_for``.
+
+
+The default automatic chunking is recommended for most uses. As with
+most heuristics, however, there are situations where controlling the
+chunk size more precisely might yield better performance.
@@ -0,0 +1,22 @@
+.. _Automically_Replacing_malloc:
+
+Automatically Replacing ``malloc`` and Other C/C++ Functions for Dynamic Memory Allocation
+==========================================================================================
+
+
+On Windows*, Linux\* operating systems, it is possible to automatically
+replace all calls to standard functions for dynamic memory allocation
+(such as ``malloc``) with the |full_name| scalable equivalents.
+Doing so can sometimes improve application performance.
+
+
+Replacements are provided by the proxy library (the library names can be
+found in platform-specific sections below). A proxy library and a
+scalable memory allocator library should be taken from the same release
+of oneTBB, otherwise the libraries may be mutually incompatible.
+
+.. toctree::
+   :maxdepth: 4
+
+   ../tbb_userguide/Windows_C_Dynamic_Memory_Interface_Replacement
+   ../tbb_userguide/Linux_C_Dynamic_Memory_Interface_Replacement
@@ -0,0 +1,113 @@
+.. _Bandwidth_and_Cache_Affinity:
+
+Bandwidth and Cache Affinity
+============================
+
+
+For a sufficiently simple function ``Foo``, the examples might not show
+good speedup when written as parallel loops. The cause could be
+insufficient system bandwidth between the processors and memory. In that
+case, you may have to rethink your algorithm to take better advantage of
+cache. Restructuring to better utilize the cache usually benefits the
+parallel program as well as the serial program.
+
+
+An alternative to restructuring that works in some cases is
+``affinity_partitioner.`` It not only automatically chooses the
+grainsize, but also optimizes for cache affinity and tries to distribute
+the data uniformly among threads. Using ``affinity_partitioner`` can
+significantly improve performance when:
+
+
+-  The computation does a few operations per data access.
+
+
+-  The data acted upon by the loop fits in cache.
+
+
+-  The loop, or a similar loop, is re-executed over the same data.
+
+
+-  There are more than two hardware threads available (and especially if
+   the number of threads is not a power of two). If only two threads are
+   available, the default scheduling in |full_name| 
+   usually provides sufficient cache affinity.
+
+
+The following code shows how to use ``affinity_partitioner``.
+
+
+::
+
+
+   #include "oneapi/tbb.h"
+    
+
+   void ParallelApplyFoo( float a[], size_t n ) {
+       static affinity_partitioner ap;
+       parallel_for(blocked_range<size_t>(0,n), ApplyFoo(a), ap);
+   }
+    
+
+   void TimeStepFoo( float a[], size_t n, int steps ) {    
+       for( int t=0; t<steps; ++t )
+           ParallelApplyFoo( a, n );
+   }
+
+
+In the example, the ``affinity_partitioner`` object ``ap`` lives between
+loop iterations. It remembers where iterations of the loop ran, so that
+each iteration can be handed to the same thread that executed it before.
+The example code gets the lifetime of the partitioner right by declaring
+the ``affinity_partitioner`` as a local static object. Another approach
+would be to declare it at a scope outside the iterative loop in
+``TimeStepFoo``, and hand it down the call chain to ``parallel_for``.
+
+
+If the data does not fit across the system’s caches, there may be little
+benefit. The following figure shows the situations.
+
+
+.. container:: fignone
+   :name: fig3
+
+
+   Benefit of Affinity Determined by Relative Size of Data Set and Cache
+   |image0|
+
+
+The next figure shows how parallel speedup might vary with the size of a
+data set. The computation for the example is ``A[i]+=B[i]`` for ``i`` in
+the range [0,N). It was chosen for dramatic effect. You are unlikely to
+see quite this much variation in your code. The graph shows not much
+improvement at the extremes. For small N, parallel scheduling overhead
+dominates, resulting in little speedup. For large N, the data set is too
+large to be carried in cache between loop invocations. The peak in the
+middle is the sweet spot for affinity. Hence ``affinity_partitioner``
+should be considered a tool, not a cure-all, when there is a low ratio
+of computations to memory accesses.
+
+
+.. container:: fignone
+   :name: fig4
+
+
+   Improvement from Affinity Dependent on Array Size
+   |image1|
+
+
+.. admonition:: Product and Performance Information 
+
+   Performance varies by use, configuration and other factors. Learn more at `www.Intel.com/PerformanceIndex <https://www.intel.com/PerformanceIndex>`_.
+   Notice revision #20201201
+
+
+
+
+.. |image0| image:: Images/image007.jpg
+   :width: 453px
+   :height: 178px
+.. |image1| image:: Images/image008.jpg
+   :width: 551px
+   :height: 192px
+
@@ -0,0 +1,16 @@
+.. _Basic_Flow_Graph_concepts:
+
+Basic Flow Graph Concepts
+=========================
+
+.. toctree::
+   :maxdepth: 4
+
+   ../tbb_userguide/Graph_Object
+   ../tbb_userguide/Nodes
+   ../tbb_userguide/Edges
+   ../tbb_userguide/Mapping_Nodes2Tasks
+   ../tbb_userguide/Flow_Graph_Message_Passing_Protocol
+   ../tbb_userguide/Flow_Graph_Single_Vs_Broadcast
+   ../tbb_userguide/Flow_Graph_Buffering_in_Nodes
+   ../tbb_userguide/Flow_Graph_Reservation
@@ -0,0 +1,46 @@
+.. _Cancellation_Without_An_Exception:
+
+Cancellation Without An Exception
+=================================
+
+
+To cancel an algorithm but not throw an exception, use the expression ``current_context()->cancel_group_execution()``.
+The part ``current_context()`` references the ``task_group_context*`` of the currently executing task if any on the current thread.
+Calling ``cancel_group_execution()`` cancels all tasks in its ``task_group_context``, which is explained in more detail in :ref:`Cancellation_and_Nested_Parallelism`.
+The method returns ``true`` if it actually causes cancellation, ``false`` if the ``task_group_context`` was already cancelled.
+
+The example below shows how to use ``current_context()->cancel_group_execution()``.
+
+::
+
+   #include "oneapi/tbb.h"
+
+   #include <vector>
+   #include <iostream>
+    
+   using namespace oneapi::tbb;
+   using namespace std;
+    
+   vector<int> Data;
+    
+   struct Update {
+       void operator()( const blocked_range<int>& r ) const {
+           for( int i=r.begin(); i!=r.end(); ++i )
+               if( i<Data.size() ) {
+                   ++Data[i];
+               } else {
+                   // Cancel related tasks.
+                   if( current_context()->cancel_group_execution() )
+                       cout << "Index " << i << " caused cancellation\n";
+                   return;
+               }
+       }
+   };
+    
+
+   int main() {
+       Data.resize(1000);
+       parallel_for( blocked_range<int>(0, 2000), Update());
+       return 0;
+   }
+
@@ -0,0 +1,136 @@
+.. _Cancellation_and_Nested_Parallelism:
+
+Cancellation and Nested Parallelism
+===================================
+
+
+The discussion so far was simplified by assuming non-nested parallelism
+and skipping details of ``task_group_context``. This topic explains
+both.
+
+
+An |full_name| algorithm executes
+by creating ``task`` objects that execute the snippets of code that you
+supply to the algorithm template. By default, these ``task`` objects are
+associated with a ``task_group_context`` created by the algorithm.
+Nested oneTBB algorithms create a tree of these ``task_group_context``
+objects. Cancelling a ``task_group_context`` cancels all of its child
+``task_group_context`` objects, and transitively all its descendants.
+Hence an algorithm and all algorithms it called can be cancelled with a
+single request.
+
+
+Exceptions propagate upwards. Cancellation propagates downwards. The
+opposition interplays to cleanly stop a nested computation when an
+exception occurs. For example, consider the tree in the following
+figure. Imagine that each node represents an algorithm and its
+``task_group_context``.
+
+
+.. container:: fignone
+   :name: fig6
+
+
+   Tree of task_group_context
+   |image0|
+
+
+Suppose that the algorithm in C throws an exception and no node catches
+the exception. oneTBB propagates the exception upwards, cancelling
+related subtrees downwards, as follows:
+
+
+#. Handle exception in C:
+
+
+   a. Capture exception in C.
+
+
+   b. Cancel tasks in C.
+
+
+   c. Throw exception from C to B.
+
+
+#. Handle exception in B:
+
+
+   a. Capture exception in B.
+
+
+   b. Cancel tasks in B and, by downwards propagation, in D.
+
+
+   c. Throw an exception out of B to A.
+
+
+#. Handle exception in A:
+
+
+   a. Capture exception in A.
+
+
+   b. Cancel tasks in A and, by downwards propagation, in E, F, and G.
+
+
+   c. Throw an exception upwards out of A.
+
+
+If your code catches the exception at any level, then oneTBB does not
+propagate it any further. For example, an exception that does not escape
+outside the body of a ``parallel_for`` does not cause cancellation of
+other iterations.
+
+
+To prevent downwards propagation of cancellation into an algorithm,
+construct an 'isolated' ``task_group_context`` on the stack and pass it
+to the algorithm explicitly. The example uses C++11 lambda expressions for brevity.
+
+
+::
+
+
+   #include "oneapi/tbb.h"
+    
+
+   bool Data[1000][1000];
+    
+
+   int main() {
+       try {
+           parallel_for( 0, 1000, 1, 
+               []( int i ) {
+                   task_group_context root(task_group_context::isolated);
+                   parallel_for( 0, 1000, 1,
+                      []( int  ) {
+                          Data[i][j] = true;
+                      },
+                      root);
+                   throw "oops";
+               });
+       } catch(...) {
+       }
+       return 0;
+   }
+
+
+The example performs two parallel loops: an outer loop over ``i`` and
+inner loop over ``j``. The creation of the isolated
+``task_group_context`` ``root`` protects the inner loop from downwards
+propagation of cancellation from the ``i`` loop. When the exception
+propagates to the outer loop, any pending ``outer`` iterations are
+cancelled, but not inner iterations for an outer iteration that started.
+Hence when the program completes, each row of ``Data`` may be different,
+depending upon whether its iteration ``i`` ran at all, but within a row,
+the elements will be homogeneously ``false`` or ``true``, not a mixture.
+
+
+Removing the blue text would permit cancellation to propagate down into
+the inner loop. In that case, a row of ``Data`` might end up with both
+``true`` and ``false`` values.
+
+
+.. |image0| image:: Images/image013.jpg
+   :width: 261px
+   :height: 131px
+
@@ -0,0 +1,108 @@
+.. _Concurrent_Queue_Classes:
+
+Concurrent Queue Classes
+========================
+
+
+Template class ``concurrent_queue<T,Alloc>`` implements a concurrent
+queue with values of type ``T``. Multiple threads may simultaneously
+push and pop elements from the queue. The queue is unbounded and has no
+blocking operations. The fundamental operations on it are ``push`` and
+``try_pop``. The ``push`` operation works just like ``push`` for a
+std::queue. The operation ``try_pop`` pops an item if it is available.
+The check and popping have to be done in a single operation for sake of
+thread safety.
+
+
+For example, consider the following serial code:
+
+
+::
+
+
+           extern std::queue<T> MySerialQueue;
+           T item;
+           if( !MySerialQueue.empty() ) {
+               item = MySerialQueue.front(); 
+               MySerialQueue.pop_front();
+               ... process item...
+           }
+
+
+Even if each std::queue method were implemented in a thread-safe manner,
+the composition of those methods as shown in the example would not be
+thread safe if there were other threads also popping from the same
+queue. For example, ``MySerialQueue.empty()`` might return true just
+before another thread snatches the last item from ``MySerialQueue``.
+
+
+The equivalent thread-safe |full_name| code is:
+
+
+::
+
+
+           extern concurrent_queue<T> MyQueue;
+           T item;
+           if( MyQueue.try_pop(item) ) {
+               ...process item...
+           }            
+
+
+In a single-threaded program, a queue is a first-in first-out structure.
+But if multiple threads are pushing and popping concurrently, the
+definition of "first" is uncertain. Use of ``concurrent_queue``
+guarantees that if a thread pushes two values, and another thread pops
+those two values, they will be popped in the same order that they were
+pushed.
+
+
+Template class ``concurrent_queue`` is unbounded and has no methods that
+wait. It is up to the user to provide synchronization to avoid overflow,
+or to wait for the queue to become non-empty. Typically this is
+appropriate when the synchronization has to be done at a higher level.
+
+
+Template class ``concurrent_bounded_queue<T,Alloc>`` is a variant that
+adds blocking operations and the ability to specify a capacity. The
+methods of particular interest on it are:
+
+
+-  ``pop(item)`` waits until it can succeed.
+
+
+-  ``push(item)`` waits until it can succeed without exceeding the
+   queue's capacity.
+
+
+-  ``try_push(item)`` pushes ``item`` only if it would not exceed the
+   queue's capacity.
+
+
+-  size() returns a *signed* integer.
+
+
+The value of concurrent_queue::size() is defined as the number of push
+operations started minus the number of pop operations started. If pops
+outnumber pushes, ``size()`` becomes negative. For example, if a
+``concurrent_queue`` is empty, and there are ``n`` pending pop
+operations, ``size()`` returns -\ ``n``. This provides an easy way for
+producers to know how many consumers are waiting on the queue. Method
+``empty()`` is defined to be true if and only if ``size()`` is not
+positive.
+
+
+By default, a ``concurrent_bounded_queue`` is unbounded. It may hold any
+number of values, until memory runs out. It can be bounded by setting
+the queue capacity with method ``set_capacity``.Setting the capacity
+causes ``push`` to block until there is room in the queue. Bounded
+queues are slower than unbounded queues, so if there is a constraint
+elsewhere in your program that prevents the queue from becoming too
+large, it is better not to set the capacity. If you do not need the
+bounds or the blocking pop, consider using ``concurrent_queue`` instead.
+
+.. toctree::
+   :maxdepth: 4
+
+   ../tbb_userguide/Iterating_Over_a_Concurrent_Queue_for_Debugging
+   ../tbb_userguide/When_Not_to_Use_Queues
@@ -0,0 +1,30 @@
+.. _Constraints:
+
+Constrained APIs
+================
+
+Starting from C++20, most of |full_name| APIs are constrained to
+enforce `named requirements <https://spec.oneapi.com/versions/latest/elements/oneTBB/source/named_requirements.html>`_ on
+template arguments types.
+
+The violations of these requirements are detected at a compile time during the template instantiation.
+
+.. rubric:: Example
+
+.. code:: cpp
+
+    // Call for body(oneapi::tbb::blocked_range) is ill-formed
+    // oneapi::tbb::parallel_for call results in constraint failure
+    auto body = [](const int& r) { /*...*/ };
+    oneapi::tbb::parallel_for(oneapi::tbb::blocked_range{1, 10}, body);
+
+    // Error example:
+    // error: no matching function to call to oneapi::tbb::parallel_for
+    // note: constraints not satisfied
+    // note: the required expression 'body(range)' is invalid
+            body(range);
+
+.. caution::
+
+    The code that violates named requirements but compiles successfully until C++20,
+    may not compile in C++20 mode due to early and strict constraints diagnostics.
@@ -0,0 +1,55 @@
+.. _Containers:
+
+Containers
+==========
+
+
+|full_name| provides highly concurrent
+container classes. These containers can be used with raw Windows\* OS or
+Linux\* OS threads, or in conjunction with task-based programming.
+
+
+A concurrent container allows multiple threads to concurrently access
+and update items in the container. Typical C++ STL containers do not
+permit concurrent update; attempts to modify them concurrently often
+result in corrupting the container. STL containers can be wrapped in a
+mutex to make them safe for concurrent access, by letting only one
+thread operate on the container at a time, but that approach eliminates
+concurrency, thus restricting parallel speedup.
+
+
+Containers provided by oneTBB offer a much higher level of concurrency,
+via one or both of the following methods:
+
+
+-  **Fine-grained locking:** Multiple threads operate on the container
+   by locking only those portions they really need to lock. As long as
+   different threads access different portions, they can proceed
+   concurrently.
+
+
+-  **Lock-free techniques:** Different threads account and correct for
+   the effects of other interfering threads.
+
+
+Notice that highly-concurrent containers come at a cost. They typically
+have higher overheads than regular STL containers. Operations on
+highly-concurrent containers may take longer than for STL containers.
+Therefore, use highly-concurrent containers when the speedup from the
+additional concurrency that they enable outweighs their slower
+sequential performance.
+
+
+.. CAUTION:: 
+   As with most objects in C++, the constructor or destructor of a
+   container object must not be invoked concurrently with another
+   operation on the same object. Otherwise the resulting race may cause
+   the operation to be executed on an undefined object.
+
+.. toctree::
+   :maxdepth: 4
+
+   ../tbb_userguide/concurrent_hash_map
+   ../tbb_userguide/concurrent_vector_ug
+   ../tbb_userguide/Concurrent_Queue_Classes
+   ../tbb_userguide/Summary_of_Containers
@@ -0,0 +1,175 @@
+.. _Controlling_Chunking:
+
+Controlling Chunking
+====================
+
+
+Chunking is controlled by a *partitioner* and a *grainsize.*\  To gain
+the most control over chunking, you specify both.
+
+
+-  Specify ``simple_partitioner()`` as the third argument to
+   ``parallel_for``. Doing so turns off automatic chunking.
+
+
+-  Specify the grainsize when constructing the range. The thread
+   argument form of the constructor is
+   ``blocked_range<T>(begin,end,grainsize)``. The default value of
+   ``grainsize`` is 1. It is in units of loop iterations per chunk.
+
+
+If the chunks are too small, the overhead may exceed the performance
+advantage.
+
+
+The following code is the last example from parallel_for, modified to
+use an explicit grainsize ``G``.
+
+
+::
+
+
+   #include "oneapi/tbb.h"
+    
+
+   void ParallelApplyFoo( float a[], size_t n ) {
+       parallel_for(blocked_range<size_t>(0,n,G), ApplyFoo(a), 
+                    simple_partitioner());
+   }
+
+
+The grainsize sets a minimum threshold for parallelization. The
+``parallel_for`` in the example invokes ``ApplyFoo::operator()`` on
+chunks, possibly of different sizes. Let *chunksize* be the number of
+iterations in a chunk. Using ``simple_partitioner`` guarantees that
+[G/2] <= *chunksize* <= G.
+
+
+There is also an intermediate level of control where you specify the
+grainsize for the range, but use an ``auto_partitioner`` and
+``affinity_partitioner``. An ``auto_partitioner`` is the default
+partitioner. Both partitioners implement the automatic grainsize
+heuristic described in :ref:`Automatic_Chunking`. An
+``affinity_partitioner`` implies an additional hint, as explained later
+in Section :ref:`Bandwidth_and_Cache_Affinity`. Though these partitioners
+may cause chunks to have more than G iterations, they never generate
+chunks with less than [G/2] iterations. Specifying a range with an
+explicit grainsize may occasionally be useful to prevent these
+partitioners from generating wastefully small chunks if their heuristics
+fail.
+
+
+Because of the impact of grainsize on parallel loops, it is worth
+reading the following material even if you rely on ``auto_partitioner``
+and ``affinity_partitioner`` to choose the grainsize automatically.
+
+
+.. container:: tablenoborder
+
+
+   .. list-table:: 
+      :header-rows: 1
+
+      * -     |image0|
+        -     |image1|
+      * -     Case A     
+        -     Case B     
+
+
+
+
+The above figure illustrates the impact of grainsize by showing the
+useful work as the gray area inside a brown border that represents
+overhead. Both Case A and Case B have the same total gray area. Case A
+shows how too small a grainsize leads to a relatively high proportion of
+overhead. Case B shows how a large grainsize reduces this proportion, at
+the cost of reducing potential parallelism. The overhead as a fraction
+of useful work depends upon the grainsize, not on the number of grains.
+Consider this relationship and not the total number of iterations or
+number of processors when setting a grainsize.
+
+
+A rule of thumb is that ``grainsize`` iterations of ``operator()``
+should take at least 100,000 clock cycles to execute. For example, if a
+single iteration takes 100 clocks, then the ``grainsize`` needs to be at
+least 1000 iterations. When in doubt, do the following experiment:
+
+
+#. Set the ``grainsize`` parameter higher than necessary. The grainsize
+   is specified in units of loop iterations. If you have no idea of how
+   many clock cycles an iteration might take, start with
+   ``grainsize``\ =100,000. The rationale is that each iteration
+   normally requires at least one clock per iteration. In most cases,
+   step 3 will guide you to a much smaller value.
+
+
+#. Run your algorithm.
+
+
+#. Iteratively halve the ``grainsize`` parameter and see how much the
+   algorithm slows down or speeds up as the value decreases.
+
+
+A drawback of setting a grainsize too high is that it can reduce
+parallelism. For example, if the grainsize is 1000 and the loop has 2000
+iterations, the ``parallel_for`` distributes the loop across only two
+processors, even if more are available. However, if you are unsure, err
+on the side of being a little too high instead of a little too low,
+because too low a value hurts serial performance, which in turns hurts
+parallel performance if there is other parallelism available higher up
+in the call tree.
+
+
+.. tip:: 
+   You do not have to set the grainsize too precisely.
+
+
+The next figure shows the typical "bathtub curve" for execution time
+versus grainsize, based on the floating point ``a[i]=b[i]*c``
+computation over a million indices. There is little work per iteration.
+The times were collected on a four-socket machine with eight hardware
+threads.
+
+
+.. container:: fignone
+   :name: fig2
+
+
+   Wall Clock Time Versus Grainsize
+   |image2|
+
+
+The scale is logarithmic. The downward slope on the left side indicates
+that with a grainsize of one, most of the overhead is parallel
+scheduling overhead, not useful work. An increase in grainsize brings a
+proportional decrease in parallel overhead. Then the curve flattens out
+because the parallel overhead becomes insignificant for a sufficiently
+large grainsize. At the end on the right, the curve turns up because the
+chunks are so large that there are fewer chunks than available hardware
+threads. Notice that a grainsize over the wide range 100-100,000 works
+quite well.
+
+
+.. tip:: 
+   A general rule of thumb for parallelizing loop nests is to
+   parallelize the outermost one possible. The reason is that each
+   iteration of an outer loop is likely to provide a bigger grain of
+   work than an iteration of an inner loop.
+
+
+.. admonition:: Product and Performance Information 
+
+   Performance varies by use, configuration and other factors. Learn more at `www.intel.com/PerformanceIndex <https://www.intel.com/PerformanceIndex>`_.
+   Notice revision #20201201
+
+
+.. |image0| image:: Images/image002.jpg
+   :width: 161px
+   :height: 163px
+.. |image1| image:: Images/image004.jpg
+   :width: 157px
+   :height: 144px
+.. |image2| image:: Images/image006.jpg
+   :width: 462px
+   :height: 193px
+
@@ -0,0 +1,84 @@
+.. _Cook_Until_Done_parallel_do:
+
+Cook Until Done: parallel_for_each
+==================================
+
+
+For some loops, the end of the iteration space is not known in advance,
+or the loop body may add more iterations to do before the loop exits.
+You can deal with both situations using the template class ``oneapi::tbb::parallel_for_each``.
+
+
+A linked list is an example of an iteration space that is not known in
+advance. In parallel programming, it is usually better to use dynamic
+arrays instead of linked lists, because accessing items in a linked list
+is inherently serial. But if you are limited to linked lists, the items
+can be safely processed in parallel, and processing each item takes at
+least a few thousand instructions, you can use ``parallel_for_each`` to
+gain some parallelism.
+
+
+For example, consider the following serial code:
+
+
+::
+
+
+   void SerialApplyFooToList( const std::list<Item>& list ) {
+       for( std::list<Item>::const_iterator i=list.begin() i!=list.end(); ++i ) 
+           Foo(*i);
+   }
+
+
+If ``Foo`` takes at least a few thousand instructions to run, you can
+get parallel speedup by converting the loop to use
+``parallel_for_each``. To do so, define an object with a ``const``
+qualified ``operator()``. This is similar to a C++ function object from
+the C++ standard header ``<functional>``, except that ``operator()``
+must be ``const``.
+
+
+::
+
+
+   class ApplyFoo {
+   public:
+       void operator()( Item& item ) const {
+           Foo(item);
+       }
+   };
+
+
+The parallel form of ``SerialApplyFooToList`` is as follows:
+
+
+::
+
+
+   void ParallelApplyFooToList( const std::list<Item>& list ) {
+       parallel_for_each( list.begin(), list.end(), ApplyFoo() ); 
+   }
+
+
+An invocation of ``parallel_for_each`` never causes two threads to act
+on an input iterator concurrently. Thus typical definitions of input
+iterators for sequential programs work correctly. This convenience makes
+``parallel_for_each`` unscalable, because the fetching of work is
+serial. But in many situations, you still get useful speedup over doing
+things sequentially.
+
+
+There are two ways that ``parallel_for_each`` can acquire work scalably.
+
+
+-  The iterators can be random-access iterators.
+
+
+-  The body argument to ``parallel_for_each``, if it takes a second
+   argument *feeder* of type ``parallel_for_each<Item>&``, can add more
+   work by calling ``feeder.add(item)``. For example, suppose processing
+   a node in a tree is a prerequisite to processing its descendants.
+   With ``parallel_for_each``, after processing a node, you could use
+   ``feeder.add`` to add the descendant nodes. The instance of
+   ``parallel_for_each`` does not terminate until all items have been
+   processed.
@@ -0,0 +1,199 @@
+.. _Data_Flow_Graph:
+
+Data Flow Graph
+===============
+
+
+In a data flow graph, nodes are computations that send and receive data
+messages. Some nodes may only send messages, others may only receive
+messages, and others may send messages in response to messages that they
+receive.
+
+
+In the following data flow graph, the left-most node generates the
+integer values from 1 to 10 and passes them to two successor nodes. One
+of the successors squares each value it receives and passes the result
+downstream. The second successor cubes each value it receives and passes
+the result downstream. The right-most node receives values from both of
+the middle nodes. As it receives each value, it adds it to a running sum
+of values. When the application is run to completion, the value of sum
+will be equal to the sum of the sequence of squares and cubes from 1 to
+10.
+
+
+.. container:: fignone
+   :name: simple_data_flow_title
+
+
+   Simple Data Flow Graph
+
+
+   .. container:: imagecenter
+
+
+      |image0|
+
+
+The following code snippet shows an implementation of the **Simple Data
+Flow Graph** shown above:
+
+
+::
+
+
+       int sum = 0;
+       graph g;
+       function_node< int, int > squarer( g, unlimited, [](const int &v) { 
+           return v*v; 
+       } );
+       function_node< int, int > cuber( g, unlimited, [](const int &v) { 
+           return v*v*v; 
+       } );
+       function_node< int, int > summer( g, 1, [&](const int &v ) -> int { 
+           return sum += v; 
+       } );
+       make_edge( squarer, summer );
+       make_edge( cuber, summer );
+
+
+       for ( int i = 1; i <= 10; ++i ) {
+         squarer.try_put(i);
+         cuber.try_put(i);
+       }
+       g.wait_for_all();
+
+
+       cout << "Sum is " << sum << "\n";
+
+
+In the implementation above, the following function_nodes are created:
+
+
+-  one to square values
+-  one to cube values
+-  one to add values to the global sum
+
+
+Since the squarer and cuber nodes are side-effect free, they are created
+with an unlimited concurrency. The summer node updates the sum through a
+reference to a global variable and therefore is not safe to execute in
+parallel. It is therefore created with a concurrency limit of 1. The
+node F from **Simple Data Flow Graph** above is implemented as a loop
+that puts messages to both the squarer and cuber node.
+
+
+A slight improvement over the first implementation is to introduce an
+additional node type, a ``broadcast_node``. A ``broadcast_node`` broadcasts any
+message it receives to all of its successors.
+
+
+This enables replacing the two ``try_put``'s in the loop with a single
+``try_put``:
+
+
+::
+
+
+       broadcast_node<int> b(g);
+       make_edge( b, squarer );
+       make_edge( b, cuber );
+       for ( int i = 1; i <= 10; ++i ) {
+         b.try_put(i);
+       }
+       g.wait_for_all();
+
+
+An even better option, which will make the implementation even more like
+the **Simple Data Flow Graph** above, is to introduce an ``input_node``. An
+``input_node``, as the name implies only sends messages and does not
+receive messages. Its constructor takes two arguments:
+
+
+::
+
+
+   template< typename Body > input_node( graph &g, Body body)
+
+The body is a function object, or lambda expression, that contains a
+function operator:
+
+
+::
+
+
+   Output Body::operator()( oneapi::tbb::flow_control &fc );
+
+
+You can replace the loop in the example with an ``input_node``
+
+
+::
+
+
+       input_node< int > src( g, src_body(10) );
+       make_edge( src, squarer );
+       make_edge( src, cuber );
+       src.activate();
+       g.wait_for_all();
+
+
+The runtime library will repeatedly invoke the function operator in
+``src_body`` until ``fc.stop()`` is invoked inside the body. You therefore
+need to create body that will act like the body of the loop in the **Simple Data Flow Graph**
+above. The final implementation after all of these changes is shown
+below:
+
+
+::
+
+
+       class src_body {
+           const int my_limit;
+           int my_next_value;
+       public:
+           src_body(int l) : my_limit(l), my_next_value(1) {}
+           int operator()( oneapi::tbb::flow_control& fc ) {
+               if ( my_next_value <= my_limit ) {
+                   return my_next_value++;
+               } else {
+                   fc.stop();
+                   return int();
+               }
+           }
+       };
+
+
+       int main() {
+         int sum = 0;
+         graph g;
+         function_node< int, int > squarer( g, unlimited, [](const int &v) { 
+             return v*v; 
+         } );
+         function_node< int, int > cuber( g, unlimited, [](const int &v) { 
+             return v*v*v; 
+         } );
+         function_node< int, int > summer( g, 1, [&](const int &v ) -> int { 
+             return sum += v; 
+         } );
+         make_edge( squarer, summer );
+         make_edge( cuber, summer );
+         input_node< int > src( g, src_body(10) );
+         make_edge( src, squarer );
+         make_edge( src, cuber );
+         src.activate();
+         g.wait_for_all();
+         cout << "Sum is " << sum << "\n";
+       }
+
+
+This final implementation has all of the nodes and edges from the
+**Simple Data Flow Graph** above. In this simple example, there is not
+much advantage in using an ``input_node`` over an explicit loop. But,
+because an ``input_node`` is able to react to the behavior of downstream
+nodes, it can limit memory use in more complex graphs. For more
+information, see `Creating a Token-Based
+System <create_token_based_system.html#create_token_based_system>`__
+
+
+.. |image0| image:: Images/flow_graph.jpg
+
@@ -0,0 +1,56 @@
+.. _Debug_Versus_Release_Libraries:
+
+Debug Versus Release Libraries
+==============================
+
+
+The following table details the |full_name| 
+dynamic shared libraries that come in debug and release
+versions.
+
+
+.. container:: tablenoborder
+
+
+   .. list-table::
+      :header-rows: 1
+
+      * -     Library 
+        -     Description    
+        -     When to Use    
+      * -    | ``tbb_debug``
+	     | ``tbbmalloc_debug``
+	     | ``tbbmalloc_proxy_debug``
+	     | ``tbbbind_debug``    
+        -     These versions have extensive internal checking for correct use of the library.   
+        -     Use with code that is compiled with the macro ``TBB_USE_DEBUG`` set to 1.    
+      * -    | ``tbb``
+	     | ``tbbmalloc``
+	     | ``tbbmalloc_proxy``
+	     | ``tbbbind``    
+        -     These versions deliver top performance. They eliminate  most checking for correct use of the library.    
+        -     Use with code compiled with ``TBB_USE_DEBUG`` undefined or set to zero.    
+
+.. tip:: 
+   Test your programs with the debug versions of the libraries first, to
+   assure that you are using the library correctly.  With the release
+   versions, incorrect usage may result in unpredictable program
+   behavior.
+
+
+oneTBB supports Intel® Inspector, Intel® VTune™ Profiler and Intel® Advisor.
+Full support of these tools requires compiling with macro ``TBB_USE_PROFILING_TOOLS=1``.
+That symbol defaults to 1 in the following conditions:
+
+-  When ``TBB_USE_DEBUG=1``.
+-  On the Microsoft Windows\* operating system, when ``_DEBUG=1``.
+
+The :ref:`reference` section explains the default values in more detail.
+
+
+.. CAUTION:: 
+   The instrumentation support for Intel® Inspector becomes live after
+   the first initialization of the task library. If the library
+   components are used before this initialization occurs, Intel® Inspector
+   may falsely report race conditions that are not really races.
+
@@ -0,0 +1,147 @@
+.. _Dependence_Graph:
+
+Dependence Graph
+================
+
+
+In a dependence graph, the nodes invoke body objects to perform
+computations and the edges create a partial ordering of these
+computations. At runtime, the library spawns and schedules tasks to
+execute the body objects when it is legal to do so according to the
+specified partial ordering. The following figure shows an example of an
+application that could be expressed using a dependence graph.
+
+
+.. container:: fignone
+   :name: dependence_graph_make_sandwitch
+
+
+   Dependence Graph for Making a Sandwich
+
+
+   .. container:: imagecenter
+
+
+      |image0|
+
+
+Dependence graphs are a special case of data flow graphs, where the data
+passed between nodes are of type oneapi::tbb::flow::continue_msg. Unlike a
+general data flow graph, nodes in a dependence graph do not spawn a task
+for each message they receive. Instead, they are aware of the number of
+predecessors they have, count the messages they receive and only spawn a
+task to execute their body when this count is equal to the total number
+of their predecessors.
+
+
+The following figure shows another example of a dependence graph. It has
+the same topology as the figure above, but with simple functions
+replacing the sandwich making steps. In this partial ordering, function
+A must complete executing before any other computation starts executing.
+Function B must complete before C and D start executing; and E must
+complete before D and F start executing. This is a partial ordering
+because, for example, there is no explicit ordering requirement between
+B and E or C and F.
+
+
+.. container:: fignone
+   :name: simple_dependence_graph
+
+
+   Simple Dependence Graph
+
+
+   .. container:: imagecenter
+
+
+      |image1|
+
+
+To implement this as a flow graph, continue_node objects are used for
+the nodes and continue_msg objects as the messages. A continue_node
+constructor takes two arguments:
+
+
+::
+
+
+   template< typename Body > continue_node( graph &g, Body body)
+
+
+The first argument is the graph it belongs to and the second is a
+function object or lambda expression. Unlike a function_node, a
+continue_node is always assumed to have unlimited concurrency and will
+immediately spawn a task whenever its dependencies are met.
+
+
+The following code snippet is an implementation of the example in this
+figure.
+
+
+::
+
+
+   typedef continue_node< continue_msg > node_t;
+   typedef const continue_msg & msg_t;
+
+
+   int main() {
+     oneapi::tbb::flow::graph g;
+     node_t A(g, [](msg_t){ a(); } );
+     node_t B(g, [](msg_t){ b(); } );
+     node_t C(g, [](msg_t){ c(); } );
+     node_t D(g, [](msg_t){ d(); } );
+     node_t E(g, [](msg_t){ e(); } );
+     node_t F(g, [](msg_t){ f(); } );
+     make_edge(A, B);
+     make_edge(B, C);
+     make_edge(B, D);
+     make_edge(A, E);
+     make_edge(E, D);
+     make_edge(E, F);
+     A.try_put( continue_msg() );
+     g.wait_for_all();
+     return 0;
+   }
+
+
+One possible execution of this graph is shown below. The execution of D
+does not start until both B and E are finished. While a task is waiting
+in the wait_for_all, its thread can participate in executing other tasks
+from the oneTBB work pool.
+
+
+.. container:: fignone
+
+
+   Execution Timeline for a Dependence Graph
+
+
+   .. container:: imagecenter
+
+
+      |image2|
+
+
+Again, it is important to note that all execution in the flow graph
+happens asynchronously. The call to A.try_put returns control to the
+calling thread quickly, after incrementing the counter and spawning a
+task to execute the body of A. Likewise, the body tasks execute the
+lambda expressions and then put a continue_msg to all successor nodes,
+if any. Only the call to wait_for_all blocks, as it should, and even in
+this case the calling thread may be used to execute tasks from the
+oneTBB work pool while it is waiting.
+
+
+The above timeline shows the sequence when there are enough threads to
+execute all of the tasks that can be executed concurrently in parallel.
+If there are fewer threads, then some tasks that are spawned will need
+to wait until a thread is available to execute them.
+
+
+.. |image0| image:: Images/flow_graph_complex.jpg
+   :width: 440px
+   :height: 337px
+.. |image1| image:: Images/dependence_graph.jpg
+.. |image2| image:: Images/execution_timeline_dependence.jpg
+
@@ -0,0 +1,46 @@
+.. _Edges:
+
+Flow Graph Basics: Edges
+========================
+
+
+Most applications contain multiple nodes with edges connecting them to
+each other. In the flow graph interface, edges are directed channels
+over which messages are passed. They are created by calling the function
+``make_edge( p, s )`` with two arguments: ``p``, the predecessor, and ``s``, the
+successor. You can modify the example used in the **Nodes** topic to
+include a second node that squares the value it receives before printing
+it and then connect that to the first node with an edge.
+
+
+::
+
+
+       graph g;
+       function_node< int, int > n( g, unlimited, []( int v ) -> int { 
+           cout << v;
+           spin_for( v );
+           cout << v;
+           return v;
+       } );
+       function_node< int, int > m( g, 1, []( int v ) -> int {
+           v *= v;
+           cout << v;
+           spin_for( v );
+           cout << v;
+           return v;
+       } );
+       make_edge( n, m );
+       n.try_put( 1 );
+       n.try_put( 2 );
+       n.try_put( 3 );
+       g.wait_for_all();
+
+
+Now there are two ``function_node``s, ``n`` and ``m``. The call to ``make_edge`` creates
+an edge from ``n`` to ``m``. The node n is created with unlimited concurrency,
+while ``m`` has a concurrency limit of 1. The invocations of ``n`` can all
+proceed in parallel, while the invocations of ``m`` will be serialized.
+Because there is an edge from ``n`` to ``m``, each value ``v``, returned by ``n``, will
+be automatically passed to node ``m`` by the runtime library.
+
@@ -0,0 +1,79 @@
+.. _Exceptions_and_Cancellation:
+
+Exceptions and Cancellation
+===========================
+
+
+|full_name| supports exceptions and
+cancellation. When code inside an oneTBB algorithm throws an exception,
+the following steps generally occur:
+
+
+#. The exception is captured. Any further exceptions inside the
+   algorithm are ignored.
+
+
+#. The algorithm is cancelled. Pending iterations are not executed. If
+   there is oneTBB parallelism nested inside, the nested parallelism may
+   also be cancelled as explained in :ref:`Cancellation_and_Nested_Parallelism`.
+
+
+#. Once all parts of the algorithm stop, an exception is thrown on the
+   thread that invoked the algorithm.
+
+
+The exception thrown in step 3 might be the original exception, or might
+merely be a summary of type ``captured_exception``. The latter usually
+occurs on current systems because propagating exceptions between threads
+requires support for the C++ ``std::exception_ptr`` functionality. As
+compilers evolve to support this functionality, future versions of
+oneTBB might throw the original exception. So be sure your code can
+catch either type of exception. The following example demonstrates
+exception handling.
+
+
+::
+
+
+   #include "oneapi/tbb.h"
+   #include <vector>
+   #include <iostream>
+    
+
+   using namespace oneapi::tbb;
+   using namespace std;
+    
+
+   vector<int> Data;
+    
+
+   struct Update {
+       void operator()( const blocked_range<int>& r ) const {
+           for( int i=r.begin(); i!=r.end(); ++i )
+               Data.at(i) += 1;
+       }
+   };
+    
+
+   int main() {
+       Data.resize(1000);
+       try {
+           parallel_for( blocked_range<int>(0, 2000), Update());
+       } catch( out_of_range& ex ) {
+          cout << "out_of_range: " << ex.what() << endl;
+       }
+       return 0;
+   }
+
+
+The ``parallel_for`` attempts to iterate over 2000 elements of a vector
+with only 1000 elements. Hence the expression ``Data.at(i)`` sometimes
+throws an exception ``std::out_of_range`` during execution of the
+algorithm. When the exception happens, the algorithm is cancelled and an
+exception thrown at the call site to ``parallel_for``.
+
+.. toctree::
+   :maxdepth: 4
+
+   ../tbb_userguide/Cancellation_Without_An_Exception
+   ../tbb_userguide/Cancellation_and_Nested_Parallelism
@@ -0,0 +1,14 @@
+.. _Flow_Graph:
+
+Parallelizing Data Flow and Dependence Graphs
+=============================================
+
+.. toctree::
+   :maxdepth: 4
+
+   ../tbb_userguide/Parallelizing_Flow_Graph
+   ../tbb_userguide/Basic_Flow_Graph_concepts
+   ../tbb_userguide/Graph_Main_Categories
+   ../tbb_userguide/Predefined_Node_Types
+   ../tbb_userguide/Flow_Graph_Tips
+   ../tbb_userguide/estimate_flow_graph_performance
@@ -0,0 +1,42 @@
+.. _Flow_Graph_Buffering_in_Nodes:
+
+Flow Graph Basics: Buffering and Forwarding
+===========================================
+
+
+|full_name| flow graph nodes use messages
+to communicate data and to enforce dependencies. If a node passes a
+message successfully to any successor, no further action is taken with
+the message by that node. As noted in the section on Single-push vs.
+Broadcast-push, a message may be passed to one or to multiple
+successors, depending on the type of the node, how many successors are
+connected to the node, and whether the message is pushed or pulled.
+
+
+There are times when a node cannot successfully push a message to any
+successor. In this case what happens to the message depends on the type
+of the node. The two possibilities are:
+
+
+-  The node stores the message to be forwarded later.
+-  The node discards the message.
+
+
+If a node discards messages that are not forwarded, and this behavior is
+not desired, the node should be connected to a buffering node that does
+store messages that cannot be pushed.
+
+
+If a message has been stored by a node, there are two ways it can be
+passed to another node:
+
+
+-  A successor to the node can pull the message using ``try_get()`` or
+   ``try_reserve()``.
+-  A successor can be connected using ``make_edge()``.
+
+
+If a ``try_get()`` successfully forwards a message, it is removed from
+the node that stored it. If a node is connected using ``make_edge`` the
+node will attempt to push a stored message to the new successor.
+
@@ -0,0 +1,47 @@
+.. _Flow_Graph_Message_Passing_Protocol:
+
+Flow Graph Basics: Message Passing Protocol
+===========================================
+
+
+|full_name| flow graph operates by passing
+messages between nodes. A node may not be able to receive and process a
+message from its predecessor. For a graph to operate most-efficiently,
+if this occurs the state of the edge between the nodes can change its
+state to pull so when the successor is able to handle a message it can
+query its predecessor to see if a message is available. If the edge did
+not reverse from push to pull, the predecessor node would have to
+repeatedly attempt to forward its message until the successor accepts
+it. This would consume resources needlessly.
+
+
+Once the edge is in pull mode, when the successor is not busy, it will
+try to pull a message from a predecessor.
+
+
+#. If a predecessor has a message, the successor will process it and the
+   edge will remain in pull mode.
+#. If the predecessor has no message, the edge between the nodes will
+   switch from pull to push mode.
+
+
+The state diagram of this Push-Pull protocol is:
+
+
+.. container:: fignone
+   :name: basic_push_pull
+
+
+   **The dynamic push / pull protocol.**
+
+
+   .. container:: imagecenter
+
+
+      |image0|
+
+
+.. |image0| image:: Images/flow_graph_message_passing_protocol.jpg
+   :width: 442px
+   :height: 196px
+
@@ -0,0 +1,286 @@
+.. _Flow_Graph_Reservation:
+
+Flow Graph Basics: Reservation
+==============================
+
+
+|full_name| flow graph
+``join_node`` has four possible policies: ``queueing``, ``reserving``,
+``key_matching`` and ``tag_matching``. ``join_nodes`` need messages at
+every input before they can create an output message. The reserving
+``join_node`` does not have internal buffering, and it does not pull
+messages from its inputs until it has a message at each input. To create
+an output message it temporarily reserves a message at each input port,
+and only if all input ports succeed reserving messages will an output
+message be created. If any input port fails to reserve a message, no
+message will be pulled by the ``join_node``.
+
+
+To support the reserving ``join_node`` some nodes support
+**reservation** of their outputs. The way reservation works is:
+
+
+-  When a node connected to a reserving ``join_node`` in push state
+   tries to push a message, the ``join_node`` always rejects the push
+   and the edge connecting the nodes is switched to pull mode.
+-  The reserving input port calls ``try_reserve`` on each edge in pull
+   state. This may fail; if so, the reserving input port switches that
+   edge to push state, and tries to reserve the next node connected by
+   an edge in pull state. While the input port's predecessor is in
+   reserved state, no other node can retrieve the reserved value.
+-  If each input port successfully reserves an edge in pull state, the
+   reserving ``join_node`` will create a message using the reserved
+   messages and try to push the resulting message to any nodes connected
+   to it.
+-  If the message is successfully pushed to a successor, the
+   predecessors that were reserved are signaled that the messages were
+   used (by calling ``try_consume()``.) Those messages will be discarded
+   by the predecessor nodes, because they have been successfully pushed.
+-  If the message was not successfully pushed to any successor, the
+   predecessors that were reserved are signaled that the messages were
+   not used (by calling ``try_release()``.) At this point, the messages
+   may be pushed to or pulled by other nodes.
+
+
+Because the reserving ``join_node`` will only attempt to push when each
+input port has at least one edge in a pull state, and will only attempt
+to create and push a message if all input ports succeed reserving
+messages, at least one of the predecessors to each of the reserving
+``join_node`` input ports must be reservable.
+
+
+The following example demonstrates a reserving ``join_node``'s behavior.
+``buffer_nodes`` buffer their output, so they accept a switch of their
+output edge from push to pull mode. ``broadcast_nodes`` do not buffer
+messages and do not support ``try_get()`` or ``try_reserve()``.
+
+
+::
+
+
+   void run_example2() {  // example for Flow_Graph_Reservation.xml
+       graph g;
+       broadcast_node<int> bn(g);
+       buffer_node<int> buf1(g);
+       buffer_node<int> buf2(g);
+       typedef join_node<tuple<int,int> reserving> join_type;
+       join_type jn(g);
+       buffer_node<join_type::output_type> buf_out(g);
+       join_type::output_type tuple_out;
+       int icnt;
+
+
+       // join_node predecessors are both reservable buffer_nodes
+       make_edge(buf1,input_port<0>jn));
+       make_edge(bn,input_port<0>jn));      // attach a broadcast_node
+       make_edge(buf2,input_port<1>jn));
+       make_edge(jn, buf_out);
+       bn.try_put(2);
+       buf1.try_put(3);
+       buf2.try_put(4);
+       buf2.try_put(7);
+       g.wait_for_all();
+       while (buf_out.try_get(tuple_out)) {
+           printf("join_node output == (%d,%d)\n",get<0>tuple_out), get<1>tuple_out) );
+       }
+       if(buf1.try_get(icnt)) printf("buf1 had %d\n", icnt);
+       else printf("buf1 was empty\n");
+       if(buf2.try_get(icnt)) printf("buf2 had %d\n", icnt);
+       else printf("buf2 was empty\n");
+   }
+
+
+In the example above, port 0 of the reserving ``join_node`` ``jn`` has
+two predecessors: a ``buffer_node`` ``buf1`` and a ``broadcast_node``
+``bn``. Port 1 of the ``join_node`` has one predecessor, ``buffer_node``
+``buf2``.
+
+
+.. container:: fignone
+   :name: reserve_step1
+
+
+   .. container:: imagecenter
+
+
+      |image0|
+
+
+We will discuss one possible execution sequence (the scheduling of tasks
+may differ slightly, but the end result will be the same.)
+
+
+::
+
+
+       bn.try_put(2);
+
+
+``bn`` attempts to forward 2 to ``jn``. ``jn`` does not accept the value
+and the arc from ``bn`` to ``jn`` reverses. Because neither bn nor jn
+buffer messages, the message is dropped. Because not all the inputs to
+``jn`` have available predecessors, ``jn`` does nothing further.
+
+
+.. CAUTION:: 
+   Any node which does not support reservation will not work correctly
+   when attached to a reserving ``join_node``. This program demonstrates
+   why this occurs; connecting non-reserving nodes to nodes requiring
+   support for reservation is **not** recommended practice.
+
+
+.. container:: fignone
+   :name: reserve_step2
+
+
+   .. container:: imagecenter
+
+
+      |image1|
+
+
+::
+
+
+       buf1.try_put(3);
+
+
+``buf1`` attempts to forward 3 to ``jn``. ``jn`` does not accept the
+value and the arc from ``buf1`` to ``jn`` reverses. Because not all the
+inputs to ``jn`` have available predecessors, ``jn`` does nothing
+further.
+
+
+.. container:: fignone
+   :name: reserve_step3
+
+
+   .. container:: imagecenter
+
+
+      |image2|
+
+
+::
+
+
+       buf2.try_put(4);
+
+
+``buf2`` attempts to forward 4 to ``jn``. ``jn`` does not accept the
+value and the arc from ``buf2`` to ``jn`` reverses. Now both inputs of
+``jn`` have predecessors, a task to build and forward a message from
+``jn`` will be spawned. We assume that task is not yet executing.
+
+
+.. container:: fignone
+   :name: reserve_step4
+
+
+   .. container:: imagecenter
+
+
+      |image3|
+
+
+::
+
+
+       buf2.try_put(7);
+
+
+``buf2`` has no successor (because the arc to ``jn`` is reversed,) so it
+stores the value 7.
+
+
+.. container:: fignone
+   :name: reserve_step5
+
+
+   .. container:: imagecenter
+
+
+      |image4|
+
+
+Now the task spawned to run ``jn`` runs.
+
+
+-  ``jn`` tries to reserve ``bn``, which fails. The arc to ``bn``
+   switches back to the forward direction.
+-  ``jn`` tries to reserve ``buf1``, which succeeds (reserved nodes are
+   colored grey.) ``jn`` receives the value 3 from ``buf1``, but it
+   remains in ``buf1`` (in case the attempt to forward a message from
+   ``jn`` fails.)
+-  ``jn`` tries to reserve ``buf2``, which succeeds. ``jn`` receives the
+   value 4 from ``buf2``, but it remains in ``buf2``.
+-  ``jn`` constructs the output message ``tuple<3,4>``.
+
+
+.. container:: fignone
+   :name: reserve_step6
+
+
+   .. container:: imagecenter
+
+
+      |image5|
+
+
+Now ``jn`` pushes its message to ``buf_out``, which accepts it. Because
+the push succeeded, ``jn`` signals ``buf1`` and ``buf2`` that the
+reserved values were used, and the buffers discard those values. Now
+``jn`` attempts to reserve again.
+
+
+-  No attempt to pull from ``bn`` is made, because the edge from ``bn``
+   to ``jn`` is in push state.
+-  ``jn`` tries to reserve ``buf1``, which fails. The arc to ``buf1``
+   switches back to the forward direction.
+-  ``jn`` does not try any further actions.
+
+
+.. container:: fignone
+   :name: reserve_step7
+
+
+   .. container:: imagecenter
+
+
+      |image6|
+
+
+No further activity occurs in the graph, and the ``wait_for_all()`` will
+complete. The output of this code is
+
+
+::
+
+
+   join_node output == (3,4)
+   buf1 was empty
+   buf2 had 7
+
+
+.. |image0| image:: Images/flow_graph_reserve_buffers_1.png
+   :width: 400px
+   :height: 222px
+.. |image1| image:: Images/flow_graph_reserve_buffers_2.png
+   :width: 400px
+   :height: 222px
+.. |image2| image:: Images/flow_graph_reserve_buffers_3.png
+   :width: 400px
+   :height: 222px
+.. |image3| image:: Images/flow_graph_reserve_buffers_4.png
+   :width: 400px
+   :height: 222px
+.. |image4| image:: Images/flow_graph_reserve_buffers_5.png
+   :width: 400px
+   :height: 222px
+.. |image5| image:: Images/flow_graph_reserve_buffers_6.png
+   :width: 400px
+   :height: 222px
+.. |image6| image:: Images/flow_graph_reserve_buffers_7.png
+   :width: 400px
+   :height: 222px
+
@@ -0,0 +1,112 @@
+.. _Flow_Graph_Single_Vs_Broadcast:
+
+Flow Graph Basics: Single-push vs. Broadcast-push
+=================================================
+
+
+Nodes in the |full_name| flow graph
+communicate by pushing and pulling messages. Two policies for pushing
+messages are used, depending on the type of the node:
+
+
+-  **single-push**: No matter how many successors to the node exist and
+   are able to accept a message, each message will be only sent to one
+   successor.
+-  **broadcast-push**: A message will be pushed to every successor which
+   is connected to the node by an edge in push mode, and which accepts
+   the message.
+
+
+The following code demonstrates this difference:
+
+
+::
+
+
+   using namespace oneapi::tbb::flow;
+
+
+   std::atomic<size_t> g_cnt;
+
+
+   struct fn_body1 {
+       std::atomic<size_t> &body_cnt;
+       fn_body1(std::atomic<size_t> &b_cnt) : body_cnt(b_cnt) {}
+       continue_msg operator()( continue_msg /*dont_care*/) {
+           ++g_cnt;
+           ++body_cnt;
+           return continue_msg();
+       }
+   };
+
+
+   void run_example1() {  // example for Flow_Graph_Single_Vs_Broadcast.xml
+       graph g;
+       std::atomic<size_t> b1;  // local counts
+       std::atomic<size_t> b2;  // for each function _node body
+       std::atomic<size_t> b3;  //
+       function_node<continue_msg> f1(g,serial,fn_body1(b1));
+       function_node<continue_msg> f2(g,serial,fn_body1(b2));
+       function_node<continue_msg> f3(g,serial,fn_body1(b3));
+       buffer_node<continue_msg> buf1(g);
+       //
+       // single-push policy
+       //
+       g_cnt = b1 = b2 = b3 = 0;
+       make_edge(buf1,f1);
+       make_edge(buf1,f2);
+       make_edge(buf1,f3);
+       buf1.try_put(continue_msg());
+       buf1.try_put(continue_msg());
+       buf1.try_put(continue_msg());
+       g.wait_for_all();
+       printf( "after single-push test, g_cnt == %d, b1==%d, b2==%d, b3==%d\n", (int)g_cnt, (int)b1, (int)b2, (int)b3);
+       remove_edge(buf1,f1);
+       remove_edge(buf1,f2);
+       remove_edge(buf1,f3);
+       //
+       // broadcast-push policy
+       //
+       broadcast_node<continue_msg> bn(g);
+       g_cnt = b1 = b2 = b3 = 0;
+       make_edge(bn,f1);
+       make_edge(bn,f2);
+       make_edge(bn,f3);
+       bn.try_put(continue_msg());
+       bn.try_put(continue_msg());
+       bn.try_put(continue_msg());
+       g.wait_for_all();
+       printf( "after broadcast-push test, g_cnt == %d, b1==%d, b2==%d, b3==%d\n", (int)g_cnt, (int)b1, (int)b2, (int)b3);
+   }
+
+
+The output of this code is
+
+
+::
+
+
+   after single-push test, g_cnt == 3, b1==3, b2==0, b3==0
+   after broadcast-push test, g_cnt == 9, b1==3, b2==3, b3==3
+
+
+The single-push test uses a ``buffer_node``, which has a "single-push"
+policy for forwarding messages. Putting three messages to the
+``buffer_node`` results in three messages being pushed. Notice also only
+the first ``function_node`` is sent to; in general there is no policy
+for which node is pushed to if more than one successor can accept.
+
+
+The broadcast-push test uses a ``broadcast_node``, which will push any
+message it receives to all accepting successors. Putting three messages
+to the ``broadcast_node`` results in a total of nine messages pushed to
+the ``function_nodes``.
+
+
+Only nodes designed to buffer (hold and forward received messages) have
+a "single-push" policy; all other nodes have a "broadcast-push" policy.
+
+Please see the :ref:`broadcast_or_send` section of
+:ref:`Flow_Graph_Tips`, and :ref:`Flow_Graph_Buffering_in_Nodes` for more
+information.
+
@@ -0,0 +1,13 @@
+.. _Flow_Graph_Tips:
+
+Flow Graph Tips and Tricks
+==========================
+
+.. toctree::
+   :maxdepth: 4
+
+   ../tbb_userguide/Flow_Graph_waiting_tips
+   ../tbb_userguide/Flow_Graph_making_edges_tips
+   ../tbb_userguide/Flow_Graph_nested_parallelism_tips
+   ../tbb_userguide/Flow_Graph_resource_tips
+   ../tbb_userguide/Flow_Graph_exception_tips
@@ -0,0 +1,17 @@
+.. _Flow_Graph_exception_tips:
+
+Flow Graph Tips for Exception Handling and Cancellation
+=======================================================
+
+
+The execution of a flow graph can be canceled directly or as a result of
+an exception that propagates beyond a node's body. You can then
+optionally reset the graph so that it can be re-executed.
+
+.. toctree::
+   :maxdepth: 4
+
+   ../tbb_userguide/catching_exceptions
+   ../tbb_userguide/cancel_a_graph
+   ../tbb_userguide/use_graph_reset
+   ../tbb_userguide/cancelling_nested_parallelism
@@ -0,0 +1,13 @@
+.. _Flow_Graph_making_edges_tips:
+
+Flow Graph Tips on Making Edges
+===============================
+
+.. toctree::
+   :maxdepth: 4
+
+   ../tbb_userguide/use_make_edge
+   ../tbb_userguide/broadcast_or_send
+   ../tbb_userguide/communicate_with_nodes
+   ../tbb_userguide/use_input_node
+   ../tbb_userguide/avoiding_data_races
@@ -0,0 +1,10 @@
+.. _Flow_Graph_nested_parallelism_tips:
+
+Flow Graph Tips on Nested Parallelism
+=====================================
+
+.. toctree::
+   :maxdepth: 4
+
+   ../tbb_userguide/use_nested_algorithms
+   ../tbb_userguide/use_nested_flow_graphs
@@ -0,0 +1,17 @@
+.. _Flow_Graph_resource_tips:
+
+Flow Graph Tips for Limiting Resource Consumption
+=================================================
+
+
+You may want to control the number of messages allowed to enter parts of
+your graph, or control the maximum number of tasks in the work pool.
+There are several mechanisms available for limiting resource consumption
+in a flow graph.
+
+.. toctree::
+   :maxdepth: 4
+
+   ../tbb_userguide/use_limiter_node
+   ../tbb_userguide/use_concurrency_limits
+   ../tbb_userguide/create_token_based_system
@@ -0,0 +1,11 @@
+.. _Flow_Graph_waiting_tips:
+
+Flow Graph Tips for Waiting for and Destroying a Flow Graph
+===========================================================
+
+.. toctree::
+   :maxdepth: 4
+
+   ../tbb_userguide/always_use_wait_for_all
+   ../tbb_userguide/avoid_dynamic_node_removal
+   ../tbb_userguide/destroy_graphs_outside_main_thread
@@ -0,0 +1,21 @@
+.. _Graph_Main_Categories:
+
+Graph Application Categories
+============================
+
+
+Most flow graphs fall into one of two categories:
+
+
+-  **Data flow graphs.** In this type of graph, data is passed along the
+   graph's edges. The nodes receive, transform and then pass along the
+   data messages.
+-  **Dependence graphs.** In this type of graph, the data operated on by
+   the nodes is obtained through shared memory directly and is not
+   passed along the edges.
+
+.. toctree::
+   :maxdepth: 4
+
+   ../tbb_userguide/Data_Flow_Graph
+   ../tbb_userguide/Dependence_Graph
@@ -0,0 +1,27 @@
+.. _Graph_Object:
+
+Flow Graph Basics: Graph Object
+===============================
+
+
+Conceptually a flow graph is a collection of nodes and edges. Each node
+belongs to exactly one graph and edges are made only between nodes in
+the same graph. In the flow graph interface, a graph object represents
+this collection of nodes and edges, and is used for invoking whole graph
+operations such as waiting for all tasks related to the graph to
+complete, resetting the state of all nodes in the graph, and canceling
+the execution of all nodes in the graph.
+
+
+The code below creates a graph object and then waits for all tasks
+spawned by the graph to complete. The call to ``wait_for_all`` in this
+example returns immediately since this is a trivial graph with no nodes
+or edges, and therefore no tasks are spawned.
+
+
+::
+
+
+   graph g;
+   g.wait_for_all();
+
--- a/Show More
+++ b/Show More