added triangulator fuzzer (#620)

* added triangulator fuzzer * added easy to use minimizer script * update readme * added rounding
2023-11-25 21:51:07 +08:00
parent 1f8e8037f1
commit f683e8c9c0
7 changed files with 4931 additions and 4691 deletions
@@ -21,12 +21,25 @@ set(MANIFOLD_PAR "NONE" CACHE STRING "Parallel backend, either \"TBB\" or \"NONE
 set(MANIFOLD_FLAGS "" CACHE STRING "Manifold compiler flags")
 option(MANIFOLD_EXPORT "Build mesh export (via assimp) utility library" OFF)
 option(MANIFOLD_TEST "Enable testing suite" ON)
+# fuzztest is a rather large dependency
+option(MANIFOLD_FUZZ "Enable fuzzing tests" OFF)
 option(MANIFOLD_DEBUG "Enable debug tracing/timing" OFF)
-
 option(MANIFOLD_PYBIND "Build python bindings" ON)
 option(MANIFOLD_CBIND "Build C (FFI) bindings" OFF)
 option(MANIFOLD_JSBIND "Build js binding" ${EMSCRIPTEN})

+if(MANIFOLD_FUZZ)
+  if(NOT CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
+    message(FATAL_ERROR "fuzztest only supports clang" )
+  endif()
+  # we should enable debug checks
+  set(MANIFOLD_DEBUG ON)
+  # enable fuzztest fuzzing mode
+  set(FUZZTEST_FUZZING_MODE ON)
+  # address sanitizer required
+  set(CMAKE_CXX_FLAGS "$CMAKE_CXX_FLAGS -fsanitize=address")
+endif()
+
 option(TRACY_ENABLE "Use tracy profiling" OFF) 
 option(TRACY_MEMORY_USAGE "Track memory allocation with tracy (expensive)" OFF)
 option(BUILD_TEST_CGAL "Build CGAL performance comparisons" OFF)
@@ -140,11 +140,22 @@ For more detailed documentation, please refer to the C++ API.

 Contributions are welcome! A lower barrier contribution is to simply make a PR that adds a test, especially if it repros an issue you've found. Simply name it prepended with DISABLED_, so that it passes the CI. That will be a very strong signal to me to fix your issue. However, if you know how to fix it yourself, then including the fix in your PR would be much appreciated!

-## Profiling
+### Profiling

 There is now basic support for the [Tracy profiler](https://github.com/wolfpld/tracy) for our tests.
 To enable tracing, compile with `-DTRACY_ENABLE=on` cmake option, and run the test with Tracy server running.

+### Fuzzing Support
+
+We use https://github.com/google/fuzztest for fuzzing the triangulator.
+
+To enable fuzzing, make sure that you are using clang compiler (`-DCMAKE_CXX_COMPILER=clang -DCMAKE_C_COMPILER=clang`), running Linux, and enable fuzzing support by setting `-DMANIFOLD_FUZZ=ON`.
+
+To run the fuzzer and minimize testcase, do
+```
+../minimizer.sh ./test/polygon_fuzz --fuzz=PolygonFuzz.TriangulationNoCrash
+```
+
 ## About the author

 This library was started by [Emmett Lalish](https://elalish.blogspot.com/). I am currently a Google employee and this is my 20% project, not an official Google project. At my day job I'm the maintainer of [\<model-viewer\>](https://modelviewer.dev/). I was the first employee at a 3D video startup, [Omnivor](https://www.omnivor.io/), and before that I worked on 3D printing at Microsoft, including [3D Builder](https://www.microsoft.com/en-us/p/3d-builder/9wzdncrfj3t6?activetab=pivot%3Aoverviewtab). Originally an aerospace engineer, I started at a small DARPA contractor doing seedling projects, one of which became [Sea Hunter](https://en.wikipedia.org/wiki/Sea_Hunter). I earned my doctorate from the University of Washington in control theory and published some [papers](https://www.researchgate.net/scientific-contributions/75011026_Emmett_Lalish).
@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+
+# Modified from https://github.com/google/fuzztest/blob/main/tools/minimizer.sh
+# If you want to use a specific reproducer, pass it in FUZZTEST_MINIMIZE_REPRODUCER
+# environment variable.
+# Otherwise, just run it as
+# $ ./minimizer.sh ./build/test/polygon_fuzz --fuzz=PolygonFuzz.TriangulationNoCrash
+# If you want to kill it, try SIGKILL, SIGTERM is not very useful
+# If you want to end the loop early, rename $(pwd)/reproducers into something
+# else...
+
+if [ -z "${FUZZTEST_MINIMIZE_REPRODUCER}"]; then
+  mkdir -p $(pwd)/reproducers
+  FUZZTEST_REPRODUCERS_OUT_DIR=$(pwd)/reproducers "$@"
+  wildcard="$(pwd)/reproducers/*"
+  reproducers=($wildcard)
+  FUZZTEST_MINIMIZE_REPRODUCER="${reproducers[0]}"
+fi
+readonly ORIGINAL_REPRODUCER="${FUZZTEST_MINIMIZE_REPRODUCER}"
+
+for i in {0001..9999}; do
+  echo
+  echo "╔════════════════════════════════════════════════╗"
+  echo "║ Minimization round: ${i}                       ║"
+  echo "╚════════════════════════════════════════════════╝"
+  echo "Note that to terminate early, simply move $ORIGINAL_REPRODUCER to somewhere else..."
+  echo
+
+  if [ ! -f $ORIGINAL_REPRODUCER ]; then
+    echo "Terminated by the user"
+    break
+  fi
+
+  TEMP_DIR=$(mktemp -d)
+  FUZZTEST_REPRODUCERS_OUT_DIR="${TEMP_DIR}" \
+  FUZZTEST_MINIMIZE_REPRODUCER="${FUZZTEST_MINIMIZE_REPRODUCER}" \
+  "$@"
+
+  if [ $? -eq 130 ]; then
+    echo
+    echo "╔═══════════════════════════════════════════════╗"
+    echo "║ Minimization terminated.                      ║"
+    echo "╚═══════════════════════════════════════════════╝"
+    echo
+    echo "Find the smallest reproducer at:"
+    echo
+    echo "${FUZZTEST_MINIMIZE_REPRODUCER}"
+
+    rm -rf "${TEMP_DIR}"
+    break
+  fi
+
+  SMALLER_REPRODUCER=$(find "${TEMP_DIR}" -type f)
+  NEW_NAME="${ORIGINAL_REPRODUCER}-min-${i}"
+  mv "${SMALLER_REPRODUCER}" "${NEW_NAME}"
+  FUZZTEST_MINIMIZE_REPRODUCER="${NEW_NAME}"
+
+  rm -rf "${TEMP_DIR}"
+done
@@ -14,6 +14,7 @@

 project(manifold_test)

+set(CMAKE_CXX_STANDARD 17)
 enable_testing()

 set(SOURCE_FILES polygon_test.cpp cross_section_test.cpp manifold_test.cpp boolean_test.cpp sdf_test.cpp samples_test.cpp test_main.cpp)
@@ -44,6 +45,21 @@ endif()
 add_executable(${PROJECT_NAME} ${SOURCE_FILES})
 target_link_libraries(${PROJECT_NAME} polygon GTest::gtest_main manifold sdf samples cross_section)

+if(MANIFOLD_FUZZ)
+  FetchContent_Declare(fuzztest
+    GIT_REPOSITORY https://github.com/google/fuzztest.git
+    # note that if commit hash is used, it cannot be a shallow clone
+    GIT_TAG        2606e04a43e5a7730e437a849604a61f1cb0ff28
+    GIT_PROGRESS   TRUE
+  )
+  FetchContent_MakeAvailable(fuzztest)
+  fuzztest_setup_fuzzing_flags()
+  add_executable(polygon_fuzz polygon_fuzz.cpp)
+  target_link_libraries(polygon_fuzz PUBLIC polygon)
+  link_fuzztest(polygon_fuzz)
+  gtest_discover_tests(polygon_fuzz)
+endif()
+
 if(MANIFOLD_CBIND)
  target_link_libraries(${PROJECT_NAME} manifoldc)
 endif()
@@ -0,0 +1,123 @@
+// Copyright 2023 The Manifold Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <atomic>
+#include <future>
+
+#include "fuzztest/fuzztest.h"
+#include "gtest/gtest.h"
+#include "polygon.h"
+
+using namespace fuzztest;
+
+void TriangulationNoCrash(
+    std::vector<std::vector<std::pair<float, float>>> input, float precision) {
+  if (precision < 0) precision = -1;
+  manifold::PolygonParams().intermediateChecks = true;
+  manifold::Polygons polys;
+  size_t size = 0;
+  for (const auto &simplePoly : input) {
+    polys.emplace_back();
+    for (const auto &p : simplePoly) {
+      polys.back().emplace_back(p.first, p.second);
+      size++;
+    }
+  }
+  std::atomic<pid_t> tid;
+  std::atomic<bool> faulted(true);
+  auto asyncFuture =
+      std::async(std::launch::async, [&polys, &faulted, &tid, precision]() {
+        tid.store(gettid());
+        try {
+          manifold::Triangulate(polys, precision);
+          faulted.store(false);
+        } catch (manifold::geometryErr e) {
+          // geometryErr is fine
+          faulted.store(false);
+        } catch (...) {
+          printf("got unexpected error\n");
+        }
+      });
+  if (asyncFuture.wait_for(std::chrono::milliseconds(
+          std::max<size_t>(size, 10000))) == std::future_status::timeout) {
+    printf("timeout after %ldms...\n", std::max<size_t>(size, 10000));
+    pthread_cancel(tid.load());
+  }
+
+  EXPECT_FALSE(faulted.load());
+}
+
+void TriangulationNoCrashRounded(
+    std::vector<std::vector<std::pair<float, float>>> input, float precision) {
+  TriangulationNoCrash(std::move(input), precision);
+}
+
+using Polygons = std::vector<std::vector<std::pair<float, float>>>;
+using TestCase = std::tuple<Polygons, float>;
+
+std::vector<TestCase> SeedProvider();
+
+auto PolygonDomain =
+    ContainerOf<Polygons>(ContainerOf<std::vector<std::pair<float, float>>>(
+                              PairOf(Finite<float>(), Finite<float>()))
+                              .WithMinSize(3)
+                              .WithMaxSize(10000))
+        .WithMinSize(1)
+        .WithMaxSize(10000);
+
+FUZZ_TEST(PolygonFuzz, TriangulationNoCrash)
+    .WithDomains(PolygonDomain, InRange<float>(-1.0, 0.1))
+    .WithSeeds(SeedProvider);
+
+FUZZ_TEST(PolygonFuzz, TriangulationNoCrashRounded)
+    .WithDomains(ReversibleMap(
+                     [](auto input) {
+                       for (auto &poly : input) {
+                         for (auto &pair : poly) {
+                           pair.first = std::round(pair.first);
+                           pair.second = std::round(pair.second);
+                         }
+                       }
+                       return input;
+                     },
+                     [](auto input) {
+                       return std::optional{std::tuple{input}};
+                     },
+                     PolygonDomain),
+                 InRange<float>(-1.0, 0.1))
+    .WithSeeds(SeedProvider);
+
+static std::vector<TestCase> TestCases;
+void TestPoly(Polygons polys, int _unused, float precision = -1.0f) {
+  TestCases.push_back({polys, precision});
+}
+
+// ----------------------------------------------------------------------------
+//                        UGLY HACK FOR SEED REUSE
+// ----------------------------------------------------------------------------
+
+#undef TEST
+#define TEST(_unused1, _unused2)
+
+std::vector<TestCase> SeedProvider() {
+  std::vector<TestCase> TestCases;
+  auto TestPoly = [&TestCases](Polygons polys, int _unused,
+                               float precision = -1.0f) {
+    TestCases.push_back({polys, precision});
+  };
+
+#include "polygon_corpus.cpp"
+
+  return TestCases;
+}