fixed folded normals smoothing (#842)

src/manifold/src/impl.h

@@ -174,6 +174,7 @@ struct Manifold::Impl {
   bool IsInsideQuad(int halfedge) const;
   bool IsMarkedInsideQuad(int halfedge) const;
   glm::vec3 GetNormal(int halfedge, int normalIdx) const;
+  glm::vec4 TangentFromNormal(const glm::vec3& normal, int halfedge) const;
   std::vector<Smoothness> UpdateSharpenedEdges(
       const std::vector<Smoothness>&) const;
   Vec<bool> FlatFaces() const;

src/manifold/src/smoothing.cpp

@@ -64,14 +64,7 @@ struct SmoothBezier {
       return;
     }
 
-    const glm::vec3 edgeVec =
-        impl->vertPos_[edge.endVert] - impl->vertPos_[edge.startVert];
-    const glm::vec3 edgeNormal =
-        impl->faceNormal_[edge.face] +
-        impl->faceNormal_[impl->halfedge_[edge.pairedHalfedge].face];
-    glm::vec3 dir =
-        glm::cross(glm::cross(edgeNormal, edgeVec), vertNormal[edge.startVert]);
-    tangent = CircularTangent(dir, edgeVec);
+    tangent = impl->TangentFromNormal(vertNormal[edge.startVert], edgeIdx);
   }
 };
 
@@ -294,6 +287,20 @@ glm::vec3 Manifold::Impl::GetNormal(int halfedge, int normalIdx) const {
   return normal;
 }
 
+/**
+ * Returns a circular tangent for the requested halfedge, orthogonal to the
+ * given normal vector, and avoiding folding.
+ */
+glm::vec4 Manifold::Impl::TangentFromNormal(const glm::vec3& normal,
+                                            int halfedge) const {
+  const Halfedge edge = halfedge_[halfedge];
+  const glm::vec3 edgeVec = vertPos_[edge.endVert] - vertPos_[edge.startVert];
+  const glm::vec3 edgeNormal =
+      faceNormal_[edge.face] + faceNormal_[halfedge_[edge.pairedHalfedge].face];
+  glm::vec3 dir = glm::cross(glm::cross(edgeNormal, edgeVec), normal);
+  return CircularTangent(dir, edgeVec);
+}
+
 /**
  * Returns true if this halfedge should be marked as the interior of a quad, as
  * defined by its two triangles referring to the same face, and those triangles
@@ -819,15 +826,15 @@ void Manifold::Impl::CreateTangents(int normalIdx) {
                 }
               }
               // calculate tangents
-              const glm::vec3 edge = vertPos_[halfedge_[halfedge].endVert] -
-                                     vertPos_[halfedge_[halfedge].startVert];
               if (differentNormals) {
+                const glm::vec3 edgeVec =
+                    vertPos_[halfedge_[halfedge].endVert] -
+                    vertPos_[halfedge_[halfedge].startVert];
                 const glm::vec3 dir = glm::cross(here.normal, next.normal);
-                tangent[halfedge] =
-                    CircularTangent(glm::sign(glm::dot(dir, edge)) * dir, edge);
+                tangent[halfedge] = CircularTangent(
+                    glm::sign(glm::dot(dir, edgeVec)) * dir, edgeVec);
               } else {
-                tangent[halfedge] =
-                    CircularTangent(OrthogonalTo(edge, here.normal), edge);
+                tangent[halfedge] = TangentFromNormal(here.normal, halfedge);
               }
             });
 

test/smooth_test.cpp

@@ -77,14 +77,22 @@ TEST(Smooth, RefineQuads) {
 }
 
 TEST(Smooth, TruncatedCone) {
-  Manifold cone = Manifold::Cylinder(5, 10, 5, 12).SmoothOut();
-  Manifold smooth = cone.RefineToLength(0.5).CalculateNormals(0);
+  Manifold cone = Manifold::Cylinder(5, 10, 5, 12);
+  Manifold smooth = cone.SmoothOut().RefineToLength(0.5).CalculateNormals(0);
   auto prop = smooth.GetProperties();
   EXPECT_NEAR(prop.volume, 1158.61, 0.01);
   EXPECT_NEAR(prop.surfaceArea, 768.12, 0.01);
   MeshGL out = smooth.GetMeshGL();
   CheckGL(out);
 
+  Manifold smooth1 = cone.SmoothOut(180, 1).RefineToLength(0.5);
+  auto prop1 = smooth1.GetProperties();
+
+  Manifold smooth2 = cone.SmoothOut(180, 0).RefineToLength(0.5);
+  auto prop2 = smooth2.GetProperties();
+  EXPECT_NEAR(prop2.volume, prop1.volume, 0.01);
+  EXPECT_NEAR(prop2.surfaceArea, prop1.surfaceArea, 0.01);
+
 #ifdef MANIFOLD_EXPORT
   ExportOptions options2;
   options2.faceted = false;