Compare commits

...
11 Commits
Author SHA1 Message Date
Patrick Hodoul 26cde0976a Improve CLF reader
Signed-off-by: Patrick Hodoul <Patrick.Hodoul@autodesk.com>
2021-11-02 18:35:01 -04:00
Patrick Hodoul 5f038e0992 Improve MacOS support
Signed-off-by: Patrick Hodoul <Patrick.Hodoul@autodesk.com>
2021-11-02 17:27:40 -04:00
Patrick Hodoul eddc21dd01 Floa conversion implementations
Signed-off-by: Patrick Hodoul <Patrick.Hodoul@autodesk.com>
2021-11-02 12:43:15 -04:00
Patrick Hodoul 61f3bfd3e5 Merge branch 'master' into amyspark/locale 2021-10-27 10:29:10 -04:00
amyspark 0181658a97 Merge branch 'master' into amyspark/locale 2021-10-26 17:33:36 -03:00
Patrick Hodoul 423abed6af Merge branch 'master' into amyspark/locale 2021-10-15 16:36:00 -04:00
L. E. Segovia 970c5958dc Upgrade XMLReaderUtils to use fast_float::from_chars
Signed-off-by: L. E. Segovia <13498015+amyspark@users.noreply.github.com>
2021-10-13 21:08:36 +00:00
L. E. Segovia e4c8161b41 Use fast_float::from_chars in Spi1D LUT parsing
Signed-off-by: L. E. Segovia <13498015+amyspark@users.noreply.github.com>
2021-10-13 13:43:11 +00:00
L. E. Segovia 26cf74fba0 Apply formatting to FileFormatSpi3D
Signed-off-by: L. E. Segovia <13498015+amyspark@users.noreply.github.com>
2021-10-13 11:39:34 +00:00
L. E. Segovia c4dddb7380 Apply the C locale to all parsing streams
Signed-off-by: L. E. Segovia <13498015+amyspark@users.noreply.github.com>
2021-10-09 02:38:40 +00:00
L. E. Segovia 6b6ce605a9 Use the fast_float library to parse floating-point numbers
This commit adds Daniel Lemire's fast_float library as an external dependency,
and applies it to our uses of sscanf with %f as well as istringstream.

This library is locale-free, and does not incur the performance penalty of
using a thread locale RAII class or imbuing a locale into a
stringstream.

See:
    Daniel Lemire, Number Parsing at a Gigabyte per Second, Software:
    Pratice and Experience 51 (8), 2021.
    <https://arxiv.org/abs/2101.11408>

Fixes #297
Fixes #379
Fixes #1322

Signed-off-by: L. E. Segovia <13498015+amyspark@users.noreply.github.com>
2021-10-09 02:38:40 +00:00
11 changed files with 380 additions and 61 deletions
@@ -56,6 +56,10 @@ else()
set(OCIO_USE_IMATH_HALF "0" CACHE STRING "Whether 'half' type will be sourced from the Imath library (>=v3.0)" FORCE)
endif()
# fast_float
# https://github.com/fastfloat/fast_float
find_package(FastFloat 3.2.0 REQUIRED)
if(OCIO_BUILD_APPS)
# NOTE: Depending of the compiler version lcms2 2.2 does not compile with
+128
View File
@@ -0,0 +1,128 @@
# SPDX-License-Identifier: BSD-3-Clause
# Copyright Contributors to the OpenColorIO Project.
#
# Locate or install the fast_float library
#
# Variables defined by this module:
# fast_float_FOUND - If FALSE, do not try to include fast_float.h
# fast_float_INCLUDE_DIR - Where to find fast_float.h
# fast_float_VERSION - The version of the library (if available)
#
# Targets defined by this module:
# fast_float::fast_float - IMPORTED target, if found
#
# The library is include-only, there is no associated binary.
#
# If fast_float is not installed in a standard path, you can use the
# fast_float_ROOT variable to tell CMake where to find it. If it is not found
# and OCIO_INSTALL_EXT_PACKAGES is set to MISSING or ALL, fast_float will be
# downloaded, built, and statically-linked into libOpenColorIO at build time.
#
###############################################################################
### Try to find package ###
if(NOT OCIO_INSTALL_EXT_PACKAGES STREQUAL ALL)
set(_FastFloat_REQUIRED_VARS FastFloat_INCLUDE_DIR)
if(NOT DEFINED FastFloat_ROOT)
# Search for FastFloatConfig.cmake
# Do notice that the CMake Config module is "FastFloat"
# while the library is "fast_float"
# (capital letters replaced by lowercase and underscore)
find_package(FastFloat ${FastFloat_FIND_VERSION} CONFIG QUIET)
endif()
# There is no pkg-config support on fast_float.
# Override REQUIRED if package can be installed
if(OCIO_INSTALL_EXT_PACKAGES STREQUAL MISSING)
set(FastFloat_FIND_REQUIRED FALSE)
endif()
include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(FastFloat
REQUIRED_VARS
${_FastFloat_REQUIRED_VARS}
VERSION_VAR
FastFloat_VERSION
)
endif()
###############################################################################
### Create target
if (NOT TARGET fast_float::fast_float)
add_library(fast_float::fast_float INTERFACE IMPORTED GLOBAL)
set(_FastFloat_TARGET_CREATE TRUE)
endif()
###############################################################################
### Install package from source ###
if(NOT FastFloat_FOUND)
include(ExternalProject)
include(GNUInstallDirs)
set(_EXT_DIST_ROOT "${CMAKE_BINARY_DIR}/ext/dist")
set(_EXT_BUILD_ROOT "${CMAKE_BINARY_DIR}/ext/build")
# Set find_package standard args
set(FastFloat_FOUND TRUE)
set(FastFloat_VERSION ${FastFloat_FIND_VERSION})
set(FastFloat_INCLUDE_DIR "${_EXT_DIST_ROOT}/${CMAKE_INSTALL_INCLUDEDIR}")
# This library is include-only. No need to add further flags.
if(_FastFloat_TARGET_CREATE)
set(FastFloat_CMAKE_ARGS
${FastFloat_CMAKE_ARGS}
-DCMAKE_POSITION_INDEPENDENT_CODE=ON
-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
-DCMAKE_CXX_FLAGS=${FastFloat_CXX_FLAGS}
-DCMAKE_CXX_STANDARD=${CMAKE_CXX_STANDARD}
-DCMAKE_INSTALL_MESSAGE=${CMAKE_INSTALL_MESSAGE}
-DCMAKE_INSTALL_PREFIX=${_EXT_DIST_ROOT}
-DCMAKE_OBJECT_PATH_MAX=${CMAKE_OBJECT_PATH_MAX}
-DFASTFLOAT_TEST=OFF
-DFASTFLOAT_SANITIZE=OFF
)
if(CMAKE_TOOLCHAIN_FILE)
set(FastFloat_CMAKE_ARGS
${FastFloat_CMAKE_ARGS} -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE})
endif()
if(APPLE)
set(FastFloat_CMAKE_ARGS
${FastFloat_CMAKE_ARGS} -DCMAKE_OSX_DEPLOYMENT_TARGET=${CMAKE_OSX_DEPLOYMENT_TARGET})
endif()
# Hack to let imported target be built from ExternalProject_Add
file(MAKE_DIRECTORY ${FastFloat_INCLUDE_DIR})
ExternalProject_Add(FastFloat_install
GIT_REPOSITORY "https://github.com/fastfloat/fast_float.git"
GIT_TAG "v${FastFloat_VERSION}"
GIT_CONFIG advice.detachedHead=false
GIT_SHALLOW TRUE
PREFIX "${_EXT_BUILD_ROOT}/fast_float"
CMAKE_ARGS ${FastFloat_CMAKE_ARGS}
EXCLUDE_FROM_ALL TRUE
)
add_dependencies(fast_float::fast_float FastFloat_install)
message(STATUS "Installing fast_float: ${FastFloat_INCLUDE_DIR} (version \"${FastFloat_VERSION}\")")
endif()
endif()
###############################################################################
### Configure target ###
if(_FastFloat_TARGET_CREATE)
set_target_properties(fast_float::fast_float PROPERTIES
INTERFACE_INCLUDE_DIRECTORIES ${FastFloat_INCLUDE_DIR}
)
mark_as_advanced(FastFloat_INCLUDE_DIR FastFloat_VERSION)
endif()
+2 -1
View File
@@ -185,7 +185,7 @@ add_library(OpenColorIO ${SOURCES})
# Require at least a C++11 compatible compiler for consumer projects.
target_compile_features(OpenColorIO
PUBLIC cxx_std_11
PUBLIC cxx_std_17
)
if(BUILD_SHARED_LIBS AND WIN32)
@@ -226,6 +226,7 @@ target_link_libraries(OpenColorIO
${OCIO_HALF_LIB}
pystring::pystring
sampleicc::sampleicc
fast_float::fast_float
utils::strings
yaml-cpp
)
+12 -6
View File
@@ -524,6 +524,7 @@ static constexpr int DOUBLE_DECIMALS = 16;
std::string FloatToString(float value)
{
std::ostringstream pretty;
pretty.imbue(std::locale::classic());
pretty.precision(FLOAT_DECIMALS);
pretty << value;
return pretty.str();
@@ -534,6 +535,7 @@ std::string FloatVecToString(const float * fval, unsigned int size)
if(size<=0) return "";
std::ostringstream pretty;
pretty.imbue(std::locale::classic());
pretty.precision(FLOAT_DECIMALS);
for(unsigned int i=0; i<size; ++i)
{
@@ -548,9 +550,9 @@ bool StringToFloat(float * fval, const char * str)
{
if(!str) return false;
std::istringstream inputStringstream(str);
float x;
if(!(inputStringstream >> x))
const bool result = ocio_from_chars(str, str + strlen(str), x);
if (!result)
{
return false;
}
@@ -565,6 +567,7 @@ bool StringToInt(int * ival, const char * str, bool failIfLeftoverChars)
if(!ival) return false;
std::istringstream i(str);
i.imbue(std::locale::classic());
char c=0;
if (!(i >> *ival) || (failIfLeftoverChars && i.get(c))) return false;
return true;
@@ -573,6 +576,7 @@ bool StringToInt(int * ival, const char * str, bool failIfLeftoverChars)
std::string DoubleToString(double value)
{
std::ostringstream pretty;
pretty.imbue(std::locale::classic());
pretty.precision(DOUBLE_DECIMALS);
pretty << value;
return pretty.str();
@@ -583,6 +587,7 @@ std::string DoubleVecToString(const double * val, unsigned int size)
if (size <= 0) return "";
std::ostringstream pretty;
pretty.imbue(std::locale::classic());
pretty.precision(DOUBLE_DECIMALS);
for (unsigned int i = 0; i<size; ++i)
{
@@ -593,15 +598,16 @@ std::string DoubleVecToString(const double * val, unsigned int size)
return pretty.str();
}
bool StringVecToFloatVec(std::vector<float> &floatArray, const StringUtils::StringVec &lineParts)
bool StringVecToFloatVec(std::vector<float> & floatArray, const StringUtils::StringVec & lineParts)
{
floatArray.resize(lineParts.size());
for(unsigned int i=0; i<lineParts.size(); i++)
{
std::istringstream inputStringstream(lineParts[i]);
float x;
if(!(inputStringstream >> x))
const char *str = lineParts[i].c_str();
const bool result = ocio_from_chars(str, str + strlen(str), x);
if (!result)
{
return false;
}
@@ -614,7 +620,7 @@ bool StringVecToFloatVec(std::vector<float> &floatArray, const StringUtils::Stri
// This will resize intArray to the size of lineParts.
// Returns true if all lineParts have been recognized as int.
// Content of intArray will be unknown if function returns false.
bool StringVecToIntVec(std::vector<int> &intArray, const StringUtils::StringVec &lineParts)
bool StringVecToIntVec(std::vector<int> & intArray, const StringUtils::StringVec & lineParts)
{
intArray.resize(lineParts.size());
+116
View File
@@ -7,6 +7,122 @@
// Platform-specific includes.
#if defined(_MSC_VER)
#define really_inline __forceinline
#else
#define really_inline inline __attribute__((always_inline))
#endif
#if defined(__has_include) && __has_include(<version>)
#include <version> // from C++20 all feature macros should be defined here
#else
#include <utility>
#endif
//#define USE_FAST_FLOAT
#ifdef USE_FAST_FLOAT
#include <fast_float/fast_float.h>
template <typename T>
really_inline bool ocio_from_chars(const char * first, const char * last, T & value) noexcept
{
// By design that's not supported by from_chars() so handle it here.
if (first && *first == '+') ++first;
const auto ret = fast_float::from_chars(first, last, value);
return ret.ec == std::errc();
}
#elif __cpp_lib_to_chars >= 201611
#include <charconv>
template <typename T>
really_inline bool ocio_from_chars(const char * first, const char * last, T & value) noexcept
{
// By design that's not supported by from_chars() so handle it here.
if (first && *first == '+') ++first;
const auto ret = std::from_chars(first, last, value);
return ret.ec == std::errc();
}
#else
#include <locale>
#include <stdlib.h>
#include <type_traits>
struct Locale
{
#ifdef _WIN32
Locale() : local(_create_locale(LC_ALL, "C")) {}
~Locale() { _free_locale(local); }
_locale_t local;
#else
Locale() : local(newlocale(LC_ALL_MASK, "C", NULL)) {}
~Locale() { freelocale(local); }
locale_t local;
#endif
};
static const Locale loc;
really_inline bool ocio_from_chars(const char * first, const char * last, double & value) noexcept
{
errno = 0;
if (!first || !last || first == last)
{
return false;
}
#ifdef _WIN32
value = _strtod_l(first, nullptr, loc.local);
#else
value = ::strtod_l(first, nullptr, loc.local);
#endif
return errno == 0;
}
really_inline bool ocio_from_chars(const char * first, const char * last, float & value) noexcept
{
errno = 0;
if (!first || !last || first == last)
{
return false;
}
#ifdef _WIN32
value = _strtof_l(first, nullptr, loc.local);
#elif __APPLE__
// On OSX, strtod_l is for some reason drastically faster than strtof_l.
value = static_cast<float>(::strtod_l(first, nullptr, loc.local));
#else
value = ::strtof_l(first, nullptr, loc.local);
#endif
return errno == 0;
}
really_inline bool ocio_from_chars(const char * first, const char * last, long int & value) noexcept
{
errno = 0;
if (!first || !last || first == last)
{
return false;
}
#ifdef _WIN32
value = _strtol_l(first, nullptr, 0, loc.local);
#else
value = ::strtol_l(first, nullptr, 0, loc.local);
#endif
return errno == 0;
}
#endif
#if defined(_WIN32)
#define NOMINMAX 1
@@ -1,6 +1,7 @@
// SPDX-License-Identifier: BSD-3-Clause
// Copyright Contributors to the OpenColorIO Project.
#include <cmath>
#include <cstdio>
#include <sstream>
@@ -124,10 +125,26 @@ CachedFileRcPtr LocalFileFormat::read(std::istream & istream,
}
else if(StringUtils::StartsWith(headerLine, "From"))
{
if (sscanf(lineBuffer, "From %f %f", &from_min, &from_max) != 2)
char fromMinS[64] = "";
char fromMaxS[64] = "";
#ifdef _WIN32
if (sscanf(lineBuffer, "From %s %s", fromMinS, 64, fromMaxS, 64) != 2)
#else
if (sscanf(lineBuffer, "From %s %s", fromMinS, fromMaxS) != 2)
#endif
{
ThrowErrorMessage("Invalid 'From' Tag", currentLine, headerLine);
}
else
{
const auto fromMinAnswer = ocio_from_chars(fromMinS, fromMinS +64, from_min);
const auto fromMaxAnswer = ocio_from_chars(fromMaxS, fromMaxS + 64, from_max);
if (!fromMinAnswer || !fromMaxAnswer)
{
ThrowErrorMessage("Invalid 'From' Tag", currentLine, headerLine);
}
}
}
else if(StringUtils::StartsWith(headerLine, "Components"))
{
@@ -179,7 +196,6 @@ CachedFileRcPtr LocalFileFormat::read(std::istream & istream,
int lineCount=0;
StringUtils::StringVec inputLUT;
std::vector<float> values;
while (istream.good())
@@ -192,10 +208,17 @@ CachedFileRcPtr LocalFileFormat::read(std::istream & istream,
if (line.length() != 0)
{
inputLUT = StringUtils::SplitByWhiteSpaces(StringUtils::Trim(lineBuffer));
values.clear();
if (!StringVecToFloatVec(values, inputLUT)
|| components != (int)values.size())
char inputLUT[4][64] = {"", "", "", ""};
#ifdef _WIN32
if (sscanf(lineBuffer, "%s %s %s %63s", inputLUT[0], 64,
inputLUT[1], 64, inputLUT[2], 64, inputLUT[3],
64) != components)
#else
if (sscanf(lineBuffer, "%s %s %s %63s", inputLUT[0],
inputLUT[1], inputLUT[2], inputLUT[3]) != components)
#endif
{
std::ostringstream os;
os << "Malformed LUT line. Expecting a ";
@@ -209,6 +232,27 @@ CachedFileRcPtr LocalFileFormat::read(std::istream & istream,
ThrowErrorMessage("Too many entries found", currentLine, "");
}
values.resize(components);
for (int i = 0; i < components; i++)
{
float v = NAN;
const auto result
= ocio_from_chars(inputLUT[i], inputLUT[i] + strlen(inputLUT[i]), v);
if (!result)
{
std::ostringstream os;
os << "Malformed LUT line. Could not convert component";
os << i << " to a floating point number.";
ThrowErrorMessage("Malformed LUT line", currentLine,
line);
}
values[i] = v;
}
// If 1 component is specified, use x1 x1 x1.
if (components == 1)
{
@@ -141,10 +141,40 @@ CachedFileRcPtr LocalFileFormat::read(std::istream & istream,
{
istream.getline(lineBuffer, MAX_LINE_SIZE);
if (sscanf(lineBuffer, "%d %d %d %f %f %f",
char redValueS[64] = "";
char greenValueS[64] = "";
char blueValueS[64] = "";
#ifdef _WIN32
if (sscanf(lineBuffer,
"%d %d %d %s %s %s",
&rIndex, &gIndex, &bIndex,
&redValue, &greenValue, &blueValue) == 6)
redValueS, 64,
greenValueS, 64,
blueValueS, 64) == 6)
#else
if (sscanf(lineBuffer, "%d %d %d %s %s %s",
&rIndex, &gIndex, &bIndex,
redValueS, greenValueS, blueValueS) == 6)
#endif
{
const auto redValueAnswer = ocio_from_chars(redValueS, redValueS + 64, redValue);
const auto greenValueAnswer = ocio_from_chars(greenValueS, greenValueS + 64, greenValue);
const auto blueValueAnswer = ocio_from_chars(blueValueS, blueValueS + 64, blueValue);
if (!redValueAnswer || !greenValueAnswer || !blueValueAnswer)
{
std::ostringstream os;
os << "Error parsing .spi3d file (";
os << fileName;
os << "). ";
os << "Data is invalid. ";
os << "A color value is specified (";
os << redValueS << " " << greenValueS << " " << blueValueS;
os << ") that cannot be parsed as a floating-point triplet.";
throw Exception(os.str().c_str());
}
bool invalidIndex = false;
if (rIndex < 0 || rIndex >= rSize
|| gIndex < 0 || gIndex >= gSize
@@ -13,6 +13,7 @@
#include <OpenColorIO/OpenColorIO.h>
#include "MathUtils.h"
#include "utils/StringUtils.h"
#include "Platform.h"
@@ -152,23 +153,36 @@ void ParseNumber(const char * str, size_t startPos, size_t endPos, T & value)
{
throw Exception("ParseNumber: nothing to parse.");
}
/*
const char * startParse = str + startPos;
const char * adjustedParse = startParse;
*/
double val = 0.0f;
char * endParse = nullptr;
// The strtod expects a C string and str might not be null terminated.
// However since strtod will stop parsing when it encounters characters
// that it cannot convert to a number, in practice it does not need to
// be null terminated.
// C++11 version of strtod processes NAN & INF ASCII values.
val = strtod(startParse, &endParse);
value = (T)val;
if (endParse == startParse)
/*
// charconv's from_chars expects a C string and str might not be null terminated.
// However, since from_chars also takes a pointer to the last character,
// the string does not need to be null terminated explicitly.
// Do note, however, that plus signs and leading whitespace
// need to be skipped manually -- this
// was left implicit in the usage of strtod.
// TODO: allow hex numbers -- this will probably mean we need another from_chars library
const std::string parsedStr = StringUtils::LeftTrim({startParse, endPos - startPos});
if (startPos + parsedStr.size() != endPos) {
adjustedParse = str + endPos - parsedStr.size();
}
if (StringUtils::StartsWith(parsedStr, "+")) {
adjustedParse += 1;
}
*/
const bool result = ocio_from_chars(str + startPos, str + endPos, val);
if (!result)
{
std::string fullStr(str, endPos);
std::string parsedStr(startParse, endPos - startPos);
std::string parsedStr(str + startPos, endPos - startPos);
std::ostringstream oss;
oss << "ParserNumber: Characters '"
<< parsedStr
@@ -176,29 +190,8 @@ void ParseNumber(const char * str, size_t startPos, size_t endPos, T & value)
<< TruncateString(fullStr.c_str(), endPos, 100) << "'.";
throw Exception(oss.str().c_str());
}
else if (!IsValid(value, val))
{
std::string fullStr(str, endPos);
std::string parsedStr(startParse, endPos - startPos);
std::ostringstream oss;
oss << "ParserNumber: Characters '"
<< parsedStr
<< "' are illegal in '"
<< TruncateString(fullStr.c_str(), endPos, 100) << "'.";
throw Exception(oss.str().c_str());
}
else if (endParse != str + endPos)
{
// Number is followed by something.
std::string fullStr(str, startPos + (endParse - startParse));
std::string parsedStr(startParse, endPos - startPos);
std::ostringstream oss;
oss << "ParserNumber: '"
<< parsedStr
<< "' number is followed by unexpected characters in '"
<< TruncateString(fullStr.c_str(), endPos, 100) << "'.";
throw Exception(oss.str().c_str());
}
value = (T) val;
}
// Extract the next number contained in the string.
+7 -1
View File
@@ -367,7 +367,13 @@ int main(int argc, const char **argv)
OCIO::ConstProcessorRcPtr processor;
if (!transformFile.empty())
{
OCIO::ConstConfigRcPtr config = OCIO::Config::Create();
OCIO::ConfigRcPtr config = OCIO::Config::Create();
if (nocache)
{
config = config->createEditableCopy();
config->setProcessorCacheFlags(nocache ? OCIO::PROCESSOR_CACHE_OFF
: OCIO::PROCESSOR_CACHE_DEFAULT);
}
// Get the transform.
OCIO::FileTransformRcPtr transform = OCIO::FileTransform::Create();
+1
View File
@@ -19,6 +19,7 @@ function(add_ocio_test NAME SOURCES PRIVATE_INCLUDES)
${OCIO_HALF_LIB}
pystring::pystring
sampleicc::sampleicc
fast_float::fast_float
unittest_data
utils::strings
yaml-cpp
@@ -43,16 +43,6 @@ OCIO_ADD_TEST(XMLReaderHelper, string_to_float_failure)
// 2 characters are parsed and this is the length required.
OCIO_CHECK_NO_THROW(OCIO::ParseNumber(str1, 0, 2, value));
const char str2[] = "12345";
const size_t len2 = std::strlen(str2);
// All characters are parsed and this is more than the required length.
// The string to double function strtod does not stop at a given length,
// but we detect that strtod did read too many characters.
OCIO_CHECK_THROW_WHAT(OCIO::ParseNumber(str2, 0, len2 - 2, value),
OCIO::Exception,
"followed by unexpected characters");
const char str3[] = "123XX";
const size_t len3 = std::strlen(str3);
// Strtod will stop after parsing 123 and this happens to be the
@@ -388,7 +378,7 @@ OCIO_ADD_TEST(XMLReaderHelper, parse_number)
{
std::string buffer(" 123 ");
OCIO_CHECK_THROW_WHAT(OCIO::ParseNumber(buffer.c_str(),
0, 3, data),
0, buffer.size(), data),
OCIO::Exception,
"followed by unexpected characters");
}