Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 (no author)
|
dd2b2c0fc3 |
@@ -1,9 +0,0 @@
|
||||
# © 2021 and later: Unicode, Inc. and others.
|
||||
# License & terms of use: http://www.unicode.org/copyright.html
|
||||
|
||||
# .bazeliskrc is the configuration file for the wrapper program for the Bazel
|
||||
# build system, called Bazelisk. Bazelisk can be used as a drop-in replacement
|
||||
# for running Bazel commands while ensuring, through configuration, that only a
|
||||
# specific version of Bazel is executed.
|
||||
|
||||
USE_BAZEL_VERSION=4.1.0
|
||||
@@ -1,74 +0,0 @@
|
||||
# Azure Pipelines configuration for Exhaustive Tests for ICU.
|
||||
#
|
||||
# Note: The exhaustive test configuration is in a separate file
|
||||
# so that it can be run independently from the regular builds.
|
||||
#
|
||||
# Only run the exhaustive tests on the main and maint branches, and
|
||||
# also batch up any pending changes so that we will only have at most
|
||||
# one build running at a given time (since they take a long time).
|
||||
trigger:
|
||||
batch: true
|
||||
branches:
|
||||
include:
|
||||
- main
|
||||
- maint/maint-*
|
||||
paths:
|
||||
include:
|
||||
- '*'
|
||||
exclude:
|
||||
- .github/*
|
||||
- .ci-builds/.azure-pipelines.yml
|
||||
- .ci-builds/.azure-valgrind.yml
|
||||
- docs/*
|
||||
- tools/*
|
||||
- vendor/*
|
||||
- .cpyskip.txt
|
||||
- .travis.yml
|
||||
- KEYS
|
||||
- README.md
|
||||
|
||||
jobs:
|
||||
#-------------------------------------------------------------------------
|
||||
# Note: The exhaustive tests can take more than 1 hour to complete.
|
||||
- job: ICU4J_OpenJDK_Ubuntu_1804
|
||||
displayName: 'J: Linux OpenJDK (Ubuntu 18.04)'
|
||||
timeoutInMinutes: 120
|
||||
pool:
|
||||
vmImage: 'ubuntu-18.04'
|
||||
demands: ant
|
||||
steps:
|
||||
- checkout: self
|
||||
lfs: true
|
||||
fetchDepth: 10
|
||||
- script: |
|
||||
echo "Building ICU4J" && cd icu4j && ant init && ant exhaustiveCheck
|
||||
displayName: 'Build and Exhaustive Tests'
|
||||
env:
|
||||
BUILD: ICU4J
|
||||
- script: |
|
||||
[ -d icu4j/out/junit-results ] && cd icu4j && cat `find out/junit-results -name "*.txt" -exec grep -l FAILED {} \;`
|
||||
condition: failed() # only run if the build fails.
|
||||
displayName: 'List failures (if any)'
|
||||
timeoutInMinutes: 2
|
||||
#-------------------------------------------------------------------------
|
||||
# Note: The exhaustive tests can take more than 1 hour to complete.
|
||||
- job: ICU4C_Clang_Exhaustive_Ubuntu_1804
|
||||
displayName: 'C: Linux Clang Exhaustive Tests (Ubuntu 18.04)'
|
||||
timeoutInMinutes: 120
|
||||
pool:
|
||||
vmImage: 'ubuntu-18.04'
|
||||
steps:
|
||||
- checkout: self
|
||||
lfs: true
|
||||
fetchDepth: 10
|
||||
- script: |
|
||||
cd icu4c/source && ./runConfigureICU Linux && make -j2
|
||||
displayName: 'Build'
|
||||
- script: |
|
||||
cd icu4c/source && make check-exhaustive
|
||||
displayName: 'Exhaustive Tests'
|
||||
env:
|
||||
CC: clang
|
||||
CXX: clang++
|
||||
#-------------------------------------------------------------------------
|
||||
|
||||
@@ -1,631 +0,0 @@
|
||||
# Azure Pipelines (VSTS) configuration for CI builds for ICU.
|
||||
|
||||
jobs:
|
||||
#-------------------------------------------------------------------------
|
||||
- job: ICU4J_OpenJDK_Ubuntu_1804
|
||||
displayName: 'J: Linux OpenJDK (Ubuntu 18.04)'
|
||||
timeoutInMinutes: 20
|
||||
pool:
|
||||
vmImage: 'ubuntu-18.04'
|
||||
demands: ant
|
||||
steps:
|
||||
- checkout: self
|
||||
lfs: true
|
||||
fetchDepth: 10
|
||||
- script: |
|
||||
echo "Building ICU4J" && cd icu4j && ant init && ant check && ant localespiCheck
|
||||
displayName: 'Build and Test'
|
||||
env:
|
||||
BUILD: ICU4J
|
||||
# exit with a non-zero status in order to make this step show as a red X in the UI.
|
||||
- script: |
|
||||
[ -d icu4j/out/junit-results ] && cd icu4j && cat `find out/junit-results -name "*.txt" -exec grep -l FAILED {} \;` && exit 1
|
||||
condition: failed() # only run if the build fails.
|
||||
displayName: 'List failures (if any)'
|
||||
#-------------------------------------------------------------------------
|
||||
- job: ICU4C_Clang_Ubuntu_1804
|
||||
displayName: 'C: Linux Clang (Ubuntu 18.04)'
|
||||
timeoutInMinutes: 30
|
||||
pool:
|
||||
vmImage: 'ubuntu-18.04'
|
||||
steps:
|
||||
- checkout: self
|
||||
lfs: true
|
||||
fetchDepth: 10
|
||||
- script: |
|
||||
cd icu4c/source && ./runConfigureICU Linux && make -j2 check
|
||||
displayName: 'Build and Test'
|
||||
env:
|
||||
CC: clang
|
||||
CXX: clang++
|
||||
- script: |
|
||||
cd icu4c/source/test/cintltst && LANG=C LD_LIBRARY_PATH=../../lib:../../tools/ctestfw ./cintltst /tsutil/cloctst/TestCDefaultLocale
|
||||
displayName: 'Test C Default locale'
|
||||
- script: |
|
||||
cd icu4c/source/test/cintltst && LANG=C.UTF-8 LD_LIBRARY_PATH=../../lib:../../tools/ctestfw ./cintltst /tsutil/cloctst/TestCDefaultLocale
|
||||
displayName: 'Test C.UTF-8 Default locale'
|
||||
#-------------------------------------------------------------------------
|
||||
- job: ICU4C_Clang13_Ubuntu_2004_WarningsAsErrors
|
||||
displayName: 'C: Linux Clang-13 WarningsAsErrors (Ubuntu 20.04)'
|
||||
timeoutInMinutes: 30
|
||||
pool:
|
||||
vmImage: 'ubuntu-20.04'
|
||||
steps:
|
||||
- checkout: self
|
||||
lfs: true
|
||||
fetchDepth: 10
|
||||
# Install Clang-13 from https://apt.llvm.org/
|
||||
- script: |
|
||||
curl -Ls https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
|
||||
sudo apt-add-repository 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-13 main'
|
||||
sudo apt update
|
||||
sudo apt install -qy clang-13 lld-13 libc++-13-dev libc++abi-13-dev
|
||||
displayName: 'Install Clang-13'
|
||||
- script: |
|
||||
export CPPFLAGS="-Werror -Wall -Wextra -Wextra-semi -Wundef -Wnon-virtual-dtor -Wsuggest-override" && cd icu4c/source && ./runConfigureICU Linux && make -j2 tests
|
||||
displayName: 'Build only (WarningsAsErrors)'
|
||||
env:
|
||||
CC: clang-13
|
||||
CXX: clang++-13
|
||||
#-------------------------------------------------------------------------
|
||||
- job: ICU4C_Clang_Ubuntu_DataFilter_1804
|
||||
displayName: 'C: Linux Clang DataFilter (Ubuntu 18.04)'
|
||||
timeoutInMinutes: 30
|
||||
pool:
|
||||
vmImage: 'ubuntu-18.04'
|
||||
steps:
|
||||
- checkout: self
|
||||
lfs: true
|
||||
fetchDepth: 10
|
||||
- script: |
|
||||
cd icu4c/source && \
|
||||
ICU_DATA_FILTER_FILE=../../.ci-builds/data-filter.json ./runConfigureICU Linux && \
|
||||
make -j2 tests && \
|
||||
\[ ! -d data/out/build/icudt66l/translit \] && \
|
||||
(cd test/intltest && LD_LIBRARY_PATH=../../lib:../../tools/ctestfw ./intltest translit/TransliteratorTest/TestBasicTransliteratorEvenWithoutData)
|
||||
displayName: 'Build with Data Filter'
|
||||
env:
|
||||
CC: clang
|
||||
CXX: clang++
|
||||
#-------------------------------------------------------------------------
|
||||
- job: ICU4C_Clang_Cpp14_Debug_Ubuntu_1804
|
||||
displayName: 'C: Linux Clang C++17 Debug (Ubuntu 18.04)'
|
||||
timeoutInMinutes: 30
|
||||
pool:
|
||||
vmImage: 'ubuntu-18.04'
|
||||
steps:
|
||||
- checkout: self
|
||||
lfs: true
|
||||
fetchDepth: 10
|
||||
- script: |
|
||||
export CXXFLAGS="-std=c++17 -Winvalid-constexpr" && cd icu4c/source && ./runConfigureICU --enable-debug --disable-release Linux && make -j2 check
|
||||
displayName: 'Build and Test C++17'
|
||||
env:
|
||||
CC: clang
|
||||
CXX: clang++
|
||||
#-------------------------------------------------------------------------
|
||||
- job: ICU4C_GCC_Ubuntu_2004
|
||||
displayName: 'C: Linux GCC (Ubuntu 20.04)'
|
||||
timeoutInMinutes: 30
|
||||
pool:
|
||||
vmImage: 'ubuntu-20.04'
|
||||
steps:
|
||||
- checkout: self
|
||||
lfs: true
|
||||
fetchDepth: 10
|
||||
- script: |
|
||||
cd icu4c/source && ./runConfigureICU Linux && make -j2 check
|
||||
displayName: 'Build and Test'
|
||||
env:
|
||||
CC: gcc
|
||||
CXX: g++
|
||||
#-------------------------------------------------------------------------
|
||||
- job: ICU4C_GCC11_Cpp20_Ubuntu_2004
|
||||
displayName: 'C: Linux GCC-11 C++20 (Ubuntu 20.04)'
|
||||
timeoutInMinutes: 30
|
||||
pool:
|
||||
vmImage: 'ubuntu-20.04'
|
||||
steps:
|
||||
- checkout: self
|
||||
lfs: true
|
||||
fetchDepth: 10
|
||||
- script: |
|
||||
sudo add-apt-repository ppa:ubuntu-toolchain-r/test
|
||||
sudo apt update
|
||||
sudo apt install gcc-11 g++-11
|
||||
displayName: 'Install GCC-11'
|
||||
- script: |
|
||||
cd icu4c/source && ./runConfigureICU Linux && make -j2 check
|
||||
displayName: 'Build and Test'
|
||||
env:
|
||||
CC: gcc-11
|
||||
CXX: g++-11
|
||||
CXXFLAGS: '-std=c++20 -fext-numeric-literals'
|
||||
#-------------------------------------------------------------------------
|
||||
- job: ICU4C_Clang_Ubuntu_2004_LANG
|
||||
displayName: 'C: Linux Clang (Ubuntu 20.04) - LANG has extension tags'
|
||||
timeoutInMinutes: 30
|
||||
pool:
|
||||
vmImage: 'ubuntu-20.04'
|
||||
steps:
|
||||
- checkout: self
|
||||
lfs: true
|
||||
fetchDepth: 10
|
||||
- script: |
|
||||
cd icu4c/source && ./runConfigureICU Linux && make -j2 check
|
||||
displayName: 'Build and Test'
|
||||
env:
|
||||
CC: clang
|
||||
CXX: clang++
|
||||
LANG: "en_US@calendar=gregorian;hours=h12"
|
||||
#-------------------------------------------------------------------------
|
||||
- job: ICU4C_Clang13_Cpp20_Ubuntu_2004
|
||||
displayName: 'C: Linux Clang-13 C++20 (Ubuntu 20.04)'
|
||||
timeoutInMinutes: 30
|
||||
pool:
|
||||
vmImage: 'ubuntu-20.04'
|
||||
steps:
|
||||
- checkout: self
|
||||
lfs: true
|
||||
fetchDepth: 10
|
||||
# Install Clang-13 from https://apt.llvm.org/
|
||||
- script: |
|
||||
curl -Ls https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
|
||||
sudo apt-add-repository 'deb http://apt.llvm.org/focal/ llvm-toolchain-focal-13 main'
|
||||
sudo apt update
|
||||
sudo apt install -qy clang-13 lld-13 libc++-13-dev libc++abi-13-dev
|
||||
displayName: 'Install Clang-13'
|
||||
- script: |
|
||||
cd icu4c/source && ./runConfigureICU Linux && make -j2 check
|
||||
displayName: 'Build and Test'
|
||||
env:
|
||||
CC: clang-13
|
||||
CXX: clang++-13
|
||||
CXXFLAGS: '-std=c++20'
|
||||
#-------------------------------------------------------------------------
|
||||
# VS 2019 Builds
|
||||
#-------------------------------------------------------------------------
|
||||
- job: ICU4C_MSVC_x64_Release_Distrelease
|
||||
displayName: 'C: MSVC 64-bit Release (VS 2019) + Distrelease'
|
||||
timeoutInMinutes: 30
|
||||
pool:
|
||||
vmImage: 'windows-2019'
|
||||
demands:
|
||||
- msbuild
|
||||
- visualstudio
|
||||
- Cmd
|
||||
steps:
|
||||
- checkout: self
|
||||
lfs: true
|
||||
fetchDepth: 10
|
||||
- task: VSBuild@1
|
||||
displayName: 'Build Solution'
|
||||
inputs:
|
||||
solution: icu4c/source/allinone/allinone.sln
|
||||
platform: x64
|
||||
configuration: Release
|
||||
- task: BatchScript@1
|
||||
displayName: 'Run Tests (icucheck.bat)'
|
||||
inputs:
|
||||
filename: icu4c/source/allinone/icucheck.bat
|
||||
arguments: 'x64 Release'
|
||||
- task: PowerShell@2
|
||||
displayName: 'PowerShell: Distrelease script (x64)'
|
||||
inputs:
|
||||
targetType: filePath
|
||||
filePath: 'icu4c/packaging/distrelease.ps1'
|
||||
arguments: '-arch x64'
|
||||
workingDirectory: icu4c
|
||||
- task: PublishBuildArtifacts@1
|
||||
displayName: 'Publish x64 Artifacts: icu-windows.zip'
|
||||
inputs:
|
||||
PathtoPublish: 'icu4c/source/dist/icu-windows.zip'
|
||||
ArtifactName: '$(Build.BuildNumber)_ICU4C_MSVC_x64_Release'
|
||||
#-------------------------------------------------------------------------
|
||||
- job: ICU4C_MSVC_x64_Release_DataFilter
|
||||
displayName: 'C: MSVC 64-bit Release DataFilter (VS 2019)'
|
||||
timeoutInMinutes: 30
|
||||
pool:
|
||||
vmImage: 'windows-2019'
|
||||
demands:
|
||||
- msbuild
|
||||
- visualstudio
|
||||
- Cmd
|
||||
steps:
|
||||
- powershell: |
|
||||
$filterPath = $Env:BUILD_SOURCESDIRECTORY + "\.ci-builds\data-filter.json"
|
||||
$vstsCommandString = "vso[task.setvariable variable=ICU_DATA_FILTER_FILE]" + $filterPath
|
||||
Write-Host "##$vstsCommandString"
|
||||
- task: VSBuild@1
|
||||
displayName: 'Build Solution with Data Filter'
|
||||
inputs:
|
||||
solution: icu4c/source/allinone/allinone.sln
|
||||
platform: x64
|
||||
configuration: Release
|
||||
msbuildArgs: '/p:SkipUWP=true'
|
||||
#-------------------------------------------------------------------------
|
||||
- job: ICU4C_MSVC_x86_Release_Distrelease
|
||||
displayName: 'C: MSVC 32-bit Release (VS 2019) + Distrelease'
|
||||
timeoutInMinutes: 30
|
||||
pool:
|
||||
vmImage: 'windows-2019'
|
||||
demands:
|
||||
- msbuild
|
||||
- visualstudio
|
||||
- Cmd
|
||||
steps:
|
||||
- checkout: self
|
||||
lfs: true
|
||||
fetchDepth: 10
|
||||
- task: VSBuild@1
|
||||
displayName: 'Build Solution'
|
||||
inputs:
|
||||
solution: icu4c/source/allinone/allinone.sln
|
||||
platform: Win32
|
||||
configuration: Release
|
||||
- task: BatchScript@1
|
||||
displayName: 'Run Tests (icucheck.bat)'
|
||||
inputs:
|
||||
filename: icu4c/source/allinone/icucheck.bat
|
||||
arguments: 'x86 Release'
|
||||
- task: PowerShell@2
|
||||
displayName: 'PowerShell: Distrelease script (x86)'
|
||||
inputs:
|
||||
targetType: filePath
|
||||
filePath: 'icu4c/packaging/distrelease.ps1'
|
||||
arguments: '-arch x86'
|
||||
workingDirectory: icu4c
|
||||
- task: PublishBuildArtifacts@1
|
||||
displayName: 'Publish x86 Artifacts: icu-windows.zip'
|
||||
inputs:
|
||||
PathtoPublish: 'icu4c/source/dist/icu-windows.zip'
|
||||
ArtifactName: '$(Build.BuildNumber)_ICU4C_MSVC_x86_Release'
|
||||
#-------------------------------------------------------------------------
|
||||
- job: ICU4C_MSVC_x64_ARM32_ARM64_Release_Distrelease
|
||||
displayName: 'C: MSVC x64 ARM32 ARM64 Release (VS 2019) + Distrelease ARM64'
|
||||
timeoutInMinutes: 60
|
||||
pool:
|
||||
vmImage: 'windows-2019'
|
||||
demands:
|
||||
- msbuild
|
||||
- visualstudio
|
||||
- Cmd
|
||||
steps:
|
||||
- checkout: self
|
||||
lfs: true
|
||||
fetchDepth: 10
|
||||
- task: VSBuild@1
|
||||
displayName: 'Build Solution'
|
||||
inputs:
|
||||
solution: icu4c/source/allinone/allinone.sln
|
||||
platform: x64
|
||||
configuration: Release
|
||||
- task: VSBuild@1
|
||||
displayName: 'Build ARM32'
|
||||
inputs:
|
||||
solution: icu4c/source/allinone/allinone.sln
|
||||
platform: ARM
|
||||
configuration: Release
|
||||
- task: VSBuild@1
|
||||
displayName: 'Build ARM64'
|
||||
inputs:
|
||||
solution: icu4c/source/allinone/allinone.sln
|
||||
platform: ARM64
|
||||
configuration: Release
|
||||
- task: PowerShell@2
|
||||
displayName: 'PowerShell: Distrelease script (ARM64)'
|
||||
inputs:
|
||||
targetType: filePath
|
||||
filePath: 'icu4c/packaging/distrelease.ps1'
|
||||
arguments: '-arch ARM64'
|
||||
workingDirectory: icu4c
|
||||
- task: PublishBuildArtifacts@1
|
||||
displayName: 'Publish ARM64 Artifacts: icu-windows.zip'
|
||||
inputs:
|
||||
PathtoPublish: 'icu4c/source/dist/icu-windows.zip'
|
||||
ArtifactName: '$(Build.BuildNumber)_ICU4C_MSVC_ARM64_Release'
|
||||
#-------------------------------------------------------------------------
|
||||
- job: ICU4C_MSVC_x64_ARM64_Debug
|
||||
displayName: 'C: MSVC x64 ARM64 Debug (VS 2019)'
|
||||
timeoutInMinutes: 45
|
||||
pool:
|
||||
vmImage: 'windows-2019'
|
||||
demands:
|
||||
- msbuild
|
||||
- visualstudio
|
||||
- Cmd
|
||||
steps:
|
||||
- checkout: self
|
||||
lfs: true
|
||||
fetchDepth: 10
|
||||
- task: VSBuild@1
|
||||
displayName: 'Build Solution'
|
||||
inputs:
|
||||
solution: icu4c/source/allinone/allinone.sln
|
||||
platform: x64
|
||||
configuration: Debug
|
||||
- task: BatchScript@1
|
||||
displayName: 'Run x64 Debug Tests (icucheck.bat)'
|
||||
inputs:
|
||||
filename: icu4c/source/allinone/icucheck.bat
|
||||
arguments: 'x64 Debug'
|
||||
- task: VSBuild@1
|
||||
displayName: 'Build ARM64 Debug'
|
||||
inputs:
|
||||
solution: icu4c/source/allinone/allinone.sln
|
||||
platform: ARM64
|
||||
configuration: Debug
|
||||
#-------------------------------------------------------------------------
|
||||
# There is no /std:c++20 yet, so use c++latest as that includes many of
|
||||
# the C++20 compiler and library changes.
|
||||
- job: ICU4C_MSVC_x64_Release_CppLatest
|
||||
displayName: 'C: MSVC 64-bit Release (VS 2019) /std:c++latest'
|
||||
timeoutInMinutes: 30
|
||||
pool:
|
||||
vmImage: 'windows-2019'
|
||||
demands:
|
||||
- msbuild
|
||||
- visualstudio
|
||||
- Cmd
|
||||
steps:
|
||||
- checkout: self
|
||||
lfs: true
|
||||
fetchDepth: 10
|
||||
- task: VSBuild@1
|
||||
displayName: 'Build Solution'
|
||||
inputs:
|
||||
solution: icu4c/source/allinone/allinone.sln
|
||||
platform: x64
|
||||
configuration: Release
|
||||
msbuildArgs: '/p:LanguageStandard="stdcpplatest"'
|
||||
- task: BatchScript@1
|
||||
displayName: 'Run Tests (icucheck.bat)'
|
||||
inputs:
|
||||
filename: icu4c/source/allinone/icucheck.bat
|
||||
arguments: 'x64 Release'
|
||||
#-------------------------------------------------------------------------
|
||||
# VS2019 using the VS2017 toolset builds
|
||||
#-------------------------------------------------------------------------
|
||||
- job: ICU4C_MSVC_x64_Release_VS2017
|
||||
displayName: 'C: MSVC 64-bit Release (VS 2017)'
|
||||
timeoutInMinutes: 30
|
||||
pool:
|
||||
vmImage: 'windows-2019'
|
||||
demands:
|
||||
- msbuild
|
||||
- visualstudio
|
||||
- Cmd
|
||||
steps:
|
||||
- checkout: self
|
||||
lfs: true
|
||||
fetchDepth: 10
|
||||
- task: VSBuild@1
|
||||
displayName: 'Build Solution'
|
||||
inputs:
|
||||
solution: icu4c/source/allinone/allinone.sln
|
||||
platform: x64
|
||||
configuration: Release
|
||||
msbuildArgs: /p:DefaultPlatformToolset=v141
|
||||
- task: BatchScript@1
|
||||
displayName: 'Run Tests (icucheck.bat)'
|
||||
inputs:
|
||||
filename: icu4c/source/allinone/icucheck.bat
|
||||
arguments: 'x64 Release'
|
||||
#-------------------------------------------------------------------------
|
||||
- job: ICU4C_MSVC_x86_Debug_VS2017
|
||||
displayName: 'C: MSVC 32-bit Debug (VS 2017)'
|
||||
timeoutInMinutes: 60
|
||||
pool:
|
||||
vmImage: 'windows-2019'
|
||||
demands:
|
||||
- msbuild
|
||||
- visualstudio
|
||||
- Cmd
|
||||
steps:
|
||||
- checkout: self
|
||||
lfs: true
|
||||
fetchDepth: 10
|
||||
- task: VSBuild@1
|
||||
displayName: 'Build Solution'
|
||||
inputs:
|
||||
solution: icu4c/source/allinone/allinone.sln
|
||||
platform: Win32
|
||||
configuration: Debug
|
||||
msbuildArgs: /p:DefaultPlatformToolset=v141
|
||||
- task: BatchScript@1
|
||||
displayName: 'Run Tests (icucheck.bat)'
|
||||
inputs:
|
||||
filename: icu4c/source/allinone/icucheck.bat
|
||||
arguments: 'x86 Debug'
|
||||
#-------------------------------------------------------------------------
|
||||
# VS2019 using the VS2015 toolset
|
||||
#-------------------------------------------------------------------------
|
||||
- job: ICU4C_MSVC_x64_Release_VS2015
|
||||
displayName: 'C: MSVC 64-bit Release (VS 2015)'
|
||||
timeoutInMinutes: 30
|
||||
pool:
|
||||
vmImage: 'windows-2019'
|
||||
demands:
|
||||
- msbuild
|
||||
- visualstudio
|
||||
- Cmd
|
||||
steps:
|
||||
- checkout: self
|
||||
lfs: true
|
||||
fetchDepth: 10
|
||||
- task: VSBuild@1
|
||||
displayName: 'Build Solution'
|
||||
inputs:
|
||||
solution: icu4c/source/allinone/allinone.sln
|
||||
platform: x64
|
||||
configuration: Release
|
||||
msbuildArgs: /p:SkipUWP=true /p:DefaultPlatformToolset=v140 /p:AutodetectWin10SDK=true
|
||||
- task: BatchScript@1
|
||||
displayName: 'Run Tests (icucheck.bat)'
|
||||
inputs:
|
||||
filename: icu4c/source/allinone/icucheck.bat
|
||||
arguments: 'x64 Release'
|
||||
#-------------------------------------------------------------------------
|
||||
- job: ICU4C_MSYS2_GCC_x86_64_Release
|
||||
displayName: 'C: MSYS2 GCC x86_64 Release'
|
||||
timeoutInMinutes: 45
|
||||
pool:
|
||||
vmImage: 'windows-2019'
|
||||
demands:
|
||||
- Cmd
|
||||
steps:
|
||||
- checkout: self
|
||||
lfs: true
|
||||
fetchDepth: 10
|
||||
- script: |
|
||||
choco install -y msys2
|
||||
rem refreshenv
|
||||
displayName: 'Install MSYS2'
|
||||
- script: |
|
||||
c:\tools\msys64\usr\bin\bash.exe -lc "echo 'Hello World' && uname -a"
|
||||
c:\tools\msys64\usr\bin\bash.exe -lc "pacman --noconfirm -S mingw-w64-x86_64-toolchain"
|
||||
c:\tools\msys64\usr\bin\bash.exe -lc "pacman --noconfirm -S make"
|
||||
exit
|
||||
displayName: 'Install mingw-w64-x86_64-toolchain'
|
||||
- script: |
|
||||
set MSYSTEM=MINGW64
|
||||
c:\tools\msys64\usr\bin\bash.exe -lc "cd $BUILD_SOURCESDIRECTORY && cd icu4c/source && ./runConfigureICU MinGW --prefix='/tmp/build-icu' && make -j2 check"
|
||||
displayName: 'Build and Test'
|
||||
- script: |
|
||||
set MSYSTEM=MINGW64
|
||||
c:\tools\msys64\usr\bin\bash.exe -lc "cd $BUILD_SOURCESDIRECTORY && cd icu4c/source && make install && cd /tmp/build-icu && echo 'Recursive ls' && ls -lR"
|
||||
displayName: 'make install'
|
||||
- script: |
|
||||
set MSYSTEM=MINGW64
|
||||
c:\tools\msys64\usr\bin\bash.exe -lc "echo 'Run icuinfo from MSYS shell' && cd /tmp/build-icu/bin && ./icuinfo.exe"
|
||||
echo 'Run icuinfo from CMD shell' && cd /d C:\tools\msys64\tmp\build-icu\bin && set PATH=C:\tools\msys64\mingw64\bin;%PATH% && icuinfo.exe"
|
||||
displayName: 'run icuinfo'
|
||||
#-------------------------------------------------------------------------
|
||||
- job: ICU4C_Clang_MacOSX_WarningsAsErrors
|
||||
displayName: 'C: macOSX Clang WarningsAsErrors (Catalina 10.15)'
|
||||
timeoutInMinutes: 30
|
||||
pool:
|
||||
vmImage: 'macOS-10.15'
|
||||
steps:
|
||||
- checkout: self
|
||||
lfs: true
|
||||
fetchDepth: 10
|
||||
- script: |
|
||||
export CPPFLAGS="-Werror -Wall -Wextra -Wextra-semi -Wundef -Wnon-virtual-dtor" && cd icu4c/source && ./runConfigureICU MacOSX && make -j2 check
|
||||
displayName: 'Build and Test (WarningsAsErrors)'
|
||||
env:
|
||||
CC: clang
|
||||
CXX: clang++
|
||||
#-------------------------------------------------------------------------
|
||||
- job: ICU4C_Cygwin_GCC_x86_64_Release
|
||||
displayName: 'C: Cygwin GCC x86_64 Release'
|
||||
timeoutInMinutes: 180
|
||||
pool:
|
||||
vmImage: 'windows-2019'
|
||||
variables:
|
||||
ICU_CI_CACHE: c:\icu-ci-cache
|
||||
CYG_URL: https://cygwin.com/setup-x86_64.exe
|
||||
CYG_MIRROR: http://mirrors.kernel.org/sourceware/cygwin/
|
||||
CYG_PACKAGES: automake,gcc-core,gcc-g++,make,pkg-config,perl,python3
|
||||
CYG_ROOT: c:\cygwin-root
|
||||
CYG_CACHE: '$(ICU_CI_CACHE)\cygwin64-v3'
|
||||
CYG_CACHED_SETUP: '$(CYG_CACHE)\setup.exe'
|
||||
CYG_VERSION_KEY: cygwin-3.2
|
||||
steps:
|
||||
# Use 'autocrlf input' since checked-in files may already be using CRLF.
|
||||
- script: |
|
||||
git config --global core.autocrlf input
|
||||
displayName: 'Configure Git to checkout with Unix line endings (LF)'
|
||||
- checkout: self
|
||||
lfs: true
|
||||
fetchDepth: 10
|
||||
# Cache expires after 7 days of no activity.
|
||||
- task: Cache@2
|
||||
displayName: 'Restore Cygwin cache'
|
||||
inputs:
|
||||
# Use the contents of the file ".azure-pipelines.yml" as part of the key, as that contains the list of CYG_PACKAGES.
|
||||
# Also include the Cygwin version as part of the key. If we want to use a newer version of Cygwin, we can update CYG_VERSION_KEY.
|
||||
# Note: CYG_VERSION_KEY may become out of sync with (older than) the version we actually use if we update this file
|
||||
# without updating CYG_VERSION_KEY. Any updates to this file guarantees that we're using the latest version.
|
||||
key: '"$(CYG_VERSION_KEY)" | .ci-builds/.azure-pipelines.yml'
|
||||
path: "$(CYG_CACHE)"
|
||||
- task: PowerShell@2
|
||||
displayName: 'Download Cygwin setup'
|
||||
inputs:
|
||||
targetType: inline
|
||||
script: |
|
||||
if ( !(Test-Path "${env:CYG_CACHED_SETUP}" -NewerThan (Get-Date).AddDays(-7)) )
|
||||
{
|
||||
Write-Host "Cached Cygwin setup does not exist or is older than 7 days, downloading from external site."
|
||||
|
||||
New-Item -Force -Type Directory $env:CYG_CACHE
|
||||
Write-Host "Downloading Cygwin setup..."
|
||||
|
||||
$start_time = Get-Date
|
||||
(New-Object System.Net.WebClient).DownloadFile($env:CYG_URL, $env:CYG_CACHED_SETUP)
|
||||
|
||||
Write-Output "Download took: $((Get-Date).Subtract($start_time).Seconds) second(s)."
|
||||
}
|
||||
- script: |
|
||||
%CYG_CACHED_SETUP% --no-verify --quiet-mode --no-shortcuts --no-startmenu --no-desktop --upgrade-also --only-site --site "%CYG_MIRROR%" --root "%CYG_ROOT%" --local-package-dir "%CYG_CACHE%" --packages "%CYG_PACKAGES%"
|
||||
displayName: 'Install Cygwin'
|
||||
- script: |
|
||||
%CYG_ROOT%\\bin\\sh -lc 'echo Hello' && %CYG_ROOT%\\bin\\sh -lc 'uname -a'
|
||||
displayName: 'Check Cygwin environment'
|
||||
- script: |
|
||||
%CYG_ROOT%\\bin\\bash -lc "cd $(cygpath \"$(Build.SourcesDirectory)\") && cd icu4c/source && ./runConfigureICU Cygwin && make tests"
|
||||
displayName: 'Build ICU (source and test)'
|
||||
env:
|
||||
CC: gcc
|
||||
CXX: g++
|
||||
- script: |
|
||||
%CYG_ROOT%\\bin\\bash -lc "cd $(cygpath \"$(Build.SourcesDirectory)\") && cd icu4c/source && make check"
|
||||
displayName: 'Run Tests'
|
||||
env:
|
||||
CC: gcc
|
||||
CXX: g++
|
||||
#-------------------------------------------------------------------------
|
||||
- job: ICU4C_icuexportdata_Ubuntu_1804
|
||||
displayName: 'C: Create Artifacts from icuexportdata'
|
||||
timeoutInMinutes: 30
|
||||
pool:
|
||||
vmImage: 'ubuntu-18.04'
|
||||
steps:
|
||||
- checkout: self
|
||||
lfs: true
|
||||
fetchDepth: 10
|
||||
- script: |
|
||||
cd icu4c/source && ./runConfigureICU Linux && make -j2
|
||||
displayName: 'Build'
|
||||
env:
|
||||
CC: clang
|
||||
CXX: clang++
|
||||
- script: |
|
||||
mkdir -p icu4c/source/icuexportdata_uprops_full/fast
|
||||
mkdir -p icu4c/source/icuexportdata_uprops_full/small
|
||||
cd icu4c/source
|
||||
LD_LIBRARY_PATH=lib ./bin/icuexportdata --mode uprops --index --copyright --destdir icuexportdata_uprops_full/fast --trie-type fast --all
|
||||
LD_LIBRARY_PATH=lib ./bin/icuexportdata --mode uprops --index --copyright --destdir icuexportdata_uprops_full/small --trie-type small --all
|
||||
displayName: 'Build Unicode property data export file (Full)'
|
||||
# In the sample file, include:
|
||||
# - Basic binary properties: AHex WSpace
|
||||
# - Property of strings: Basic_Emoji
|
||||
# - POSIX binary property: blank
|
||||
# - Enumerated properties: gc nt sc
|
||||
- script: |
|
||||
mkdir -p icu4c/source/icuexportdata_uprops_sample/fast
|
||||
mkdir -p icu4c/source/icuexportdata_uprops_sample/small
|
||||
cd icu4c/source
|
||||
LD_LIBRARY_PATH=lib ./bin/icuexportdata --mode uprops --index --copyright --destdir icuexportdata_uprops_sample/fast --trie-type fast AHex gc nt Basic_Emoji sc WSpace blank
|
||||
LD_LIBRARY_PATH=lib ./bin/icuexportdata --mode uprops --index --copyright --destdir icuexportdata_uprops_sample/small --trie-type small AHex gc nt Basic_Emoji sc WSpace blank
|
||||
displayName: 'Build Unicode property data export file (Sample)'
|
||||
- task: PublishBuildArtifacts@1
|
||||
displayName: 'Publish Artifact: icuexportdata_uprops_full'
|
||||
inputs:
|
||||
PathtoPublish: 'icu4c/source/icuexportdata_uprops_full'
|
||||
ArtifactName: 'icuexportdata_uprops_full'
|
||||
- task: PublishBuildArtifacts@1
|
||||
displayName: 'Publish Artifact: icuexportdata_uprops_sample'
|
||||
inputs:
|
||||
PathtoPublish: 'icu4c/source/icuexportdata_uprops_sample'
|
||||
ArtifactName: 'icuexportdata_uprops_sample'
|
||||
@@ -1,70 +0,0 @@
|
||||
# Azure Pipelines configuration for Valgrind for ICU4C.
|
||||
#
|
||||
# Note: The valgrind test configuration is in a separate file
|
||||
# as it used to be run independently from the regular CI builds.
|
||||
#
|
||||
# The Ubuntu images don't have valgrind installed by default, so we need
|
||||
# install it first.
|
||||
#
|
||||
# Run Valgrind for every Pull Request that goes into main
|
||||
# or a maint- branch. This lets us catch any issues sooner rather than later.
|
||||
pr:
|
||||
branches:
|
||||
include:
|
||||
- main
|
||||
- maint/maint-*
|
||||
paths:
|
||||
include:
|
||||
- '*'
|
||||
exclude:
|
||||
- docs/*
|
||||
- icu4j/*
|
||||
- tools/*
|
||||
- vendor/*
|
||||
- .cpyskip.txt
|
||||
- .travis.yml
|
||||
- KEYS
|
||||
- README.md
|
||||
|
||||
jobs:
|
||||
#-------------------------------------------------------------------------
|
||||
- job: ICU4C_Clang_Valgrind_Ubuntu_1804
|
||||
displayName: 'C: Linux Clang Valgrind (Ubuntu 18.04)'
|
||||
timeoutInMinutes: 60
|
||||
pool:
|
||||
vmImage: 'ubuntu-18.04'
|
||||
steps:
|
||||
- checkout: self
|
||||
lfs: true
|
||||
fetchDepth: 10
|
||||
- script: |
|
||||
set -ex
|
||||
sudo apt -y update
|
||||
sudo apt install -y valgrind
|
||||
displayName: 'Install valgrind'
|
||||
timeoutInMinutes: 5
|
||||
- script: |
|
||||
cd icu4c/source && ./runConfigureICU --enable-debug Linux --disable-renaming && make -j2 tests
|
||||
displayName: 'Build'
|
||||
timeoutInMinutes: 10
|
||||
env:
|
||||
CC: clang
|
||||
CXX: clang++
|
||||
- script: |
|
||||
cd icu4c/source/test/intltest && LD_LIBRARY_PATH=../../lib:../../stubdata:../../tools/ctestfw:$LD_LIBRARY_PATH valgrind --tool=memcheck --error-exitcode=1 --leak-check=full --show-reachable=yes ./intltest
|
||||
displayName: 'Valgrind intltest'
|
||||
timeoutInMinutes: 45
|
||||
- script: |
|
||||
cd icu4c/source/test/cintltst && LD_LIBRARY_PATH=../../lib:../../stubdata:../../tools/ctestfw:$LD_LIBRARY_PATH valgrind --tool=memcheck --error-exitcode=1 --leak-check=full --show-reachable=yes ./cintltst
|
||||
displayName: 'Valgrind cintltst'
|
||||
timeoutInMinutes: 15
|
||||
- script: |
|
||||
cd icu4c/source/test/iotest && LD_LIBRARY_PATH=../../lib:../../stubdata:../../tools/ctestfw:$LD_LIBRARY_PATH valgrind --tool=memcheck --error-exitcode=1 --leak-check=full --show-reachable=yes ./iotest
|
||||
displayName: 'Valgrind iotest'
|
||||
timeoutInMinutes: 5
|
||||
- script: |
|
||||
cd icu4c/source/tools/icuinfo && LD_LIBRARY_PATH=../../lib:../../stubdata:../../tools/ctestfw:$LD_LIBRARY_PATH valgrind --tool=memcheck --error-exitcode=1 --leak-check=full --show-reachable=yes ./icuinfo
|
||||
displayName: 'Valgrind icuinfo'
|
||||
timeoutInMinutes: 5
|
||||
|
||||
#-------------------------------------------------------------------------
|
||||
@@ -1,49 +0,0 @@
|
||||
{
|
||||
"localeFilter": {
|
||||
"filterType": "locale",
|
||||
"whitelist": [
|
||||
"en",
|
||||
"de",
|
||||
"zh"
|
||||
]
|
||||
},
|
||||
// Test mixed feature filter and resource filter
|
||||
// Exclude translit data so we can run test for ICU-20673
|
||||
// Also test for "whitelist" versus "includelist"
|
||||
"featureFilters": {
|
||||
"misc": {
|
||||
"whitelist": ["supplementalData"]
|
||||
},
|
||||
"translit": "exclude",
|
||||
"curr_tree": {
|
||||
"filterType": "locale",
|
||||
"includelist": ["my"]
|
||||
},
|
||||
"brkitr_rules": {
|
||||
"excludelist": ["line"]
|
||||
},
|
||||
"brkitr_dictionaries": {
|
||||
"blacklist": ["cjdict"]
|
||||
}
|
||||
},
|
||||
"resourceFilters": [
|
||||
{
|
||||
"categories": ["misc"],
|
||||
"files": {
|
||||
"includelist": ["supplementalData"]
|
||||
},
|
||||
"rules": ["+/*"]
|
||||
}
|
||||
],
|
||||
// Test fileReplacements (relative path)
|
||||
"fileReplacements": {
|
||||
"directory": "$SRC",
|
||||
"replacements": [
|
||||
{
|
||||
"src": "brkitr/rules/line.txt",
|
||||
"dest": "brkitr/rules/line_normal.txt"
|
||||
},
|
||||
"misc/dayPeriods.txt"
|
||||
]
|
||||
}
|
||||
}
|
||||
-119
@@ -1,119 +0,0 @@
|
||||
#
|
||||
# Copyright (C) 2017 and later: Unicode, Inc. and others.
|
||||
# License & terms of use: http://www.unicode.org/copyright.html
|
||||
#
|
||||
# Copyright (c) 2005-2011 International Business Machines
|
||||
# Corporation and others. All Rights Reserved.
|
||||
## see NOTE below.
|
||||
## Also see: http://icu-project.org/copyright-scan.html
|
||||
#
|
||||
# git stuff
|
||||
# .git is ignored by code.
|
||||
#.git/*
|
||||
.github/*
|
||||
.gitignore
|
||||
.gitattributes
|
||||
.travis.yml
|
||||
.ci-builds/*
|
||||
# PGP Keys
|
||||
KEYS
|
||||
# suffix matches - start with '*'. They are turned into as RE, '.brk$'
|
||||
*.brk
|
||||
*.bz2
|
||||
*.classpath
|
||||
*.csproj
|
||||
*.cvsignore
|
||||
*.dat
|
||||
*.DS_Store
|
||||
*.doc
|
||||
*.fuzz
|
||||
*.gif
|
||||
*.gz
|
||||
*.ico
|
||||
*.icu
|
||||
*.intaglio
|
||||
*.jar
|
||||
*.jpg
|
||||
*.launch
|
||||
*.nrm
|
||||
*.odp
|
||||
*.otf
|
||||
*.pdf
|
||||
*.png
|
||||
*.ppt
|
||||
*.prefs
|
||||
*.project
|
||||
*.res
|
||||
*.rtf
|
||||
*.sln
|
||||
*.sxd
|
||||
*.sxg
|
||||
*.sxw
|
||||
*.tri2
|
||||
*.vcproj
|
||||
*.vcxproj
|
||||
*.vcxproj.filters
|
||||
*.zip
|
||||
|
||||
# UnicodeData.txt does not have any header.
|
||||
*/UnicodeData.txt
|
||||
|
||||
#
|
||||
# ICU4C
|
||||
#
|
||||
icu4c/source/aclocal.m4
|
||||
icu4c/source/config.guess
|
||||
icu4c/source/config.log
|
||||
icu4c/source/config.status
|
||||
icu4c/source/config.sub
|
||||
icu4c/source/install-sh
|
||||
icu4c/source/extra/uconv/samples/*
|
||||
icu4c/source/samples/layout/Sample.txt
|
||||
icu4c/source/samples/ucnv/data01.txt
|
||||
icu4c/source/samples/ufortune/resources/res-file-list.txt
|
||||
icu4c/source/test/fuzzer/*_seed_corpus.txt
|
||||
icu4c/source/test/testdata/ConverterSelectorTestUTF8.txt
|
||||
icu4c/source/test/testdata/encoded.utf16be
|
||||
icu4c/source/test/testdata/idna_conf.txt
|
||||
icu4c/source/test/testdata/ra.xlf
|
||||
icu4c/source/test/testdata/re_tests.txt
|
||||
icu4c/source/test/thaitest/space.txt
|
||||
icu4c/source/tools/tzcode/asctime.c
|
||||
icu4c/source/tools/tzcode/ialloc.c
|
||||
icu4c/source/tools/tzcode/localtime.c
|
||||
icu4c/source/tools/tzcode/private.h
|
||||
icu4c/source/tools/tzcode/scheck.c
|
||||
icu4c/source/tools/tzcode/tzfile.h
|
||||
icu4c/source/tools/tzcode/tzselect.ksh
|
||||
icu4c/source/tools/tzcode/zdump.c
|
||||
icu4c/source/tools/tzcode/zic.c
|
||||
#
|
||||
# ICU4J
|
||||
#
|
||||
icu4j/eclipse-build/pdebuild/allElements.xml
|
||||
icu4j/eclipse-build/pdebuild/customTargets.xml
|
||||
icu4j/main/tests/core/src/com/ibm/icu/dev/data/IDNATestInput.txt
|
||||
icu4j/main/tests/core/src/com/ibm/icu/dev/data/unicode/confusablesWholeScript.txt
|
||||
icu4j/main/tests/core/src/com/ibm/icu/dev/data/unicode/UnicodeData.txt
|
||||
icu4j/main/tests/core/src/com/ibm/icu/dev/test/duration/testdata/*
|
||||
icu4j/perf-tests/data/conversion/*
|
||||
icu4j/perf-tests/data/udhr/*
|
||||
icu4j/perf-tests/data/collation/thesis.txt
|
||||
icu4j/perf-tests/data/collation/ulyss10.txt
|
||||
icu4j/perf-tests/data/collation/2drvb10.txt
|
||||
icu4j/perf-tests/data/collation/vfear11a.txt
|
||||
icu4j/perf-tests/data/collation/nvsbl10.txt
|
||||
#
|
||||
# docs
|
||||
#
|
||||
docs/ide4c/vscode/*.json
|
||||
docs/Gemfile.lock
|
||||
#
|
||||
# tools
|
||||
#
|
||||
tools/trac/IcuCodeTools/*
|
||||
tools/unicodetools/*
|
||||
#
|
||||
# vendor
|
||||
#
|
||||
vendor/double-conversion/upstream/*
|
||||
@@ -0,0 +1,5 @@
|
||||
build
|
||||
*-stamp
|
||||
bin
|
||||
include
|
||||
lib
|
||||
+26
-46
@@ -1,49 +1,32 @@
|
||||
# The line "* text=auto !eol" means the following:
|
||||
# For text files (auto-detected), EOLs will be converted on checkout to OS-dependent EOL (LF for Unix, CRLF for Windows).
|
||||
# No EOL (end of line) conversion for binary files.
|
||||
* text=auto !eol
|
||||
|
||||
# Note:
|
||||
# "text" tells git the file is text.
|
||||
# "-text" tells git the file is binary.
|
||||
# The only difference between the two is that git will do EOL conversion for text files.
|
||||
# "!eol" is the equivalent of "svneol=native".
|
||||
|
||||
*.bat text !eol
|
||||
*.c text !eol diff=cpp
|
||||
*.cc text !eol diff=cpp
|
||||
*.c text !eol
|
||||
*.cc text !eol
|
||||
*.classpath text !eol
|
||||
*.cmd text eol=crlf
|
||||
*.cpp text !eol diff=cpp
|
||||
*.css text !eol diff=css
|
||||
*.cpp text !eol
|
||||
*.css text !eol
|
||||
*.dsp text !eol
|
||||
*.dsw text !eol
|
||||
*.dtd text !eol
|
||||
*.el text !eol
|
||||
*.filters text !eol
|
||||
*.h text !eol diff=cpp
|
||||
*.htm text !eol diff=html
|
||||
*.html text !eol diff=html
|
||||
*.h text !eol
|
||||
*.htm text !eol
|
||||
*.html text !eol
|
||||
*.in text !eol
|
||||
*.java text !eol diff=java
|
||||
*.java text !eol
|
||||
*.launch text !eol
|
||||
*.m4 text !eol
|
||||
*.mak text !eol
|
||||
*.md text !eol
|
||||
*.MF text !eol
|
||||
*.mk text !eol
|
||||
*.pl text !eol diff=perl
|
||||
*.pm text !eol diff=perl
|
||||
*.pl text !eol
|
||||
*.pm text !eol
|
||||
*.project text !eol
|
||||
*.properties text !eol
|
||||
*.props text !eol
|
||||
*.py text !eol diff=python
|
||||
*.py text !eol
|
||||
*.rc text !eol
|
||||
*.sh text eol=lf
|
||||
*.sed text eol=lf
|
||||
*.sln text !eol
|
||||
*.stub text !eol
|
||||
*.targets text !eol
|
||||
*.txt text !eol
|
||||
*.ucm text !eol
|
||||
*.vcproj text !eol
|
||||
@@ -51,18 +34,11 @@
|
||||
*.xml text !eol
|
||||
*.xsl text !eol
|
||||
*.xslt text !eol
|
||||
AUTHORS text !eol
|
||||
BUILD text !eol
|
||||
COPYING text !eol
|
||||
Changelog text !eol
|
||||
LICENSE text !eol
|
||||
Makefile text !eol
|
||||
README text !eol
|
||||
SConscript text !eol
|
||||
SConstruct text !eol
|
||||
configure text !eol
|
||||
LICENSE text !eol
|
||||
README text !eol
|
||||
|
||||
# Explicitly set the following file types as binary files.
|
||||
*.bin -text
|
||||
*.brk -text
|
||||
*.cnv -text
|
||||
@@ -71,16 +47,20 @@ configure text !eol
|
||||
*.nrm -text
|
||||
*.spp -text
|
||||
*.tri2 -text
|
||||
*.otf -text
|
||||
*.utf16be -text
|
||||
|
||||
as_is/win32/msvc7.zip -text
|
||||
source/data/brkitr/thaidict.brk -text
|
||||
source/data/unidata/UCARules.txt -text
|
||||
source/samples/ucnv/data02.bin -text
|
||||
source/test/testdata/importtest.bin -text
|
||||
source/test/testdata/iscii.bin -text
|
||||
source/test/testdata/uni-text.bin -text
|
||||
|
||||
# The following file types are stored in Git-LFS.
|
||||
# Only .jars
|
||||
*.jar filter=lfs diff=lfs merge=lfs -text
|
||||
|
||||
# Enable syntax highlighting on GitHub.com
|
||||
.cpyskip.txt linguist-language=Ignore-List
|
||||
|
||||
# Use JSONC for syntax highlighting on GitHub.com
|
||||
*.json linguist-language=jsonc
|
||||
*.dat filter=lfs diff=lfs merge=lfs -text
|
||||
*.zip filter=lfs diff=lfs merge=lfs -text
|
||||
*.gz filter=lfs diff=lfs merge=lfs -text
|
||||
*.bz2 filter=lfs diff=lfs merge=lfs -text
|
||||
*.gif filter=lfs diff=lfs merge=lfs -text
|
||||
|
||||
|
||||
@@ -1,15 +0,0 @@
|
||||
// © 2021 and later: Unicode, Inc. and others.
|
||||
// License & terms of use: http://www.unicode.org/copyright.html
|
||||
// Generated using tools/cldr/cldr-to-icu/build-icu-data.xml
|
||||
//
|
||||
// Include Burmese and Thai lstm models.
|
||||
{
|
||||
"featureFilters": {
|
||||
"brkitr_lstm": {
|
||||
"includelist": [
|
||||
"Thai_graphclust_model4_heavy",
|
||||
"Burmese_graphclust_model5_heavy"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,20 +0,0 @@
|
||||
<!--
|
||||
Thank you for your pull request!
|
||||
|
||||
Please see http://site.icu-project.org/processes/contribute for general
|
||||
information on contributing to ICU.
|
||||
|
||||
You will be automatically asked to sign the contributors license agreement (CLA) before the PR is accepted.
|
||||
- sign: https://cla-assistant.io/unicode-org/icu
|
||||
- license: http://www.unicode.org/copyright.html
|
||||
-->
|
||||
|
||||
##### Checklist
|
||||
|
||||
- [ ] Required: Issue filed: https://unicode-org.atlassian.net/browse/ICU-_____
|
||||
- [ ] Required: The PR title must be prefixed with a JIRA Issue number. <!-- For example: "ICU-1234 Fix xyz" -->
|
||||
- [ ] Required: The PR description must include the link to the Jira Issue, for example by completing the URL in the first checklist item
|
||||
- [ ] Required: Each commit message must be prefixed with a JIRA Issue number. <!-- For example: "ICU-1234 Fix xyz" -->
|
||||
- [ ] Issue accepted (done by Technical Committee after discussion)
|
||||
- [ ] Tests included, if applicable
|
||||
- [ ] API docs and/or User Guide docs changed or added, if applicable
|
||||
@@ -1,442 +0,0 @@
|
||||
# Copyright (C) 2016 and later: Unicode, Inc. and others.
|
||||
# License & terms of use: http://www.unicode.org/copyright.html
|
||||
#
|
||||
# GitHub Action configuration script for ICU continuous integration tasks.
|
||||
|
||||
name: GHA CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- 'maint/maint*'
|
||||
pull_request:
|
||||
branches: '**'
|
||||
|
||||
jobs:
|
||||
|
||||
# ICU4C docs build using doxygen..
|
||||
icu4c-docs-build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: ICU4C doc
|
||||
run: |
|
||||
sudo apt-get -y install doxygen;
|
||||
cd icu4c/source;
|
||||
./runConfigureICU Linux --disable-renaming;
|
||||
# Fail if 'warning:' appears in doxygen's output, but ignore warnings from file Doxyfile.
|
||||
# Regex note: (?! ... ) is a negative lookahead. Succeed if the pattern is not present.
|
||||
set +o pipefail && make doc 2>&1 | tee doxygen.log && ( ! grep -P 'warning:(?! .* file .?Doxyfile)' doxygen.log )
|
||||
|
||||
# Java7 ICU4J build and unit test
|
||||
java7-icu4j-build-and-test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout and setup
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
lfs: true
|
||||
- name: Checkout lfs objects
|
||||
run: git lfs pull
|
||||
- uses: actions/setup-java@v1
|
||||
with:
|
||||
java-version: '8'
|
||||
- name: Cache for Java 7 tarball
|
||||
id: cache-java7
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: java7-tarball
|
||||
key: ${{ runner.os }}-java7-tarball
|
||||
- name: Download Java 7
|
||||
if: steps.cache-java7.outputs.cache-hit != 'true'
|
||||
run: |
|
||||
mkdir -p java7-tarball
|
||||
download_url="https://download.java.net/openjdk/jdk7u75/ri/openjdk-7u75-b13-linux-x64-18_dec_2014.tar.gz"
|
||||
wget -O java7-tarball/java_package.tar.gz $download_url
|
||||
pushd java7-tarball
|
||||
gunzip java_package.tar.gz
|
||||
tar xvf java_package.tar
|
||||
popd
|
||||
- name: Configure Ant build to use Java 7 JRE
|
||||
run: |
|
||||
echo "java7.bootclasspath" > $RUNNER_TEMP/draft
|
||||
ls java7-tarball/java-se-7u75-ri/jre/lib/*.jar|paste -sd ":" - >> $RUNNER_TEMP/draft
|
||||
paste -sd "=" < $RUNNER_TEMP/draft > icu4j/build-local.properties
|
||||
- name: ICU4J
|
||||
run: |
|
||||
cd icu4j;
|
||||
ant init;
|
||||
ant check;
|
||||
ant localespiCheck
|
||||
- name: List failures (if any)
|
||||
run: |
|
||||
[ -d icu4j/out/junit-results ] && cd icu4j && cat `find out/junit-results -name "*.txt" -exec grep -l FAILED {} \;`;
|
||||
if: ${{ failure() }}
|
||||
|
||||
|
||||
# ICU4J build and unit test under Java11
|
||||
java11-icu4j-build-and-test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout and setup
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
lfs: true
|
||||
- name: Checkout lfs objects
|
||||
run: git lfs pull
|
||||
- uses: actions/setup-java@v1
|
||||
with:
|
||||
java-version: '11'
|
||||
- name: ICU4J
|
||||
run: |
|
||||
cd icu4j;
|
||||
ant init;
|
||||
ant check;
|
||||
ant localespiCheck
|
||||
- name: List failures (if any)
|
||||
run: |
|
||||
[ -d icu4j/out/junit-results ] && cd icu4j && cat `find out/junit-results -name "*.txt" -exec grep -l FAILED {} \;`;
|
||||
if: ${{ failure() }}
|
||||
|
||||
# ICU4J build and unit test under Java16
|
||||
java16-icu4j-build-and-test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout and setup
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
lfs: true
|
||||
- name: Checkout lfs objects
|
||||
run: git lfs pull
|
||||
- uses: actions/setup-java@v1
|
||||
with:
|
||||
java-version: '16'
|
||||
- name: ICU4J
|
||||
run: |
|
||||
cd icu4j;
|
||||
ant init;
|
||||
ant check;
|
||||
ant localespiCheck
|
||||
- name: List failures (if any)
|
||||
run: |
|
||||
[ -d icu4j/out/junit-results ] && cd icu4j && cat `find out/junit-results -name "*.txt" -exec grep -l FAILED {} \;`;
|
||||
if: ${{ failure() }}
|
||||
|
||||
# ICU4J build and unit test under lstm
|
||||
lstm-icu4j-build-and-test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout and setup
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
lfs: true
|
||||
- name: Checkout lfs objects
|
||||
run: git lfs pull
|
||||
- uses: actions/setup-java@v1
|
||||
with:
|
||||
java-version: '11'
|
||||
- name: Config LSTM and Rebuild data jar
|
||||
run: |
|
||||
cd icu4c/source;
|
||||
ICU_DATA_BUILDTOOL_OPTS=--include_uni_core_data ICU_DATA_FILTER_FILE=../../.github/lstm_for_th_my.json ./runConfigureICU --enable-debug --disable-release Linux -disable-layoutex;
|
||||
make clean;
|
||||
make -j2 ICU4J_ROOT=../../../icu4j icu4j-data-install;
|
||||
cd ../..
|
||||
- name: ICU4J
|
||||
run: |
|
||||
cd icu4j;
|
||||
ant init;
|
||||
ant check;
|
||||
ant localespiCheck
|
||||
- name: List failures (if any)
|
||||
run: |
|
||||
[ -d icu4j/out/junit-results ] && cd icu4j && cat `find out/junit-results -name "*.txt" -exec grep -l FAILED {} \;`;
|
||||
if: ${{ failure() }}
|
||||
|
||||
# gcc debug build.
|
||||
# Includes dependency checker.
|
||||
# Note - the dependency checker needs to be run on both a debug and an optimized build.
|
||||
# This one (gcc) for debug, and linux clang (see job below) for optimized.
|
||||
#
|
||||
# Test both out-of-source and in-source builds. This one (gcc) for out-of-source,
|
||||
# and linux clang (below) for in-source.
|
||||
#
|
||||
# Invokes test/hdrtst to check public headers compliance.
|
||||
gcc-debug-build-and-test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: ICU4C with gcc
|
||||
env:
|
||||
PREFIX: /tmp/icu-prefix
|
||||
run: |
|
||||
mkdir build;
|
||||
cd build;
|
||||
../icu4c/source/runConfigureICU --enable-debug --disable-release Linux/gcc --prefix=$PREFIX --enable-tracing;
|
||||
make -j2;
|
||||
make -j2 check;
|
||||
( cd ../icu4c/source/test/depstest && ./depstest.py ../../../../build/ );
|
||||
make install;
|
||||
PATH=$PREFIX/bin:$PATH make -C test/hdrtst check
|
||||
|
||||
# clang release build with some options to enforce useful constraints.
|
||||
# Includes dependency checker on an in-source, optimized build.
|
||||
# Includes checking @draft etc. API tags vs. ifndef guards like
|
||||
# U_HIDE_DRAFT_API and U_FORCE_HIDE_DRAFT_API.
|
||||
# (FORCE guards make this tool pass but won't compile to working code.
|
||||
# See the testtagsguards.sh script for details.)
|
||||
clang-release-build-and-test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: ICU4C with clang
|
||||
env:
|
||||
CPPFLAGS: -DU_NO_DEFAULT_INCLUDE_UTF_HEADERS=1
|
||||
CFLAGS: -Wimplicit-fallthrough
|
||||
CXXFLAGS: -Wimplicit-fallthrough
|
||||
run: |
|
||||
sudo apt-get -y install doxygen;
|
||||
cd icu4c/source;
|
||||
./runConfigureICU Linux;
|
||||
make -j 2;
|
||||
make -j 2 check;
|
||||
( cd test/depstest && python3 depstest.py ../../../source/ );
|
||||
( cd .. && source/test/hdrtst/testtagsguards.sh );
|
||||
make dist
|
||||
|
||||
# Out of source build with gcc 10, c++14, and extra warnings; executes icuinfo.
|
||||
gcc-10-stdlib14:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: ICU4C with gcc 10 and c++14 and extra warnings.
|
||||
env:
|
||||
PREFIX: /tmp/icu-prefix
|
||||
CC: gcc-10
|
||||
CXX: g++-10
|
||||
CXXFLAGS: -std=c++14 -Wextra
|
||||
run: |
|
||||
mkdir build;
|
||||
cd build;
|
||||
../icu4c/source/runConfigureICU Linux --disable-layout --disable-layoutex --prefix=$PREFIX;
|
||||
make -j2;
|
||||
make -j2 check;
|
||||
make install;
|
||||
cd $PREFIX/bin;
|
||||
LD_LIBRARY_PATH=../lib ./icuinfo
|
||||
|
||||
# Clang Linux with address sanitizer.
|
||||
clang-asan:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: ICU4C with clang and asan
|
||||
run: |
|
||||
cd icu4c/source;
|
||||
./runConfigureICU --enable-debug --disable-release Linux --disable-renaming --enable-tracing;
|
||||
make -j2;
|
||||
make -j2 check
|
||||
env:
|
||||
CPPFLAGS: -fsanitize=address
|
||||
LDFLAGS: -fsanitize=address
|
||||
|
||||
# Clang Linux with thread sanitizer.
|
||||
clang-tsan:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: ICU4C with clang and tsan
|
||||
run: |
|
||||
cd icu4c/source;
|
||||
./runConfigureICU --enable-debug --disable-release Linux --disable-renaming
|
||||
make -j2;
|
||||
make -j2 -C test;
|
||||
make -j2 -C test/intltest check
|
||||
env:
|
||||
INTLTEST_OPTS: utility/MultithreadTest
|
||||
CPPFLAGS: -fsanitize=thread
|
||||
LDFLAGS: -fsanitize=thread
|
||||
|
||||
# MacOS with clang
|
||||
macos-clang:
|
||||
runs-on: macos-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: ICU4C with clang on MacOS
|
||||
run: |
|
||||
cd icu4c/source;
|
||||
PYTHON=python3 ./runConfigureICU MacOSX;
|
||||
make -j2 check
|
||||
|
||||
# Run ICU4C tests with stubdata.
|
||||
run-with-stubdata:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: ICU4C tests with stubdata
|
||||
run: |
|
||||
cd icu4c/source;
|
||||
./runConfigureICU Linux;
|
||||
make -j2 check;
|
||||
rm lib/libicudata.so*;
|
||||
cp -P stubdata/libicudata.so* lib;
|
||||
|
||||
cd test/cintltst;
|
||||
echo 'Running ICU4C cintltst with stubdata.';
|
||||
# Note: 'Elapsed Time: ' is printed by makefile upon final success.
|
||||
CINTLTST_OPTS=-w make check 2>&1 | tee stubdata_ctest.log;
|
||||
if ! grep 'Elapsed Time: ' stubdata_ctest.log
|
||||
then
|
||||
echo
|
||||
echo cintltst run with stubdata failed
|
||||
echo
|
||||
echo See
|
||||
echo https://unicode-org.github.io/icu/processes/release/tasks/integration.html#run-tests-without-icu-data
|
||||
echo for how to reproduce and debug the failure
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cd ../intltest;
|
||||
echo 'Running ICU4C intltest with stubdata.';
|
||||
INTLTEST_OPTS=-w make check 2>&1 | tee stubdata_intltest.log;
|
||||
if ! grep 'Elapsed Time: ' stubdata_intltest.log
|
||||
then
|
||||
echo
|
||||
echo intltest run with stubdata failed
|
||||
echo
|
||||
echo See
|
||||
echo https://unicode-org.github.io/icu/processes/release/tasks/integration.html#run-tests-without-icu-data
|
||||
echo for how to reproduce and debug the failure
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Test U_CHARSET_IS_UTF8
|
||||
u-charset-is-utf8-test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- run: |
|
||||
cd icu4c/source;
|
||||
./runConfigureICU Linux CPPFLAGS="-DU_CHARSET_IS_UTF8=1";
|
||||
make -j2 check
|
||||
|
||||
# Test U_OVERRIDE_CXX_ALLOCATION-is-0-test
|
||||
u-override-cxx-allocation-is-0-test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- run: |
|
||||
cd icu4c/source;
|
||||
./runConfigureICU Linux CPPFLAGS="-DU_OVERRIDE_CXX_ALLOCATION=0";
|
||||
make clean;
|
||||
make -j2 check
|
||||
|
||||
# Test LSTM
|
||||
lstm-test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- run: |
|
||||
cd icu4c/source;
|
||||
ICU_DATA_FILTER_FILE=../../.github/lstm_for_th_my.json ./runConfigureICU --enable-debug --disable-release Linux -disable-layoutex;
|
||||
make clean;
|
||||
make -j2 check
|
||||
|
||||
# Build and run testmap
|
||||
testmap:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- run: |
|
||||
cd icu4c/source;
|
||||
./runConfigureICU Linux;
|
||||
make -j2 check;
|
||||
CONFIG_FILES=test/testmap/Makefile ./config.status;
|
||||
cd test/testmap;
|
||||
make check | grep '*** PASS PASS PASS, test PASSED!!!!!!!!'
|
||||
|
||||
# Copyright scan
|
||||
copyright-scan:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- run: perl tools/scripts/cpysearch/cpyscan.pl
|
||||
|
||||
# Check compilation of internal headers.
|
||||
internal-header-compilation:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- run: cd icu4c/source; test/hdrtst/testinternalheaders.sh
|
||||
|
||||
# Check source files for valid UTF-8 and for absence of BOM.
|
||||
valid-UTF-8-and-no-BOM-check:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- run: tools/scripts/icu-file-utf8-check.py
|
||||
|
||||
# Verify icu4c release tools buildability.
|
||||
icu4c-release-tools:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- run: mvn -f tools/release/java/pom.xml package dependency:analyze
|
||||
|
||||
# Run unit tests with UCONFIG_NO_XXX variations.
|
||||
uconfig_variation-check-unit-tests:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- run: python3 tools/scripts/uconfig_vars_test.py -u
|
||||
|
||||
# Run header tests with UCONFIG_NO_XXX variations.
|
||||
uconfig_variation-check-all-header-tests:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- run: python3 tools/scripts/uconfig_vars_test.py -p
|
||||
|
||||
# Build Unicode update tools
|
||||
unicode-update-tools:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
- uses: bazelbuild/setup-bazelisk@v1
|
||||
- name: Mount bazel cache
|
||||
uses: actions/cache@v2
|
||||
with:
|
||||
path: "~/.cache/bazel"
|
||||
key: bazel
|
||||
|
||||
- name: Generate the data
|
||||
run: |
|
||||
export ICU_SRC=`pwd`;
|
||||
icu4c/source/data/unidata/generate.sh;
|
||||
if $?
|
||||
then
|
||||
echo
|
||||
echo Build of Unicode update tools failed.
|
||||
echo See
|
||||
echo https://unicode-org.github.io/icu/processes/unicode-update#bazel-build-process
|
||||
echo for how to reproduce and debug the failure
|
||||
exit 1
|
||||
fi
|
||||
|
||||
git diff --exit-code;
|
||||
if $?
|
||||
then
|
||||
echo
|
||||
echo ICU unicode data has changed!
|
||||
echo Did you forget to include the changed data files in this PR?
|
||||
exit 1
|
||||
fi
|
||||
@@ -1,38 +0,0 @@
|
||||
# Copyright (C) 2021 and later: Unicode, Inc. and others.
|
||||
# License & terms of use: http://www.unicode.org/copyright.html
|
||||
#
|
||||
# GitHub Action continuous integration script for ICU post-merge tasks.
|
||||
|
||||
name: GHA ICU Merge CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- 'maint/maint*'
|
||||
|
||||
jobs:
|
||||
|
||||
# Test ICU4J with little-endian ICU4C data only
|
||||
icu4j-little-endian-data-test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout and setup
|
||||
uses: actions/checkout@v2
|
||||
with:
|
||||
lfs: true
|
||||
- name: Checkout lfs objects
|
||||
run: git lfs pull
|
||||
- uses: actions/setup-java@v1
|
||||
with:
|
||||
java-version: '11'
|
||||
- name: ICU4J little-endian
|
||||
run: |
|
||||
cd icu4c/source;
|
||||
ICU_DATA_BUILDTOOL_OPTS=--include_uni_core_data ./runConfigureICU Linux;
|
||||
make -j2 check;
|
||||
mkdir /tmp/icu4j_data_test;
|
||||
cp -P data/out/tmp/icudt*l* /tmp/icu4j_data_test/
|
||||
cd ../../icu4j;
|
||||
ant clean;
|
||||
ant -Dicu4c.data.path=/tmp/icu4j_data_test check;
|
||||
@@ -1,40 +0,0 @@
|
||||
name: Publish icu4j.jar/utilities.jar to GH Maven
|
||||
on:
|
||||
release:
|
||||
types: [created]
|
||||
jobs:
|
||||
publish:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
with:
|
||||
lfs: true
|
||||
- uses: actions/setup-java@v1
|
||||
with:
|
||||
java-version: 1.8
|
||||
- name: icu4j ant releaseCLDR
|
||||
run: ant -noinput releaseCLDR -f icu4j/build.xml
|
||||
- name: deploy it
|
||||
run: |
|
||||
MYVERSION=$(sed -n -e 's%^\s*maven.pom.ver[ =]*%%p' < icu4j/build.properties)-$(echo ${GITHUB_REF} | cut -d/ -f3- | tr ./ _-)
|
||||
echo Github Ref ${GITHUB_REF} @ ${GITHUB_SHA}, version ${MYVERSION}
|
||||
mvn deploy:deploy-file -DgroupId=com.ibm.icu \
|
||||
-DartifactId=icu4j-for-cldr \
|
||||
-Dversion=${MYVERSION} \
|
||||
-Dpackaging=jar \
|
||||
-Dfile=icu4j/release_cldr/icu4j.jar \
|
||||
-Dsources=icu4j/release_cldr/icu4j-src.jar \
|
||||
-DrepositoryId=github \
|
||||
-Durl=https://maven.pkg.github.com/${GITHUB_REPOSITORY}
|
||||
mvn deploy:deploy-file -DgroupId=com.ibm.icu \
|
||||
-DartifactId=utilities-for-cldr \
|
||||
-Dversion=${MYVERSION} \
|
||||
-Dpackaging=jar \
|
||||
-Dfile=icu4j/release_cldr/utilities.jar \
|
||||
-Dsources=icu4j/release_cldr/utilities-src.jar \
|
||||
-DrepositoryId=github \
|
||||
-Durl=https://maven.pkg.github.com/${GITHUB_REPOSITORY}
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
# Note: can use 0.0.0-${GITHUB_SHA} as the version for hash-based.
|
||||
# i.e. build on every commit
|
||||
+313
-280
@@ -1,280 +1,313 @@
|
||||
# Wildcards
|
||||
*.1
|
||||
*.8
|
||||
*.a
|
||||
*.ao
|
||||
*.d
|
||||
*.dSYM
|
||||
*.db
|
||||
*.dll
|
||||
*.dylib
|
||||
*.exp
|
||||
*.lib
|
||||
*.lnk
|
||||
*.ncb
|
||||
*.o
|
||||
*.obj
|
||||
*.opendb
|
||||
*.opendf
|
||||
*.opensdf
|
||||
*.opt
|
||||
*.out
|
||||
*.pc
|
||||
*.pdb
|
||||
*.plg
|
||||
*.pyc
|
||||
*.s
|
||||
*.sdf
|
||||
*.so
|
||||
*.suo
|
||||
*.tgz
|
||||
*.user
|
||||
*.vcproj.*.*.user
|
||||
*.vcxproj.user
|
||||
*_debug
|
||||
*_release
|
||||
bazel-*
|
||||
cygicudata*
|
||||
libicu*
|
||||
libicutest*
|
||||
libsicu*
|
||||
libsicutest*
|
||||
output-*.cpp
|
||||
perf-*.xml
|
||||
test-*.xml
|
||||
|
||||
# Directories
|
||||
.idea/
|
||||
.vs/
|
||||
.vscode/
|
||||
ARM/
|
||||
ARM64/
|
||||
Debug/
|
||||
Generated[!!-~]Files/
|
||||
Release/
|
||||
__pycache__/
|
||||
_site/
|
||||
arm/
|
||||
arm64/
|
||||
bin/
|
||||
debug/
|
||||
doc/
|
||||
lib/
|
||||
!tools/cldr/lib
|
||||
out/
|
||||
release/
|
||||
!docs/processes/release/
|
||||
tmp/
|
||||
x64/
|
||||
x86/
|
||||
|
||||
# Files
|
||||
Makefile
|
||||
Makefile.local
|
||||
build-local.properties
|
||||
icupkg.inc
|
||||
ipch
|
||||
pkgdata.inc
|
||||
pkgdataMakefile
|
||||
rules.mk
|
||||
.DS_Store
|
||||
|
||||
!icu4c/source/samples/csdet/Makefile
|
||||
|
||||
# Directory specific wildcards
|
||||
icu4c/source/extra/uconv/resources/*.res
|
||||
icu4c/source/perf-*.xml
|
||||
icu4c/source/samples/udata/*.dat
|
||||
icu4c/source/samples/ufortune/resources/*.dat
|
||||
icu4c/source/samples/ufortune/resources/*.res
|
||||
icu4c/source/samples/uresb/*.res
|
||||
icu4c/source/test-*.xml
|
||||
icu4c/source/test/hdrtst/ht_*
|
||||
icu4c/source/test/perf/howExpensiveIs/*.xml
|
||||
icu4c/source/tools/genrb/derb_*
|
||||
icu4c/source/tools/genren/urename.*
|
||||
icu4j/*.jar
|
||||
icu4j/lib/*.jar
|
||||
tools/multi/packages/*.jar
|
||||
tools/release/java/APIChangeReport*
|
||||
|
||||
icu4c/source/tools/icuinfo/icuplugins*.txt
|
||||
!icu4c/source/tools/icuinfo/icuplugins_windows_sample.txt
|
||||
|
||||
# Specific file names
|
||||
docs/Gemfile.lock
|
||||
icu4c/binARM
|
||||
icu4c/bin32uwp
|
||||
icu4c/bin64
|
||||
icu4c/binARM64
|
||||
icu4c/bin64uwp
|
||||
icu4c/binARMUWP
|
||||
icu4c/binARM64UWP
|
||||
icu4c/commondata
|
||||
icu4c/include
|
||||
icu4c/libARM
|
||||
icu4c/lib32uwp
|
||||
icu4c/lib64
|
||||
icu4c/libARM64
|
||||
icu4c/lib64uwp
|
||||
icu4c/libARMuwp
|
||||
icu4c/libARM64uwp
|
||||
icu4c/source/Doxyfile
|
||||
icu4c/source/README
|
||||
icu4c/source/autom4te.cache
|
||||
icu4c/source/common/common.res
|
||||
icu4c/source/common/icucfg.h
|
||||
icu4c/source/common/svchook.mk
|
||||
icu4c/source/config.cache
|
||||
icu4c/source/config.log
|
||||
icu4c/source/config.status
|
||||
icu4c/source/config/Makefile.inc
|
||||
icu4c/source/config/icu-config
|
||||
icu4c/source/config/icu.pc
|
||||
icu4c/source/config/icucross.inc
|
||||
icu4c/source/config/icucross.mk
|
||||
icu4c/source/config/icuinfo.xml
|
||||
icu4c/source/configure-local.mk
|
||||
icu4c/source/data/build-dir
|
||||
icu4c/source/data/build-local
|
||||
icu4c/source/data/data
|
||||
icu4c/source/data/obj
|
||||
icu4c/source/data/packagedata
|
||||
icu4c/source/data/uni-core-data
|
||||
icu4c/source/dist
|
||||
icu4c/source/extra/scrptrun/scrptrun.d
|
||||
icu4c/source/extra/scrptrun/scrptrun.o
|
||||
icu4c/source/extra/scrptrun/srtest
|
||||
icu4c/source/extra/scrptrun/srtest.d
|
||||
icu4c/source/extra/scrptrun/srtest.o
|
||||
icu4c/source/extra/scrptrun/srtest.exe
|
||||
icu4c/source/extra/uconv/pkgdatain.txt
|
||||
icu4c/source/extra/uconv/uconv
|
||||
icu4c/source/extra/uconv/uconv.plg
|
||||
icu4c/source/extra/uconv/uconvmsg
|
||||
icu4c/source/i18n/i18n.res
|
||||
icu4c/source/icudefs.local
|
||||
icu4c/source/icudefs.mk
|
||||
icu4c/source/io/io.res
|
||||
icu4c/source/layoutex/layoutex.res
|
||||
icu4c/source/samples/all/all.ncb
|
||||
icu4c/source/samples/all/all.suo
|
||||
icu4c/source/samples/break/break
|
||||
icu4c/source/samples/break/break.exe
|
||||
icu4c/source/samples/cal/icucal
|
||||
icu4c/source/samples/cal/icucal.exe
|
||||
icu4c/source/samples/case/case
|
||||
icu4c/source/samples/case/case.exe
|
||||
icu4c/source/samples/citer/citer
|
||||
icu4c/source/samples/citer/citer.exe
|
||||
icu4c/source/samples/csdet/csdet
|
||||
icu4c/source/samples/csdet/csdet.exe
|
||||
icu4c/source/samples/date/icudate
|
||||
icu4c/source/samples/date/icudate.exe
|
||||
icu4c/source/samples/datefmt/datefmt
|
||||
icu4c/source/samples/msgfmt/msgfmt
|
||||
icu4c/source/samples/numfmt/numfmt
|
||||
icu4c/source/samples/numfmt/numfmt.exe
|
||||
icu4c/source/samples/props/props
|
||||
icu4c/source/samples/props/props.exe
|
||||
icu4c/source/samples/translit/translit
|
||||
icu4c/source/samples/translit/translit.exe
|
||||
icu4c/source/samples/uciter8/uciter8
|
||||
icu4c/source/samples/uciter8/uciter8.exe
|
||||
icu4c/source/samples/ucnv/convsamp
|
||||
icu4c/source/samples/ucnv/data40.utf16
|
||||
icu4c/source/samples/udata/reader
|
||||
icu4c/source/samples/udata/reader.exe
|
||||
icu4c/source/samples/udata/writer
|
||||
icu4c/source/samples/udata/writer.exe
|
||||
icu4c/source/samples/ufortune/ufortune
|
||||
icu4c/source/samples/ufortune/ufortune.exe
|
||||
icu4c/source/samples/ugrep/ugrep
|
||||
icu4c/source/samples/ugrep/ugrep.exe
|
||||
icu4c/source/samples/uresb/uresb
|
||||
icu4c/source/samples/uresb/uresb.exe
|
||||
icu4c/source/samples/ustring/ustring
|
||||
icu4c/source/samples/ustring/ustring.exe
|
||||
icu4c/source/stubdata/stubdatabuilt.txt
|
||||
icu4c/source/test/cintltst/cintltst
|
||||
icu4c/source/test/cintltst/cintltst.exe
|
||||
icu4c/source/test/intltest/intltest
|
||||
icu4c/source/test/intltest/intltest.exe
|
||||
icu4c/source/test/iotest/iotest
|
||||
icu4c/source/test/iotest/iotest.exe
|
||||
icu4c/source/test/letest/letest
|
||||
icu4c/source/test/letest/letest.exe
|
||||
icu4c/source/test/perf/DateFmtPerf/DateFmtPerf
|
||||
icu4c/source/test/perf/DateFmtPerf/DateFmtPerf.d
|
||||
icu4c/source/test/perf/charperf/charperf
|
||||
icu4c/source/test/perf/collationperf/collperf
|
||||
icu4c/source/test/perf/collationperf/collperf.exe
|
||||
icu4c/source/test/perf/collperf/collperf
|
||||
icu4c/source/test/perf/collperf2/collperf2
|
||||
icu4c/source/test/perf/collperf2/collperf2.d
|
||||
icu4c/source/test/perf/howExpensiveIs/howExpensiveIs
|
||||
icu4c/source/test/perf/normperf/normperf
|
||||
icu4c/source/test/perf/perldriver/Common.pl
|
||||
icu4c/source/test/perf/ubrkperf/ubrkperf
|
||||
icu4c/source/test/perf/unisetperf/unisetperf
|
||||
icu4c/source/test/perf/usetperf/usetperf
|
||||
icu4c/source/test/perf/ustrperf/charperf
|
||||
icu4c/source/test/perf/ustrperf/stringperf
|
||||
icu4c/source/test/perf/utfperf/utfperf
|
||||
icu4c/source/test/perf/utrie2perf/utrie2perf
|
||||
icu4c/source/test/perf/utrie2perf/utrie2perf.d
|
||||
icu4c/source/test/testmap/testmap
|
||||
icu4c/source/test/testmap/testmap.plg
|
||||
icu4c/source/tools/genbrk/genbrk
|
||||
icu4c/source/tools/genbrk/genbrk.1
|
||||
icu4c/source/tools/genccode/genccode
|
||||
icu4c/source/tools/genccode/genccode.8
|
||||
icu4c/source/tools/gencfu/gencfu
|
||||
icu4c/source/tools/gencfu/gencfu.1
|
||||
icu4c/source/tools/gencfu/gencfu.exe
|
||||
icu4c/source/tools/gencmn/gencmn
|
||||
icu4c/source/tools/gencmn/gencmn.8
|
||||
icu4c/source/tools/gencmn/gencmn.[0-9]
|
||||
icu4c/source/tools/gencnval/gencnval
|
||||
icu4c/source/tools/gencnval/gencnval.1
|
||||
icu4c/source/tools/gencnval/gencnval.[0-9]
|
||||
icu4c/source/tools/gencolusb/extract_unsafe_backwards
|
||||
icu4c/source/tools/gencolusb/verify_uset
|
||||
icu4c/source/tools/gennorm2/gennorm2
|
||||
icu4c/source/tools/gennorm2/gennorm2.[0-9]
|
||||
icu4c/source/tools/genrb/derb
|
||||
icu4c/source/tools/genrb/derb.[0-9]
|
||||
icu4c/source/tools/genrb/genrb
|
||||
icu4c/source/tools/genrb/genrb.[0-9]
|
||||
icu4c/source/tools/genrb/temp
|
||||
icu4c/source/tools/genren/ICUunrenamed
|
||||
icu4c/source/tools/gensprep/gensprep
|
||||
icu4c/source/tools/gensprep/gensprep.[0-9]
|
||||
icu4c/source/tools/gensprep/gensprep.plg
|
||||
icu4c/source/tools/gentest/gentest
|
||||
icu4c/source/tools/gentest/gentest.exe
|
||||
icu4c/source/tools/icuinfo/icuinfo
|
||||
icu4c/source/tools/icuinfo/icuinfo.exe
|
||||
icu4c/source/tools/icupkg/icupkg
|
||||
icu4c/source/tools/icupkg/icupkg.[0-9]
|
||||
icu4c/source/tools/icuswap/icuswap
|
||||
icu4c/source/tools/icuswap/icuswap.[0-9]
|
||||
icu4c/source/tools/makeconv/makeconv
|
||||
icu4c/source/tools/makeconv/makeconv.[0-9]
|
||||
icu4c/source/tools/pkgdata/pkgdata
|
||||
icu4c/source/tools/pkgdata/pkgdata.[0-9]
|
||||
icu4c/source/uconfig.h.prepend
|
||||
icu4j/.project
|
||||
icu4j/eclipse
|
||||
tools/multi/proj/icu4cscan/cooked
|
||||
tools/multi/proj/icu4cscan/dtdcache
|
||||
tools/multi/proj/icu4cscan/jxml
|
||||
tools/multi/proj/icu4cscan/xml
|
||||
tools/release/c/uconfigtest
|
||||
tools/release/c/uconfigtest.local
|
||||
tools/release/java/classes
|
||||
tools/unicode/c/genprops/genprops
|
||||
tools/unicode/c/genprops/genprops.[0-9]
|
||||
tools/unicode/c/genuca/genuca
|
||||
tools/unicode/c/genuca/genuca.8
|
||||
tools/unicode/c/icudefs.txt
|
||||
/*-stamp
|
||||
/bin
|
||||
/build
|
||||
debian/*.debhelper
|
||||
debian/*.substvars
|
||||
debian/*files
|
||||
debian/*libicu[0-9]*
|
||||
debian/control
|
||||
debian/docs
|
||||
debian/icu
|
||||
debian/icu-data
|
||||
debian/icu-doc
|
||||
debian/icu-i18ndata
|
||||
debian/icu-locales
|
||||
debian/icu.postinst
|
||||
debian/icu.prerm
|
||||
debian/libicu-dev
|
||||
debian/postinst
|
||||
debian/prerm
|
||||
debian/tmp
|
||||
/include
|
||||
/lib
|
||||
source/Makefile
|
||||
source/README
|
||||
source/allinone/*.ncb
|
||||
source/allinone/*.opt
|
||||
source/common/*.ao
|
||||
source/common/*.d
|
||||
source/common/*.plg
|
||||
source/common/Debug
|
||||
source/common/Makefile
|
||||
source/common/Release
|
||||
source/common/icucfg.h
|
||||
source/common/libicu*.*
|
||||
source/common/unicode/platform.h
|
||||
source/config.cache
|
||||
source/config.log
|
||||
source/config.status
|
||||
source/config/Makefile.inc
|
||||
source/data/*.cnv
|
||||
source/data/*.dat
|
||||
source/data/*.dll
|
||||
source/data/*.exp
|
||||
source/data/*.lib
|
||||
source/data/*.obj
|
||||
source/data/*.plg
|
||||
source/data/.links
|
||||
source/data/Makefile
|
||||
source/data/icupkg.inc
|
||||
source/data/out
|
||||
source/extra/Makefile
|
||||
source/extra/scrptrun/Makefile
|
||||
source/extra/uconv/*.d
|
||||
source/extra/uconv/Debug
|
||||
source/extra/uconv/Makefile
|
||||
source/extra/uconv/Release
|
||||
source/extra/uconv/pkgdata.inc
|
||||
source/extra/uconv/pkgdatain.txt
|
||||
source/extra/uconv/resources/*.res
|
||||
source/extra/uconv/uconv
|
||||
source/extra/uconv/uconv.1
|
||||
source/extra/uconv/uconv.ncb
|
||||
source/extra/uconv/uconv.opt
|
||||
source/extra/uconv/uconv.plg
|
||||
source/extra/uconv/uconvmsg
|
||||
source/extra/ustdio/*.d
|
||||
source/extra/ustdio/Debug
|
||||
source/extra/ustdio/Makefile
|
||||
source/extra/ustdio/Release
|
||||
source/extra/ustdio/libustdio.*
|
||||
source/i18n/*.ao
|
||||
source/i18n/*.d
|
||||
source/i18n/Debug
|
||||
source/i18n/Makefile
|
||||
source/i18n/Release
|
||||
source/i18n/libicu*.*
|
||||
source/icudefs.mk
|
||||
source/layout/*.d
|
||||
source/layout/*.pdb
|
||||
source/layout/.#*
|
||||
source/layout/Debug
|
||||
source/layout/Makefile
|
||||
source/layout/Release
|
||||
source/layout/libicule.so.20.0
|
||||
source/layout/tmp
|
||||
source/samples/Makefile
|
||||
source/samples/cal/*.d
|
||||
source/samples/cal/*.pdb
|
||||
source/samples/cal/Debug
|
||||
source/samples/cal/Makefile
|
||||
source/samples/cal/Release
|
||||
source/samples/cal/icucal
|
||||
source/samples/date/*.d
|
||||
source/samples/date/*.pdb
|
||||
source/samples/date/Debug
|
||||
source/samples/date/Makefile
|
||||
source/samples/date/Release
|
||||
source/samples/date/icudate
|
||||
source/samples/layout/*.d
|
||||
source/samples/layout/*.pdb
|
||||
source/samples/layout/.#*
|
||||
source/samples/layout/Debug
|
||||
source/samples/layout/Makefile
|
||||
source/samples/layout/Release
|
||||
source/samples/layout/tmp
|
||||
source/stubdata/*.ao
|
||||
source/stubdata/*.d
|
||||
source/stubdata/*.plg
|
||||
source/stubdata/Debug
|
||||
source/stubdata/Makefile
|
||||
source/stubdata/Release
|
||||
source/stubdata/libicu*.*
|
||||
source/test/Makefile
|
||||
source/test/cintltst/*.d
|
||||
source/test/cintltst/*.plg
|
||||
source/test/cintltst/Debug
|
||||
source/test/cintltst/Makefile
|
||||
source/test/cintltst/Release
|
||||
source/test/cintltst/cintltst
|
||||
source/test/collperf/Makefile
|
||||
source/test/ieeetest/*.d
|
||||
source/test/ieeetest/Debug
|
||||
source/test/ieeetest/Makefile
|
||||
source/test/ieeetest/Release
|
||||
source/test/ieeetest/ieeetest
|
||||
source/test/intltest/*.d
|
||||
source/test/intltest/*.plg
|
||||
source/test/intltest/Debug
|
||||
source/test/intltest/Makefile
|
||||
source/test/intltest/Release
|
||||
source/test/intltest/intltest
|
||||
source/test/iotest/*.d
|
||||
source/test/iotest/*.ncb
|
||||
source/test/iotest/*.opt
|
||||
source/test/iotest/*.plg
|
||||
source/test/iotest/Debug
|
||||
source/test/iotest/Makefile
|
||||
source/test/iotest/Release
|
||||
source/test/letest/*.d
|
||||
source/test/letest/*.pdb
|
||||
source/test/letest/.#*
|
||||
source/test/letest/Debug
|
||||
source/test/letest/Makefile
|
||||
source/test/letest/Release
|
||||
source/test/letest/tmp
|
||||
source/test/testdata/Makefile
|
||||
source/test/testdata/out
|
||||
source/test/testmap/*.d
|
||||
source/test/testmap/Debug
|
||||
source/test/testmap/Makefile
|
||||
source/test/testmap/Release
|
||||
source/test/testmap/testmap
|
||||
source/test/testmap/testmap.plg
|
||||
source/test/thaitest/Makefile
|
||||
source/test/threadtest/Makefile
|
||||
source/tools/*.d
|
||||
source/tools/*.pdb
|
||||
source/tools/*.plg
|
||||
source/tools/Debug
|
||||
source/tools/Makefile
|
||||
source/tools/Release
|
||||
source/tools/ctestfw/*.d
|
||||
source/tools/ctestfw/*.pdb
|
||||
source/tools/ctestfw/Debug
|
||||
source/tools/ctestfw/Makefile
|
||||
source/tools/ctestfw/Release
|
||||
source/tools/ctestfw/libicuctestfw*
|
||||
source/tools/ctestfw/pkgdata
|
||||
source/tools/ctestfw/tmp
|
||||
source/tools/dumpce/*.css
|
||||
source/tools/dumpce/*.d
|
||||
source/tools/dumpce/*.html
|
||||
source/tools/dumpce/*.o
|
||||
source/tools/dumpce/Makefile
|
||||
source/tools/dumpce/dumpce
|
||||
source/tools/genbrk/*.d
|
||||
source/tools/genbrk/*.pdb
|
||||
source/tools/genbrk/*.plg
|
||||
source/tools/genbrk/Debug
|
||||
source/tools/genbrk/Makefile
|
||||
source/tools/genbrk/Release
|
||||
source/tools/genbrk/genbrk
|
||||
source/tools/genccode/*.d
|
||||
source/tools/genccode/*.pdb
|
||||
source/tools/genccode/*.plg
|
||||
source/tools/genccode/Debug
|
||||
source/tools/genccode/Makefile
|
||||
source/tools/genccode/Release
|
||||
source/tools/genccode/genccode
|
||||
source/tools/genccode/genccode.8
|
||||
source/tools/genccode/tmp
|
||||
source/tools/gencmn/*.d
|
||||
source/tools/gencmn/*.pdb
|
||||
source/tools/gencmn/*.plg
|
||||
source/tools/gencmn/Debug
|
||||
source/tools/gencmn/Makefile
|
||||
source/tools/gencmn/Release
|
||||
source/tools/gencmn/decmn
|
||||
source/tools/gencmn/decmn.[0-9]
|
||||
source/tools/gencmn/gencmn
|
||||
source/tools/gencmn/gencmn.[0-9]
|
||||
source/tools/gencmn/tmp
|
||||
source/tools/gencnval/*.d
|
||||
source/tools/gencnval/*.pdb
|
||||
source/tools/gencnval/Debug
|
||||
source/tools/gencnval/Makefile
|
||||
source/tools/gencnval/Release
|
||||
source/tools/gencnval/convrtrs.txt.[0-9]
|
||||
source/tools/gencnval/gencnval
|
||||
source/tools/gencnval/gencnval.[0-9]
|
||||
source/tools/gencnval/tmp
|
||||
source/tools/gennames/*.d
|
||||
source/tools/gennames/*.pdb
|
||||
source/tools/gennames/Debug
|
||||
source/tools/gennames/Makefile
|
||||
source/tools/gennames/Release
|
||||
source/tools/gennames/gennames
|
||||
source/tools/gennames/gennames.8
|
||||
source/tools/gennames/tmp
|
||||
source/tools/gennorm/*.d
|
||||
source/tools/gennorm/*.pdb
|
||||
source/tools/gennorm/*.plg
|
||||
source/tools/gennorm/Debug
|
||||
source/tools/gennorm/Makefile
|
||||
source/tools/gennorm/Release
|
||||
source/tools/gennorm/gennorm
|
||||
source/tools/gennorm/gennorm.[0-9]
|
||||
source/tools/gennorm/tmp
|
||||
source/tools/genpname/*.d
|
||||
source/tools/genpname/*.pdb
|
||||
source/tools/genpname/Debug
|
||||
source/tools/genpname/Makefile
|
||||
source/tools/genpname/Release
|
||||
source/tools/genpname/genpname
|
||||
source/tools/genprops/*.d
|
||||
source/tools/genprops/*.pdb
|
||||
source/tools/genprops/Debug
|
||||
source/tools/genprops/Makefile
|
||||
source/tools/genprops/Release
|
||||
source/tools/genprops/genprops
|
||||
source/tools/genprops/genprops.[0-9]
|
||||
source/tools/genprops/tmp
|
||||
source/tools/genrb/*.d
|
||||
source/tools/genrb/*.pdb
|
||||
source/tools/genrb/*.plg
|
||||
source/tools/genrb/Debug
|
||||
source/tools/genrb/Makefile
|
||||
source/tools/genrb/Release
|
||||
source/tools/genrb/derb
|
||||
source/tools/genrb/derb.[0-9]
|
||||
source/tools/genrb/derb_*
|
||||
source/tools/genrb/genjp/Debug
|
||||
source/tools/genrb/genjp/genjp.ncb
|
||||
source/tools/genrb/genjp/genjp.opt
|
||||
source/tools/genrb/genjp/genjp.plg
|
||||
source/tools/genrb/genjp/genjp.positions
|
||||
source/tools/genrb/genrb
|
||||
source/tools/genrb/genrb.[0-9]
|
||||
source/tools/genrb/tmp
|
||||
source/tools/gentest/*.d
|
||||
source/tools/gentest/*.pdb
|
||||
source/tools/gentest/Debug
|
||||
source/tools/gentest/Makefile
|
||||
source/tools/gentest/Release
|
||||
source/tools/gentest/gentest
|
||||
source/tools/gentest/pkgdata
|
||||
source/tools/gentest/tmp
|
||||
source/tools/gentz/*.d
|
||||
source/tools/gentz/*.pdb
|
||||
source/tools/gentz/Debug
|
||||
source/tools/gentz/Makefile
|
||||
source/tools/gentz/Release
|
||||
source/tools/gentz/gentz
|
||||
source/tools/gentz/gentz.[0-9]
|
||||
source/tools/gentz/tmp
|
||||
source/tools/genuca/*.d
|
||||
source/tools/genuca/*.pdb
|
||||
source/tools/genuca/Debug
|
||||
source/tools/genuca/Makefile
|
||||
source/tools/genuca/Release
|
||||
source/tools/genuca/genuca
|
||||
source/tools/genuca/genuca.8
|
||||
source/tools/genuca/tmp
|
||||
source/tools/icupkg.inc
|
||||
source/tools/makeconv/*.d
|
||||
source/tools/makeconv/*.pdb
|
||||
source/tools/makeconv/*.plg
|
||||
source/tools/makeconv/Debug
|
||||
source/tools/makeconv/Makefile
|
||||
source/tools/makeconv/Release
|
||||
source/tools/makeconv/makeconv
|
||||
source/tools/makeconv/makeconv.[0-9]
|
||||
source/tools/makeconv/tmp
|
||||
source/tools/pkgdata/*.d
|
||||
source/tools/pkgdata/*.ncb
|
||||
source/tools/pkgdata/*.opt
|
||||
source/tools/pkgdata/*.pdb
|
||||
source/tools/pkgdata/Debug
|
||||
source/tools/pkgdata/Makefile
|
||||
source/tools/pkgdata/Release
|
||||
source/tools/pkgdata/icupkg.inc
|
||||
source/tools/pkgdata/pkgdata
|
||||
source/tools/pkgdata/pkgdata.[0-9]
|
||||
source/tools/pkgdata/tmp
|
||||
source/tools/tmp*
|
||||
source/tools/toolutil/*.d
|
||||
source/tools/toolutil/*.pdb
|
||||
source/tools/toolutil/Debug
|
||||
source/tools/toolutil/Makefile
|
||||
source/tools/toolutil/Release
|
||||
source/tools/toolutil/libicu*
|
||||
source/tools/toolutil/pkgdata
|
||||
source/tools/toolutil/tmp
|
||||
|
||||
@@ -1,14 +0,0 @@
|
||||
# Contributing to ICU
|
||||
|
||||
Thank you for wanting to contribute to ICU!
|
||||
|
||||
Please see our instructions at <https://icu.unicode.org/processes/contribute>
|
||||
|
||||
### License
|
||||
|
||||
Please see [./icu4c/LICENSE](./icu4c/LICENSE) (C and J are under an identical license file.)
|
||||
|
||||
> Copyright © 2021 and later Unicode, Inc. and others. All Rights Reserved.
|
||||
Unicode and the Unicode Logo are registered trademarks
|
||||
of Unicode, Inc. in the U.S. and other countries.
|
||||
[Terms of Use and License](http://www.unicode.org/copyright.html)
|
||||
@@ -1,962 +0,0 @@
|
||||
This file contains the PGP keys of ICU developers.
|
||||
Compare to http://www.apache.org/dist/ant/KEYS and read https://www.apache.org/dev/openpgp.html for background
|
||||
|
||||
To import these keys:
|
||||
gpg --import KEYS
|
||||
|
||||
To update this file:
|
||||
replace $KEYID is your key's ID:
|
||||
(gpg --list-sigs $KEYID && gpg --armor --export $KEYID ) >> KEYS
|
||||
|
||||
|
||||
pub 2048R/825EA2CD 2010-11-02
|
||||
uid ICU Project @IBM (International Components for Unicode Development at IBM) <icuintl@us.ibm.com>
|
||||
sig 3 825EA2CD 2010-11-02 ICU Project @IBM (International Components for Unicode Development at IBM) <icuintl@us.ibm.com>
|
||||
sig 3 FD8FABF1 2013-12-12 Steven R. Loomis (ICU Project) <srl@icu-project.org>
|
||||
sig 0F0DE47D 2014-09-17 Steven R. Loomis (CODE SIGNING KEY) <srl@icu-project.org>
|
||||
sub 2048R/1D06B1BF 2010-11-02
|
||||
sig 825EA2CD 2010-11-02 ICU Project @IBM (International Components for Unicode Development at IBM) <icuintl@us.ibm.com>
|
||||
|
||||
-----BEGIN PGP PUBLIC KEY BLOCK-----
|
||||
Version: GnuPG v2.0.14 (GNU/Linux)
|
||||
|
||||
mQENBEzQQMUBCACbwbw7tuTWgwPsDAdQTWGO355jP75oBLHwGgEwV+OCKtxkNXNw
|
||||
wrJqXst83vmD1dEJyHflQww+d+Olj90IefQGfR+K7O005C2nky7eNGIomxaP52Y/
|
||||
90+tmw8qtsI4nsPWPuVj4WdFvlFgUwIZ0SmX4CauVzg0Ris8f0taxg7PH9zEvICs
|
||||
G/WAXdB9em08WDD6ruhMAvDF4W8Yy7mpGmdWiFD+B9OC006tv+GzYAvUHRFeCnnT
|
||||
SoKRiBeLejW+t4kpdMnEfC9ILAYBEEjNYvBIyPdPKBwNfy0yjRebsUf0eNmjGTpk
|
||||
VPlfofjVaUaOZytUOQvntYpocMX+377DGQIdABEBAAG0X0lDVSBQcm9qZWN0IEBJ
|
||||
Qk0gKEludGVybmF0aW9uYWwgQ29tcG9uZW50cyBmb3IgVW5pY29kZSBEZXZlbG9w
|
||||
bWVudCBhdCBJQk0pIDxpY3VpbnRsQHVzLmlibS5jb20+iQE4BBMBAgAiBQJM0EDF
|
||||
AhsDBgsJCAcDAgYVCAIJCgsEFgIDAQIeAQIXgAAKCRBEznvygl6izdyHB/0c++b/
|
||||
5s355bj5P7AuMxLsMCIu60hkBN3O4IBkI5zRah+rdChAlbDLIDbqepKTyTNYIKPT
|
||||
m3szJAy3g8e8C7Spn+p8I2HFNrLg1uKmeiO1bJoorseRQXb0tJuwgpmkAaXWzCX/
|
||||
ZQFPygWzJXKeREdI8gTyFjGMwUyfpIfAGD8cz0f+AYgCiLGM9VXa4jXPXOLco8R/
|
||||
TeHe09Pu914RWNSQIFsI+dGyzdrHG9taHJ5K0q71iiZOEGoWBQWlCMq8bHrImvjP
|
||||
ipHyWdwowYg3bDNrs25GMexiS7O5rrJyezchIWbcV/54vTcNQy4co7hZCad/Ar4B
|
||||
SYb1E6lBZnFf3FmSiQEcBBMBAgAGBQJSqRp0AAoJEKyl2+H9j6vxhG0IAKsbN6WA
|
||||
07JZTqY9oyaidLWMuBUYjb4ETSCsy1jvrlv1WJbjNOvS59ilDmnxA3sRKQBY0CSI
|
||||
IhBjc+Pm3478F28w0bSj3a4lw3T6EwX1UJMqkzNP5vGrkO8ZOMa561uTtQ6rCc8p
|
||||
/QS2+elXSAbbn7ytg7dAjEdCgViC99b/0Q7Ixi/UgXpIkaXYJ8DWU+TMtbCfXceK
|
||||
cKTLT7VlYnmlHgCwHOBs51TzqOsB91rXcvW+4LA3EZ2ITRQYAkWq5POn7cy7bVsd
|
||||
DS8FyHuU8qgUHCp4m9SHkb0gokQlIkwBjFA18TZJ1WKyGQQRZQVEUoPHoXSjFC42
|
||||
il70JdaiGTAtRDqJAhwEEAECAAYFAlQZ8l4ACgkQqqmunA8N5H3MIw/+K4G0Zce6
|
||||
7a1CBTM0YyIwnkWBST98IxcqOLJfrq5rQlKEwn7J+CLlwekDq5C3rlYxHtVQ/6/V
|
||||
Ncj3Uz/LEURrZZFr5dylTkFtkKnvDgVvJdyUA+RcGnQSM5q+6JthDul1NvPTZE9M
|
||||
rmKwK1DGrSJ1lsszFvk2tS3mq1IZOP3KH74o22Ks7ONP3GM8TDgGxy+pfGOzfEap
|
||||
Q0kwwG9wnybH4suYKUtpPm5BR1yQkktCnOoJLV5QYEoYQLAF2ez74jVjffEByl6S
|
||||
GVr6CaqscaRLKtm7sp6p/hCKOkV7rdIy0UEtMGluJCY8y9My+ij+SaTM2F0nl71F
|
||||
8zjY0SoKtXKo8GsucIMT1Ow9+igGDto4RzNnNd8MkhoMWKizGlOWrFR+TK/zpaj8
|
||||
AVx2Cw7OPPh4vB8yPa3x7u9cXJcejzryMStADkQg3fFSKdcELNOjIKTAe/+BEAiK
|
||||
DAs6Ka1/yRpjz7esWqYjI1hU70wwjXIPPMwAig4IYfvTBE02ilUO0NETUPWSiwfz
|
||||
8ZONAFjMCkxb7Vm0dhDVzVt4vCPWUop2Ip3QjnXCmDCj1KxoolYaw4fqKh+1khHM
|
||||
1Aspk9nSfLG4VrXHTCYjLYD1OpPByOWiAqOJnbW/wqFhjKrrAHR5AyjsremtaQlv
|
||||
r3lmoQLs9qmdP4oeQG4nR4UqzhjG2bNLV9+5AQ0ETNBAxQEIANp35mwgILWWQI36
|
||||
F2SyWFTFF8zupjzqlcF4Vx1Mjs2rQYErv1qs09TJfO+uxPRAva0fKik9PFmZ0vLM
|
||||
7UAveZgaknzjTQveOaAJuT5OjudoyptG81ilHO66+e9RnuyV6gSINnt6wnq4n8KR
|
||||
PoEmI/nxMhvmBCPw+YmxG3ZcU5Zko4GtjLA4J01nag7jY+LH08qFfPcKsADmx7Et
|
||||
CRuBcXgQ202kF3o7BjULcLV3R8vni3jp5nljH0vJG5USSY2ZR1bFiLtTCMO8cm+j
|
||||
Iuay25iWil5DaejASIXzbzTtSYg9Skxet29AWjlrt1+zKZsMSLPbKJ6qhdW5XNU4
|
||||
Qo9ycx0AEQEAAYkBHwQYAQIACQUCTNBAxQIbDAAKCRBEznvygl6izZwgB/4gOqGH
|
||||
c0Cc165PZggwbf1nyJZ+af0QiKWasgoUzo4a7OSIISmBYPGnW/VP4EMsaNEhnqWK
|
||||
3Tu+cuidW6z7MWHRsqLcvYUKeiZE2IhnMP85ad2S14UOHE8cbkLwfIGe+rIu+zYp
|
||||
zyc5bcRYqeLvd5Dk1H/eW0/hsadW9qpSWQRGxWPEsMqaAhiret0ntlGH2igswZ7S
|
||||
YNYLNtK78wppFqqFHfFvnVu1XNmiTDDuk2IokPIGRjeWO59eSR/kotg0zox4S+CJ
|
||||
bRdQdPq2ofDNqKCIEtg6y4yM+dc7c7n1Txk53fMBMzXfGnLGfbTFLo1hpYdC5sep
|
||||
AZA1XWLRgZ3+5+vD
|
||||
=NkHr
|
||||
-----END PGP PUBLIC KEY BLOCK-----
|
||||
|
||||
|
||||
pub 4096R/0F0DE47D 2014-09-17 [expires: 2019-09-16]
|
||||
uid Steven R. Loomis (CODE SIGNING KEY) <srl@icu-project.org>
|
||||
sig 3 0F0DE47D 2014-09-17 Steven R. Loomis (CODE SIGNING KEY) <srl@icu-project.org>
|
||||
sig FD8FABF1 2014-09-17 Steven R. Loomis (ICU Project) <srl@icu-project.org>
|
||||
|
||||
-----BEGIN PGP PUBLIC KEY BLOCK-----
|
||||
Version: GnuPG v2.0.14 (GNU/Linux)
|
||||
|
||||
mQINBFQZ3doBEACpnA2AbRRVxu9TeDTnjHtz6h5NyMyGZIPZOX7Iv2bNS0JSAS/H
|
||||
9f2WP9+3ANHuKI32rYh0snKpWjYsgUCxZ8x2Wp4cu8YnTvvpL6a5fn5n04dTyd4s
|
||||
/gPLb51qHIL8L6MDfK2ARgukntzfLR2CoL+RwGLDOjNlnoGNw65XEdjkzZ65s05Q
|
||||
/YHdlMYJE6gQ7VzJf7JYO6z6MpclLSj106TDs0hmOSetRm66nt+G7q1iY/jMXSMm
|
||||
bs3CFe4MNoVlRnl2THFZWjGSrVNYkenKexrjJ5HD9sm2nFF5pXgMQBoyOE5gic/b
|
||||
H5hMeh3v4vmhxRL5GPmI9Ir25LwqYCaWZDu6TC8mDDJj82YQMOzvElpRU/5V8KVc
|
||||
R7e4FN7D6tXDPb8IMgHb1U7zEnKYAc+uAS5NQ6Vq/S3NoueEQNk+rwqy44yIigtD
|
||||
iOJOnLvf1JFUPNFUxRu2JUy7YzvvcyGG9PY7bxQEb/MLwa424GWRTHV4Yw+hHP6D
|
||||
+XgylCyd36wfwgF48+gFeDza9gMPPDgGqJu059PjEEG9ELgBFkTVjM0+rqq1M6Jk
|
||||
+WPFRgM9ioccwdVIvfVZNnkp+EUjpl/PHLYlFvP0orkK3SGc1jaqW5m3Y6P60RzM
|
||||
7OovZqtqUDTEfMZ4bQKT4EdLayBPFALvz4TgndxrZwoPi2QGmFU9Dy+snwARAQAB
|
||||
tDlTdGV2ZW4gUi4gTG9vbWlzIChDT0RFIFNJR05JTkcgS0VZKSA8c3JsQGljdS1w
|
||||
cm9qZWN0Lm9yZz6JAj4EEwECACgFAlQZ3doCGwMFCQlmAYAGCwkIBwMCBhUIAgkK
|
||||
CwQWAgMBAh4BAheAAAoJEKqprpwPDeR9St0P/RJzlQDOgtD/rq8ThV5clhcgzCQU
|
||||
J34bC9+SdvQVtG6cOu+/mE9fjPcRuMSC2saj3e6KdfdCF51ZUelFgg7Nf4VjzKXK
|
||||
egYFe11CJSYFt0YQyT3CwlX5a6lKv9BH4RJPdAjeACSNCLvuZYlkxfAFPAwJFMAH
|
||||
CeiY6P7Llm2sHHIIubK+yFviZiD1ic92ADt2llM06VwniCS2Cs0kx4NAl70jd/vW
|
||||
B/j9+k+PphZFA2IUuB80qnTdrEb+nVLQ9n0SMFwFm89bEm8IhXe0Lb//jl5ZbZCr
|
||||
afn02GvdSqU73jqOfEHS7hHemB1KSM4hEdWGFRhUf0+Xpc31aOsnKY5Wu89A8Sz/
|
||||
sKp554qAhGmBGeEjkKbw0t+k4liP+LW2efmVBt+/6msF+TGq0Rr5vFQEUNCYeIzz
|
||||
DS5nKDlKvRwgx/41RnSUYiLJS9GkCB6skxIQZvW2G7rnkGQkgmP3/tkTTX0EP0rT
|
||||
GlW4ivfp+AKoYozkZ5FNLXC06yzCSKYL0z9uoL38uR6opXo4768byzzkBOvSut3Q
|
||||
V9gbvAtZvwEibwOt0A4UhbIyv96+Oi2WI66q37Wl33F/QIsxpC9UjXju1RrAajOv
|
||||
0gY8VMxwuWHN8ewslykX5rohxgA7mkTf/YeELSiYEM1WpmKmj30uJapgDwVRpsTC
|
||||
dfSvIc/nOLji4u0iiQEcBBABAgAGBQJUGfLZAAoJEKyl2+H9j6vx+58IAMIzjJTv
|
||||
9n7Yq2iyncFyg7fmPmlKo28l6cIrmveGtzLWgO7pXw9hZy8mTwUAhg9nMnNbnjDw
|
||||
qSA+YH8rNy0MB/AaTj46faE/RlqB8bCUlnLlRk9b6IxKODOVAarFk0ltCUeNt1IP
|
||||
w8YziwUaJewmb2//tEjwG4RuliQeo8EPcg2MejsxctNJ1gWQxZFOSMk2M6I9NwW6
|
||||
xMUmqGzzKdPChdwzOBaHK2f4lB3IlPqpJVqAWuXdINzi91tCIKfo508hJSpzGm/n
|
||||
Lq4yQHJPjQjxTldFBxC8Ji5yYIQ2ZcWLaYndWH4aVbAkEelU3eIp44mU4tqhqCP6
|
||||
zJJjdh87G81yUoE=
|
||||
=pu4w
|
||||
-----END PGP PUBLIC KEY BLOCK-----
|
||||
pub 2048R/FD8FABF1 2013-12-12 [expires: 2020-10-03]
|
||||
uid Steven R. Loomis (ICU Project) <srl@icu-project.org>
|
||||
sig 3 FD8FABF1 2017-10-04 Steven R. Loomis (ICU Project) <srl@icu-project.org>
|
||||
uid [jpeg image of size 10375]
|
||||
sig 3 FD8FABF1 2017-10-04 Steven R. Loomis (ICU Project) <srl@icu-project.org>
|
||||
sig 3 FD8FABF1 2013-12-12 Steven R. Loomis (ICU Project) <srl@icu-project.org>
|
||||
sig 3 FD8FABF1 2013-12-12 Steven R. Loomis (ICU Project) <srl@icu-project.org>
|
||||
sub 2048R/A6EB1469 2013-12-12 [expires: 2019-10-04]
|
||||
sig FD8FABF1 2017-10-04 Steven R. Loomis (ICU Project) <srl@icu-project.org>
|
||||
|
||||
-----BEGIN PGP PUBLIC KEY BLOCK-----
|
||||
|
||||
mQENBFKpD3cBCADklZIzuPsYrEy08rASB+MRnFF8hqDDUkDHuvS0bJwEDge2EEHT
|
||||
l9rnUOt3SGtSrokeniQbwut26vuY/V59SSpwClytbHovIRJ/ovjq3nUasCRPOxDL
|
||||
mlmftkRGrO6WYH6hSzmRy2x3NTgQOlE/YRtOya3CAVlkiybpYXL3L1mJA4e80NpF
|
||||
V4UG7vyfw44OHFeO6mInqkcxVgKkxwp3wBbck3FEySEZT0DTT+sQHI52fgUftb+P
|
||||
xBH1s58lLAub4o2z1hnImPN/BR0lZpiNj59vzNlqmcnRHTpa6KIz+Qeep9dwLL7i
|
||||
71lMfCsEgVyDZxkQxFpNoRYRHRkfvBEY7JLpABEBAAG0NFN0ZXZlbiBSLiBMb29t
|
||||
aXMgKElDVSBQcm9qZWN0KSA8c3JsQGljdS1wcm9qZWN0Lm9yZz6JAT4EEwECACgC
|
||||
GwMGCwkIBwMCBhUIAgkKCwQWAgMBAh4BAheABQJZ1XNBBQkMz/5KAAoJEKyl2+H9
|
||||
j6vxKIEH/0aFGlFOtgXNHKEG7F1mVxhjNyFlZD5O/9t8oI46vYCgVfwsn6BsHFpv
|
||||
9yAtzA6z26Zt2NsPjH6dVRfc3sGCojBIKQ/lxnyQqjAYaEbVVQ/3jhAhgnm+uAF4
|
||||
3Rr54g+6FL7aHB33nDy9FmRtmQsyVf1OySR7VXbHpwEVzWeLrgmPKR1t8P1hdiSo
|
||||
YfroXVAPy2pynXUWgWbMbuqAcTj0iAQQ7LuVOVZ8bYA922LRgUxg/ZPxiGGm2g3M
|
||||
LF/QVWMH6FAHtSf9DOaFjZqQuypkUoOZZuFfNMvKaYIe+XviZRtWls5M6H8wOz6c
|
||||
bFrLBV5XBBwKbOXULpDv/3+oehDO22vR/wAAKJ3/AAAomAEQAAEBAAAAAAAAAAAA
|
||||
AAAA/9j/4AAQSkZJRgABAQEASABIAAD/4hscSUNDX1BST0ZJTEUAAQEAABsMYXBw
|
||||
bAIQAABtbnRyUkdCIFhZWiAH3QALAAUADwAzADFhY3NwQVBQTAAAAAAAAAAAAAAA
|
||||
AAAAAAAAAAAAAAAAAAAA9tYAAQAAAADTLWFwcGwAAAAAAAAAAAAAAAAAAAAAAAAA
|
||||
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABFkZXNjAAABUAAAAGJkc2NtAAAB
|
||||
tAAABA5jcHJ0AAAFxAAAACN3dHB0AAAF6AAAABRyWFlaAAAF/AAAABRnWFlaAAAG
|
||||
EAAAABRiWFlaAAAGJAAAABRyVFJDAAAGOAAACAxhYXJnAAAORAAAACB2Y2d0AAAO
|
||||
ZAAABhJuZGluAAAUeAAABj5jaGFkAAAauAAAACxtbW9kAAAa5AAAAChiVFJDAAAG
|
||||
OAAACAxnVFJDAAAGOAAACAxhYWJnAAAORAAAACBhYWdnAAAORAAAACBkZXNjAAAA
|
||||
AAAAAAhEaXNwbGF5AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
|
||||
AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAbWx1
|
||||
YwAAAAAAAAAhAAAADGhySFIAAAAUAAABnGtvS1IAAAAMAAABsG5iTk8AAAASAAAB
|
||||
vGlkAAAAAAASAAABzmh1SFUAAAAUAAAB4GNzQ1oAAAAWAAAB9GRhREsAAAAcAAAC
|
||||
CnVrVUEAAAAcAAACJmFyAAAAAAAUAAACQml0SVQAAAAUAAACVnJvUk8AAAASAAAC
|
||||
am5sTkwAAAAWAAACfGhlSUwAAAAWAAACkmVzRVMAAAASAAACamZpRkkAAAAQAAAC
|
||||
qHpoVFcAAAAMAAACuHZpVk4AAAAOAAACxHNrU0sAAAAWAAAC0npoQ04AAAAMAAAC
|
||||
uHJ1UlUAAAAkAAAC6GZyRlIAAAAWAAADDG1zAAAAAAASAAADImNhRVMAAAAYAAAD
|
||||
NHRoVEgAAAAMAAADTGRlREUAAAAQAAADWGVuVVMAAAASAAADaHB0QlIAAAAYAAAD
|
||||
enBsUEwAAAASAAADkmVsR1IAAAAiAAADpHN2U0UAAAAQAAADxnRyVFIAAAAUAAAD
|
||||
1mphSlAAAAAOAAAD6nB0UFQAAAAWAAAD+ABMAEMARAAgAHUAIABiAG8AagBpzuy3
|
||||
7AAgAEwAQwBEAEYAYQByAGcAZQAtAEwAQwBEAEwAQwBEACAAVwBhAHIAbgBhAFMA
|
||||
egDtAG4AZQBzACAATABDAEQAQgBhAHIAZQB2AG4A/QAgAEwAQwBEAEwAQwBEAC0A
|
||||
ZgBhAHIAdgBlAHMAawDmAHIAbQQaBD4EOwRMBD4EQAQ+BDIEOAQ5ACAATABDAEQg
|
||||
DwBMAEMARAAgBkUGRAZIBkYGKQBMAEMARAAgAGMAbwBsAG8AcgBpAEwAQwBEACAA
|
||||
YwBvAGwAbwByAEsAbABlAHUAcgBlAG4ALQBMAEMARCAPAEwAQwBEACAF5gXRBeIF
|
||||
1QXgBdkAVgDkAHIAaQAtAEwAQwBEX2mCcgAgAEwAQwBEAEwAQwBEACAATQDgAHUA
|
||||
RgBhAHIAZQBiAG4A6QAgAEwAQwBEBCYEMgQ1BEIEPQQ+BDkAIAQWBBoALQQ0BDgE
|
||||
QQQ/BDsENQQ5AEwAQwBEACAAYwBvAHUAbABlAHUAcgBXAGEAcgBuAGEAIABMAEMA
|
||||
RABMAEMARAAgAGUAbgAgAGMAbwBsAG8AcgBMAEMARAAgDioONQBGAGEAcgBiAC0A
|
||||
TABDAEQAQwBvAGwAbwByACAATABDAEQATABDAEQAIABDAG8AbABvAHIAaQBkAG8A
|
||||
SwBvAGwAbwByACAATABDAEQDiAOzA8cDwQPJA7wDtwAgA78DuAPMA70DtwAgAEwA
|
||||
QwBEAEYA5AByAGcALQBMAEMARABSAGUAbgBrAGwAaQAgAEwAQwBEMKsw6TD8ACAA
|
||||
TABDAEQATABDAEQAIABhACAAQwBvAHIAZQBzAAB0ZXh0AAAAAENvcHlyaWdodCBB
|
||||
cHBsZSBJbmMuLCAyMDEzAABYWVogAAAAAAAA81IAAQAAAAEWz1hZWiAAAAAAAABq
|
||||
egAAPKoAAAYFWFlaIAAAAAAAAGhwAACq+AAAH7dYWVogAAAAAAAAI+wAABheAACt
|
||||
cWN1cnYAAAAAAAAEAAAAAAUACgAPABQAGQAeACMAKAAtADIANgA7AEAARQBKAE8A
|
||||
VABZAF4AYwBoAG0AcgB3AHwAgQCGAIsAkACVAJoAnwCjAKgArQCyALcAvADBAMYA
|
||||
ywDQANUA2wDgAOUA6wDwAPYA+wEBAQcBDQETARkBHwElASsBMgE4AT4BRQFMAVIB
|
||||
WQFgAWcBbgF1AXwBgwGLAZIBmgGhAakBsQG5AcEByQHRAdkB4QHpAfIB+gIDAgwC
|
||||
FAIdAiYCLwI4AkECSwJUAl0CZwJxAnoChAKOApgCogKsArYCwQLLAtUC4ALrAvUD
|
||||
AAMLAxYDIQMtAzgDQwNPA1oDZgNyA34DigOWA6IDrgO6A8cD0wPgA+wD+QQGBBME
|
||||
IAQtBDsESARVBGMEcQR+BIwEmgSoBLYExATTBOEE8AT+BQ0FHAUrBToFSQVYBWcF
|
||||
dwWGBZYFpgW1BcUF1QXlBfYGBgYWBicGNwZIBlkGagZ7BowGnQavBsAG0QbjBvUH
|
||||
BwcZBysHPQdPB2EHdAeGB5kHrAe/B9IH5Qf4CAsIHwgyCEYIWghuCIIIlgiqCL4I
|
||||
0gjnCPsJEAklCToJTwlkCXkJjwmkCboJzwnlCfsKEQonCj0KVApqCoEKmAquCsUK
|
||||
3ArzCwsLIgs5C1ELaQuAC5gLsAvIC+EL+QwSDCoMQwxcDHUMjgynDMAM2QzzDQ0N
|
||||
Jg1ADVoNdA2ODakNww3eDfgOEw4uDkkOZA5/DpsOtg7SDu4PCQ8lD0EPXg96D5YP
|
||||
sw/PD+wQCRAmEEMQYRB+EJsQuRDXEPURExExEU8RbRGMEaoRyRHoEgcSJhJFEmQS
|
||||
hBKjEsMS4xMDEyMTQxNjE4MTpBPFE+UUBhQnFEkUahSLFK0UzhTwFRIVNBVWFXgV
|
||||
mxW9FeAWAxYmFkkWbBaPFrIW1hb6Fx0XQRdlF4kXrhfSF/cYGxhAGGUYihivGNUY
|
||||
+hkgGUUZaxmRGbcZ3RoEGioaURp3Gp4axRrsGxQbOxtjG4obshvaHAIcKhxSHHsc
|
||||
oxzMHPUdHh1HHXAdmR3DHeweFh5AHmoelB6+HukfEx8+H2kflB+/H+ogFSBBIGwg
|
||||
mCDEIPAhHCFIIXUhoSHOIfsiJyJVIoIiryLdIwojOCNmI5QjwiPwJB8kTSR8JKsk
|
||||
2iUJJTglaCWXJccl9yYnJlcmhya3JugnGCdJJ3onqyfcKA0oPyhxKKIo1CkGKTgp
|
||||
aymdKdAqAio1KmgqmyrPKwIrNitpK50r0SwFLDksbiyiLNctDC1BLXYtqy3hLhYu
|
||||
TC6CLrcu7i8kL1ovkS/HL/4wNTBsMKQw2zESMUoxgjG6MfIyKjJjMpsy1DMNM0Yz
|
||||
fzO4M/E0KzRlNJ402DUTNU01hzXCNf02NzZyNq426TckN2A3nDfXOBQ4UDiMOMg5
|
||||
BTlCOX85vDn5OjY6dDqyOu87LTtrO6o76DwnPGU8pDzjPSI9YT2hPeA+ID5gPqA+
|
||||
4D8hP2E/oj/iQCNAZECmQOdBKUFqQaxB7kIwQnJCtUL3QzpDfUPARANER0SKRM5F
|
||||
EkVVRZpF3kYiRmdGq0bwRzVHe0fASAVIS0iRSNdJHUljSalJ8Eo3Sn1KxEsMS1NL
|
||||
mkviTCpMcky6TQJNSk2TTdxOJU5uTrdPAE9JT5NP3VAnUHFQu1EGUVBRm1HmUjFS
|
||||
fFLHUxNTX1OqU/ZUQlSPVNtVKFV1VcJWD1ZcVqlW91dEV5JX4FgvWH1Yy1kaWWlZ
|
||||
uFoHWlZaplr1W0VblVvlXDVchlzWXSddeF3JXhpebF69Xw9fYV+zYAVgV2CqYPxh
|
||||
T2GiYfViSWKcYvBjQ2OXY+tkQGSUZOllPWWSZedmPWaSZuhnPWeTZ+loP2iWaOxp
|
||||
Q2maafFqSGqfavdrT2una/9sV2yvbQhtYG25bhJua27Ebx5veG/RcCtwhnDgcTpx
|
||||
lXHwcktypnMBc11zuHQUdHB0zHUodYV14XY+dpt2+HdWd7N4EXhueMx5KnmJeed6
|
||||
RnqlewR7Y3vCfCF8gXzhfUF9oX4BfmJ+wn8jf4R/5YBHgKiBCoFrgc2CMIKSgvSD
|
||||
V4O6hB2EgITjhUeFq4YOhnKG14c7h5+IBIhpiM6JM4mZif6KZIrKizCLlov8jGOM
|
||||
yo0xjZiN/45mjs6PNo+ekAaQbpDWkT+RqJIRknqS45NNk7aUIJSKlPSVX5XJljSW
|
||||
n5cKl3WX4JhMmLiZJJmQmfyaaJrVm0Kbr5wcnImc951kndKeQJ6unx2fi5/6oGmg
|
||||
2KFHobaiJqKWowajdqPmpFakx6U4pammGqaLpv2nbqfgqFKoxKk3qamqHKqPqwKr
|
||||
davprFys0K1ErbiuLa6hrxavi7AAsHWw6rFgsdayS7LCszizrrQltJy1E7WKtgG2
|
||||
ebbwt2i34LhZuNG5SrnCuju6tbsuu6e8IbybvRW9j74KvoS+/796v/XAcMDswWfB
|
||||
48JfwtvDWMPUxFHEzsVLxcjGRsbDx0HHv8g9yLzJOsm5yjjKt8s2y7bMNcy1zTXN
|
||||
tc42zrbPN8+40DnQutE80b7SP9LB00TTxtRJ1MvVTtXR1lXW2Ndc1+DYZNjo2WzZ
|
||||
8dp22vvbgNwF3IrdEN2W3hzeot8p36/gNuC94UThzOJT4tvjY+Pr5HPk/OWE5g3m
|
||||
lucf56noMui86Ubp0Opb6uXrcOv77IbtEe2c7ijutO9A78zwWPDl8XLx//KM8xnz
|
||||
p/Q09ML1UPXe9m32+/eK+Bn4qPk4+cf6V/rn+3f8B/yY/Sn9uv5L/tz/bf//cGFy
|
||||
YQAAAAAAAwAAAAJmZgAA8qcAAA1ZAAAT0AAACg52Y2d0AAAAAAAAAAAAAwEAAAIA
|
||||
AAAIACQAVwCkAQYBRgGTAe8CTAK0AycDoAQnBLoFWAYABrUHeAhGCRsJ+ArhC9IM
|
||||
zg3RDxQQaBG8ExkUdhXZF0kYuBotG6YdKx6vIDghyiNgJPomkygwKdUrZSzsLnUv
|
||||
9zF5Mvk0bzXjN084sToMO2M8tD3+P0VAhUHFQvlEHkVFRmpHkUi2SdxLAUwmTUlO
|
||||
bU+PULFR01L2VBlVPFZSV2pYf1mWWq1bw1zZXe5fAmAWYSliPGNOZGFldGaBZ4Bo
|
||||
gmmDaoRrhmyJbYpujG+OcJFxk3KVc5d0m3WVdnJ3UHgteQt56nrIe6Z8hn1nfkh/
|
||||
KoANgPGB1oK7g5+EhYVxhmOHX4hniXuKoYvXjR2Ob4/PkTySkJOylNaV+JcbmECZ
|
||||
ZpqOm7mc6Z4cn1KgjaHKovikDqUmpj+nWah3qZiqvKvlrRKuQ694sLGx67MftFC1
|
||||
frart9W4/boku0q8b72Vvr2/5cEPwjbDUcRhxWPGVsc7yBLI3MmbylXLDcvFzILN
|
||||
Rs4Rzt/PrdB80UrSGdLn07PUf9VJ1hLW2teZ2FHZBtm82nTbLtvo3KXdYt4h3uDf
|
||||
n+Bf4R/h3+Kc41bkDuTD5XPmIObL53LoGOi86WTqD+rC63XsKOzd7ZLuSO7/77jw
|
||||
c/Ev8ezywfOp9JL1dvZa9z/4Kvkf+iP7QvyG/gv//wAAAAcAHgBKAIsA4wEqAW4B
|
||||
wAIZAnQC2gNHA70EQQTSBW0GEQa9B3QINgkACdUKrQuODIENlg7MEAARPBJ6E7oV
|
||||
AxZPF6MY+BpUG7QdGh6GH/IhZCLWJE0lxycvKJMp9ytXLLMuCy9gMLAx+DM8NHc1
|
||||
rTbeOAo5MDpWO3c8kD2dPqw/ukDKQdlC6UP5RQpGGkcqSD1JTkpgS3FMg02UTp5P
|
||||
qVCyUbtSxlPQVNlV4lbqV/NY+1oDWwtcFF0dXiFfIWAfYR1iG2MZZBZlE2YPZwpo
|
||||
BWkAaftq9WvwbOdt2G7Ib7ZwpHGQcntzZHRLdTF2Fnb6d954wnmmeod7Z3xKfS5+
|
||||
FX8Cf/OA6oHpgvCD/YUShi2HTYhuiYuKp4vEjOKOAY8hkEORZpKNk7WU4JYNlzyY
|
||||
ZZmHmqqbzJzxnhefP6BpoZaixqP5pS6mZqegqNqqFKtOrIatvq72sCyxZLKbs9W1
|
||||
D7ZLt4m4x7oBuzq8cL2jvtG/+cEbwjbDTMRcxWnGcsdtyFrJQ8oryxPL+8zizcjO
|
||||
rs+T0HjRXtJD0yTUAdTd1bnWlNdv2ErZJdn/2tnbstyM3WbeQN8b3/fg1OGx4o3j
|
||||
auRH5STmAebd57nolOlp6j3rEevl7Lvtku5q70TwIPD98dzyu/Oc9I71lPas99X5
|
||||
C/pG+3n8mf2c/oD/Sf//AAAABQAWADUAZACkAPUBLQFqAbACAAJTArADFgOBA/UE
|
||||
cAT1BYMGHAa+B2UIEwjGCX4KQwsqDCYNJg4lDy4QNxFHElcTbhSGFaUWxxfxGRka
|
||||
SBt5HK0d4x8bIEQhaiKNI7Ak0yXwJwsoISk1KkArSCxKLUcuQS82MCoxHDIGMukz
|
||||
yTSqNY02bzdQODM5FDn1Otc7uDyaPXw+Xj9AQCJBBUHpQs5Ds0SYRX5GY0dISCxJ
|
||||
EEn1StlLvEyhTYZOaU9IUCdRBlHlUsRTolSBVV5WO1cZV/ZY01mvWo1baVxCXRtd
|
||||
817MX6Rge2FSYihi/mPTZKllfmZTZyhn+mjLaZxqbWs9bA1s3G2rbnlvRXAScN5x
|
||||
qXJ1cztz/3TBdYN2Q3cCd794e3kyeed6m3tKe/h8rH1xflp/R4A0gSKCEIL/g+2E
|
||||
2oXHhrOHnYiIiXCKaotpjGiNaI5pj2mQaJFmkmKTXZRVlUyWQZdEmFOZZ5p7m5Gc
|
||||
pp26ns2f3aDrofejAKQHpR6mU6ePqMyqC6tNrJGt2q8osHmxzrMntIO13LckuGi5
|
||||
prrcvAu9Mr5Sv2zAgMGRwp/DrcS4xcLGy8fWyOLJ78r+zA/NI846z1PQb9GN0rLT
|
||||
2dUB1irXU9h92ana1twF3TXeZ9+b4O/iheQo5d/nv+nQ7CHuuPGn9N/4XPwV//8A
|
||||
AG5kaW4AAAAAAAAGNgAAmj0AAFllAABRRwAAjYQAACXWAAAXCgAAUA0AAFQ5AAKK
|
||||
PQACRR4AAYzMAAMBAAACAAAACwAiAD0AWgB3AJUAswDRAPABEAEwAVEBcgGUAa8B
|
||||
ywHoAgYCJAJDAmMCgwKjAsQC5gMIAysDTgNxA5UDugPfBAQEKgRQBHcEngTGBO8F
|
||||
GAVBBWsFmAXHBfYGJgZXBooGvQbyBycHXweXB9IIDghMCI0IzwkTCVkJoQnsCjgK
|
||||
hwrXCykLgQveDDsMmgz6DVwNwA4lDowO9Q9gD8wQOhCqERwRjxIEEnoS8hNyE/IU
|
||||
dRT5FX8WBxaRFx0XrBg8GM8ZZBn7GpQbLxvLHGwdGR3GHnYfKB/cIJEhSiIEIsAj
|
||||
fiQ/JQIlxyaNJ1koRyk3KisrISwaLRYuFC8TMBUxGDIdMyM0LDU6Nkc3UThWOVI6
|
||||
RTsvPA885j22PoA/RkAIQMVBgEJRQ0JEM0UnRh1HE0gLSQNJ+0rzS+lM303UTshP
|
||||
u1CuUaZSwFPaVPVWEVctWEhZYlp7W5JcqF27Xs5f32DwYgBjFmQzZVJmdmecaMdp
|
||||
9WsmbFptkW7JcANxPXJ5c7R08HY6d5F4+np5fBR90H+2gcGD5YYNiCqKLYwnjiSQ
|
||||
JJIolDKWQ5hbmn2cpp7voV6jzKY4qKKrCq1yr9yySbS5ty65sLxFvvHBt8SXx43K
|
||||
lM2L0F/TN9YQ2Orbwt6W4WjkOOa26PrrVO258B7yevTC9uv46fq6/Fj9uv7p//8A
|
||||
AAANACcARgBnAIcAqADKAOwBDwEyAVcBfAGfAb4B3QH+Ah8CQQJkAogCrALRAvYD
|
||||
HANDA2oDkgO6A+MEDAQ2BGEEjAS4BOUFEgVABW4FoQXUBgkGPgZ1Bq4G5wcjB2AH
|
||||
nwffCCMIaAiwCPoJRwmWCegKPQqTCuwLRwurDA8MdgzeDUcNsw4gDo8O/w9yD+YQ
|
||||
WxDTEUwRxxJEEsITRhPMFFQU3xVrFfkWiRccF7EYSBjhGXwaGhq5G1ob/RymHVQe
|
||||
BB63H2wgJCDeIZsiWyMeI+MkqiV0JkEnDyfnKMQppSqJK3IsXi1PLkUvPzA9MT8y
|
||||
RDNMNFo1bDZ+N5E4oTmuOrc7uzy5PbM+qD+aQIpBd0JoQ11EVUVORkhHREhBST5K
|
||||
PEs6TDhNNk40TzNQMVEvUjlTSFRYVWpWfFePWKNZtlrJW9xc7l4AXxJgI2E1YkZj
|
||||
WWRuZYRmnme5aNdp9msYbDttYG6Gb6xw03H6cyJ0SnV0dqN31XkKekN7gXzFfhF/
|
||||
ZIDBgiaDlIULhomIDomzi3KNNI76kMOSkpRlljyYGJn3m9mdv5+6obujv6XIp9Wp
|
||||
56v9rhiwNrJZtH62pLjMuvW9Ib9RwYPDusX0yDTKd8zDzybRi9Pz1lrYwdsl3Yjf
|
||||
6eJK5KrnCukv6y7tFu7n8KryZ/Qr9gT4BPpD/OP//wAAABIANQBbAIMArADVAP4B
|
||||
KQFVAYMBqwHRAfgCIQJKAnUCoALNAvoDKQNYA4gDuQPqBB0EUASEBLkE7wUmBV4F
|
||||
mgXYBhgGWgadBuIHKgd0B8EIEAhjCLkJEwlxCdIKNwqfCwoLewvzDG4M6w1qDe0O
|
||||
cQ75D4MQEBCgETIRxxJeEvgTlBQyFNIVdRYaFsIXbRgbGMwZfxo2Gu8bqhxpHS8d
|
||||
+B7DH5IgZCE5IhIi7iPNJK8llSZ9J2koXClSKkwrSSxLLVAuWi9nMHgxjTKlM8A0
|
||||
5TYNNzk4aTmdOtY8FD1YPqA/7UE+QptEBEVxRuZIY0nrS4BNIU7RUIxSF1NkVLJW
|
||||
AldUWKhZ/1tbXLleHV+FYPFiYmO6ZRBmaGfBaRxqe2vdbURur3AgcZdzE3SUdfp3
|
||||
VniyehB7b3zQfjV/nYEKgnyD9IVyhvWIfonTix2MaI2zjv+QSZGTktyUI5VplqyX
|
||||
75kvmm+brpzsni6fgqDbojqjoaUQpoioC6mZqzGs1K6AsDKx6rOmtWi3Lrj3usC8
|
||||
i75VwB/B6MOvxXTHN8j5yrrMec4uz+TRm9NU1Q3WyNiD2j7b+d20327hKeLj5J7m
|
||||
Ked+6M7qF+tZ7JHtuu7V7+fw4/Hb8r/zoPRw9UH1//a993P4I/jU+Xj6HfrA+1z7
|
||||
+PyS/Sf9vP5Q/uD/b///AABzZjMyAAAAAAABDEIAAAXe///zJgAAB5IAAP2R///7
|
||||
ov///aMAAAPcAADAbG1tb2QAAAAAAAAGEAAAnKUAAAAAw7GoAAAAAAAAAAAAAAAA
|
||||
AAAAAAD/4QCARXhpZgAATU0AKgAAAAgABQESAAMAAAABAAEAAAEaAAUAAAABAAAA
|
||||
SgEbAAUAAAABAAAAUgEoAAMAAAABAAIAAIdpAAQAAAABAAAAWgAAAAAAAABIAAAA
|
||||
AQAAAEgAAAABAAKgAgAEAAAAAQAAADugAwAEAAAAAQAAAFMAAAAA/9sAQwACAQEC
|
||||
AQECAgECAgICAgMFAwMDAwMGBAQDBQcGBwcHBgYGBwgLCQcICggGBgkNCQoLCwwM
|
||||
DAcJDQ4NDA4LDAwL/9sAQwECAgIDAgMFAwMFCwgGCAsLCwsLCwsLCwsLCwsLCwsL
|
||||
CwsLCwsLCwsLCwsLCwsLCwsLCwsLCwsLCwsLCwsLCwsL/8AAEQgAUwA7AwEiAAIR
|
||||
AQMRAf/EAB8AAAEFAQEBAQEBAAAAAAAAAAABAgMEBQYHCAkKC//EALUQAAIBAwMC
|
||||
BAMFBQQEAAABfQECAwAEEQUSITFBBhNRYQcicRQygZGhCCNCscEVUtHwJDNicoIJ
|
||||
ChYXGBkaJSYnKCkqNDU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3
|
||||
eHl6g4SFhoeIiYqSk5SVlpeYmZqio6Slpqeoqaqys7S1tre4ubrCw8TFxsfIycrS
|
||||
09TV1tfY2drh4uPk5ebn6Onq8fLz9PX29/j5+v/EAB8BAAMBAQEBAQEBAQEAAAAA
|
||||
AAABAgMEBQYHCAkKC//EALURAAIBAgQEAwQHBQQEAAECdwABAgMRBAUhMQYSQVEH
|
||||
YXETIjKBCBRCkaGxwQkjM1LwFWJy0QoWJDThJfEXGBkaJicoKSo1Njc4OTpDREVG
|
||||
R0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoKDhIWGh4iJipKTlJWWl5iZmqKj
|
||||
pKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uLj5OXm5+jp6vLz9PX2
|
||||
9/j5+v/aAAwDAQACEQMRAD8A/P8A8Xz3DW1tiaQAyn+I88fWuv0qzuJlT97Icjn5
|
||||
jWJ4r0gyWtpwf9Yf5CvRNXW2+Hnge41nWYy0VpGCsYIDTOcBUGeMkn+tfzhjsXKG
|
||||
FoRhrKTaS7u6sfsmGo3rVXLZWv8AcVLTRpgpaVyoA6lsAfrXO6r8aPCOg6zJp+p+
|
||||
JbJLqA7ZQjs4jbrhmUEA+2e9ed6nceK/2gRcTv8A8ecBUJZw5W3hB4BxnMh4OWPb
|
||||
GBXR+Ef+Cdeva/ZRalb6hYxM6h47MxFcA9sZxmvUwOT0Zz5MbXal2jbR+rvf7l6s
|
||||
MRHGTo+2wdC8e76r8Lfeei6XHF4h0tbvw7fRXttL92WGTejH0yP5Gqt3pdwhHzyH
|
||||
Puaw7T9lv4k/s33/APb/AIQtYdesZGMt5p8UrLvQAbhtPBcAHaR/+v2DTtDj8ZeE
|
||||
dN1rSYpY7bVLSO6i3r8wV13fN6EdK4c4w88lqxanzU5bP9Gv6uThJSxkWqlPlmt1
|
||||
/kzyecXETnfJIfxNJ+9blppAf9411niTw09rIxK9OuKxPsboSCoOD3FXhsT7RXRh
|
||||
UpcujQuuaCTa6eQOWlb+QroP2p/CUt/pfhLRywitbl5bu4JHyuEVVVT68ufxrR17
|
||||
SQbfSRgYMzA84/u10Xx0hn8a6tbw6HNp9ufDsj290lwH8x4x5LvJHjg4ZlXafqeO
|
||||
K+Rni6nLgqsdo+1bfbon63eh9hlGUzzCtiIwjeyg2l2MT4Z/C+0+HWkNNOY2jVVL
|
||||
bV3c46dOT6V6b4a8daDHLFajxHa6ddykGOGdTEWx2AYD1HSsHUvhxfwXWn3uk3t7
|
||||
MbQFpQZMgEptBEX3CRnIyD271SvvFDeB7ezXxNquo6vm5kujb6pEsjuuAUjj+Vcb
|
||||
SPlx1LcnFfacG0aePw08dXrJta/8B/5ntcSzxWVzp4HD4V8j0779f+ArnuIsdUuR
|
||||
9nuWsJIhyDvGW4ycevXFcj4S8CRaTpGo6XbJ5UNlfSFF252rLiXaPYF2xXmst3qP
|
||||
ivVdLfU9M0HW4o/LaWS7uJLWSFSpMuzysc7wOeOCPTn3vQ9Ih0rTdLi8O2ki219b
|
||||
vc3fnX32lrU5URKrElmyNwwccLn69/iDg1/Y0a9Oa+JPfayfl2+/ZanxWDxTePlh
|
||||
50nFxutt7/1f5HjHxC8MfZ9x2feyQcV53Lo7CVwAPvHtXvnxG04O7jAATk5rye80
|
||||
ljdSbQpG418BkuKcqerNMwpRjIm8U22yLRQuPmlYfqldH+0p8KINMtNS8YWKXAv7
|
||||
nTP7K3b18lFlki3HaeUdig+YdRwegrnvFF4uNDBCnMx/9CSvY/jxap4r+A+rWSkJ
|
||||
LLEhTno+8bf1xXgYzFVcBRwDi7KTmpf4XNXPr+F6tsbWiknzcq+9aP1TKXxG8QL8
|
||||
HvgjrXiXUbOXUo9FsHu/KhwDO6qMKD2zuHPQAZr478R/tGfEb4weKEv10f8Asi9h
|
||||
iFjFcWksSPZjeDui+0IyqQBtLbTkZ/Dv/j/+2Wyfs2T+Gb2CW01y62WWoQSrgqFX
|
||||
5yOxVhjr61hfsxftoaPL4dfQ/ivZNBPBlre7hiEu125O5T+dfp/COGlhMsVL2d7y
|
||||
96+zXlfRpWXk0z0c7zCnis0VOtiXCKi+W3fze6e6+XQ6jxh+1hP8NPEb2fxN0+9m
|
||||
023t0UTixh8qSUEM4WSNiSGBIyeARngdPrH4d6zY6/4Etr7w8kkdldIs8KyJtIR1
|
||||
DLkep/pXxH+1n+0H4K+I/hbTfCfhfTTqGq61eQW13ffYlthbwtMqHYo5JIPQde/W
|
||||
vuO5kj0nQY7e2jFuBBEiRDgQoiBFX64GT7mtfEWVCOVwwyespLlXlHf5LS780fIU
|
||||
cVOpiqsVLmitb26vp/WljifHzYL4PLfe5968qvIke6kLgZ3HORk13fi7UPML89f8
|
||||
a4K4kPnvww56AjFfD5PQVKnY4cfU5pHK+MdXAl0gO2F8wnPr8yV9a/Aj4GyftJ6B
|
||||
rVjeXj6bBFbCO1umQsou+Gj3LxuUY+YDqGFQfCH4Y+FPCEqyvpn2+5jUlZp41dl+
|
||||
hbpzivb/AIJa3FperyRqgt7aY+cPmBLPgBs+5wD+Nfpdbwsr1sPhViJJuDd4rzd9
|
||||
/LTofL4XjqOFr1KmHum7NSdt1tpqfnX+2N+yLJL4nuNO8eWb6J4oslIEgyYr6EHh
|
||||
s9JADjDjnacHkED448ZeAvEnw212VNV02WJ1G1GVTJHMOgZWAwRz9fav6HPjJ8I/
|
||||
C37SngtdK+INqrT226TT9QiCi702UjAkiY/XBQ5VgSCOa+HfjD8FvEn7NXiIWniX
|
||||
T4tR0y4YCx1O2iJtrj/Y28mOT1jOT/dLc19Pg+Eq2UpU6cual5/Z8v609NjvrcV4
|
||||
biO86i5K/W20n3T/AKa89z4w/YR/Zi1Xx/8AFOy8T/FGxu7bw/oEgvYxcQmP7fMp
|
||||
zFGqtzsVvnLccqoGea+4fFHio3Urkt+YrtvB37OHizVfhTqutavEttqWtJa/2Zpc
|
||||
v7qZIo3Ls8hPCM4YBUIBAHzEFsDxvxWl94e1KSx8RWtzY3kJ+eGdCjr7jPUe4yK/
|
||||
PuPsjxlHMadatF+y5Vyu3up3d1fv172aNMjzLDVaM6VKaclJ31u3tr6dOxk+JtQ3
|
||||
k7cVyk13vlYgkZNX9d1IYzwT1Fc5LfN5jbSQM9K8rBUOWJrXqan2hoNpClwAEC7s
|
||||
jpmuy0fSI54wIZfJYchgOh9a42DMcikdR6V0Wkam8OMHg1/Wfs0z8K9o0dt4Q1S+
|
||||
vbJZYvPgHIbzBtwQcHg9elfnf/wWrj+JXhbxpofjJPG+qv4WtZ/stnptnM9k2jzh
|
||||
dwnR4yPMZsjLtll2gLgE1+g2neJN0QBbjFfGf/BbadZP2btNmg2+b/wkFsMk8DMb
|
||||
HPPH/LOuXH4enHCVGt0r/ce5w1XqPNcOlazklZ+ej9NOu66Hr/7D/wC1f4j+NX7N
|
||||
/hO7+MV/Bf6tqdqUTVljCJfSISjxS4+VLhGUhhwHI3KBkqO++JfhaLxzoZt/F1mm
|
||||
o26j5JU4mhOOq55U/Q/hXxp/wRA+IEWo+AfH/g7Vx9ottNvLTU4YZm8xYzJGY5Cn
|
||||
YDfAG47nPevtnUtXW1k3Id1s/wAjdzEff2rSlClmWEiq0VJSVmnqnbR7nPnWHnk+
|
||||
Z1aVN25XdNaaOzW22j2Wh8c/Gz4V3fw+ujPaym+0pjgT7CrwHssq9vZhwfY8V5v9
|
||||
o/zivuLxd4ei1aKWC5jSSKVSDlQQQexz2rxjVv2LrHUtRln03VZ7CGU7lt1QOsXH
|
||||
IBPOM5+nSvzHOfD+pTqueWq8X9m+q9G916u68+n0OX8VxnDlxj95de/3dfwPaFuH
|
||||
UfK7fnUi6ncJjbKwxRRX3iqzX2n958tyrsTR+Ir2IYS4cflXJ/GT4YaF+0D4bh0j
|
||||
4wWC61p1vcLdRwSSvEFlUEK2Y2UnAZuM45oopSqSmuWTuiofupKcNGuq3ML4N/sz
|
||||
eCP2ftfm1T4P6M2i3txbGzldL64lWSIvv2lJJGU/NyDjIycHk16I2q3DuWaZyWGD
|
||||
6EfSiilCTpR5YOy8jSvVnipupWk5SfVu7+9h/atx5YTzWKqMAHmmG+lY8uaKKv29
|
||||
T+Z/ezHkj2P/2YkBPgQTAQIAKAIbAwYLCQgHAwIGFQgCCQoLBBYCAwECHgECF4AF
|
||||
AlnVc0QFCQzP/koACgkQrKXb4f2Pq/FdFQgArUxV0TaJWfM/ymf2HCnh2LlqOBod
|
||||
whKXzwnwCAzVUWX4QKm2MlWhcSfPbafVQlc4na5ecW7QHqp24vs3kAlzVKP9nUpA
|
||||
dJHP76Sc4UdOCmRBJqfL4OHDRbFLIp27ZnVF8RbYF74fqSPSWVefwya9Pb/FjS50
|
||||
rnZRaCTEm2aRx+Pm+SD5X1levN1T4YC+fLxECZ8ERoRQiuIED9YEUCE1E68M1IFy
|
||||
xvy+mFz9Ck3DCBdL45v1IDVQz+IRw4j1PJzwnI8sHnyoXyJpmVvyinoXiqqQWMUR
|
||||
tG7HMkTg8d8PlWWllZxp6E5jItP+g60NbsgAkOFNpkB8vhXC8SHRgguoMokBPgQT
|
||||
AQIAKAUCUqkWjQIbAwUJB4YfgAYLCQgHAwIGFQgCCQoLBBYCAwECHgECF4AACgkQ
|
||||
rKXb4f2Pq/HGdggAqnLiZBfqSUVtrMPzoHyDNwGl6szKWLHRoJ+x0z7smiGI4DUe
|
||||
oaUL7VrKcQuj1Ww6fNLya7YqLTxLobG91nY1HfEFyBaycUHhYfjj87HG+hRYuQRR
|
||||
W4l441rEFSZlm42Ui7ZAlAECZ2WLOnxRNR/i5b/18B7R3T7DIChgtSEeSUoNV7Le
|
||||
C9vWOgQ5LHmxQWy9u/QDpVBGIJgMgycjFNyOfF/x9EB2Edxa1kpPIkzIRJeXrj30
|
||||
RbRTMfifn0iBVoUKNnkDtNARP/SV+y3d7YeiQWqBgh4SB3Sw3yKL4LbUZ5qrzTjw
|
||||
TGMQb8q2apwlHzJM6L8HeYPlvDCXZh3JMskRTokBPgQTAQIAKAUCUqkWFQIbAwUJ
|
||||
B4YfgAYLCQgHAwIGFQgCCQoLBBYCAwECHgECF4AACgkQrKXb4f2Pq/Fm8Af/RZ/P
|
||||
IYzw0q3za636DfepvXQTKC7aCemUgktxGp4qoKBOv0380cA8ERWUo1ajs93ea69i
|
||||
qNAkZFCFB6DblmDNbKFut89nBK4TlJlehMdM1C4F+XexX/9Tds/ToncO04Q9V/dM
|
||||
efVnqAvlPZPjhffBd+est+66xYMUxR4+Apby7TWfl27LKEzl3MBElt+e8Ki43Z6W
|
||||
SSCDDHChNKWLD5EzYoVv9HJV6CRgGUmg5Y94Yoo4vSVzBttjQ4JeED9mtd41OhDV
|
||||
7jwOjf5XWJcW5DXPzIyAt/bWJu+09WbX4qNiJ51xQ+bom91M3uaj/Mq7Hy+qO802
|
||||
airAH8j7Mckj2yOuEbkBDQRSqQ93AQgA+pgk1zYDFRi6wsRyujz6w4JPvU+YKMVH
|
||||
nr9c5agZsdG+8rQqVQ9WoFKKde2UiicVjAEPHsl73FI1CiJYU7H7dfgbebq5kxbg
|
||||
k9h685ea+LVLwSA+Z1OURlmpPNbU1QcYKxvC5L4ET5ExlQ7ngBVhxFd5I51bIb7o
|
||||
/NQFd+fZkHQmGVu+ejkjx7yH3soPlXLmvEIRKKD2Rkcz8G0ugnfAfDL66NAoWGuv
|
||||
y9wgLhqLlrXZFJG5s9cHliAH7cCU8DJ47sUDo8UP841Z4KcBfYZSFoOomjLdfpHz
|
||||
isiLp/hlUSk6Ay7VBYr1w136uSpfVgrr9voypi4iOMs8PSZl8wUlaQARAQABiQEl
|
||||
BBgBAgAPAhsMBQJZ1XNQBQkK7srZAAoJEKyl2+H9j6vxfK0IALXlPtgZPyMgwDOQ
|
||||
FnyMbZiUNI0AVNaKlkuP3AxpRA9WJ3Y9S8Mm9Ns+4u5e8GieE6rvde3MjD2RcpQL
|
||||
sS1ZyWB+EeB8YUrWeSHnKHFF50LaE4hDzeBYHJpNc1UelK9u7DqpMnL2oIR0XO5i
|
||||
6EUl1ASJOdEBy3IOiBss0R54WsMslRpcR8UCBRS9Ea9zpgd0S+FjMWGGRkytn6P2
|
||||
bzIVXe0kzXNgvWkOztdga2kp/r4VlG83TpOh0MQ+0+qcLzR90aHLEbLT1gKmV8XW
|
||||
xH+rnw2J4OH3SKWbGFyP1ZTpEYRM9vQ4U40UQtiXF5gNJ8Bh9rqzh2actPo9/Q+W
|
||||
DyXQOcA=
|
||||
=x0WA
|
||||
-----END PGP PUBLIC KEY BLOCK-----
|
||||
|
||||
pub rsa4096 2015-11-15 [SC]
|
||||
FFA9129A180D765B7A5BEA1C9B432B27D1BA20D7
|
||||
uid [desconocida] Fredrik Roubert <fredrik@roubert.name>
|
||||
sig C191C1EE0DED33EA 2015-11-23 [ID de usuario no encontrado]
|
||||
sig 216D7E92EB61819E 2015-11-25 [ID de usuario no encontrado]
|
||||
sig 1C2C9D2A7564DC9F 2015-12-02 [ID de usuario no encontrado]
|
||||
sig A42898C74BF155DB 2015-12-03 [ID de usuario no encontrado]
|
||||
sig 3 C44AB8B850327DF9 2015-11-15 [ID de usuario no encontrado]
|
||||
sig 4719E4FC1E16C930 2015-12-02 [ID de usuario no encontrado]
|
||||
sig 4484F5552D1E8BB4 2015-12-02 [ID de usuario no encontrado]
|
||||
sig 6C6580E77BD756C4 2015-12-02 [ID de usuario no encontrado]
|
||||
sig 7F61756177978F76 2015-12-08 [ID de usuario no encontrado]
|
||||
sig 47712171F2ED62FB 2016-04-06 [ID de usuario no encontrado]
|
||||
sig 2 80D88B22D4330331 2015-11-21 [ID de usuario no encontrado]
|
||||
sig 3 9C187BA29B2157F8 2015-12-01 [ID de usuario no encontrado]
|
||||
sig 3 6DFD2ACE211BAAD0 2015-11-23 [ID de usuario no encontrado]
|
||||
sig 3 4445C665FFAD9AE0 2015-11-23 [ID de usuario no encontrado]
|
||||
sig 3 2DFF526BB17F76C6 2016-06-11 [ID de usuario no encontrado]
|
||||
sig 3 9B432B27D1BA20D7 2015-11-15 Fredrik Roubert <fredrik@roubert.name>
|
||||
uid [desconocida] Fredrik Roubert <roubert@google.com>
|
||||
sig C191C1EE0DED33EA 2015-11-23 [ID de usuario no encontrado]
|
||||
sig 216D7E92EB61819E 2015-11-25 [ID de usuario no encontrado]
|
||||
sig 1C2C9D2A7564DC9F 2015-12-02 [ID de usuario no encontrado]
|
||||
sig A42898C74BF155DB 2015-12-03 [ID de usuario no encontrado]
|
||||
sig 3 C44AB8B850327DF9 2015-11-15 [ID de usuario no encontrado]
|
||||
sig 4719E4FC1E16C930 2015-12-02 [ID de usuario no encontrado]
|
||||
sig 4484F5552D1E8BB4 2015-12-02 [ID de usuario no encontrado]
|
||||
sig 6C6580E77BD756C4 2015-12-02 [ID de usuario no encontrado]
|
||||
sig 7F61756177978F76 2015-12-08 [ID de usuario no encontrado]
|
||||
sig 47712171F2ED62FB 2016-04-06 [ID de usuario no encontrado]
|
||||
sig 2 80D88B22D4330331 2015-11-21 [ID de usuario no encontrado]
|
||||
sig 3 9C187BA29B2157F8 2015-12-01 [ID de usuario no encontrado]
|
||||
sig 3 6DFD2ACE211BAAD0 2015-11-23 [ID de usuario no encontrado]
|
||||
sig 3 4445C665FFAD9AE0 2015-11-23 [ID de usuario no encontrado]
|
||||
sig 3 2DFF526BB17F76C6 2016-06-11 [ID de usuario no encontrado]
|
||||
sig 3 9B432B27D1BA20D7 2015-11-15 Fredrik Roubert <fredrik@roubert.name>
|
||||
sub rsa4096 2015-11-15 [E]
|
||||
sig 9B432B27D1BA20D7 2015-11-15 Fredrik Roubert <fredrik@roubert.name>
|
||||
|
||||
-----BEGIN PGP PUBLIC KEY BLOCK-----
|
||||
|
||||
mQINBFZIiQoBEADCWPLK62Qyz2AkaXLY8N2SSv4iFL3SMdJW6JAlb4ELrO67w81L
|
||||
GliVieSnu+vq0kU1xOhuB2Jrsy8g7MuMZ2Dy+/ySVPmgK3EllNqan3OdSPzg6tk6
|
||||
x1kG/DOCDqbu9d8y5i9CoW+h4hYGd+NFFu/Vs0osKyNV2O8+XPJe+26ihwCALnz5
|
||||
VKeSmZeSyf/FjIyYvEPFhk3LESJuCJzoGKdcoQlSKUy39kBy0+yYvzr4e6mUnicO
|
||||
InppVgeon4lWxytHVVfaVGnvp5A72q718xHaFrfhDGrb6CPh8pzwlKcsic6fV5UH
|
||||
oxSdXfxu7KBUOyY28kEBAzRx/qAmX5O4jr1FZxsWMG75X7AxOiH/33NdE6flGpId
|
||||
JQvk4AOT/mG0mmLce3NU39FQBR+jZtASdeHFeyCflcUdUOYzLSeq1HVj8FbUAExM
|
||||
sjAzvXcNZURM42iDLrcp1sc1Z/4PIEJy9pFnm0tlda70CQKU5Fzf6lj0WxNgCuyH
|
||||
gs7SaRepSlFydAPafQ7Qe4oC7fwrkazqNEf/NAEWanONB+fsDU4vWHwT9xy7Sz+m
|
||||
wpnwwUgjUMjT01Itnf9HpgRCc+VVTG4EE0L01zPb7bOdYVhc8QIXbR9g7lzdjw2q
|
||||
yGh/IOvrU48BEQW8iv7NLjWCOpZpGfgD78GZy7Eod8O5IF9J4ZkvT1RebQARAQAB
|
||||
tCRGcmVkcmlrIFJvdWJlcnQgPHJvdWJlcnRAZ29vZ2xlLmNvbT6IXgQQEQgABgUC
|
||||
VlLOagAKCRDBkcHuDe0z6kKfAPwIfmy5ZTkyAbT3c6xsAhL14KPYgrGVLRWy+niK
|
||||
m3+ATwD7B5TqgHT9pCEHvzN8xthIxYxFHahqe+b/PeUEvhtpkPuIXgQQEQgABgUC
|
||||
VlVmNgAKCRAhbX6S62GBnrVjAP9r92ru0lzxBPiVTJfUdLI7BaLkSRzvLxPomlbo
|
||||
x+0FQAEAv7hhYHviW5WRBuXHIv/CQRssuxse/ZxUEd8ibyyz9NGIXgQQEQgABgUC
|
||||
Vl63GgAKCRAcLJ0qdWTcnyvJAP9zpjevsS/CeXBAm4/aA21BlRxMxu5ayvWs701+
|
||||
XkOQxgD+Kx8RRTvVM9Gju1XIsdPzbYelpkaZuWI18nFz7rnJIGeIXgQQEQgABgUC
|
||||
VmDKAgAKCRCkKJjHS/FV2yS1AP9+iz/HwRNIOLuHmWhWUkU6EksKrZk5DqjDIjgZ
|
||||
HW6izAEAn3wEe5lrzUzZDFFxzcJj5mDRXgcLfrFYmyDhT0nimM+InAQTAQgABgUC
|
||||
VkiKJgAKCRDESri4UDJ9+a9LA/4tWMpIORjCYM4ymxepz7NOrFbIpTRxUf9PnBq8
|
||||
2faj7tXkgtoqj2sKQ5Gudmq/tDtboltF2dyLg086gPcAKwiwJ1rfyWtu8Ue5xqHE
|
||||
VlerY150C7NlUNmYUuI08MmlAEByL00Ub89u1CdFsTBrAsYEHVJXOZdbJpC1ukHi
|
||||
bHKG+IkBHAQQAQoABgUCVl7q3wAKCRBHGeT8HhbJMImECACnRNM01tl+Iiuv3Ek5
|
||||
CpFrqF04t5qsrr1GJZ0xyiYF8ekrz5yhcUgi5S1UEDQxnPV2YjvYMU7uAg9+tAw1
|
||||
Nuqp9VF3O3gpIgNFOrEwSe8h5tPwTYlkGV/FTjfSksbTI6TJ4sQ/IqHp5Qm3MuXE
|
||||
EfroRXU7SCeKh33LeCjHleJl6FXGwiGD6nuUrgFD8i+rvszpKA9Yh5bahZbn1wrc
|
||||
lTEsPZFFNIaAURkm/p8mNuKz7sDNqmyocAtxb9keXh9lJyzWhOUe0m+R9M8ju+H8
|
||||
SGCNVO+lZN4axMojL0+QBKq4vNJF1ojvvSylgV3tIA6g7Kfi6shgaE/U3YzmdiEM
|
||||
aTqwiQIcBBABAgAGBQJWXszMAAoJEESE9VUtHou0DNgQAJBLjQkZjXKAmbIejdID
|
||||
XuUXVPIfs9Yj8Gddmt9PQJ1G0WdT+jChs5jXp1NBdIJzP0B31XDVIWOfhDXcRmnS
|
||||
dbTllBgPPhSMPpgkKAZ8L6/IDOVXWnnDZjk791v5WGIDWUtVhTEjDFLZ3CL0Wcyk
|
||||
6aApPzsDSrf9vH+71KQbkxDUesp8aUBU4spFBPVWsvT+ZFzQcB6iNrqzDHIu1qw1
|
||||
q6idaWuEkaFVloj3qhnBKufxx0ivS+CUaIzbO3Gbq+9FmajTKXcFeUMv3Ua7/bfE
|
||||
+rQIVf7G9m5fHm1yZm+/1Fh3IPBEf9ES16bg9N+IcJokuzw+pTQz2IvoaiQuZPz6
|
||||
j/VmSMUb/KHDMbJ60LuqKRZHZmy7344V4EkBFJsrJXFMfEGwmXjdIUPsxUhSq0Dm
|
||||
RAD8z3ytdAsREsdoddXKk6/snQ18IeqzxNCrzIhC0OjqszFttj/dnb7n9dGSuJL1
|
||||
sAH4oyyOH+mPCc2841Dpybvwlm4n2rhhYDSWuI4iMRwWhzKx8RigFk1CQvoXRqdT
|
||||
Vsg3Zy6430iOLOoWs2tohn2sSvoyflo01U9xNq2Kw6q/L8/7kYmFqP6HXtWjU6dC
|
||||
EJFmc826Zx3kiNWvANbOp5pvvKK3gm1Rv+xNh/FPFAIwYbD70y3GwB5HgHrvu6Q4
|
||||
RKCwUlMZp6VWyWCT5bKvdBQ8iQIcBBABCAAGBQJWXy8IAAoJEGxlgOd711bEolwP
|
||||
/ROlu9gf+af7rEoVgHvgVJWJjaetXHiydL/+vBojBi+h7EyOTOdur/T2VF9HyJPF
|
||||
jsy18VV3grK85YZBV1BdlidfEPDcUuCzNrZA2y4SDQvpIcQEggkd2TnbKRjhczHQ
|
||||
RHu2CFsTo0Nl1VowrLVI7e7rpt4Ih2yeVLtiiBZkekPOHrAKhZXlErOIZL9eWm8m
|
||||
z8mcRlaI0BxXJO0F5uStYAP//6fEU+zNePzwkRx26Iz1qTyJ3ZQB6ZxYoKFqR4Xp
|
||||
C98Rv+cSHyUny0UJRrM3Y63PpOpW0sUMRxXR6k8PpJAClSfOMW0L/b+mrBQmV1Ph
|
||||
lbmWZ8Np4pxLxPXr2xieKqSev6QTqYPleH3y0Oc7MYciZP6hTxPM2TFRfrsJYWEh
|
||||
0zgLZUs9OlYqGnI3hrHhkeXnbQQ3pOD8YGTifkHAiImB1yOMCxqIB7HtRE1gNC/V
|
||||
cBKBsbSUqLDACrXimNf0zz4cRaSm4bFnDllvfI0BrYDeqvWidzAMsV4DFI8G9xUA
|
||||
bC6qt9Rg/bxOFFP7yfD08Nvxjog6oDtFlTAo30zpwWyGSJjXBFbpN9qUUANUzG2d
|
||||
cWs0KIj+gdHcuXcmyvptGfHsuD4312T35jTA1w//N6OOBZ4N+Tv2OHSj4DdREQAO
|
||||
sNebJu1PuF8fB8kXSVxpgXTSsl+LZr/iFxvP8ydBWs6ziQIcBBABCAAGBQJWZqgb
|
||||
AAoJEH9hdWF3l492718P/Aq8dtvyWFE7Zm09VUKZbadh9mEc28MxjZKruM326jp/
|
||||
TpxM3sAGlemlvqN22ZyKofuhhVV2gE+QEWhh3dflZ7OYiR+2UY8hBGOUoBhAX3fu
|
||||
lpdxZm2DIsJusxcPngeowHm0EF6+CxYX0FAhxGbodVezNxEo05SSTIGRuqyJa8vI
|
||||
VzuKGio5lR/uFd+XSa+NZK1f6kRpVmF08F2N4cGPFqUCTppO+eXseXxN8Zg65mLI
|
||||
z7jcEHeIHpkAo2pysaK+u9uc2EQ8YvJxdq3pqUEPSFagU0C5mC8YMnjt8LOpZU5v
|
||||
ggCUM1Z9z0Jjexd6/ndTKjVw0Z4N0pMxmyz0XS8NPTfNqYa9u32csmq7TIxVL4EP
|
||||
TSOkSjUnGAbH7nzR+43Ajc+8wObJKGey42padYAmXPs8BteG8H67jt2+xwzPORJ1
|
||||
P7k7i8aK7pfji45qJxTL6lLAA9V3PXLhOgAX6w8J8h2brt3wjDBYNgKacza2j6CM
|
||||
2Ua1sZrSHFDxWwPjv7bHJgj1hEBQwbG+6bBlkqq8DO1UHOLLoN3sW6yl63F+jwZR
|
||||
Gijssocd/ew5HdR/Weo5H/K4+5XbgADWUK8ZJxOFvTscdS9cQVmDfh0YezL0Otrb
|
||||
qMHnuOQ5FobhHpRuklisxEtXMcM0a2XEkkxZ5LwuZC8lTZh3s8YroUf6OyInSOs5
|
||||
iQIcBBABCAAGBQJXBLX7AAoJEEdxIXHy7WL7CEsP/0QyXwVhS5UHOq4jVnTe+2Nf
|
||||
2BbBb4qg3GvCMVL8ZGpVbKZS58J1fveTGaXjiHoBO272W8/B36+nO4PR7Uz5AXlM
|
||||
vpIPVbOvPwCHmjWwq6Wx0x+vcA9X72fTrbGTYVvJrTKwspO6Ba4Wct0FD3Gs0VsX
|
||||
JR3sNud0lhx9GOlFkxSsRS9bBshEZJr76yIp++r4jiDCEuZ8VMFtLzQEGRY9DN9f
|
||||
8seXBD/m/R3dFY4NAk0WXMmP/H21Yn7/oEqZIM4PRb3xaUC1eQvMT1UtPzscPduI
|
||||
D49+kOYWvyZmm5Qn9+p6Twvy8qAKh3VQh7HKQRl2FqKgdaTWJr71Iowiv9eDpCNU
|
||||
XiEXZC7dlS63twssn9aunUfPD89ImtKtfULK2rXG/BHPUI7XFJ7MllT1P66KkikH
|
||||
/6Y27ugXjD7DRfEm1OJy73Q1vtdIxMqNhbc1U/BpY/IJcFQ9m4bH6E4VgjqIpoBd
|
||||
AZOsCYA+dJ9MP47LAt86aI+Ea2aE9oUKA29HpnN7Ci5Gh6L+Jjf04ZrrE2+8vz3X
|
||||
VaHejfUoGP6dPPh6VI86g3r+a1YtqZYZwRt/IochTHP1HxLTbrXng0N8ukrX1CgI
|
||||
RueNaQGpZ0XVsPPC0xfbhbSPHqp+kO2rnTOLh8gN0ZuDZwHd0UYxsZd2SPoywvAh
|
||||
QeKte0lGYfwvC9NHi1QIiQIcBBIBCgAGBQJWUOxRAAoJEIDYiyLUMwMxjpIQAKKL
|
||||
r5TTwCG3j94sN2NzNcs6U34Y5YXACVmlmxG0HqJOUbio7IytwwHEP+YgF/wTF0TK
|
||||
EBPzuuvlg60yiO65zrK+QYTClFuT5NodkKNPOy6VXsqL9njPBbDTW/chfWdrewK5
|
||||
tW5Ddhkma5GhEtNRwHOpHwjpt0CZV+1OTPM/tt+jFk5zEVXUphNRgi2upPcbxGI2
|
||||
bpEVsuXjUgJmcQqxl64OECnPGpzxxobRFqzuRzQUoU2s2jOaKghI7Ma9WLfW7j6d
|
||||
J/T1D3dlV+2+ysYHimA7ZMFz4+3qrJLNT2mwWUK27U97Ifgz4wkemq9Z0d/gAxNI
|
||||
OJJu2zVmVtuSVY9HsbMvL7hidDFkljwk2lTu0bDhRwqQ2Wiqh/Ca7YySvJJiyTCE
|
||||
Cc3xMnsJ5+ntGSREsk6XFRKT2qCXddccfMpfTw1P++V9bUY7Nbs8eM59JPl60/hE
|
||||
w5DiUHuLFukbYAW7KQBF/9elxO4B4haZ8PoZ9mVvEj9+c8hEQEBWbx27t7Ahifkv
|
||||
rDy1JCUszAnkEsIg9hk/hFfCBfE+t2ap6j7jl6pcnAGYS2WmxbUUOfko3VbPVqVI
|
||||
Fz0lSfvcQJpkL2vrTHlH++g/YnwseALIQ91IBMq/LHd4uixaYyC0PT/u5jExnJ2O
|
||||
I9adEFSBRelzo/NC//Y1RhVZCsE2o5pJkQwVOxoUiQIcBBMBCAAGBQJWXY6aAAoJ
|
||||
EJwYe6KbIVf4fjoP/iWGzELg5ymLiJ4PRiCjTsF4RI8DHW1edGP6NumjLyRHgt52
|
||||
GZMgVNuVRmT5nag1V0NGbaha1kBExoZLhQ2So1C8hth4sKJA3gfw/cP0ugggFEhE
|
||||
+/+w6AkTxs28oyH1p1FbWQn/GTyTtnKarLxWzbSFvrWuaYS28fnxIY0E24/T1/Qb
|
||||
YZsmBDJeZbuMpaDlIox0hS5KDt8o/++q1F5hTayON7MzPnCAwxC7TR1XCzXzlAME
|
||||
N2quE0y2nztfU539KW44iHbq8LPZLx2rw9kTjp6oIzFl+Grs4h8ZQdor86f1OrwY
|
||||
esB65VrQ1ljvKJomdVeP/p/NFLcDAoKZLDyoZ/cFYstOBRwUQjfDVE08DN0+qith
|
||||
HKNQi/FaK8pPkHL2ZLWSfw6NQpiGSLNlGqXVUt0Oeme/vshynUYbbQcws3PlRh9W
|
||||
+RvMk1+x6fmmtshUzdmhO/RCzsKpGiqWN1ki5y3Skt29DBpDMIozT4ItkPRs0dJ+
|
||||
4YxpJXJ7GaLJ4GtZUgDAI0xsSbx/n7OCCNbzo+R49fm8RnbNa+hOiHs9SO+M2WRT
|
||||
hxX0d4Yda+4dtf7DJFjj/bOOP00RF+z4OK8sMJe4i9kN+VVn3s72pp6xZ/7Gw+Wg
|
||||
4o+3I6PQDPbRhNUzPiSRFIUaE6zCBIhzNnJrQQk0Hlco64n1t4VZL+DnGEKbiQIc
|
||||
BBMBCgAGBQJWUx/xAAoJEG39Ks4hG6rQOGsP/2E5PUOKDKQP0871XZcghtRA17AG
|
||||
iNcxzliNO8u+yBnHHXQ7pPoLM8M309H+YNSnSm0B8QHD02upvSPTr1iYA4BdZTwv
|
||||
KaIf/f4+q1nU5JeQAdnU7bbQNLhCqFUIc/qQKN47jdWmelGXp0WuYFw1Nm6VZ4B1
|
||||
wK9+piCF1/BchpdQZaXMyiZo7fzlqTwvgwGh8jQKJQQiHiaS5kk33p8ZnxY0anWN
|
||||
3DxUyAHz5TaiJf3MOny5aYkJgRSFiuX1wqjIuXMTtSrhbJaiCFGvfClzuB4q6/XC
|
||||
4O610dKztHSqit2BM6Rfxq/zD+R5eFkZWw4dRPD1uzbqckltOyMKt1kjRiGPoq7W
|
||||
yqA37GLOcxcVQBa7/8us34qY4VdRqiIHdy6OsyxnrNWWC67zEo33rXhiH52f/iUm
|
||||
8Yd3Ccf73HAGTK6YRUDnHmw2LGrNSa9jMNnLa9NpC/dyf3uHxhzBCRt911drGPeS
|
||||
+VSCrlKuvfrhrgJ96lBaPvRHNYXeVt8OkGrvmxzdHZ6AJmOTLXM8apMnF61WX2/y
|
||||
V1MTmHG7Wgco3FX0j7oS6g0CYdsz1Frb8XXTcPUfQGqneqJTIwFfIav1tToV5bIE
|
||||
fjj2xL14RHDDCbWwrRjSiGe3GzVYt/EgvqxV7L4Jq7Fz5Cidqrn7uoOmdMP91FIF
|
||||
qKLfw3mXO4KEjqLdiQIcBBMBCgAGBQJWUyYEAAoJEERFxmX/rZrg0NIP/R5eG16k
|
||||
LGofi+zSojLHd2rwLXBsqHXg4oGnmdOYHOZIksBz14tytElgM1p4RU2RMy4BhyCz
|
||||
EZuRBnKeSmsXfjGBUpes7h0BtQljLx2nRSnSWpbTKdmZ4OHVM7dDrYBvHRxz00Qm
|
||||
F+XexCJ0Ach6epv5DO5EUEEsT/4RYL6IkDg3ln1WRJmNu7M7YOEjxS4NLoGZjYLB
|
||||
jHk2vEaK0jlLWS/vnrFpnSUEkh2SlqtiCzMgXzkVbrJDvQCaZvRJTggygLckNvyw
|
||||
A0DitEpviP01yFkJqXi+tItyJO6e0JrO6mfRyFZX5KzMqS6Y5+UV+zhdiiAodk6+
|
||||
bOBB01xsmrpeVEGg5uWalusUQV3dCMBwGkYrTCc/32EAnl91ei2U/aDPgtWdVKhP
|
||||
s+9CpeBbrsK42ErC3kdOQe6Gml5L5IipPbl1rjLed1VeQ4dAUG4odyFQGRScTIBr
|
||||
djzvxPDu0/HA+Ts7LuYOziHsPmyr9FduArLcF6vnZBg/O2cGtcMVOVGOhSGjYjqA
|
||||
vbZsYU5TWgNWpnEXJn/o7DOot7RIC3/xkQf0847U+S+do45P0C7BCPmT2eZz33Ph
|
||||
+WS76NFMgvH0GPCBkeeT66xW3BgFyvdRbmx6BDYlzM9cu7MmQfbMLQvMlM9pwejC
|
||||
YmhAKRGbyAGXyrCToU1WHk39uJB6rpLIhb6HiQIcBBMBCgAGBQJXXCHxAAoJEC3/
|
||||
Umuxf3bG9zMQAIFgyOuQmEY/qLrEEcXL223tRWSWPhvQbGzzFBdC0pXnhLE5JU+p
|
||||
kCPnz9PNO25cO2ayS5p8Phwu2d7npzwHcjju1hzFVPEOjjtyKSaUsRe77CI7R0Cs
|
||||
9egNxm+VV7kzjpYVgPlON85qN6UvgLNVmB3SGRAC9CcsWgtiUJT4PqtUtUvG3e4q
|
||||
DQcfTFhF9+pE91mgKtCFUOXYj/HJS0oEpY+8SzML+lNxE7VSBGCYmDeWgX0m5hpt
|
||||
1wDblTKHCR5m0XKtVmTnLcNQIx/Vc7yJ6hAo5eUHkCxQCe4uWNGD7HNz5Xzxfe9g
|
||||
QUDv7ruxbOW9dALqLckQFJxdiIJAi/RGlUey+GXfxjJjYv9QWZLiAyrDEr1hq3aO
|
||||
vdkkG9KidclMv4dSPDU/CUnZlvxUps4z2eJHmYw6pwHJpb5Bq/eQAcx9fPPrrdmn
|
||||
KkaduYM5dEe4F6Zyg6iLY5Q65waRwY4qqc3DwsvViqGTwsBcFkeRc6FMdmflumHS
|
||||
EM5BYVT/6aire3SUog7D427BkzklLsTOi/XtySiPb2Js2uwQ4V7ijamytvTqQ8oE
|
||||
4MUxuJLPxb7sSPjiWW+1HkXQcc+4y4xHSOnHv8L4IIWNMs2ebY6GcqYznUaQKhYT
|
||||
OCgigPDye8/B/IEUYqgI0MX5+jcP2AqYANodemClOGmbuQ9D4P/0L/3piQI3BBMB
|
||||
CAAhBQJWSIk5AhsDBQsJCAcDBRUKCQgLBRYCAwEAAh4BAheAAAoJEJtDKyfRuiDX
|
||||
/38P/2/zksU/2Vmcsf4BOS0lbTMjrg1T+PYBrGcXRcmg8prDUEOPDMobLEgHNlvI
|
||||
mCo6mt6D2fjTOs5v9IatzuOM1B6c+fasEFtQoP9jsU59/y0QV7Vy3iz1n16sBjZ1
|
||||
iL68lTWsqnfSQMmRjnn8sXJdA3R8cTx7p5NYE9qyn5WR9K+v8nWDwPpfG0nbu8Lr
|
||||
Y1Jio1GlZX0JWFxX5Vq72id8k48IHCG7+7pIpSBak0Ot1TDYSpZgrPFbmpR7oPPL
|
||||
h8/V3PqvCRnxHgLYyDSX/XLJwAsYzz3lHsieWF5cnk5dhVcgCrum4OLhz+OVTUmV
|
||||
Tj3yRZatibUCknk6+QJK/0NTfH43K2hNGJUKPk+B50ZkjqZfYWwfW0KBd60AGesI
|
||||
TvP4S4P9DAGNPYyhcvr2wrN/4DGnB0Z3/V52q2suz9SEWHBDBBfATmDQ0M6YZ8Ww
|
||||
RJOl6vaOKFeWHIaOsSQfI/QdjNjt74RPqf5E70uAal//gQCoS97rF/3dkpdssTZC
|
||||
O69fploQYycqWpEA2O7iPmhopQSuZy9K35JbSpqVyNZmomi1vGV2npBQauD4QnI8
|
||||
g6QkREI/VcWgsYmQjIUPXNp7K7JstbDkLyUdssmiMBOQZnFErKjJQ93wcO16lTE/
|
||||
5kQavdXUZXhqnBgY7FZrPWBxEJte8yoAgx4KGybYgi5Iy/TPtCZGcmVkcmlrIFJv
|
||||
dWJlcnQgPGZyZWRyaWtAcm91YmVydC5uYW1lPoheBBARCAAGBQJWUs5qAAoJEMGR
|
||||
we4N7TPqKvYA/2FasdKjS77IiKpJIQivgeCFxYckXe21catm6gYGI/EgAP9oxJ+Q
|
||||
mVXFHWYpEeAkWBCaMJ+F8t6uMpKN2vW2MAVHAoheBBARCAAGBQJWVWY2AAoJECFt
|
||||
fpLrYYGekwYA/0sgp7t6tkVxXSaZBLjt5osc8gOoqqGaLa/EAIzNCnF0AQC8M7LG
|
||||
9Ar15M46A0mf6+22iIcaCI9xOCIFNqNMBm34noheBBARCAAGBQJWXrcXAAoJEBws
|
||||
nSp1ZNyf/HUBAIaeJna5WuuyWrMKxS1CD2VRJ/PphOCgAO3o/YhzOC3zAP9beddv
|
||||
YRH74yUaxG3Ca+1zShOWujeSQBpKDB4bV3/ZoYheBBARCAAGBQJWYMoCAAoJEKQo
|
||||
mMdL8VXb8XIA/3vsPgosoEbSRheu9lEvVrFC++2LA6Tp4F+01yqNOopgAP4k4edY
|
||||
9cBl0713VXIgHnFNW2Bj6Bu4mKAEJyjHi2K0OoicBBMBCAAGBQJWSIomAAoJEMRK
|
||||
uLhQMn35btQD/i1zaMjfRakgj2ze8ngTcLsv0HWU5kq5qWIfdJcNg+G1HEpkBTYD
|
||||
pFEQd40OzevAEezkR+pCbL65G6HDYoBUNaJ/bJCBdYX0B5Wlu2oinA6ZRW70DlHf
|
||||
/xGd9HsXlfTihJiW5swTxM/xiVlS9qu32ifrcP4S3E21hIi97FIbgfjgiQEcBBAB
|
||||
CgAGBQJWXurfAAoJEEcZ5PweFskwtgQH/1ZBH38wrP5Cjrrf83l0FImlxML27kQF
|
||||
BQ+EmXXE/DC2XkDglHE6cIOM+PB0Ibp+mCN+7F3SS4kWQe+wwLxWgaWon0PzZmbu
|
||||
IHbZU7IgeUspu3q5945YRqbVC6O7K6bsZiofaegOSeCJmkXvieI5aMoqJmBP7RQi
|
||||
fCxXCjH+5NwLMhNNZIUHKscNDf8AcyJMrlkoceE6wtHYj7qgO9raq/Sqcf8DvpDn
|
||||
qcxOdDaHJIbiA2Ok78OPwR71NyoUAtC/SwhLzBiALiK6oGItaV3mkMvZRXw55xVe
|
||||
hSo1NNdkRS/DjegRWvfxm2hMZ+wV3XS+lGKMDsHqUkMDRIFXL50OZUCJAhwEEAEC
|
||||
AAYFAlZezMwACgkQRIT1VS0ei7RCJRAAuuGFPGYzHtkq+9gHTqb8JvU8rGGf3EJR
|
||||
dDIK+5E1Ca/QZdq9ZTucLVu/O3b7edsxugPfjeXVK/M1HO9IcJmD00znR33L0YwN
|
||||
ZfaWIHTXlygzqGZkx1TlIZi+p74cunKrZcQCsTmd2GwVtFvdS1llcjALBJsQ6iGT
|
||||
lyAiTPzrVRsF8xtGmTxxVXojxS9Ji4crKxyVWrf4j1KB8whs8580b1PULr+75DSo
|
||||
Qi4WVvIoa7ajcQ4K8xt+gJhyNUs5u8lEojYa18TQznsYFs8pKhdsBPlYzK0L2qXa
|
||||
TGX8H58PU7CrufzyuSALhV5SW2ZgaHyl1Dob+GSktARSNue95BEdFusCeJkFrPpS
|
||||
N6f3RDohJvO4Nm5kFLJewhep/YAttjrKdtKPKW7NeSh9CDjn2z4WQyXKmc4kx0yu
|
||||
sm0ywnIu5wPe1jUbjl6ku0FhCFl6uJ97rjjLF/krTtriKqzYX9mP6Gp1xcQ9XvUw
|
||||
q4TXmBlLuHf9AQQqc0e/M5JCS/oqAMh8JP0gm0PRAbtRWgy9BlQUdB5A3j1tdSMw
|
||||
vpzuaE0CAYIMB4jqzkrNbfDBhK9IK0XaWLLVZHXSWhdVcg7d+W70b87PEyN2Ag8u
|
||||
hur7se/c1rE+IfB+aGY41Ne5Y5PupG8gB351icFbzsP76OPu1JLdc2W2wYSgSsjj
|
||||
HOdRBxUFtJ2JAhwEEAEIAAYFAlZfLwgACgkQbGWA53vXVsRAaA//Z9YiXWfY/VQu
|
||||
qNDuRU/9kaRHp6TqyGNEnspSB4vAuVGWuQVQtaji0Vbu7spakvLDG/jwWx3757V5
|
||||
jTlfa5v3Y8l9ieDFdQLrSUjqTYvBkKGXHmrWDDsj9j3PN2tNc6wC4qucGM7KkuMH
|
||||
o8ikfvoWsik+sx3y5SZPwYehQd1An4XjqQNDK2IYnVbpZbj34xiz3EiaKhMLbWgS
|
||||
XstFhXIxPoEOeZJCDM3qnPKDS7+lkayDQ/WV+s41WkZN3N4stHnIRDl03v3hksXp
|
||||
v7Q0uG3GYvEKDZA1bBfGgjkrzaLc3JboQtON6pgm/jkLx2Goruam+m/O98glRXVO
|
||||
9xOFAYfUvmxP0NBLNocuoG1ZpQQasYmG9pa5C6wrKY1h3YCm6/87U7LPLdkfkJuS
|
||||
H5o5MasVkInN/Iz6yjqkdD+PK4k/JnrsEKpG3rx7vI17sCTxGTfd+uoADlXAax8x
|
||||
wNTfeVxRhaq20I+fq2H8Z9rnn39BHCp92cDKi/Q4VHPKNEuPQgJg5VdT/lzvfaZj
|
||||
akE8hym5KUKo/ArNR76zwqD6j1DgmDpL7uEd7a1K2BzjJC4nnHFjfJinN/enosWd
|
||||
gKhYUGE6sT6ggpbDVFQCxujpgBVn6Xc9jshdksP37YrrhoRDiU40nGxmJsM42cPJ
|
||||
r07ZZAghx1UDcGE2cw2phnS0hX5sknyJAhwEEAEIAAYFAlZmqBsACgkQf2F1YXeX
|
||||
j3bJSBAAi0LAUXOVdLAnoctRIyFRabhxMoIpKywIbuE7hvROo0hbvFecZJk9iIIy
|
||||
vhoZR1HNFGKbOdYYWg6mbl6q2KumV3Oy0Q91ydeQr7s1gycodsJ3rUgvQGidiC1C
|
||||
ajTWxchS0t2lrKTYnHTQ9mhTXb+B528Vm+MZavajqv8gQDsl2AZF1XfG9o6HkcGX
|
||||
HvHirWC0MsedZaVXvKfqFSNN3JYBZT4BKoUMk3I8Tae2bHOVKEEEUje1NrGxg/xr
|
||||
Ic0/7UAxccvblZkeezi4V9KNMuDX3/DydVEwTL1M8/mbYroYbxR1rhOdEWlTP4nW
|
||||
LNLsOmHCuvRfwwia+vrMFJbBBWWfh/SJlSJBnP+n6tqeIIKCl7eAYYLcOReDO7U7
|
||||
ZMdgPqGr5fZdaR6+rIuzfdwIorjKQWxyskgDGeWzoSFu1KVdyOS0+c0jbja5HGcy
|
||||
TQPE+8RdLuLjk0in9xn1yMhuLOfH6mx7plA8e8xpxU5BnVgduI6BJhV4cqRhh5zl
|
||||
87dhzRh0EV6cxEF5XdS2gmj8aw7OaFbb/da75hRGfT3eRFkIiDxMKyFmvpwgUbkz
|
||||
kFFU+88ZViM987mp+2guSvvEQKqIba8qOVujgNSL2EBYug3HpWSjsLMS6Mz6HXsH
|
||||
Mvx8IdXH6xpt4nOFEmOJmJRgLitAFwpIJ4R5WYXSTtC/U+v9/XeJAhwEEAEIAAYF
|
||||
AlcEtfsACgkQR3EhcfLtYvscDw/7BREMLSbp8OF6cFWd0/4fj3kEYjTaq+ZdsV8w
|
||||
p9lE6OxryoCD1iE2hOqQOaWAqx3jI8x/eaEJfqZtHxwtNndCSma5ZWzMKL8hvJI5
|
||||
cJGRINiLF/04V1yqhWsuOsIzNZCyv8lQBQEbB8zB5uYmbof4ayLW+Zz9nyvg9f6T
|
||||
8HBt8iJ1x1zbmwM9mCCLpk+DYtDRCYCMXrYwHrV7fxkWX8Kinm9prSJ8ctqMsu+a
|
||||
17XIJCp3flgsWKzIJPHogaqZWHyRJ3uuJ0awzZ52k//dLIp360ElcqRhT2IA8sFd
|
||||
cd6CElRbe5ZmHkTU8lYwh046V7b133kusKCIvd1wVgqMPBzN+Yd781QL9BkxgK6t
|
||||
a1xlw/q8/D9ZyUbvBu61eX+e7QrSpNyyFpoOpSRTfkbNcUIYFUpsCRVFedbvw87P
|
||||
FXC4c3UDSMkU8DkHlKSq1LsfjPaFjGJ1uftWOPX/v2l2B/bEYnu0orLJTD86Ynag
|
||||
HIWxjDye1WpdQLkHd0stCX7Ohp3QD2u8mi52xU+GTSgQOaymHomKMx0YlOouBXqJ
|
||||
debQ8us/AmVrpWtcOzVPtYK8W6pn4S3k/S6YC2v/ZHTgFkdEpz1Ffo16WAm3+xbp
|
||||
AEqmjh62QLWz2K2UsBeZFfzsrpGBG27jPSYQEZvvfsettiH4HOpcg0izBqio+Vdy
|
||||
IYmi256JAhwEEgEKAAYFAlZQ7FEACgkQgNiLItQzAzGfcw/9FvM+Hluo/6BQCqZi
|
||||
2pwLE5r4ZmihLmQ+hkP1IHP2xZUd8308tjNSgNNe3vaU4aPCk2SJtXHpeEPIfv42
|
||||
x//yZEcolupoHKZK4xzQHDbjFvSLfh8jQSlN1h43ZpeLhmDX0Na/PgrrTbjbh9jP
|
||||
Q0hrVXh3/w6TKFg6qOW3y+gbpBJbruuri2lheDo6J1OK2GDlZwNDAsWkDQ3Dcsx+
|
||||
R+LtmmNXM2NW0Jmv5sTiQAO+6B5Iod7oRahXpm4J2GbxdL7WYE7/1dzbXoxvcvxk
|
||||
axqFuC99BkdCl7fL31PBPZBXjLTljGUFGIyh/jlbBMqoj5WhSv3nSkV2J8FCwcKu
|
||||
SH0x40tE76/9cHCKpHrl9Wt9jyUWKy/FaFhHYc6WzCpPSgtVBiFFZJMIFrzrs/Er
|
||||
8o7saKJSTnPejg9ilsdGrTvVDRvxLDh7e0tCWKINoMnFFHW76p2v+gwUiNpTpr6M
|
||||
y/eM/lR8ZvFqsMO+exOI8hVbLTl73bbEHkdoTW6Rgqv9ChqBUq2KDzY1f9NaT07X
|
||||
FLv1Z+eqD0jawQ10CMsG8HXPoF8t95VQaJ9gjSgiXOd0q0uVAMOLUhxRoBpCOgAK
|
||||
nx1qOLYKjjUHRtn7Npunh/I7AojGn6KXIqHrxzyFm+PaFYdxFyR5RFPRDyNOa5EH
|
||||
GLFlvgArUxwDZxe+SWzEmGxdaZCJAhwEEwEIAAYFAlZdjpQACgkQnBh7opshV/gT
|
||||
bw//QACSLTFy0nXxLjA1QsLBl6kmCY6itdiGNU3ko98vwtCK63sxQJR7BlU2Js/f
|
||||
6UVOAvPK2a5wfz/mhRcd7OCjBc5fz4EjuWWofmC6YPsyTspAkzNFdAiWj20Oh/7E
|
||||
jTbQSUb2R2F31y94pWVCI1iPhwYsaT69zc6QH7F5xG5YSpuZPQsmLX50YGmst7zh
|
||||
4oL6o6dGNiMpKzzmEqKi80/5F5897TSn/AFJz8wMiantSMhaRR0ZYb1+r9DkbOwk
|
||||
jSdTgMY//50ZEF2VODSrGkZnQLnmKBVC3LKbvbyJWHevhCu+SvscPwwAoTt4eBiu
|
||||
Ig+0H1qY4WQGdOTShrmpQd6bMRAad/e7dO6dztJm3ok73mduISA0RbB9hDgf5jnF
|
||||
0VkIeelQ1Il1fGfiwUZp+NIXVbmLuJiGlfxzYkj6G9DWiDRH7YDzc2pij9bpDx91
|
||||
Lp5IcuBvjHQzj1UCTt5Z/UUcNG0rdxUqLG6Ehb3iVa6clFyUvI6P7P5ws2dkzP3t
|
||||
zigTFKgI8q4Io8EZlmIrQ1hMuX4C97PD5MPGJzz7cZidvg5LIBACg5OjLdZUGhux
|
||||
APXjLLDawHzUK13EJK2pe2THLepm3fXQhFwATYn6fDOdzZIwhf0+NSj9xgKuWfoo
|
||||
+D51x/ktumf3eJBLT4WEae4cSd1a45so9JBczVNj4Qrs8s6JAhwEEwEKAAYFAlZT
|
||||
H/EACgkQbf0qziEbqtBo8A/+KXrMctxDrqm5xAqhBVkhg8i36F83KPZTcNbe5/iu
|
||||
e0PbSHzVQZJliFgtM014SIe+3AtiFEvloFuneBsV6D0/3Qh5QBp/VK8sCMp5JFKz
|
||||
5vfpCViGdmgU7iVu1jBHwqAoYLi74Fzdp9wamkexsC7iJmqRcEZ8sFioL48+Gn9q
|
||||
UmRbdxH6te2w2gEP27dMo0plJQAEpyED/8bm7w6aPikAk9/yGUOBsSMan0fFRW7k
|
||||
jja7qmdqfRKWa6CnxFdYWuNaHoxQumGgJLRhqOlZWeLyhV9E3ma3sUaqeGr6Uudt
|
||||
MKrUfLcnTCk8iPH92OYXVUB5uA2KWvGFuTNQyMu9omh0bch5bLWThlxSugnw0NxP
|
||||
oQg/yQSK7w0sYuNBRHdoW5WycJiQ/a5RPsNaOVxYG8JRQ4lqOow23mWSROBx5j3n
|
||||
CWf4p8IPg2evsT+qM4JfEZriz5S4NCOFei1C58L1Lx/7QrozjPNOmXGf7XitAb0N
|
||||
iMd6IUQB0VZdLTWt5gPo0W52m0yjRYhq9VWUYhxbwLbaVn9EXwS7r6GXkHKK7SH+
|
||||
5smbj/aOobrtB/F1lPzXk88kpQP9KtONBQaqfmpcN7+trPm8Y2M9f184hKk9tqVl
|
||||
TcxaYOc5P2yxan841plt4OuMufxQO2/ToB+1KLVIuzWnY5y2i8YgxfFXxSWveBtc
|
||||
mkSJAhwEEwEKAAYFAlZTJgQACgkQREXGZf+tmuDKZA//ZY2Ske05BygW7aHxNWZZ
|
||||
tdfXViFGcZznRTrLZLYO2HJLPIpz9odlob2XeME4jiF6mrrMQghkp75nqGHY99vu
|
||||
TolJElYjnApU5XWhR5/wgjDAWmL+36C64lX+B9ehayw3SJ0TEx8zBK7t5HZ/AbfL
|
||||
sCmut76DGPHNLMgJfi66XsPaVraygg7801XDNFT5R2U0T9gjy7HzAC2khM7qO7zP
|
||||
1tB/31tNrCvyDk9BaQPzdaD69rM6Bf6NJcjpCsbf0OmaNsElGuMaWdrOkMfFoxW3
|
||||
D5kFTn3pfLdqqMWtT2NHHs/uIDmu9BRdSXaPGY+pPB/TySSEt2peMa1t+fkVAzLy
|
||||
3Od7fyoyEbMaQ2ji22MyrOovifUWzsMUGpyvFXsnP+I15grYxlO1zhJZxWjlPScx
|
||||
TGy+1mSCPHbUvQxF9+6AbuoEEMyHW1XQ441t4JrNuzoVRcloh06z0F/bHkzU2PRA
|
||||
1Rnssd80oHys/hIjlbADo7K7Y/6Z9WXz/B8qdaNSZBKnLk+ME7ODAijZN8VoJ6bA
|
||||
K4eFEMdBx/YZhy0wkxfaV1OQ1pLLE3xD/uw+AdmuE5J6hM1W8dJRmtLiL70mnkM/
|
||||
3M6Ou+tK9PjyNsOhbKP2K8sRfum7yk3DLUKmIzNeXSSZKDs699DeUn2sSVC6VKfJ
|
||||
nEg6iZEFnWhVpnowcv2tBQWJAhwEEwEKAAYFAldcIfEACgkQLf9Sa7F/dsahjhAA
|
||||
jc+hNnZSmva927b1ctGVPESgaiXy5FOwq56zwYtHSR350ioVR2xxQRApSZYKcwU9
|
||||
Ol/1Xz6EfVQ/pnsr4QUulAobLxFWKNpKhHzAIHocOUaynVTJm0AnLD+As4IYA4B4
|
||||
65vmCExY4kDgGjRKuUc/hk/Yj/6mPtxTT0de+CfbDNvij/1J/WVo04pKkjaPJ4D9
|
||||
fXijKuVgZjRineP2NrMnvqdF7ERqCOOFp0ITIyFhDk8ueBrQ3SDKiZLUfTwSO/HM
|
||||
MBY0i514kZDFxzO+f2xorOCpO5SEiYiSyxc2+fTLQi7g2pZ9PAGwD/XVIbDZz6W+
|
||||
2k03scBx1Kj8D/p9UjejWFyLrsvSihx1/pZL7ZZMqsrPbz9WkxzvSupP5q4z60ZH
|
||||
YI8sewG1cWbw/2/QPj0WdKurYVWgsHFTl+OovSPTbA98NAqGlgkAavUvLUyBderg
|
||||
kRRsqj3LLrU+6QDsfR1ZxIMZ4WS0nJmMeGHn3G2wkVe25/sz5+UbO2cmLomgFszV
|
||||
YJiX0EyIPAk6JdIRnfSbgtncT3sLQ1YwWJH1FD58SjIaseubSlalqyzj6erCmL75
|
||||
F2GLZ+pIwPhTAeRE2d0VC3ynQeg0NPCeRcMZwvHAWceVulGBtPCz08onYY+QLNtx
|
||||
6uF72qT4KW9q4LAZ8jIQ+qsPM5omfi8XFZ+rHFf1IX+JAjoEEwEIACQCGwMFCwkI
|
||||
BwMFFQoJCAsFFgIDAQACHgECF4AFAlZIiWACGQEACgkQm0MrJ9G6INcG7Q/8DBOJ
|
||||
b4RuXv7Ch9FDNQIkZnwsKd8xiRT6BIzSHyPm9fgswP2vSAf/KKrbHcpXPKH8QBOl
|
||||
NtY7SEdfPQ5aA3ObGEs2VNt/oe4VN7Fewik18Ye0evletdSLzNsctp+UjSD9+TVY
|
||||
mH6SMc9jbun0DPcdqpD+ztcmJmiDn2K8L5JcRHspV+uSQ9b0onPMDxs7mAY1xPiR
|
||||
XWKYWEflQadfp9cBtnaTeWPtB08nYBjUFa21o7/qoK6Ys91pspSPvDMcJb0GNdcY
|
||||
dlbzjlWvVuJBlI2dhwXak9MVapZrT1h0IFa//PS7RHd7FaDaqM4+lVmNDMkAq6MJ
|
||||
MXLtVZmTd62DhSD6JPXpQqE3+6RF9SbOa+yqm2DYVpUpMq2anrt3BQXq4rZK3Mgm
|
||||
B+Iy+cyWDLOqukRwv1KIf7NFEcXybd8T6YkqgMWEavC/G4TCZ4AfH78+aVv//tHH
|
||||
s2Bh4ChTJJtrqsK6hEcy/oXvZ40LngrUyFHTc4WwxIZdz4t2AoGG4zDH1xeIfzlq
|
||||
BnzYhneftAH8Dhypmmi5ETQyr/BH2nM9FgNOxBd2PBWwpbmLASljQQSNbYOGr1J8
|
||||
y+9B/GyFPd1yO4WjbcXaAZuF7UwYo66NwxbrPwNqyCPMBNc97hGZLwaNFo0ogC/2
|
||||
jrleS5CNobvCLj6lljZXE1CFbcndAAHMLinn9HC5Ag0EVkiJmQEQALu0m1PMC+pJ
|
||||
JTHXnbsVXdrFMXbXPNVqE9/v/wlVrhUuvKXVrQVkLvdq3Rc3Y4iBVbYFTGH42c0Q
|
||||
WovGeDLGIhtvGwvr0VGNnS9WousHWn/9iyMmyjhhTAwOYuu5z7Cutx2TtCz3ppjL
|
||||
Mm5DWH3oEtFnIghqYvyLSW6hiDCPgRJXF7rTXV1aLjYAGScoNkIjSjY3LObNbFmO
|
||||
leMxZRKsRf5fw8QrQs/Uw50icGmVSDbV3pVVCvL458ilmhI2CNwQ3k1OifxwDaVK
|
||||
dFwY7ICIu8s30FhqivVGKOx1QxcxRTOeFx7AnSz4j98BeIoF7raP8T54ad3BTTLK
|
||||
6G4wY7Aq8SVKQcH15XuHaZ5jZcr4U7DgzqsoQ2GkGLzj3u/2u09ogrQ9jcR8LsgZ
|
||||
KaWU2FPy1xlvaLdGglJlu3eFgIoM8iKQGTm4iRf6PlZ8RJ4d9lB47NdFLS2PPA/M
|
||||
HE3TUosWz/iP2mJbqdeH20w5sGxrtjkHqKY3UxtKYgbFAn3+26E/u8BwmArJOXoF
|
||||
ti4iXHaj1/VQ4zR0TmJYpLUrloI2Whh1IvVasB2RFvnJ9SKAjbJbXSzPxRH+yLmi
|
||||
bRw2LTzPTPgqnH4abZ2PFFt5Gy+QHblqUlRmd+/Iqy50VedDCHp3au1bW6+F51eT
|
||||
qYV1x5XejnaT5FFotG44DhD9fKLE7dKZABEBAAGJAh8EGAEIAAkFAlZIiZkCGwwA
|
||||
CgkQm0MrJ9G6INeq+hAAkdQTQA1SZ6ToY2p5v4yJa/UeYfEn0YcEEG/nwtYiItTZ
|
||||
DZ5CaEj8AlRTy6il+dNqbr1dYx1GZhPBQLJpfF9cA4Z9is1vFN6+lKorK90q6Dyc
|
||||
2BfCqoV5O05gINV48e29wik1FreitDZCan/J/ld3KSk4qpJPnEml62t7D54V5Z+G
|
||||
WjZGUHBiHGyrPXVK2gqQLJLkgRX97Lt/MffkVLR12ehDBQMncG4Atcug4XJj4tPs
|
||||
tNIy59/qL7Gcah3N7wDVZi9EPZZXoxczXYEEaouh1/lWr6Wb79uu5R37I1h+0t6d
|
||||
d181TLTL5ItgFooMrYq/+V6rHkjLZ7qhqVhzTDMbC+0v/sXQ+aaxXFRSirIyW58Q
|
||||
Q7V5L2HNEOA2lkTeFDaW9asUyrRv89L0n8u+NI+JsJCc0+1Ca3q+/eXO8NAHpP8P
|
||||
WkdVBUvXrEANR5cf7Xks09rERCMQIiBBhgyvEZpcirN9Nd/RLGMkY4+FsMEJgQf0
|
||||
VcRGEyb6FYC6OCtZjcg9FK6X5k2KM/RcAJABz7sgq2yaTuP3AXAtlZCkLxXKWaqy
|
||||
HT2H0Lho1zmFTrASANEM00mGo0nmt0NBB6UI4pPSAaGjFxlAw7/5ZcvqQipTVysj
|
||||
6O4waGnUigGXSp0ETRkfplRB8bD95syrMAEfeXelMdksbhVVdvvayjxOlaIABwM=
|
||||
=vIM2
|
||||
-----END PGP PUBLIC KEY BLOCK-----
|
||||
|
||||
pub rsa4096 2018-06-28 [SC]
|
||||
827320CB5E847C916CF49D06147473802F35F6CB
|
||||
uid [ultimate] Jeff Genovy <jefgen@microsoft.com>
|
||||
sig 3 147473802F35F6CB 2019-12-04 Jeff Genovy <jefgen@microsoft.com>
|
||||
uid [ultimate] Jeff Genovy <Jeff.Genovy@microsoft.com>
|
||||
sig 3 147473802F35F6CB 2019-12-04 Jeff Genovy <jefgen@microsoft.com>
|
||||
sub rsa4096 2018-06-28 [E]
|
||||
sig 147473802F35F6CB 2018-06-28 Jeff Genovy <jefgen@microsoft.com>
|
||||
|
||||
-----BEGIN PGP PUBLIC KEY BLOCK-----
|
||||
|
||||
mQINBFs1QYIBEACyx/qJBfSY1EbCQ4aJTY4QE2R/u2cqCa/mpPMDje+GDZ6SyBwj
|
||||
PrXx7W7kQcwTTGrylJd1A6Wnx4PwAS3cCl7vSExYOsEbxKUcZiHVweMQBDNHFfL6
|
||||
bmdsXrHD0ypWqyCmbWdSGdwXfYhlX3UlntoETgHBrwZ+qf6Uu70cDm7cKPEoPlWz
|
||||
2eg6evT0vgtD1Y/47x5zAyCVqgC+rsEJKVdK3X3QSoUerFiYRUEZDXBc9KfHrn2I
|
||||
+St9fflVz6cNFExuc2x4H3b2vOwjZJqFO/cq47ppYXmOwbPQtH2r3hFpNxOIlQih
|
||||
h9MfWNiVKRICJZLZFicnFyANJVxK3ApK2Xhw4gJUjkhyXMZcpW891ETyOyuNjuDP
|
||||
cL0U6SqggfH0aubIfqBhgZqwTCp64Ckq28tjFqR3r7YqEFpiRu5mtO0gvk3sWl69
|
||||
bxXm0OxKLhhiVu0is3tokQU4M7zl7J8MHygBCkowoun/aw9Vis6K8ibItHb4CjTi
|
||||
SGGjNRbJxcPFq2YrU/c26T4IwtL+Mv59toXO0rJuwcaommkQ+G5zlBNViciyrOC4
|
||||
OzMiOkOrtYa6qaChAPfGrWL9ZETOBtVzZGpqLhYvTwP6Q7bDFoogtCq6iCU09FRU
|
||||
Q0fe3eJCeMFmVh84SWsdIe/GasXtWhvjlcmqnpz59Tzym8ZCDxhFaQt6CwARAQAB
|
||||
tCdKZWZmIEdlbm92eSA8SmVmZi5HZW5vdnlAbWljcm9zb2Z0LmNvbT6JAk4EEwEI
|
||||
ADgCGwMFCwkIBwIGFQoJCAsCBBYCAwECHgECF4AWIQSCcyDLXoR8kWz0nQYUdHOA
|
||||
LzX2ywUCXegVDAAKCRAUdHOALzX2y8NKD/9Ac7MhdVXmaoIjTNBQA/ZbZTipzLkC
|
||||
RxCiUai2ThMHIZKSyWFszVySHSvw0Cj8ZFUuDW61HliT6uYS/t3M93Wh1fTC4+qd
|
||||
Y0u0RVffhWIYxoW5pzo5p00L6Lb3oMfY6QuvLgvQd6iP0gXMQyUr/hC1kVOFJ9TX
|
||||
16ylf4GwSpJhKPXF+Heu5jVgDw2Qx/22/xl6xSsPRqTU4rc5HOQmHr5EfU3LjmXX
|
||||
PtSBXcknV1TSA/0Nbdxm5vpYQ2tdDnq/c0RU0B7L/mxOy8gH6wWR8qM3x0vpCnV1
|
||||
hu9U7Rh6ldOpd5FEVBeDluNQZMIEQHBAZ174UCkvcS/x0RgBa3ZwjgBGYpanonJ1
|
||||
qPcEqN4qWt3uXWuVxtNyM0rFgtpJ6fhF6rGyHo7nasI68a9XEbUh5IsKcDttTZgz
|
||||
blLgHHs0IyXyHG2DYG8wr18viiLo8Vt5QpLPIQT9tAgnRoiD83kO1IUTktksz/jE
|
||||
sGzf2gbgGTYWomM11W8iMXL6flBdhY5oasiP6jwPTtpLjmqCLZkz9hEJ7NBgOi3+
|
||||
Aqeu0PMwets0WQW6S2OzlHDGqprWQvVqOSa0XRXSJT7ZSlGImMMZMiUMpVukJjCT
|
||||
N2z8lOxpw2s9AL9jfvCD7TodpfmGd1SHNTNRv/Ivt2eHrMhfJ29zabIfPb1nPChK
|
||||
b1EpNlfgeNl64bQiSmVmZiBHZW5vdnkgPGplZmZnZW5vdnlAZ21haWwuY29tPokC
|
||||
TgQTAQgAOAIbAwULCQgHAgYVCgkICwIEFgIDAQIeAQIXgBYhBIJzIMtehHyRbPSd
|
||||
BhR0c4AvNfbLBQJd6BUQAAoJEBR0c4AvNfbL1YYQAJ6MocZlLZ2107lYKOCPcCRy
|
||||
dnR4xi0t1jPOoSV6KNyC/frRppVzBodPGjAY6RfRM5jq6IVhqD6Nf9HEYydbn1oe
|
||||
FdD/mCXLiqtI2AOvVCzMJctoKo2NlxcnHQGJCvIEjMts6lWi4wDqAgTxOL6hov6Y
|
||||
HzsFjcDTZ3cVU51nFHM1tHLjYFYMALDy7CJ87k1GB3mTSAX0SIC7FVJmDWO3ooVP
|
||||
rqMofUvnlzBfHfJUfDsrOFeZZWHy4DXqHpMMebY9WLJv9W0LwgcLKugS+uxy98Zr
|
||||
QcQgHLh7Nl3yuYtftWL1bqAmBx4kTjQ3DAXDa3WYZQhV0DEZzRQP5pvlqkQoKp+x
|
||||
y//SzoFni55bJpH36F3Arp3flgFBpqy68ll3Y0/aOJ+jLD0O6/xQYPfG4Pu/Fly4
|
||||
28GQegfq6I/o+bc1k55ncT4+jpNb2a8n9TaFIs/uM2guowUisqt6WkB9jXrC7a0Z
|
||||
Cybt5iNBOuQVIZyz+BQvRMxkt7uESDPgo8cVAzr6DfX+GdBVAcZjm3piXWkaDxlE
|
||||
mOOW5HMW/3uOD2SAVH3+mPdvC37yFkPuTEoDbsgKRMIGolb3NVJVUu8+Hxyg0818
|
||||
h0G/zpE8XGNzAL52NrEsGyu8jN3L1BRFeClrP+xDas1I7lcohUXUHSnxJnA1GbKZ
|
||||
P2jq1DDDE3o06Ld3BFGntCJKZWZmIEdlbm92eSA8amVmZ2VuQG1pY3Jvc29mdC5j
|
||||
b20+iQJOBBMBCAA4AhsDBQsJCAcCBhUKCQgLAgQWAgMBAh4BAheAFiEEgnMgy16E
|
||||
fJFs9J0GFHRzgC819ssFAl3oFREACgkQFHRzgC819suQhQ//eepmASOFTA5CRegl
|
||||
Y9hzF0RP6+9kq+KTSbstbbQQuR4Vd2m0QeF4gT8vcIHEIfxVn6sfJFf2k4v/TKGE
|
||||
OFHXw8Wl4/XvquVpfp8x8pZ/XVd+/K9F1o6WZ/o2UUleEecIJENeSzJ69/WbGozE
|
||||
8HkSTaA4BR/G5bAzX79Rm0JfiS9l8ZyapkcPfSGO/Jcz1pkBfacAPyyDMxYFos6O
|
||||
y038D/2qSCLI3TxS/kS2FANpCpXwNUjvR04WfbMWnaVnOih/gqau7ZVQ20lpKZOY
|
||||
E4GcQtXVbaTR9vJ1tayA8FVpSO/4nqjD3CdI9JLkq7MGLnkHOdL/YtD0k+HtW8dr
|
||||
5p8m+tgjTmt7dAiCIXI+a8M4jjoIx/qAdHOOH/4T92s+zNYXd7CBkRp/6ipalfn+
|
||||
QnyFgL0r0z1P7hSwFckWKsbvT2CIFRgJOwoTAMJ26HsIqt9ZkMMwPwth+nYKRlw4
|
||||
WgmbZeoHZp0tpyqa7LTqNxVVxdx4rcgTL+jOOJMw+cQv4opv/pN7rWyFP6jCy5Wg
|
||||
DzmyrPZR/zEhH0h0cRLblq7Fuvl3t/pXxgWSAVYmAvTQyIwRWC0Nvec/AVRGE5uF
|
||||
koXM6nl27ZtGc7PwmqJZePi54RDIy7LCI09zdmu50JALfxkQ0Yg4EVFy+gf979jy
|
||||
ey7Fz42xM7+BTjQ6lXdmuSnIbFe5Ag0EWzVBggEQAL8RWXQwK1jupZkKCoGSSmuc
|
||||
FO9MdCS/744340u9TU0oFY8mmqOppoOiVi9EKIOWVHJ5a5DixsIdNkTAtmXM4a6u
|
||||
sr5wB555+KTkmofJ8nzHQM7gEaaHt9JErNo0f7Fzi9VEJ+G1XHFjW80HvQ9krpSj
|
||||
Lz3x0CbD4KjRRJWoUrKiBvNJjk2Xox28pA9F7lHWLgGTgZFrtCWMaViOcZqBZfAl
|
||||
pXYFR4hacddsDrRw4r4cl3NlfQ+SS4PL2kCcH1gx5AlhRkygtiJfYrHDvHpBk6rz
|
||||
r/IMojXOUAYV8TFQRyhsz/Cpx6rAun+aOXQlL4eDuNtWnxEeoHoNRAnAtUobP8iv
|
||||
cNqY1F3ZWaaY/MmNtw/FiDmYePzLz9p8FhBHKvE20Yt6UrUX+E9fPPSQDZxdtTzy
|
||||
LOWRUtOhMJIPigQRipFWng9d79Rf/goX5fcSPja/eqZaKkosYVlJW09+MxLkNO2W
|
||||
/UDmoVsvhnUK77I8BxqgrZUI8uj+d2cO+WQfL/RlGJ7a+icspycT8nTeEfCqjarQ
|
||||
wzNY5PLo6FN+N8dLVRyrQsPwwfWstA/6DJaRHgwhz6UdV7wYOBtI35xIZ8Cy71tT
|
||||
PycgOiUmatM8M5/INvqqPmQwl3VQdvrsD8MV3dP/U3edCSeVDKhqFXps47zdh/Ji
|
||||
gTssKOz6Lafg8Uo3314zABEBAAGJAjYEGAEIACAWIQSCcyDLXoR8kWz0nQYUdHOA
|
||||
LzX2ywUCWzVBggIbDAAKCRAUdHOALzX2yx0qD/9Wr1rnH2GZMwOCpwAplRegw/I5
|
||||
PNS36fOYs2YEv6/g3OdgSrGu2ZL+VqhCRTaqE7UrJDIsV9JORRptgg48nw5IGQlq
|
||||
NyyxMEJxu7KlpgRcKC6o1hZ+zqkLyQGdcd29i8AWBKU++MUunVMUplVuQa2mDRon
|
||||
KpN2lg4y2djfXvLOiKo7kLXnM8S+42gB34+X8HqFhfTXwz0Q/oroHayrU4ncTEob
|
||||
D81bOSQ2agn0Yi5GNCS6FR1Vmbto3j5HgvLbn9WkT8cM+nBr7F2zsZtG1Op+SKZK
|
||||
KJ9/eHkMJ/9XgZMcaEPJzD938+AscFUZeAL0QUS8MRlyYLFQKvQHiZDGgwfDcFGs
|
||||
7X9kZbMdx8nKRej8wqMOcro/TfogKSJj04L3qtsa5Cs71oVeUBalIlEcVkZIWUNS
|
||||
HzwDFcJtfh1lbjRm9ALLTEHwhoZ6/OxRRbixEUlGZGCn754Jt2JrcQGm+wqL/w2P
|
||||
ryG1zQFz5bcfxofFF/YxpHOsrMXb2GkAC1Wg9ypGfQ1RStGCe2q+6Qwo+csO7lyQ
|
||||
mGdooN2u+X2SXbBq+F2JxQe2X65SAWrFsmflN1vsY22IWExQE9kA63twjQu3iAF/
|
||||
mrLU2gvbaFZtV1HwYC3bCNqlx68rchjSQwOI5sdocwWsWbvJ28yi5I0PcBmyBa0Y
|
||||
TMuS5W4GIP4f7uR1uQ==
|
||||
=Veoq
|
||||
-----END PGP PUBLIC KEY BLOCK-----
|
||||
pub rsa4096 2020-02-24 [SC] [caduca: 2021-02-23]
|
||||
4569BBC09DA846FC91CBD21CE1BBA44593CF2AE0
|
||||
uid [ absoluta ] Steven R. Loomis (codesign-qormi) <srloomis@us.ibm.com>
|
||||
sig 3 E1BBA44593CF2AE0 2020-02-24 Steven R. Loomis (codesign-qormi) <srloomis@us.ibm.com>
|
||||
sig 44CE7BF2825EA2CD 2020-02-28 ICU Project @IBM (International Components for Unicode Development at IBM) <icuintl@us.ibm.com>
|
||||
uid [ absoluta ] Steven R. Loomis (qormi signing key) <srl295@gmail.com>
|
||||
sig 3 E1BBA44593CF2AE0 2020-02-24 Steven R. Loomis (codesign-qormi) <srloomis@us.ibm.com>
|
||||
sig 44CE7BF2825EA2CD 2020-02-28 ICU Project @IBM (International Components for Unicode Development at IBM) <icuintl@us.ibm.com>
|
||||
sub rsa2048 2020-02-24 [E] [caduca: 2021-02-23]
|
||||
sig E1BBA44593CF2AE0 2020-02-24 Steven R. Loomis (codesign-qormi) <srloomis@us.ibm.com>
|
||||
-----BEGIN PGP PUBLIC KEY BLOCK-----
|
||||
|
||||
mQINBF5UP8oBEADBaR5vyDozi5LbtcKnemb8Yy4p7g8Ki5gO3K+K2MxeB9wf6xhU
|
||||
Qp/4bOYgpdCmBDT8bhEENTzNFxIV3g49WxC1LuSVeklcK7j+VJKAwtysm4N/648U
|
||||
9NUr8N3sKaKbj4qEBWPc7VZigbthpFEHT4TCZ2DO8XFzZfnGbDn60SvVWsh64k5L
|
||||
hzLKt6k5PRIiQufkJBtRqF5IHGKjSH/W941HrNfLGUodcZzsDqqtql9279BRp05h
|
||||
bsc4cNZzdNrrWNSOAmY2F7ar+JjFDPPU3utyU4rLfIXJ9EbTTCLA6OKygTKPhtX2
|
||||
FnS+HBtRYOkzH3KZGpJ79agGgSAxKThsL7WmRMjssUfaaL1Ay4j4HCUHqvXoQe6h
|
||||
SzNgYhQkN7OjTLYgXXBdjBwAOJHH4tNeHkbyMF24d85MzEraauYyAFuXVldHBVBO
|
||||
q9oNJktHbVwNR7nbSz63QA/gLzqX01HFu9NQHSF7yRMDOog7gLPEtXp2ZtxzR+FI
|
||||
sReNU0UwzUrvM8zmlnbwXQMPDBCj4uhLp8zdcTmNgmkVUm9tPumGNruaV4W2vjD2
|
||||
vowin+GGvXjKYkflGO+GZoC5SzG9k3ef1J5csRD9oDswNomq6Pa22vujnzacGc8n
|
||||
r0F9TnRcH+ozbRYMz1qoklSH//XJFH3utwEyk8VRuPcmqKdAW/lz2pQIbwARAQAB
|
||||
tDdTdGV2ZW4gUi4gTG9vbWlzIChjb2Rlc2lnbi1xb3JtaSkgPHNybG9vbWlzQHVz
|
||||
LmlibS5jb20+iQJUBBMBCAA+FiEERWm7wJ2oRvyRy9Ic4bukRZPPKuAFAl5UQUMC
|
||||
GwMFCQHhM4AFCwkIBwIGFQoJCAsCBBYCAwECHgECF4AACgkQ4bukRZPPKuAnaBAA
|
||||
p6J21ohqnYERDp6H7mtwuSVVp2aAio8EmtPt5Q/1cwHg653T5DMZda0lBPOpHzhM
|
||||
3p0W0WCPpxu5GuQ0vd7MMqollyGHmL8fOGAsA/REEzyw3SAyjzLHbb3quGn9U9qE
|
||||
7/irizA2SQGAXKbyK8JNorloKAg9Fv4S6DsfUxqDBb6bMyvWqHNsNNIHbhVvLblK
|
||||
yJuk4acWowphkLRC9YuM7rAaalrOO3C9PozbHtaukJZ0sBifJRG7f6JICoDpwujW
|
||||
+zXkb5yuLQE5NcuZhd/YFo9HSKXsp3D75hz4/bKxe8KCV3POt2RJqONhJGGVU6BE
|
||||
EXr3J4lZWUic+8gGefAMxAL9WbzrOEmfesTrSkzkX60+3qQ1lzqv+f91fWpBz2/k
|
||||
RNN6UDM/jbDfUHhcEQ82DHCATY+/LTcjN54gBnhWW8jszxY9/s670nLCaGoGC5IJ
|
||||
haVaUOA4zKkgfv0Uewc2cxunYKkoK7KIu0THNHSDToivF93tN9LOmWpSah4Bx+of
|
||||
ik1+IPE9xXqcld821r3XXQPYKrdv6bXmTJLVbkKv1VQsP90ytuZ3eQJ+FZvs4Yk0
|
||||
UHqW4QDV2A5ddEfUdbtXDREUDRQtY322s9jtafGfWanZ5/BpUKnb/IjYeLYcLEwy
|
||||
uqbC8b9ljDFmHchsyKYPQQzdu3AQhkwgDWAbfaF/YoOJATMEEAEIAB0WIQTLMZDK
|
||||
eEJDnlfzcS5Eznvygl6izQUCXll7XgAKCRBEznvygl6izdOnB/9HyeOBHpQzD91t
|
||||
f84/oGFk+uSNJGG2XdUyHw1O8zEvCCeCrE/7nqstp7ZAJ5Q34zgc8Y71L+3EFN99
|
||||
dTB4WDsXpHA9cxc6ij4pA/IhMr5phorcUdl6gX1EbGSpM6yaS8FKJyD53rhklSyM
|
||||
qZSg0H94Wg4rP8i5+N/DT5A8C5l72GipkXRz6t2s/VJWfFIeXnxGFh+HzJ1PMKoi
|
||||
HXEbnSQRxd08hm+G5A85K8ptPVuDVojRLLu5oWVrMvSc/xXbQMIeoT9z+z/2DFWi
|
||||
KWfXeujwoLdibJriVyajqrYchZcCZ0gVke7CEgIPupO8ngzF2TwnCLPIFWjNvcPm
|
||||
5pZh+cs3tDdTdGV2ZW4gUi4gTG9vbWlzIChxb3JtaSBzaWduaW5nIGtleSkgPHNy
|
||||
bDI5NUBnbWFpbC5jb20+iQJUBBMBCAA+FiEERWm7wJ2oRvyRy9Ic4bukRZPPKuAF
|
||||
Al5UP8oCGwMFCQHhM4AFCwkIBwIGFQoJCAsCBBYCAwECHgECF4AACgkQ4bukRZPP
|
||||
KuAaHw/+OtiQoOP0CHwjYiOsZZdr725HDCGZ4jjWTP/oVda8Oh9yJfzG6NakMzbt
|
||||
Ek0TuqIkNiGwoxWiwiCaeNXK4QPdMIOLyyE6fIvVWCqbdNnOIGdsl6uePKq+MwIy
|
||||
YI0T/HbFqstJhLc6bmJZOtmEa2kzHcrmNAaAzbNtY6dgr+e/2ChUCUKbQesoKvBK
|
||||
R2I1VJHPfJCjaj6Jz1aHH3k56mUaI5ZLrQ5ZkEbIXnr8mRoJ1gvvn+0ktU/gWKUz
|
||||
YSEH7grkaI+gSpQgZ45otYolZ2z0mDzVj30Q0MYmDb1TBo2VonoZXBi2a+w/OSU8
|
||||
1L2a+gncbHjEqtOWHuzljM6yRDAQns+TRZ+vfqb1pI+d2NNl0EjqQowfKvfbdWt7
|
||||
HsiUZE5p7v1b7+BvxXOP4aieezbyn/eye4gJFxJVKHHd4f81rtVRO//eW6bOmpv+
|
||||
FQltv9GNJ65RsifKbaU8DCjdEWarechw5Yi6nj2X1NeHYrYgkbcntAD6bWvG6Eqt
|
||||
dgXInb5DceP0jJMqXAI453aUle283gPydOtIimJCzxiCwub1A7ozy/X8Li4OsT5C
|
||||
vqMoYIAF2jiyaAlqADcv3wT2qrKFpxGV3n1ZFPgRVOp/N23OF1Zq6JnOf5F19vJ3
|
||||
TgN16KI9oi+nq7Sr9140JqZr1KWovUHX0j33DvUgskR8YfypY+CJATMEEAEIAB0W
|
||||
IQTLMZDKeEJDnlfzcS5Eznvygl6izQUCXll7mgAKCRBEznvygl6izRefB/9jXPRk
|
||||
73N1ywqoGejXxB1dIiFlm/8r7F5IU624fc/gt8mcdqJ54rBiC3ESnIwT4wGG+8GZ
|
||||
Cb74PNmBoTlWe6D4QT4XzJlYQOYeQjtOEGm9KKCT5bIXCbu/z7Dd8eDrZTsuxuxw
|
||||
7o4pDfEIlQUeSx9QOkjLK46vSVVOiuLOjiq0XIYqbmG9MHjZ2EktwVLQOzf9biwz
|
||||
IOItOd0XzrxDGI4iC0A+0UCvL4ZHAL8yuuwuL+jGaM9RipEB+9kB/qNlEaRYZost
|
||||
XNTwPeLOln2yqFU5V0OjbWZC0GTN8PP1IsY34jXYSTqIkDyeOoUdyG8YXWhe/eKY
|
||||
hj4iq8Y+Iy3oDU1WuQENBF5UP8oBCAClbz91MxtQgqgn1jOBPh79C3tSyscIQwfX
|
||||
OxrIJxg8gOr+BAr6vDBIc+iYFO380zqGloCfWUqsVdBthzwErzjyEByvKWLfbnP7
|
||||
WHpyEWpaUTS57PYX65bLiLnNZMCmgmRbv/xdOL9rEhwLHBtjo0Q45vNXFVJlUvcu
|
||||
H4OiL7pnfP2VPRiVzlYgLRi7XpvRMyh+PWhkXE+o0P5blQ+GIuhTJ5U23PRCrFA1
|
||||
ElX06Jy2c2GEJeox692wxUY1QHUvyLujPhPWWPKN4QJbPoSi+FOCOxKakBNs2MeY
|
||||
4zBYHzjZJ2to8kBmdkTEIKy2aC0OO3QTlfmsJcKjPUeaAycaN479ABEBAAGJAjwE
|
||||
GAEIACYWIQRFabvAnahG/JHL0hzhu6RFk88q4AUCXlQ/ygIbDAUJAeEzgAAKCRDh
|
||||
u6RFk88q4GRRD/91E0HRVUu0mhJ9IL/0UQV4wRTrrS7oLNoOFFQHHdBOXBbNvH51
|
||||
UxE3dmHYMVz4Mx3QuwmiqCTHeE1oHSjuOoSfRu9V3gG1HpIBnUdGtcYm95setIeG
|
||||
yZ1Yza9P3lweWGp9ublhXqzAJ/SqYTFpFTQA2cBAEblU9hNlmBi4DJTqz7XzNXUw
|
||||
RzU9U7hRMzhi9JJybsLfDJZap+yK1YIlMqAAMtveqKqWkEyFwJRwZIy8BYAfQvoh
|
||||
N7XHvvvq53pyzOX9H/2YNFyBUiG56C1irxtJ/hlprFryYEjiilWX8UFVO/rBSc4o
|
||||
n5hSP2VE0yibJQK8uLiA+cJvOKoYMe9u0mPeUa7lf4aQSBBctwRGYZMZMjri8lI7
|
||||
+IQEXgb+Se+5DoM2nfycZF9svgwam+y92d31yklJsdUAnMUIn6XCLJmrO5I7qQQ4
|
||||
bYUInhI2r3roRIskLASta3JH++SRVeRECw0k6NdkPhXFbjkSv5TZYk8mX4bfYf2B
|
||||
+pbB6xMqy/qygfOgsnTZnA2b9i/agudx1Qg7YRZT0M2YsiWLGBcJVGLnBbirDORB
|
||||
eCVlKx7ws8kC/oPcNgwoqs8XdHPNHUr+MGWcoib4Fo+KhijYGIM0XNfc1fWRPDCm
|
||||
DH2mVgW7fxtB1JdUbJPU1i9vzAONu6ZKyrK4d6/MTGduyO+zZISqVKS7eg==
|
||||
=DEHc
|
||||
-----END PGP PUBLIC KEY BLOCK-----
|
||||
|
||||
pub rsa3072 2020-10-14 [SC]
|
||||
0E51E7F06EF719FBD072782A5F56E5AFA63CCD33
|
||||
uid [ultimate] Craig Cornelius (For use with ICU releases) <ccornelius@google.com>
|
||||
sig 3 5F56E5AFA63CCD33 2020-10-14 Craig Cornelius (For use with ICU releases) <ccornelius@google.com>
|
||||
sig 3 9B432B27D1BA20D7 2020-10-14 Fredrik Roubert <fredrik@roubert.name>
|
||||
sub rsa3072 2020-10-14 [E]
|
||||
sig 5F56E5AFA63CCD33 2020-10-14 Craig Cornelius (For use with ICU releases) <ccornelius@google.com>
|
||||
|
||||
-----BEGIN PGP PUBLIC KEY BLOCK-----
|
||||
|
||||
mQGNBF+HGz4BDAC00mXlPPgW5JI9ECxI9juSX/xpzAhM/2NvXzc4Smq7efJMzwvp
|
||||
PkfBwdfLavred6rxqjVfXJZuPL4rAY2fKBiMq2ANBRsx6t0Bl1jbZbI/MGb18Qcl
|
||||
5EQ/n1I0WslugsZpSX7crj2A+bO42BRWOgroQdpLTrdTjdeS774MaOvt7i3S2Cy/
|
||||
YT+U6ysrz0F7wWdiOMUftHEBOrXPJ0s7P3bJGDYsca0sV1o5EmefQWh7cymhroHQ
|
||||
UzUnsthmU0Qv8BQ7xc7zsCBrJR0V0ryMww8u/mLexz7i+W522olx78Vj887yC//l
|
||||
lAZYI+ORG9hTr3QvcVaF/FaZsbwrA3FPKB+LFWQr/CFa0eSuXWoY5u2g3aL5KoCy
|
||||
G/l7x7YlMx7Rw+65YIssZYv39AiZEU3nei+qsaneNc8v6HTX39mMXVdMc5/iWV6A
|
||||
pBo+1+wTgaOWMFf5zOg38DyzdQaDz6TNoAgwHbQhz7MDHqLZssy2U/ATnqAPi/Xf
|
||||
8oGmQtTh/8zupCsAEQEAAbRDQ3JhaWcgQ29ybmVsaXVzIChGb3IgdXNlIHdpdGgg
|
||||
SUNVIHJlbGVhc2VzKSA8Y2Nvcm5lbGl1c0Bnb29nbGUuY29tPokBzgQTAQoAOBYh
|
||||
BA5R5/Bu9xn70HJ4Kl9W5a+mPM0zBQJfhxs+AhsDBQsJCAcCBhUKCQgLAgQWAgMB
|
||||
Ah4BAheAAAoJEF9W5a+mPM0zJ5IL/2XJ6VHdSdMstzjpnCDjQTYuhFkwEfGSlMrh
|
||||
NTZftoQ1qYXuU8DKISgEo1OlguuK61xePn5wl8BfT6XfF2XCZpVUah2fom1m9ZAI
|
||||
2gkt/Bwn1ulBKQd3NQpAQKPSKzOVJ+0BBOoBCwoCy+hHr8oeRJGgxz+j8aux10IV
|
||||
OHviRkBizx0vnVcy3dN41QLNvtq2cZcKig68sAJPM3t4//co89pH3kjKzBFVCqxF
|
||||
+INFyaGMyhT1zEhEYB+o6fvYGS4zX9qGRpLzx64beA84DZRmv6aLvjwzhSfA74Ey
|
||||
f+FLiAMYae1opWLS+to54WEB5hK2Ak251PqOfWCuR1KVGmQaa4sabT3uQQ4bRh6Y
|
||||
yGSXD9qSRJC/W2vNRwkEnKrjwFgH6w2XgHpVLlJiC6Ir7U57AkCfUDVdRCcWQoGN
|
||||
ZTuPIHfoj7IehJgR8NZvbHAWViC3FP9DZEm+1druAbc62PngbTUlF6zJrZ/zl90s
|
||||
kDg8cNNhQtCdgDKdL/cZwBm/G8RzUIkCMwQTAQgAHRYhBP+pEpoYDXZbelvqHJtD
|
||||
KyfRuiDXBQJfhzz/AAoJEJtDKyfRuiDXegYQALsvJzFyxsYUsgr7VYTlyEmjLx2k
|
||||
S87toZTgD4zDHprjGT/mRQvzKaMqgp6cwGtnGYLbJi3W17E6ykMk523CWfrqAbqV
|
||||
RpMH6nQvxNEbjHCAOAlXZc2zk78lqHkd+UePpDW0Lzwllyf5AK/hf9AvHGZlRSij
|
||||
OZEpvRIinwczPcKCyy1RdwO2aeKyjrk98e8SKjpjdl/OXFiuWpLWUXJE/QOkIwbk
|
||||
lwUUgHQuauuKZFDDu1T4AGBEAv5yUWU7m6PMa0riGQ7m1pSvTVX9wmV0KgkQ3/sW
|
||||
VDqYf3X1VL/NVt4GUBAFTCw04Xzqdb94ERlYde5yFn/wJm7Pnv5pDIlxvGvL+uA6
|
||||
e9oGc7msHA2PQipDaAPcvmDT6IQLLuhmO9PA85vZ8egGjKKBey5FE+zRrowWfsLm
|
||||
WRXeBPohHjHY2KRto3onFJM4JEydKLtpBAGQ04NAGAhth3opd3jpaPSu6u4rhx4i
|
||||
HLiBoJzsWtRTE0lTAeA6Wm0MoH9JQQEt4rgi5kd9BcYMsDfZiR9Fsq2FwEkPwFbi
|
||||
1sSNaxW5BWYFKSBhYHOkAFzsxRrD1OAj/anPO/QYfRUu08hUyYi1StTa/0zZ+Cf3
|
||||
NY/Q3YJLe/6PJrzxv4wnE7QZs5OSUDaTqL9kOB1mDz6Z3iLnPeOnkBuL6JFF5IUv
|
||||
lc9Y1w/UOFefZgGguQGNBF+HGz4BDADO2pKzANvXFSpT5IeMCcAvOIAf8vgJpO84
|
||||
BnjQ9/PgiOZMmQW03TIdrY+9wKATOxvUTpC7G+070M438UwlGJX86zjY7CxX+xdi
|
||||
yqL9ArK5QJlvaXSvhx1qcArgLIiW+X3qs/U/Y/qXOINS2gZUSYaMBnWHuXutIXSQ
|
||||
Liqi3l662nSPvKi7WjpHgi87TV2zG+B9xF23AxMZWth/0vtIevIheoB8vk0mJoos
|
||||
8Hmx1qknknDqICT/Flnxm9f1UyDOY+IwWMRyvZZUr+r08vg7qg0TIHkYeMwRMovn
|
||||
TZcAPoNvFT1yHsxTdf7mA1yTHCymBvP5aUuXh+FJ+tfPmKGvz65bs0VmlcU10Kl+
|
||||
EntF7TniVhCp/QB5M8MA6HKM0zxORFMGZ7xnV9gu6nRPcR+8PvEFxY2wgEcf7pfj
|
||||
6wGdS+Pri/dK3+53abE8RLi/aJW+wi2XNd/aRWjpSC2KBmHUk17Zgp3Jb6Iijb+e
|
||||
7bcTf+REXgi0+PskNaeCyU4LzStZCwUAEQEAAYkBtgQYAQoAIBYhBA5R5/Bu9xn7
|
||||
0HJ4Kl9W5a+mPM0zBQJfhxs+AhsMAAoJEF9W5a+mPM0zULYL/27+ED5a8s43q24Z
|
||||
COQQPZflXy+0MnzW4Fwj9f7WsxD/fkj651p/MoRVmugytRti3T8+A4ULUuKxatCZ
|
||||
+Q+SIobXbWHD0H+hVosBohU6HmWX5Nbyi18wk5YDNWzN4n0duhv6lb4tn3M9utX9
|
||||
WdQWnudXNjKzddNiixalc5eU9vMEpEnwKLcmZ/efSimqfGXldMjTw+FIh6wHhBcn
|
||||
uVXQqj/+Ie20l4RXt6VxUltZwaQJnH6nf/4T1B4sllNo6Sz8RHjnrc+SjuIREaga
|
||||
nRNZ0BF6aAj6qk9RTlkmBGiAdmvtCuGRqqsG2i8OrTv0xArCGEs13Vv5BE6JfZI/
|
||||
YdzPuq2u0JM9dsEM6v3LLxhlfxxOmM9OZM4+GBP+JMB3TydK8t7vfLW0kDRvymVM
|
||||
uVrQ13BH7D1TWlzM7jN1dToIySXIjGRcMpqT/2LqYDwHfrJCdQZGKDBl5vNWXwNs
|
||||
OC7PFtJwtXNl8p7Jvk3lplU5rNSSkTF8hSFgtxu4cpJ5VAY8nQ==
|
||||
=FdGa
|
||||
-----END PGP PUBLIC KEY BLOCK-----
|
||||
@@ -1,41 +0,0 @@
|
||||
# International Components for Unicode
|
||||
|
||||
This is the repository for the [International Components for Unicode](https://icu.unicode.org/).
|
||||
The ICU project is under the stewardship of [The Unicode Consortium](https://www.unicode.org).
|
||||
|
||||
- Source: https://github.com/unicode-org/icu
|
||||
- Bugs: https://unicode-org.atlassian.net/projects/ICU
|
||||
- API Docs: https://unicode-org.github.io/icu-docs/
|
||||
- User Guide: https://unicode-org.github.io/icu/
|
||||
|
||||
|
||||
|
||||
### 🔴🔴🔴 Special Notice About Branch Renaming 🔴🔴🔴
|
||||
Around March 24-25, 2021 we renamed the `master` branch to `main`. You may need to rename your branch in your local git repo and change your normal git command to reflect this change. See also https://github.com/github/renaming
|
||||
|
||||
### Build Status (`main` branch)
|
||||
|
||||
Build | Status
|
||||
------|-------
|
||||
GitHub Actions | [](https://github.com/unicode-org/icu/actions?query=workflow%3A%22GHA+CI%22)
|
||||
Azure Pipelines | [](https://dev.azure.com/ms/icu/_build/latest?definitionId=360&branchName=main)
|
||||
Azure Pipelines (Exhaustive Tests) | [](https://dev.azure.com/ms/icu/_build/latest?definitionId=361&branchName=main)
|
||||
Azure Pipelines (Valgrind ICU4C) | [](https://dev.azure.com/ms/icu/_build/latest?definitionId=364&branchName=main)
|
||||
Fuzzing | [](https://bugs.chromium.org/p/oss-fuzz/issues/list?sort=-opened&can=1&q=proj:icu)
|
||||
|
||||
|
||||
### Subdirectories and Information
|
||||
|
||||
- [`icu4c/`](./icu4c/) [ICU for C/C++](./icu4c/readme.html)
|
||||
- [`icu4j/`](./icu4j/) [ICU for Java](./icu4j/readme.html)
|
||||
- [`tools/`](./tools/) Tools
|
||||
- [`vendor/`](./vendor/) Vendor dependencies
|
||||
|
||||
### License
|
||||
|
||||
Please see [./icu4c/LICENSE](./icu4c/LICENSE) (C and J are under an identical license file.)
|
||||
|
||||
> Copyright © 2016 and later Unicode, Inc. and others. All Rights Reserved.
|
||||
Unicode and the Unicode Logo are registered trademarks
|
||||
of Unicode, Inc. in the U.S. and other countries.
|
||||
[Terms of Use and License](http://www.unicode.org/copyright.html)
|
||||
@@ -1,5 +0,0 @@
|
||||
# © 2021 and later: Unicode, Inc. and others.
|
||||
# License & terms of use: http://www.unicode.org/copyright.html
|
||||
|
||||
# This file tells Bazel where the root directory of the project is.
|
||||
# "Target" and file paths in Bazel BUILD files are relative to this root directory.
|
||||
Executable
+127
@@ -0,0 +1,127 @@
|
||||
#!/bin/sh
|
||||
# Copyright (C) 2001 - 2002, International Business Machines Corporation.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Authors:
|
||||
# Ami Fixler
|
||||
# Steven R. Loomis <srl@jtcsv.com>
|
||||
# George Rhoten
|
||||
#
|
||||
# Shell script to unpax ICU and convert the files to an EBCDIC codepage.
|
||||
# After extracting to EBCDIC, binary files are re-extracted without the
|
||||
# EBCDIC conversion, thus restoring them to original codepage.
|
||||
#
|
||||
# Set the following variable to the list of binary file suffixes (extensions)
|
||||
|
||||
#binary_suffixes='ico ICO bmp BMP jpg JPG gif GIF brk BRK'
|
||||
#ICU specific binary files
|
||||
binary_suffixes='brk BRK bin BIN'
|
||||
|
||||
usage()
|
||||
{
|
||||
echo "Enter archive filename as a parameter: $0 icu-archive.tar [strip]"
|
||||
echo "(strip is an option to remove hex '0D' carraige returns)"
|
||||
}
|
||||
# first make sure we at least one arg and it's a file we can read
|
||||
if [ $# -eq 0 ]; then
|
||||
usage
|
||||
exit
|
||||
fi
|
||||
if [ ! -r $1 ]; then
|
||||
echo "$1 does not exist or cannot be read."
|
||||
usage
|
||||
exit
|
||||
fi
|
||||
# set up a few variables
|
||||
|
||||
echo ""
|
||||
echo "Extracting from $1 ..."
|
||||
echo ""
|
||||
# extract files while converting them to EBCDIC
|
||||
pax -rvf $1 -o to=IBM-1047,from=ISO8859-1
|
||||
|
||||
if [ $# -gt 1 ]; then
|
||||
if [ $2 -eq strip ]; then
|
||||
echo ""
|
||||
echo "Stripping hex 0d characters ..."
|
||||
for i in $(pax -f $1 2>/dev/null)
|
||||
do
|
||||
case $i in
|
||||
*/)
|
||||
# then this entry is a directory
|
||||
;;
|
||||
*)
|
||||
# then this entry is NOT a directory
|
||||
tr -d
|
||||
<$i >@@@icu@tmp
|
||||
chmod +w $i
|
||||
rm $i
|
||||
mv @@@icu@tmp $i
|
||||
;;
|
||||
esac
|
||||
done
|
||||
fi
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Determining binary files ..."
|
||||
echo ""
|
||||
|
||||
#for dir in `find ./icu -type d \( -name CVS -o -print \)`; do
|
||||
# if [ -f $dir/CVS/Entries ]; then
|
||||
# binary_files="$binary_files`cat $dir/CVS/Entries | fgrep -- -kb \
|
||||
# | cut -d / -f2 | sed -e "s%^%$dir/%" \
|
||||
# | sed -e "s%^\./%%" | tr '\n' ' '`"
|
||||
# fi
|
||||
#done
|
||||
#echo "Detecting Unicode files"
|
||||
for file in `find icu \( -name \*.txt -print \)`; do
|
||||
bom8=`head -c 3 $file|\
|
||||
od -t x1|\
|
||||
head -n 1|\
|
||||
sed 's/ */ /g'|\
|
||||
cut -f2-4 -d ' '|\
|
||||
tr 'A-Z' 'a-z'`;
|
||||
#Find a converted UTF-8 BOM
|
||||
if test "$bom8" = "57 8b ab"
|
||||
then
|
||||
binary_files="$binary_files $file";
|
||||
fi
|
||||
done
|
||||
|
||||
for i in $(pax -f $1 2>/dev/null)
|
||||
do
|
||||
case $i in
|
||||
*/)
|
||||
# then this entry is a directory
|
||||
;;
|
||||
*.*)
|
||||
# then this entry has a dot in the filename
|
||||
for j in $binary_suffixes
|
||||
do
|
||||
suf=${i#*.*}
|
||||
if [ "$suf" = "$j" ]
|
||||
then
|
||||
binary_files="$binary_files $i"
|
||||
break
|
||||
fi
|
||||
done
|
||||
;;
|
||||
*)
|
||||
# then this entry does not have a dot in it
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# now see if a re-extract of binary files is necessary
|
||||
if [ ${#binary_files} -eq 0 ]; then
|
||||
echo ""
|
||||
echo "There are no binary files to restore."
|
||||
else
|
||||
echo "Restoring binary files ..."
|
||||
echo ""
|
||||
rm $binary_files
|
||||
pax -rvf $1 $binary_files
|
||||
fi
|
||||
echo ""
|
||||
echo "$0 has completed extracting ICU from $1."
|
||||
+4402
File diff suppressed because it is too large
Load Diff
Executable
+137
@@ -0,0 +1,137 @@
|
||||
#!/usr/bin/qsh
|
||||
#
|
||||
# Authors:
|
||||
# Ami Fixler
|
||||
# Barry Novinger
|
||||
# Steven R. Loomis <srl@jtcsv.com>
|
||||
# George Rhoten
|
||||
#
|
||||
# Shell script to unpax ICU and convert the files to an EBCDIC codepage.
|
||||
# After extracting to EBCDIC, binary files are re-extracted without the
|
||||
# EBCDIC conversion, thus restoring them to original codepage.
|
||||
#
|
||||
# Set the following variable to the list of binary file suffixes (extensions)
|
||||
|
||||
#binary_suffixes='ico ICO bmp BMP jpg JPG gif GIF brk BRK'
|
||||
#ICU specific binary files
|
||||
binary_suffixes='brk BRK bin BIN'
|
||||
data_files='icu/source/data/brkitr/* icu/source/data/locales/* icu/source/data/mappings/* icu/source/data/misc/* icu/source/data/translit/* icu/source/data/unidata/* icu/source/test/testdata/*'
|
||||
|
||||
usage()
|
||||
{
|
||||
echo "Enter archive filename as a parameter: $0 icu-archive.tar [strip]"
|
||||
echo "(strip is an option to remove hex '0D' carraige returns)"
|
||||
}
|
||||
# first make sure we at least one arg and it's a file we can read
|
||||
if [ $# -eq 0 ]; then
|
||||
usage
|
||||
exit
|
||||
fi
|
||||
if [ ! -r $1 ]; then
|
||||
echo "$1 does not exist or cannot be read."
|
||||
usage
|
||||
exit
|
||||
fi
|
||||
# set up a few variables
|
||||
|
||||
echo ""
|
||||
echo "Extracting from $1 ..."
|
||||
echo ""
|
||||
# extract everything as iso-8859-1 except these directories
|
||||
pax -C 819 -rcvf $1 $data_files
|
||||
|
||||
# extract files while converting them to EBCDIC
|
||||
echo ""
|
||||
echo "Extracting files which must be in ibm-37 ..."
|
||||
echo ""
|
||||
pax -C 37 -rvf $1 $data_files
|
||||
|
||||
if [ $# -gt 1 ]; then
|
||||
if [ $2 -eq strip ]; then
|
||||
echo ""
|
||||
echo "Stripping hex 0d characters ..."
|
||||
for i in $(pax -f $1 2>/dev/null)
|
||||
do
|
||||
case $i in
|
||||
*/)
|
||||
# then this entry is a directory
|
||||
;;
|
||||
*)
|
||||
# then this entry is NOT a directory
|
||||
tr -d
|
||||
<$i >@@@icu@tmp
|
||||
chmod +w $i
|
||||
rm $i
|
||||
mv @@@icu@tmp $i
|
||||
;;
|
||||
esac
|
||||
done
|
||||
fi
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Determining binary files ..."
|
||||
echo ""
|
||||
|
||||
#for dir in `find ./icu -type d \( -name CVS -o -print \)`; do
|
||||
# if [ -f $dir/CVS/Entries ]; then
|
||||
# binary_files="$binary_files`cat $dir/CVS/Entries | fgrep -- -kb \
|
||||
# | cut -d / -f2 | sed -e "s%^%$dir/%" \
|
||||
# | sed -e "s%^\./%%" | tr '\n' ' '`"
|
||||
# fi
|
||||
#done
|
||||
#echo "Detecting Unicode files"
|
||||
for file in `find ./icu \( -name \*.txt -print \)`; do
|
||||
bom8=`head -n 1 $file|\
|
||||
od -t x1|\
|
||||
head -n 1|\
|
||||
sed 's/ */ /g'|\
|
||||
cut -f2-4 -d ' '|\
|
||||
tr 'A-Z' 'a-z'`;
|
||||
# echo "bom8 is" $bom8 "for" $file
|
||||
# bom8=`head -c 3 $file|od -t x1|head -n 1|cut -d ' ' -f2-4`;
|
||||
#Find a converted UTF-8 BOM
|
||||
if [ "$bom8" = "057 08b 0ab" ]
|
||||
then
|
||||
binary_files="$binary_files `echo $file | cut -d / -f2-`";
|
||||
fi
|
||||
done
|
||||
|
||||
#echo $binary_files
|
||||
|
||||
for i in $(pax -f $1 2>/dev/null)
|
||||
do
|
||||
case $i in
|
||||
*/)
|
||||
# then this entry is a directory
|
||||
;;
|
||||
*.*)
|
||||
# then this entry has a dot in the filename
|
||||
for j in $binary_suffixes
|
||||
do
|
||||
suf=${i#*.*}
|
||||
if [ "$suf" = "$j" ]
|
||||
then
|
||||
binary_files="$binary_files $i"
|
||||
break
|
||||
fi
|
||||
done
|
||||
;;
|
||||
*)
|
||||
# then this entry does not have a dot in it
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# now see if a re-extract of binary files is necessary
|
||||
if [ ${#binary_files} -eq 0 ]; then
|
||||
echo ""
|
||||
echo "There are no binary files to restore."
|
||||
else
|
||||
echo "Restoring binary files ..."
|
||||
echo ""
|
||||
rm $binary_files
|
||||
pax -C 819 -rvf $1 $binary_files
|
||||
fi
|
||||
echo ""
|
||||
echo "$0 has completed extracting ICU from $1."
|
||||
@@ -0,0 +1,3 @@
|
||||
version https://git-lfs.github.com/spec/v1
|
||||
oid sha256:443b51ab9dabbd38d947abfcc8d868cc80773827d3b266f35850d1fc50603bea
|
||||
size 48455
|
||||
Vendored
+17
@@ -0,0 +1,17 @@
|
||||
*.debhelper
|
||||
docs
|
||||
*files
|
||||
*.substvars
|
||||
libicu-dev
|
||||
*libicu[0-9]*
|
||||
icu
|
||||
icu-locales
|
||||
icu-data
|
||||
icu-i18ndata
|
||||
tmp
|
||||
control
|
||||
postinst
|
||||
prerm
|
||||
icu.postinst
|
||||
icu.prerm
|
||||
icu-doc
|
||||
Vendored
+8
@@ -0,0 +1,8 @@
|
||||
libicu-dev for Debian
|
||||
---------------------
|
||||
|
||||
The documentation in /usr/share/doc/libicu-dev/docs/ provides a high-level
|
||||
overview of ICU. Detailed API documentation is provided by the icu-docs
|
||||
package.
|
||||
|
||||
-- Yves Arrouye <yves@debian.org>, Wed, 11 Oct 2000 19:37:57 -0700
|
||||
Vendored
+113
@@ -0,0 +1,113 @@
|
||||
icu (2.1-1) unstable; urgency=low
|
||||
|
||||
* ICU 2.1 release.
|
||||
* Changed the icu package description. Closes: 142886
|
||||
* Use -O1 for CXXFLAGS for OS/390. Closes: 143021
|
||||
|
||||
-- Yves Arrouye <yves@debian.org> Mon, 15 Apr 2002 14:03:12 -0700
|
||||
|
||||
icu (2.0.2-1) unstable; urgency=low
|
||||
|
||||
* Minor release of ICU with fixes for threading and strTo/FromWCS
|
||||
|
||||
-- Yves Arrouye <yves@debian.org> Tue, 2 Apr 2002 09:06:00 -0800
|
||||
|
||||
icu (2.0-2.1pre20020318-1) unstable; urgency=low
|
||||
|
||||
* Use the library number in the development package too.
|
||||
* ICU changed to version 2.1.
|
||||
|
||||
-- Yves Arrouye <yves@debian.org> Tue, 19 Mar 2002 18:38:37 -0800
|
||||
|
||||
icu (2.0-2.1pre20020303-1) unstable; urgency=low
|
||||
|
||||
* Fixed a crash in uconv when no argument is passed to -f or -t.
|
||||
* Other upstream changes.
|
||||
* Fresh upload with an up to date orig tar file so that future
|
||||
diffs won't be 6 megabytes long!
|
||||
|
||||
-- Yves Arrouye <yves@debian.org> Sun, 3 Mar 2002 15:31:13 -0800
|
||||
|
||||
icu (2.0-2.1pre-1) unstable; urgency=low
|
||||
|
||||
* Prerelease of 2.1 with a working upgraded uconv(1).
|
||||
|
||||
-- Yves Arrouye <yves@debian.org> Fri, 1 Mar 2002 21:51:47 -0800
|
||||
|
||||
icu (2.0-3) unstable; urgency=low
|
||||
|
||||
* Renamed doc-base to icu-doc.doc-base. Closes: 127487
|
||||
|
||||
-- Yves Arrouye <yves@debian.org> Fri, 18 Jan 2002 22:33:53 -0800
|
||||
|
||||
icu (2.0-2) unstable; urgency=low
|
||||
|
||||
* Fixed a bug in uprv_uca_cloneTempTable(). Closes: 128484
|
||||
* Update Debian bugs status. Closes: 104642
|
||||
* Use the official 2.0 tarball as the original tar.
|
||||
* Added manual pages for every tool.
|
||||
|
||||
-- Yves Arrouye <yves@debian.org> Wed, 16 Jan 2002 20:45:42 -0800
|
||||
|
||||
icu (2.0-1) unstable; urgency=low
|
||||
|
||||
* Update to ICU version 2.0.
|
||||
|
||||
-- Yves Arrouye <yves@debian.org> Sat, 10 Nov 2001 21:58:19 -0800
|
||||
|
||||
icu (1.8.1-2) unstable; urgency=low
|
||||
|
||||
* Updated copyright file. Closes: 112488
|
||||
* Updated icu-locales description. Closes: 75499
|
||||
|
||||
-- Yves Arrouye <yves@debian.org> Sat, 10 Nov 2001 21:24:58 -0800
|
||||
|
||||
icu (1.8.1-1.1) unstable; urgency=low
|
||||
|
||||
* NMU to resolve build failures on ia64 and (hopefully) hppa.
|
||||
* config.{sub|guess} update
|
||||
* source/tools/ctestfw/ctest.c: Add static declaration to global
|
||||
variables local to that module to avoid @gprel relocation errors.
|
||||
Closes: 104642
|
||||
|
||||
-- Yves Arrouye <yves@debian.org> Sat, 10 Nov 2001 21:24:44 -0800
|
||||
|
||||
icu (1.8.1-1) unstable; urgency=low
|
||||
|
||||
* Update to ICU version 1.8.1.
|
||||
|
||||
-- root <yves@debian.org> Mon, 21 May 2001 15:27:36 -0700
|
||||
|
||||
icu (1.7-1) unstable; urgency=low
|
||||
|
||||
* Update to ICU version 1.7.
|
||||
|
||||
-- Yves Arrouye <yves@debian.org> Tue, 21 Nov 2000 22:54:52 -0800
|
||||
|
||||
icu (1.6.0.1-20001113-2) unstable; urgency=low
|
||||
|
||||
* New snapshot with better ISO-2022.
|
||||
|
||||
-- Yves Arrouye <yves@debian.org> Mon, 13 Nov 2000 21:05:00 -0800
|
||||
|
||||
icu (1.6.0.1-20001027-1) unstable; urgency=low
|
||||
|
||||
* Move architecture-dependent files into /usr/lib, instead of
|
||||
/usr/share.
|
||||
* Move convrtrs.txt into /etc/icu, make it a conffile, and generate
|
||||
/usr/lib/icu/1.6.0.1/cnvalias.dat from it at postinst time.
|
||||
* Manage a /usr/lib/icu/current symbolic link across installations
|
||||
of the libicuXX packages. The symlink will always point to the
|
||||
highest numbered version of ICU.
|
||||
|
||||
-- Yves Arrouye <yves@debian.org> Fri, 27 Oct 2000 15:40:12 -0700
|
||||
|
||||
icu (1.6.0.1-20001017-1) unstable; urgency=low
|
||||
|
||||
* Initial Release.
|
||||
|
||||
-- Yves Arrouye <yves@debian.org> Tue, 24 Oct 2000 16:14:12 -0700
|
||||
|
||||
Local variables:
|
||||
mode: debian-changelog
|
||||
End:
|
||||
Vendored
+89
@@ -0,0 +1,89 @@
|
||||
Source: icu
|
||||
Section: libs
|
||||
Priority: optional
|
||||
Maintainer: Yves Arrouye <yves@debian.org>
|
||||
Standards-Version: 3.5.6
|
||||
Build-Depends: debhelper, doxygen
|
||||
|
||||
Package: libicu21-dev
|
||||
Section: devel
|
||||
Architecture: any
|
||||
Depends: libicu${major} (= ${Source-Version}), libc6-dev
|
||||
Provides: libicu-dev
|
||||
Conflicts: libicu-dev
|
||||
Suggests: icu-doc
|
||||
Description: International Components for Unicode (development files)
|
||||
ICU is a C++ and C library that provides robust and full-featured Unicode
|
||||
and locale support. This package contains the development files for ICU.
|
||||
|
||||
Package: icu-doc
|
||||
Section: doc
|
||||
Architecture: all
|
||||
Description: API documentation for ICU classes and functions
|
||||
ICU is a C++ and C library that provides robust and full-featured Unicode
|
||||
and locale support. This package contains HTML files documenting the ICU
|
||||
APIs.
|
||||
|
||||
Package: libicu21
|
||||
Section: libs
|
||||
Architecture: any
|
||||
Depends: ${shlibs:Depends}
|
||||
Recommends: icu
|
||||
Description: International Components for Unicode (libraries)
|
||||
ICU is a C++ and C library that provides robust and full-featured Unicode
|
||||
and locale support. This package contains the runtime libraries for
|
||||
ICU. It does not contain any of the data files needed at runtime and
|
||||
present in the `icu' and `icu-locales` packages.
|
||||
|
||||
Package: icu
|
||||
Section: libs
|
||||
Architecture: any
|
||||
Depends: libicu${major} (>= ${Source-Version}), make
|
||||
Recommends: icu-locales
|
||||
Description: International Components for Unicode (non locale data)
|
||||
ICU is a C++ and C library that provides robust and full-featured Unicode
|
||||
and locale support. The library provides calendar support, conversions
|
||||
for more than 230 character sets, language-sensitive collation, date
|
||||
and time formatting, support for more than 140 locales, message catalogs
|
||||
and resources, message formatting, normalization, number and currency
|
||||
formatting, time zones support, transliteration, word, line and
|
||||
sentence breaking, etc.
|
||||
.
|
||||
This package contains the Unicode character database and derived
|
||||
properties, along with converters and time zones data.
|
||||
It also contains the tools needed to manipulate these data.
|
||||
|
||||
Package: icu-locales
|
||||
Section: libs
|
||||
Architecture: any
|
||||
Depends: libicu${major} (>= ${Source-Version})
|
||||
Description: Locale data for ICU
|
||||
The locale data are used by ICU to provide localization (l10n) and
|
||||
internationalization (i18n) support to applications using the ICU library.
|
||||
This package also contains break data for various languages, and
|
||||
transliteration data.
|
||||
|
||||
Package: icu-data
|
||||
Priority: extra
|
||||
Section: libs
|
||||
Architecture: all
|
||||
Depends: icu
|
||||
Description: Data for ICU (source files)
|
||||
These data describe the Unicode data (normative and informative), and
|
||||
also all the table-based converters provided in the ICU distribution.
|
||||
.
|
||||
This package contains uncompiled source data. Precompiled data are in the
|
||||
`libicu${major}' package.
|
||||
|
||||
Package: icu-i18ndata
|
||||
Priority: extra
|
||||
Section: libs
|
||||
Architecture: all
|
||||
Depends: icu
|
||||
Description: Locale data for ICU (source files)
|
||||
The locale data are used by ICU to provide localization (l10n) and
|
||||
internationalization (i18n) support to ICU applications. This package
|
||||
also contains break data for various languages, and transliteration data.
|
||||
.
|
||||
This package contains uncompiled source data. Precompiled data are in the
|
||||
`icu-locales' package.
|
||||
Vendored
+35
@@ -0,0 +1,35 @@
|
||||
ICU License - ICU 1.8.1 and later
|
||||
|
||||
COPYRIGHT AND PERMISSION NOTICE
|
||||
|
||||
Copyright (c) 1995-2001 International Business Machines Corporation and others
|
||||
All rights reserved.
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a
|
||||
copy of this software and associated documentation files (the
|
||||
"Software"), to deal in the Software without restriction, including
|
||||
without limitation the rights to use, copy, modify, merge, publish,
|
||||
distribute, and/or sell copies of the Software, and to permit persons
|
||||
to whom the Software is furnished to do so, provided that the above
|
||||
copyright notice(s) and this permission notice appear in all copies of
|
||||
the Software and that both the above copyright notice(s) and this
|
||||
permission notice appear in supporting documentation.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
||||
OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
|
||||
OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
|
||||
HOLDERS INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL
|
||||
INDIRECT OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING
|
||||
FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT,
|
||||
NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION
|
||||
WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||
|
||||
Except as contained in this notice, the name of a copyright holder
|
||||
shall not be used in advertising or otherwise to promote the sale, use
|
||||
or other dealings in this Software without prior written authorization
|
||||
of the copyright holder.
|
||||
|
||||
--------------------------------------------------------------------------------
|
||||
All trademarks and registered trademarks mentioned herein are the property of their respective owners.
|
||||
|
||||
Vendored
+13
@@ -0,0 +1,13 @@
|
||||
Document: icu-doc
|
||||
Title: ICU API Documentation
|
||||
Author: IBM Corporation and Others
|
||||
Abstract: This manual describes the APIs of the
|
||||
International Components for Unicode C/C++
|
||||
library. It is a useful reference for the
|
||||
ICU programmer.
|
||||
Section: Libs
|
||||
|
||||
Format: HTML
|
||||
Index: /usr/share/doc/icu-doc/html/index.html
|
||||
Files: /usr/share/doc/icu-doc/html/*.html
|
||||
|
||||
Vendored
+1
@@ -0,0 +1 @@
|
||||
/etc/icu/convrtrs.txt
|
||||
Vendored
+75
@@ -0,0 +1,75 @@
|
||||
#! /bin/sh
|
||||
# postinst script for foo
|
||||
#
|
||||
# see: dh_installdeb(1)
|
||||
|
||||
set -e
|
||||
|
||||
# summary of how this script can be called:
|
||||
# * <postinst> `configure' <most-recently-configured-version>
|
||||
# * <old-postinst> `abort-upgrade' <new version>
|
||||
# * <conflictor's-postinst> `abort-remove' `in-favour' <package>
|
||||
# <new-version>
|
||||
# * <deconfigured's-postinst> `abort-deconfigure' `in-favour'
|
||||
# <failed-install-package> <version> `removing'
|
||||
# <conflicting-package> <version>
|
||||
# for details, see /usr/share/doc/packaging-manual/
|
||||
#
|
||||
# quoting from the policy:
|
||||
# Any necessary prompting should almost always be confined to the
|
||||
# post-installation script, and should be protected with a conditional
|
||||
# so that unnecessary prompting doesn't happen if a package's
|
||||
# installation fails and the `postinst' is called with `abort-upgrade',
|
||||
# `abort-remove' or `abort-deconfigure'.
|
||||
|
||||
case "$1" in
|
||||
configure)
|
||||
# Adjust the current ICU link in @libdir@/@PACKAGE@
|
||||
|
||||
prefix=@prefix@
|
||||
exec_prefix=@exec_prefix@
|
||||
|
||||
if [ -d @libdir@/@PACKAGE@ ]
|
||||
then
|
||||
icucurrent=`2>/dev/null ls -dp @libdir@/@PACKAGE@/* | sed -n 's,.*/\([^/]*\)/$,\1,p'| sort -rn | head -1`
|
||||
cd @libdir@/@PACKAGE@
|
||||
rm -f current
|
||||
if test x"$icucurrent" != x
|
||||
then
|
||||
ln -s "$icucurrent" current
|
||||
fi
|
||||
|
||||
# Set ICU_DATA for ease of testing files. It is not necessary to
|
||||
# export it as we are guaranteed that the binaries we invoke will
|
||||
# use the right directory. But we're not taking chances with the
|
||||
# future :)
|
||||
|
||||
ICU_DATA=@libdir@/@PACKAGE@/@VERSION@
|
||||
export ICU_DATA
|
||||
|
||||
if test ! -f $ICU_DATA/cnvalias.dat -o @sysconfdir@/icu/convrtrs.txt -nt $ICU_DATA/cnvalias.dat
|
||||
then
|
||||
echo Compiling converters and aliases list from @sysconfdir@/@PACKAGE@/convrtrs.txt
|
||||
@sbindir@/gencnval @sysconfdir@/@PACKAGE@/convrtrs.txt
|
||||
fi
|
||||
|
||||
fi
|
||||
;;
|
||||
|
||||
abort-upgrade|abort-remove|abort-deconfigure)
|
||||
|
||||
;;
|
||||
|
||||
*)
|
||||
echo "postinst called with unknown argument \`$1'" >&2
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
|
||||
# dh_installdeb will replace this with shell code automatically
|
||||
# generated by other debhelper scripts.
|
||||
|
||||
#DEBHELPER#
|
||||
|
||||
exit 0
|
||||
|
||||
Vendored
+54
@@ -0,0 +1,54 @@
|
||||
#! /bin/sh
|
||||
# prerm script for foo
|
||||
#
|
||||
# see: dh_installdeb(1)
|
||||
|
||||
set -e
|
||||
|
||||
# summary of how this script can be called:
|
||||
# * <prerm> `remove'
|
||||
# * <old-prerm> `upgrade' <new-version>
|
||||
# * <new-prerm> `failed-upgrade' <old-version>
|
||||
# * <conflictor's-prerm> `remove' `in-favour' <package> <new-version>
|
||||
# * <deconfigured's-prerm> `deconfigure' `in-favour'
|
||||
# <package-being-installed> <version> `removing'
|
||||
# <conflicting-package> <version>
|
||||
# for details, see /usr/share/doc/packaging-manual/
|
||||
|
||||
case "$1" in
|
||||
remove|upgrade|deconfigure)
|
||||
# Adjust the current ICU link in @libdir@/@PACKAGE@
|
||||
|
||||
prefix=@prefix@
|
||||
exec_prefix=@exec_prefix@
|
||||
|
||||
if [ -d @libdir@/@PACKAGE@ ]
|
||||
then
|
||||
icucurrent=`2>/dev/null ls -dp @libdir@/@PACKAGE@/* | sed -n 's,.*/\([^/]*\)/$,\1,p'| sort -rn | head -1`
|
||||
cd @libdir@/@PACKAGE@
|
||||
rm -f current
|
||||
if test x"$icucurrent" != x
|
||||
then
|
||||
ln -s "$icucurrent" current
|
||||
fi
|
||||
fi
|
||||
|
||||
# install-info --quiet --remove /usr/info/foo.info.gz
|
||||
;;
|
||||
|
||||
failed-upgrade)
|
||||
;;
|
||||
*)
|
||||
echo "prerm called with unknown argument \`$1'" >&2
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
|
||||
# dh_installdeb will replace this with shell code automatically
|
||||
# generated by other debhelper scripts.
|
||||
|
||||
#DEBHELPER#
|
||||
|
||||
exit 0
|
||||
|
||||
|
||||
Vendored
+68
@@ -0,0 +1,68 @@
|
||||
#! /bin/sh
|
||||
# postinst script for foo
|
||||
#
|
||||
# see: dh_installdeb(1)
|
||||
|
||||
set -e
|
||||
|
||||
# summary of how this script can be called:
|
||||
# * <postinst> `configure' <most-recently-configured-version>
|
||||
# * <old-postinst> `abort-upgrade' <new version>
|
||||
# * <conflictor's-postinst> `abort-remove' `in-favour' <package>
|
||||
# <new-version>
|
||||
# * <deconfigured's-postinst> `abort-deconfigure' `in-favour'
|
||||
# <failed-install-package> <version> `removing'
|
||||
# <conflicting-package> <version>
|
||||
# for details, see /usr/share/doc/packaging-manual/
|
||||
#
|
||||
# quoting from the policy:
|
||||
# Any necessary prompting should almost always be confined to the
|
||||
# post-installation script, and should be protected with a conditional
|
||||
# so that unnecessary prompting doesn't happen if a package's
|
||||
# installation fails and the `postinst' is called with `abort-upgrade',
|
||||
# `abort-remove' or `abort-deconfigure'.
|
||||
|
||||
# Call ldconfig in a way that makes lintian happy.
|
||||
|
||||
if [ "$1" = "configure" ]; then ldconfig; fi
|
||||
|
||||
# And now do the rest of the configuration
|
||||
|
||||
case "$1" in
|
||||
configure)
|
||||
|
||||
# Adjust the current ICU link in /usr/lib/icu
|
||||
|
||||
if [ -d /usr/lib/icu ]
|
||||
then
|
||||
|
||||
icucurrent=`2>/dev/null ls -dp /usr/lib/icu/* | sed -n 's,.*/\([^/]*\)/$,\1,p'| sort -rn | head -1`
|
||||
cd /usr/lib/icu
|
||||
rm -f current
|
||||
if test x"$icucurrent" != x
|
||||
then
|
||||
ln -s "$icucurrent" current
|
||||
ln -sf current/Makefile.inc .
|
||||
fi
|
||||
|
||||
fi
|
||||
;;
|
||||
|
||||
abort-upgrade|abort-remove|abort-deconfigure)
|
||||
|
||||
;;
|
||||
|
||||
*)
|
||||
echo "postinst called with unknown argument \`$1'" >&2
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
|
||||
# dh_installdeb will replace this with shell code automatically
|
||||
# generated by other debhelper scripts.
|
||||
|
||||
#DEBHELPER#
|
||||
|
||||
exit 0
|
||||
|
||||
|
||||
Vendored
+53
@@ -0,0 +1,53 @@
|
||||
#! /bin/sh
|
||||
# prerm script for foo
|
||||
#
|
||||
# see: dh_installdeb(1)
|
||||
|
||||
set -e
|
||||
|
||||
# summary of how this script can be called:
|
||||
# * <prerm> `remove'
|
||||
# * <old-prerm> `upgrade' <new-version>
|
||||
# * <new-prerm> `failed-upgrade' <old-version>
|
||||
# * <conflictor's-prerm> `remove' `in-favour' <package> <new-version>
|
||||
# * <deconfigured's-prerm> `deconfigure' `in-favour'
|
||||
# <package-being-installed> <version> `removing'
|
||||
# <conflicting-package> <version>
|
||||
# for details, see /usr/share/doc/packaging-manual/
|
||||
|
||||
case "$1" in
|
||||
remove|upgrade|deconfigure)
|
||||
# Adjust the current ICU link in @libdir@/@PACKAGE@
|
||||
|
||||
prefix=@prefix@
|
||||
exec_prefix=@exec_prefix@
|
||||
|
||||
if [ -d @libdir@/@PACKAGE@ ]
|
||||
then
|
||||
icucurrent=`2>/dev/null ls -dp @libdir@/@PACKAGE@/* | sed -n 's,.*/\([^/]*\)/$,\1,p'| sort -rn | head -1`
|
||||
cd @libdir@/@PACKAGE@
|
||||
rm -f current
|
||||
if test x"$icucurrent" != x
|
||||
then
|
||||
ln -s "$icucurrent" current
|
||||
fi
|
||||
fi
|
||||
|
||||
# install-info --quiet --remove /usr/info/foo.info.gz
|
||||
;;
|
||||
failed-upgrade)
|
||||
;;
|
||||
*)
|
||||
echo "prerm called with unknown argument \`$1'" >&2
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
|
||||
# dh_installdeb will replace this with shell code automatically
|
||||
# generated by other debhelper scripts.
|
||||
|
||||
#DEBHELPER#
|
||||
|
||||
exit 0
|
||||
|
||||
|
||||
Vendored
+61
@@ -0,0 +1,61 @@
|
||||
#! /bin/sh
|
||||
# postinst script for foo
|
||||
#
|
||||
# see: dh_installdeb(1)
|
||||
|
||||
set -e
|
||||
|
||||
# summary of how this script can be called:
|
||||
# * <postinst> `configure' <most-recently-configured-version>
|
||||
# * <old-postinst> `abort-upgrade' <new version>
|
||||
# * <conflictor's-postinst> `abort-remove' `in-favour' <package>
|
||||
# <new-version>
|
||||
# * <deconfigured's-postinst> `abort-deconfigure' `in-favour'
|
||||
# <failed-install-package> <version> `removing'
|
||||
# <conflicting-package> <version>
|
||||
# for details, see /usr/share/doc/packaging-manual/
|
||||
#
|
||||
# quoting from the policy:
|
||||
# Any necessary prompting should almost always be confined to the
|
||||
# post-installation script, and should be protected with a conditional
|
||||
# so that unnecessary prompting doesn't happen if a package's
|
||||
# installation fails and the `postinst' is called with `abort-upgrade',
|
||||
# `abort-remove' or `abort-deconfigure'.
|
||||
|
||||
case "$1" in
|
||||
configure)
|
||||
# Adjust the current ICU link in @libdir@/@PACKAGE@
|
||||
|
||||
prefix=@prefix@
|
||||
exec_prefix=@exec_prefix@
|
||||
|
||||
if [ -d @libdir@/@PACKAGE@ ]
|
||||
then
|
||||
icucurrent=`2>/dev/null ls -dp @libdir@/@PACKAGE@/* | sed -n 's,.*/\([^/]*\)/$,\1,p'| sort -rn | head -1`
|
||||
cd @libdir@/@PACKAGE@
|
||||
rm -f current
|
||||
if test x"$icucurrent" != x
|
||||
then
|
||||
ln -s "$icucurrent" current
|
||||
fi
|
||||
fi
|
||||
;;
|
||||
|
||||
abort-upgrade|abort-remove|abort-deconfigure)
|
||||
|
||||
;;
|
||||
|
||||
*)
|
||||
echo "postinst called with unknown argument \`$1'" >&2
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
|
||||
# dh_installdeb will replace this with shell code automatically
|
||||
# generated by other debhelper scripts.
|
||||
|
||||
#DEBHELPER#
|
||||
|
||||
exit 0
|
||||
|
||||
|
||||
Vendored
+53
@@ -0,0 +1,53 @@
|
||||
#! /bin/sh
|
||||
# prerm script for foo
|
||||
#
|
||||
# see: dh_installdeb(1)
|
||||
|
||||
set -e
|
||||
|
||||
# summary of how this script can be called:
|
||||
# * <prerm> `remove'
|
||||
# * <old-prerm> `upgrade' <new-version>
|
||||
# * <new-prerm> `failed-upgrade' <old-version>
|
||||
# * <conflictor's-prerm> `remove' `in-favour' <package> <new-version>
|
||||
# * <deconfigured's-prerm> `deconfigure' `in-favour'
|
||||
# <package-being-installed> <version> `removing'
|
||||
# <conflicting-package> <version>
|
||||
# for details, see /usr/share/doc/packaging-manual/
|
||||
|
||||
case "$1" in
|
||||
remove|upgrade|deconfigure)
|
||||
# Adjust the current ICU link in @libdir@/@PACKAGE@
|
||||
|
||||
prefix=@prefix@
|
||||
exec_prefix=@exec_prefix@
|
||||
|
||||
if [ -d @libdir@/@PACKAGE@ ]
|
||||
then
|
||||
icucurrent=`2>/dev/null ls -dp @libdir@/@PACKAGE@/* | sed -n 's,.*/\([^/]*\)/$,\1,p'| sort -rn | head -1`
|
||||
cd @libdir@/@PACKAGE@
|
||||
rm -f current
|
||||
if test x"$icucurrent" != x
|
||||
then
|
||||
ln -s "$icucurrent" current
|
||||
fi
|
||||
fi
|
||||
|
||||
# install-info --quiet --remove /usr/info/foo.info.gz
|
||||
;;
|
||||
failed-upgrade)
|
||||
;;
|
||||
*)
|
||||
echo "prerm called with unknown argument \`$1'" >&2
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
|
||||
# dh_installdeb will replace this with shell code automatically
|
||||
# generated by other debhelper scripts.
|
||||
|
||||
#DEBHELPER#
|
||||
|
||||
exit 0
|
||||
|
||||
|
||||
+247
@@ -0,0 +1,247 @@
|
||||
#!/usr/bin/make -f
|
||||
# ICU debian/rules that uses debhelper.
|
||||
|
||||
# The Debian packages for ICU are created and maintained by
|
||||
# Yves Arrouye <yves@debian.org>. They are part of Debian GNU/Linux
|
||||
# since the Debian woody release.
|
||||
|
||||
# Decide whether we want to automatically manage the repackaging of
|
||||
# the common data from individual files or not.
|
||||
ICU_AUTO_COMMON=NO
|
||||
|
||||
# Uncomment this to turn on verbose mode.
|
||||
#export DH_VERBOSE=1
|
||||
|
||||
# This is the debhelper compatability version to use.
|
||||
export DH_COMPAT=2
|
||||
#firstdir := $(shell sed -n 's/Package: \(.*\)/\1/p' debian/control | head -1)else
|
||||
|
||||
# Where the source for ICU is, where to build
|
||||
srcdir = source
|
||||
builddir = build
|
||||
# How to get to $(srcdir) from $(builddir)
|
||||
builddir_relative_topdir = ..
|
||||
builddir_relative_srcdir = $(builddir_relative_topdir)/$(srcdir)
|
||||
|
||||
# Getting the ICU version numbers, stolen from configure.in (that's okay, I
|
||||
# wrote the configure.in part anyway)
|
||||
|
||||
version := $(shell sed -n -e 's/^[ ]*\#[ ]*define[ ]*U_ICU_VERSION[ ]*"\([^"]*\)".*/\1/p' $(srcdir)/common/unicode/uversion.h)
|
||||
libversion := $(shell echo $(version) | sed -e 's/\.//' -e 's/^\([^.]*\)$$/\1.0/')
|
||||
major := $(shell echo $(libversion) | sed 's/\..*//')
|
||||
|
||||
# ICU installation directories, w/o the initial slash
|
||||
pkgdatadir = usr/share/icu
|
||||
icudatadir = $(pkgdatadir)/$(version)
|
||||
pkglibdir = usr/lib/icu
|
||||
iculibdir = $(pkglibdir)/$(version)
|
||||
datasrcdir = usr/share/icu/$(version)/data
|
||||
# This is where data needs to be moved
|
||||
ifeq ($(ICU_AUTO_COMMON),YES)
|
||||
icufildir = $(iculibdir)/files
|
||||
else
|
||||
icufildir = $(iculibdir)
|
||||
endif
|
||||
|
||||
# Build flags
|
||||
|
||||
CFLAGS = -O2 # -Wall
|
||||
ifeq ($(shell uname -m),s390)
|
||||
CXXFLAGS = -O1 # -Wall
|
||||
else
|
||||
CXXFLAGS = -O2 # -Wall
|
||||
endif
|
||||
|
||||
INSTALL_PROGRAM = /usr/bin/install
|
||||
|
||||
ifneq (,$(findstring debug,$(DEB_BUILD_OPTIONS)))
|
||||
CFLAGS += -g
|
||||
CXXFLAGS += -g
|
||||
endif
|
||||
ifneq (,$(findstring nostrip,$(DEB_BUILD_OPTIONS)))
|
||||
INSTALL_PROGRAM += -s
|
||||
endif
|
||||
|
||||
all: build
|
||||
|
||||
configure: $(builddir)/configure-stamp
|
||||
$(builddir)/configure-stamp:
|
||||
# Add here commands to configure the package.
|
||||
test -d $(builddir) || mkdir $(builddir)
|
||||
cd $(builddir); CFLAGS='$(CFLAGS)' CXXFLAGS='$(CXXFLAGS)' INSTALL_PROGRAM='$(INSTALL_PROGRAM)' $(SHELL) $(builddir_relative_srcdir)/configure --srcdir=$(builddir_relative_srcdir) --prefix=/usr --mandir=/usr/share/man --sysconfdir=/etc --localstatedir=/var --with-data-packaging=files --disable-rpath --enable-shared --enable-static --disable-samples
|
||||
touch $@
|
||||
|
||||
build: $(builddir)/configure-stamp debian-files $(builddir)/build-stamp
|
||||
|
||||
# Build some Debian files
|
||||
DEBIAN_FILES=icu.conffiles postinst prerm icu.postinst icu.prerm libicu$(major).postinst README.Debian.libicu$(major)-dev
|
||||
|
||||
debian/%: debian/%.in
|
||||
cd $(builddir) \
|
||||
&& CONFIG_FILES=$(builddir_relative_topdir)/$@ CONFIG_HEADERS= $(SHELL) ./config.status
|
||||
|
||||
debian/libicu$(major)%: debian/libicu%
|
||||
cp $< $@
|
||||
debian/README.Debian.libicu$(major)%: debian/README.Debian.libicu%
|
||||
cp $< $@
|
||||
|
||||
debian-files: $(addprefix debian/,$(DEBIAN_FILES))
|
||||
|
||||
$(builddir)/build-stamp:
|
||||
dh_testdir
|
||||
|
||||
# Add here commands to compile the package.
|
||||
$(MAKE) -C $(builddir)
|
||||
|
||||
touch $@
|
||||
|
||||
clean: update-debian-files
|
||||
dh_testdir
|
||||
dh_testroot
|
||||
$(RM) $(builddir)/build-stamp $(builddir)/configure-stamp
|
||||
|
||||
# Add here commands to clean up after the build process.
|
||||
-$(MAKE) -C $(builddir) distclean
|
||||
#-$(RM) `find $(builddir)/samples $(builddir)/test -name Makefile -print`
|
||||
-$(RM) `find $(builddir) -name Makefile -print`
|
||||
dh_clean
|
||||
$(RM) $(addprefix debian/,$(DEBIAN_FILES)) debian/control.new
|
||||
|
||||
#if test -d $(builddir) -a $(builddir) != $(srcdir); then rmdir `find $(builddir) -type d -print | sort -r`; fi
|
||||
if test -d $(builddir) -a $(builddir) != $(srcdir); then $(RM) -r $(builddir); fi
|
||||
|
||||
install: build
|
||||
dh_testdir
|
||||
dh_testroot
|
||||
dh_clean -k
|
||||
dh_installdirs
|
||||
|
||||
# Add here commands to install the package into debian/tmp
|
||||
$(MAKE) -C $(builddir) install DESTDIR=`pwd`/debian/tmp
|
||||
|
||||
# Build architecture-independent files here.
|
||||
binary-indep: build install
|
||||
#
|
||||
# build icu-doc package by moving files
|
||||
#
|
||||
mv debian/tmp/usr/share/doc/icu debian/tmp/usr/share/doc/icu-doc
|
||||
dh_movefiles -picu-doc \
|
||||
usr/share/doc/icu-doc
|
||||
#
|
||||
# build icu-i18ndata from the source data
|
||||
#
|
||||
for d in locales translit brkitr; do \
|
||||
test -d debian/icu-i18ndata/$(datasrcdir)/$$d || \
|
||||
mkdir -p debian/icu-i18ndata/$(datasrcdir)/$$d; \
|
||||
install -m 644 $(srcdir)/data/$$d/*.* \
|
||||
debian/icu-i18ndata/$(datasrcdir)/$$d; \
|
||||
done
|
||||
#
|
||||
# build icu-data from the source data
|
||||
#
|
||||
for d in mappings unidata misc; do \
|
||||
test -d debian/icu-data/$(datasrcdir)/$$d || \
|
||||
mkdir -p debian/icu-data/$(datasrcdir)/$$d; \
|
||||
install -m 644 $(srcdir)/data/$$d/*.* \
|
||||
debian/icu-data/$(datasrcdir)/$$d; \
|
||||
done
|
||||
cd debian/icu-data/$(datasrcdir)/mappings && \
|
||||
$(RM) convrtrs.txt && ln -s /etc/icu/convrtrs.txt .
|
||||
|
||||
# Build architecture-dependent files here.
|
||||
binary-arch: build install
|
||||
# dh_testversion
|
||||
dh_testdir
|
||||
dh_testroot
|
||||
#
|
||||
# build libicu${major} package by moving files from icu-dev
|
||||
#
|
||||
$(RM) debian/tmp/usr/lib/libicuctestfw.*
|
||||
dh_movefiles -plibicu$(major) \
|
||||
usr/lib/*.so.*
|
||||
mkdir debian/libicu$(major)/usr/lib/icu
|
||||
#
|
||||
# massage data files
|
||||
#
|
||||
$(RM) debian/tmp/$(iculibdir)/cnvalias.dat
|
||||
ifneq ($(iculibdir),$(icufildir))
|
||||
test -d debian/tmp/$(icufildir) || mkdir debian/tmp/$(icufildir)
|
||||
mv debian/tmp/$(iculibdir)/*.* debian/tmp/$(icufildir)
|
||||
mv debian/tmp/$(icufildir)/*msg.dat debian/tmp/$(iculibdir)
|
||||
mv debian/tmp/$(icufildir)/*.inc debian/tmp/$(iculibdir)
|
||||
endif
|
||||
#
|
||||
# build icu package by moving files from icu-dev
|
||||
#
|
||||
$(RM) debian/tmp/$(iculibdir)/../Makefile.inc debian/tmp/$(iculibdir)/../current
|
||||
ifneq ($(iculibdir),$(icufildir))
|
||||
dh_movefiles -picu \
|
||||
$(iculibdir)/*msg.dat
|
||||
endif
|
||||
dh_movefiles -picu \
|
||||
etc \
|
||||
usr/bin \
|
||||
usr/sbin \
|
||||
$(icudatadir)/README \
|
||||
$(iculibdir)/*.inc \
|
||||
$(icufildir)/*.cnv \
|
||||
$(icufildir)/*.dat \
|
||||
usr/share/man
|
||||
#
|
||||
# build icu-locales package by moving files from icu-dev
|
||||
#
|
||||
dh_movefiles -picu-locales \
|
||||
$(icufildir)/*.res \
|
||||
$(icufildir)/*.brk
|
||||
#
|
||||
# move the rest of the development files
|
||||
# in their package
|
||||
#
|
||||
dh_movefiles usr
|
||||
#
|
||||
# do a bit of cleanup
|
||||
#
|
||||
$(RM) -r debian/tmp/etc debian/tmp/usr/sbin \
|
||||
debian/tmp/usr/share/man
|
||||
#
|
||||
# continue to prepare packages
|
||||
#
|
||||
# dh_installdebconf
|
||||
dh_installdocs
|
||||
dh_installexamples $(srcdir)/samples/*
|
||||
find debian/libicu$(major)-dev -name CVS -o -name .cvsignore | xargs $(RM) -r
|
||||
dh_installmenu
|
||||
# dh_installemacsen
|
||||
# dh_installpam
|
||||
# dh_installinit
|
||||
dh_installcron
|
||||
dh_installmanpages -picu
|
||||
dh_installinfo
|
||||
# dh_undocumented -picu
|
||||
dh_installchangelogs
|
||||
dh_link
|
||||
dh_strip
|
||||
dh_compress
|
||||
dh_fixperms
|
||||
# You may want to make some executables suid here.
|
||||
dh_makeshlibs
|
||||
dh_installdeb
|
||||
# dh_perl
|
||||
dh_shlibdeps -ldebian/libicu$(major)/usr/lib
|
||||
dh_gencontrol -- -Vversion=$(version) -Vmajor=$(major)
|
||||
dh_md5sums
|
||||
dh_builddeb
|
||||
|
||||
# Automatically update changelog and control
|
||||
update-debian-files:
|
||||
if [ x"$$NO_AUTO_DCH" = x ]; \
|
||||
then \
|
||||
cvers=`sed -n 's/^icu (\([^)]*\)).*/\1/p' debian/changelog | head -1`; if dpkg --compare-versions $$cvers lt $(version)-1; then \
|
||||
dch -v $(version)-1 "Update to ICU version $(version). (This comment is generated.)"; \
|
||||
sed 's/libicu[0-9][0-9]*/libicu$(major)/g' debian/control >debian/control.new && mv debian/control.new debian/control; \
|
||||
fi; \
|
||||
fi
|
||||
|
||||
binary: update-debian-files binary-indep binary-arch
|
||||
.PHONY: build clean binary-indep binary-arch binary install debian-files update-changelog
|
||||
|
||||
@@ -1,28 +0,0 @@
|
||||
---
|
||||
layout: default
|
||||
---
|
||||
<!--
|
||||
© 2020 and later: Unicode, Inc. and others.
|
||||
License & terms of use: http://www.unicode.org/copyright.html
|
||||
-->
|
||||
|
||||
<style type="text/css" media="screen">
|
||||
.container {
|
||||
margin: 10px auto;
|
||||
max-width: 600px;
|
||||
text-align: center;
|
||||
}
|
||||
h1 {
|
||||
margin: 30px 0;
|
||||
font-size: 4em;
|
||||
line-height: 1;
|
||||
letter-spacing: -1px;
|
||||
}
|
||||
</style>
|
||||
|
||||
<div class="container">
|
||||
<h1>404</h1>
|
||||
|
||||
<p><strong>Page not found :(</strong></p>
|
||||
<p>The requested page could not be found.</p>
|
||||
</div>
|
||||
@@ -1,37 +0,0 @@
|
||||
# © 2020 and later: Unicode, Inc. and others.
|
||||
# License & terms of use: http://www.unicode.org/copyright.html
|
||||
|
||||
source "https://rubygems.org"
|
||||
# Hello! This is where you manage which Jekyll version is used to run.
|
||||
# When you want to use a different version, change it below, save the
|
||||
# file and run `bundle install`. Run Jekyll with `bundle exec`, like so:
|
||||
#
|
||||
# bundle exec jekyll serve
|
||||
#
|
||||
# This will help ensure the proper Jekyll version is running.
|
||||
# Happy Jekylling!
|
||||
|
||||
# gem "jekyll", "~> 3.8.7"
|
||||
|
||||
# This is the default theme for new Jekyll sites. You may change this to anything you like.
|
||||
gem "minima", "~> 2.5"
|
||||
# If you aren't using GitHub Pages, then uncomment out the line above that starts with
|
||||
# 'gem "jekyll"' and then comment out the line below that starts with 'gem "github-pages"'.
|
||||
# To upgrade, run `bundle update github-pages`.
|
||||
gem "github-pages", "~> 213", group: :jekyll_plugins
|
||||
gem "just-the-docs"
|
||||
# If you have any plugins, put them here!
|
||||
group :jekyll_plugins do
|
||||
gem "jekyll-feed", "~> 0.12"
|
||||
end
|
||||
|
||||
# Windows and JRuby does not include zoneinfo files, so bundle the tzinfo-data gem
|
||||
# and associated library.
|
||||
platforms :mingw, :x64_mingw, :mswin, :jruby do
|
||||
gem "tzinfo", "~> 1.2"
|
||||
gem "tzinfo-data"
|
||||
end
|
||||
|
||||
# Performance-booster for watching directories on Windows
|
||||
gem "wdm", "~> 0.1.1", :platforms => [:mingw, :x64_mingw, :mswin]
|
||||
|
||||
@@ -1,99 +0,0 @@
|
||||
# © 2020 and later: Unicode, Inc. and others.
|
||||
# License & terms of use: http://www.unicode.org/copyright.html
|
||||
|
||||
title: ICU Documentation
|
||||
email: icu-support@icu-project.org
|
||||
description: >- # this means to ignore newlines until "baseurl:"
|
||||
ICU is a mature, widely used set of C/C++ and Java libraries providing Unicode
|
||||
and Globalization support for software applications. The ICU User Guide provides
|
||||
documentation on how to use ICU.
|
||||
|
||||
baseurl: "/icu" # the subpath for the site
|
||||
url: "https://unicode-org.github.io" # the base hostname & protocol for your site
|
||||
|
||||
twitter_username: unicode
|
||||
github_username: unicode-org
|
||||
|
||||
remote_theme: pmarsceill/just-the-docs
|
||||
|
||||
# GitHub uses its own markdown renderer called CommonMarkGhPages.
|
||||
#
|
||||
# To use this we'd need to set the following options:
|
||||
# markdown: CommonMarkGhPages
|
||||
# Plugins:
|
||||
# jekyll-commonmark-ghpages
|
||||
#
|
||||
# commonmark:
|
||||
# options: ["SMART", "FOOTNOTES"]
|
||||
# extensions: ["strikethrough", "autolink", "table", "tagfilter"]
|
||||
#
|
||||
# However, it doesn't work well with Jekyll and the Just-The-Docs theme,
|
||||
# and causes too many formatting problems. Additionally, it also doesn't
|
||||
# have any support for auto-generating a Table of Contents (TOC) section
|
||||
# for a page. Due to these reasons, we instead use the recommended
|
||||
# markdown renderer for Jekyll, called kramdown.
|
||||
#
|
||||
markdown: kramdown
|
||||
|
||||
# Notes on plugins:
|
||||
# - GitHub Pages ignores all plugins included in the Gemfile.
|
||||
# - Only the ones listed in the _config.yml file (this file) are used.
|
||||
# - There is a very limited set of plugins that GH Pages supports (for security
|
||||
# reasons). The listing is here: https://pages.github.com/versions
|
||||
# - The jekyll-sitemap plugin generates a "sitemap.xml" for search engines.
|
||||
# - We need the "jemoji" plugin for rendering emoji like ":point_right:".
|
||||
plugins:
|
||||
- jekyll-remote-theme
|
||||
- jekyll-sitemap
|
||||
- jemoji
|
||||
|
||||
search_enabled: true
|
||||
|
||||
# Color scheme currently only supports "dark" or nil (light).
|
||||
color_scheme: nil
|
||||
|
||||
# Aux links for the upper right navigation
|
||||
aux_links:
|
||||
"API Docs":
|
||||
- "https://unicode-org.github.io/icu-docs"
|
||||
"ICU on GitHub":
|
||||
- "https://github.com/unicode-org/icu"
|
||||
"ICU Home Page":
|
||||
- "http://site.icu-project.org/"
|
||||
"Unicode Home Page":
|
||||
- "https://www.unicode.org"
|
||||
|
||||
# Makes Aux links open in a new tab.
|
||||
aux_links_new_tab: true
|
||||
|
||||
# Add a "Back to top" link at the bottom of each page.
|
||||
back_to_top: true
|
||||
back_to_top_text: "Back to top"
|
||||
|
||||
# Enable heading anchors
|
||||
heading_anchors: true
|
||||
|
||||
# Footer content: appears at the bottom of every page after the main content.
|
||||
footer_content: "© 2016 and later: Unicode, Inc. and others. License & terms of use: <a href=\"http://www.unicode.org/copyright.html\">http://www.unicode.org/copyright.html</a>"
|
||||
|
||||
# Footer content: Add a "Edit this page on GitHub" link.
|
||||
# Note: I think this would be better at the *top* of the page, but it isn't possible to do that by default with just-the-docs.
|
||||
gh_edit_link: true # show or hide edit this page link
|
||||
gh_edit_link_text: "Edit this page on GitHub."
|
||||
gh_edit_repository: "https://github.com/unicode-org/icu"
|
||||
# NOTE: Since we serve the docs out of the "docs" folder, we need to include that here even though
|
||||
# it is technically *not* part of the branch name.
|
||||
gh_edit_branch: "main/docs"
|
||||
# Can be "tree" or "edit". Edit takes the user to jump into the editor immediately.
|
||||
gh_edit_view_mode: "tree"
|
||||
|
||||
# Exclude from processing.
|
||||
# Excluded items can be processed by explicitly listing the directories or
|
||||
# their entries' file path in the `include:` list.
|
||||
exclude:
|
||||
- Gemfile
|
||||
- Gemfile.lock
|
||||
|
||||
# Google Analytics
|
||||
ga_tracking: UA-7670256-1
|
||||
ga_tracking_anonymize_ip: true # Use GDPR compliant Google Analytics settings
|
||||
@@ -1,8 +0,0 @@
|
||||
// © 2020 and later: Unicode, Inc. and others.
|
||||
// License & terms of use: http://www.unicode.org/copyright.html
|
||||
|
||||
// Increase the side bar width as some of the page titles don't fit in the default size.
|
||||
$nav-width: 284px;
|
||||
|
||||
// Increase the main content area width as some tables assume a larger size.
|
||||
$content-width: 900px;
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 1.6 KiB |
@@ -1,23 +0,0 @@
|
||||
<!--- © 2020 and later: Unicode, Inc. and others. --->
|
||||
<!--- License & terms of use: http://www.unicode.org/copyright.html --->
|
||||
|
||||
# Configuring VS Code for ICU4C
|
||||
|
||||
- Create a `.vscode` folder in icu4c/source
|
||||
- Copy the `tasks.json`, `launch.json` and `c_cpp_properties.json` files into
|
||||
the `.vscode` folder.
|
||||
- To test only specific test targets, specify them under `args` in
|
||||
`launch.json`.
|
||||
- To adjust the parallelism when building, adjust the `args` in `tasks.json`.
|
||||
- `-l20` tells VSCode to not launch jobs if the system load average is above
|
||||
20 (note that the [system load
|
||||
average](https://en.wikipedia.org/wiki/Load_(computing)) is *not* a CPU
|
||||
usage percentage).
|
||||
- `-j24` limits the number of jobs launched in parallel to 24. The system
|
||||
load average takes a while to respond, reducing this number helps the
|
||||
initial bad system performance when a new build is launched.
|
||||
|
||||
NOTE:
|
||||
Run the
|
||||
[`./runConfigureICU` command](https://unicode-org.github.io/icu/userguide/icufaq)
|
||||
before building `icu4c` from VSCode.
|
||||
@@ -1,21 +0,0 @@
|
||||
{
|
||||
"configurations": [
|
||||
{
|
||||
"name": "Linux",
|
||||
"includePath": [
|
||||
"${workspaceFolder}/tools/toolutil",
|
||||
"${workspaceFolder}/tools/ctestfw",
|
||||
"${workspaceFolder}/common",
|
||||
"${workspaceFolder}/i18n"
|
||||
],
|
||||
"defines": [
|
||||
"U_DISABLE_RENAMING=1"
|
||||
],
|
||||
"compilerPath": "/usr/bin/clang",
|
||||
"cStandard": "c11",
|
||||
"cppStandard": "c++17",
|
||||
"intelliSenseMode": "clang-x64"
|
||||
}
|
||||
],
|
||||
"version": 4
|
||||
}
|
||||
@@ -1,102 +0,0 @@
|
||||
// © 2020 and later: Unicode, Inc. and others.
|
||||
// License & terms of use: http://www.unicode.org/copyright.html
|
||||
{
|
||||
"version": "0.2.0",
|
||||
"configurations": [
|
||||
{
|
||||
"name": "intltest Launch",
|
||||
"type": "cppdbg",
|
||||
"request": "launch",
|
||||
"targetArchitecture": "x64",
|
||||
"program": "${workspaceFolder}/test/intltest/intltest",
|
||||
"environment": [
|
||||
{
|
||||
"name": "LD_LIBRARY_PATH", // Use "DYLD_LIBRARY_PATH" for mac osx.
|
||||
"value": "${workspaceRoot}/lib:${workspaceRoot}/stubdata:${workspaceRoot}/tools/ctestfw"
|
||||
},
|
||||
{
|
||||
"name": "ICU_DATA",
|
||||
"value": "${workspaceRoot}/data/out"
|
||||
}
|
||||
],
|
||||
"args": [ // Here you add which test cases you want to run, leaving it empty will run all the test cases.
|
||||
// Examples:
|
||||
// "rbbi/RBBITest",
|
||||
// "format/NumberFormatTest"
|
||||
],
|
||||
"stopAtEntry": false, // If you want to add a break point in the beginning of the program (i.e. test cases).
|
||||
"cwd": "${workspaceFolder}/test/intltest",
|
||||
"externalConsole": false, //set to true to see output in an external terminal instead
|
||||
"MIMode": "gdb", // Use "lldb" for mac osx.
|
||||
"logging": {
|
||||
"trace": true,
|
||||
"traceResponse": true,
|
||||
"engineLogging": true,
|
||||
"programOutput": true
|
||||
},
|
||||
"preLaunchTask": "buildTest"
|
||||
},
|
||||
{
|
||||
"name": "cintltst Launch",
|
||||
"type": "cppdbg",
|
||||
"request": "launch",
|
||||
"targetArchitecture": "x64",
|
||||
"program": "${workspaceFolder}/test/cintltst/cintltst",
|
||||
"environment": [
|
||||
{
|
||||
"name": "LD_LIBRARY_PATH", // Use "DYLD_LIBRARY_PATH" for mac osx.
|
||||
"value": "${workspaceRoot}/lib:${workspaceRoot}/stubdata:${workspaceRoot}/tools/ctestfw"
|
||||
},
|
||||
{
|
||||
"name": "ICU_DATA",
|
||||
"value": "${workspaceRoot}/data/out"
|
||||
}
|
||||
],
|
||||
"args": [ // Here you add which test cases you want to run, leaving it empty will run all the test cases.
|
||||
// "/tsutil/cstrcase" // Just for example.
|
||||
],
|
||||
"stopAtEntry": false, // If you want to add a break point in the beginning of the program (i.e. test cases).
|
||||
"cwd": "${workspaceFolder}",
|
||||
"externalConsole": false, //set to true to see output in an external terminal instead
|
||||
"MIMode": "gdb", // Use "lldb" for mac osx.
|
||||
"logging": {
|
||||
"trace": true,
|
||||
"traceResponse": true,
|
||||
"engineLogging": true,
|
||||
"programOutput": true
|
||||
},
|
||||
"preLaunchTask": "buildTest"
|
||||
},
|
||||
{
|
||||
"name": "iotest Launch",
|
||||
"type": "cppdbg",
|
||||
"request": "launch",
|
||||
"targetArchitecture": "x64",
|
||||
"program": "${workspaceFolder}/test/iotest/iotest",
|
||||
"environment": [
|
||||
{
|
||||
"name": "LD_LIBRARY_PATH", // Use "DYLD_LIBRARY_PATH" for mac osx.
|
||||
"value": "${workspaceRoot}/lib:${workspaceRoot}/stubdata:${workspaceRoot}/tools/ctestfw"
|
||||
},
|
||||
{
|
||||
"name": "ICU_DATA",
|
||||
"value": "${workspaceRoot}/data/out"
|
||||
}
|
||||
],
|
||||
"args": [ // Here you add which test cases you want to run, leaving it empty will run all the test cases.
|
||||
// "/datadriv" // Just for example.
|
||||
],
|
||||
"stopAtEntry": false, // If you want to add a break point in the beginning of the program (i.e. test cases).
|
||||
"cwd": "${workspaceFolder}",
|
||||
"externalConsole": false, //set to true to see output in an external terminal instead
|
||||
"MIMode": "gdb", // Use "lldb" for mac osx.
|
||||
"logging": {
|
||||
"trace": true,
|
||||
"traceResponse": true,
|
||||
"engineLogging": true,
|
||||
"programOutput": true
|
||||
},
|
||||
"preLaunchTask": "buildTest"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,37 +0,0 @@
|
||||
// © 2020 and later: Unicode, Inc. and others.
|
||||
// License & terms of use: http://www.unicode.org/copyright.html
|
||||
{
|
||||
"version": "2.0.0",
|
||||
"tasks": [
|
||||
{
|
||||
"label": "Makefile", // You can choose your own label.
|
||||
"type": "shell",
|
||||
"command": "make",
|
||||
// Make this the default build command.
|
||||
"group": {
|
||||
"kind": "build",
|
||||
"isDefault": true
|
||||
},
|
||||
"presentation": {
|
||||
"reveal": "always"
|
||||
},
|
||||
// You can pass all the building arguments here.
|
||||
"args": [
|
||||
"all",
|
||||
"--debug", // Enable debug mode.
|
||||
"-j24", // Launch no more than 24 jobs in parallel
|
||||
"-l20" // Stop launching jobs if system load average is too high
|
||||
]
|
||||
},
|
||||
{
|
||||
"label": "buildTest", // for building test cases before running it.
|
||||
"command": "make", // Could be any other shell command
|
||||
"args": [
|
||||
"tests",
|
||||
"-j24", // Launch no more than 24 jobs in parallel
|
||||
"-l20" // Stop launching jobs if system load average is too high
|
||||
],
|
||||
"type": "shell"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,26 +0,0 @@
|
||||
---
|
||||
layout: default
|
||||
title: ICU Documentation
|
||||
nav_order: 100
|
||||
description: ICU Documentation
|
||||
---
|
||||
|
||||
<!--
|
||||
© 2020 and later: Unicode, Inc. and others.
|
||||
License & terms of use: http://www.unicode.org/copyright.html
|
||||
-->
|
||||
|
||||
# ICU Documentation
|
||||
|
||||
[](https://icu.unicode.org/)
|
||||
|
||||
## ICU User Guide
|
||||
|
||||
The [ICU User Guide](./userguide) provides information on i18n topics for which ICU has services, and
|
||||
includes details that go beyond the C, C++, and Java API docs (and avoids some duplication between them).
|
||||
|
||||
This is the new home of the User Guide (since 2020 August).
|
||||
|
||||
## ICU team member pages
|
||||
|
||||
Other documentation pages here are written by and for team members.
|
||||
@@ -1,156 +0,0 @@
|
||||
<!--
|
||||
© 2019 and later: Unicode, Inc. and others.
|
||||
License & terms of use: http://www.unicode.org/copyright.html
|
||||
-->
|
||||
|
||||
Developing Fuzzer Targets for ICU APIs
|
||||
======================================
|
||||
|
||||
This documents describes how to develop a [fuzzer](https://opensource.google.com/projects/oss-fuzz)
|
||||
target for an ICU API and its integration into the ICU build process.
|
||||
|
||||
### Directory and naming conventions
|
||||
|
||||
Fuzzer targets are exclusively in directory
|
||||
[`source/test/fuzzer/`](https://github.com/unicode-org/icu/tree/main/icu4c/source/test/fuzzer)
|
||||
and end with `_fuzzer.cpp`. Only files with such ending are recognized and executed as fuzzer
|
||||
targets by the OSS-Fuzz system.
|
||||
|
||||
### General structure of a fuzzer target
|
||||
|
||||
As a minimum, a fuzzer target contains the function
|
||||
|
||||
|
||||
```
|
||||
extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
|
||||
...
|
||||
}
|
||||
```
|
||||
|
||||
This function is expected and invoked by the fuzzer system. The `data` parameter contains the
|
||||
fuzzer-controlled data of size `size` bytes. Part or all of this data is then passed into the
|
||||
ICU API under test.
|
||||
|
||||
Fuzzer target
|
||||
[`collator_rulebased_fuzzer.cpp`](https://github.com/unicode-org/icu/blob/main/icu4c/source/test/fuzzer/collator_rulebased_fuzzer.cpp)
|
||||
illustrates the basic elements.
|
||||
|
||||
```
|
||||
// © 2019 and later: Unicode, Inc. and others.
|
||||
// License & terms of use: http://www.unicode.org/copyright.html
|
||||
|
||||
#include <cstring>
|
||||
|
||||
#include "fuzzer_utils.h"
|
||||
#include "unicode/coll.h"
|
||||
#include "unicode/localpointer.h"
|
||||
#include "unicode/locid.h"
|
||||
#include "unicode/tblcoll.h"
|
||||
|
||||
IcuEnvironment* env = new IcuEnvironment();
|
||||
|
||||
extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
|
||||
UErrorCode status = U_ZERO_ERROR;
|
||||
|
||||
size_t unistr_size = size/2;
|
||||
std::unique_ptr<char16_t[]> fuzzbuff(new char16_t[unistr_size]);
|
||||
std::memcpy(fuzzbuff.get(), data, unistr_size * 2);
|
||||
icu::UnicodeString fuzzstr(false, fuzzbuff.get(), unistr_size);
|
||||
|
||||
icu::LocalPointer<icu::RuleBasedCollator> col1(
|
||||
new icu::RuleBasedCollator(fuzzstr, status));
|
||||
|
||||
return 0;
|
||||
}
|
||||
```
|
||||
|
||||
The ICU API under test is the `RuleBasedCollator(const UnicodeString &rules, UErrorCode &status)`
|
||||
constructor. The code interprets the fuzzer data as UnicodeString and passes it to the constructor.
|
||||
And that is all. Specific error handling or return value verification is not required because the
|
||||
fuzzer will detect all memory issues by means of memory/address sanitizer findings.
|
||||
|
||||
### Makefile.in changes
|
||||
|
||||
ICU fuzzer targets are built and executed by the OSS-Fuzz project. On side of ICU they are compiled
|
||||
to assure that the code is syntactically correct and, as a sanity check, executed in the most basic
|
||||
manner, i.e. with minimal testdata and without ASAN or MSAN analysis.
|
||||
|
||||
Add the new fuzzer target to the list of targets in the `FUZZER_TARGETS` variable in
|
||||
[`Makefile.in`](https://github.com/unicode-org/icu/blob/main/icu4c/source/test/fuzzer/Makefile.in).
|
||||
The new fuzzer target will then be built and executed as part of a normal ICU4C unit test run. Note
|
||||
that each fuzzer target becomes executable on its own. As such it is linked with the code in
|
||||
`fuzzer_driver.cpp`, which contains the `main()` function.
|
||||
|
||||
### Fuzzer seed corpus
|
||||
|
||||
Any fuzzer seed data for a fuzzer target goes into a file with name `<fuzzer_target>_seed_corpus.txt`.
|
||||
In many cases the input parameter of the ICU API under test is of type `UnicodeString`, in case
|
||||
of which the seed data should be in UTF-16 format. As an example,see
|
||||
[collator_rulebased_fuzzer_seed_corpus.txt](https://github.com/unicode-org/icu/blob/main/icu4c/source/test/fuzzer/collator_rulebased_fuzzer_seed_corpus.txt).
|
||||
|
||||
### Guidelines and tips
|
||||
|
||||
* Leave all randomness to the fuzzer. If a random selection of any kind is needed (e.g., of a
|
||||
locale), then use bytes from the fuzzer data to make the selection
|
||||
([example](https://github.com/unicode-org/icu/blob/main/icu4c/source/test/fuzzer/break_iterator_fuzzer.cpp)).
|
||||
* In many cases ICU unit tests can provide seed data or at least ideas for seed data. If the API
|
||||
under test requires a Unicode string then make sure that the seed data is in UTF-16 encoding.
|
||||
This can be achieved with e.g. the 'iconv' command or using an editor that saves text in UTF-16.
|
||||
|
||||
### How to locally reproduce fuzzer findings
|
||||
|
||||
At this time reproduction of fuzzer findings requires Docker installed on the local machine and the
|
||||
OSS-Fuzz project downloaded in a local git client.
|
||||
|
||||
1. Install Docker (Ubuntu):
|
||||
|
||||
```
|
||||
sudo apt install docker
|
||||
```
|
||||
2. Download OSS-Fuzz, switch into directory oss-fuzz/
|
||||
|
||||
In a git client directory, download the fuzzer system.
|
||||
|
||||
```
|
||||
git clone https://github.com/google/oss-fuzz.git
|
||||
cd oss-fuzz/
|
||||
```
|
||||
3. Build the Docker image for ICU.
|
||||
In some setups root permissions may be required to connect to the Docker.
|
||||
|
||||
```
|
||||
[sudo] python infra/helper.py build_image icu
|
||||
```
|
||||
A prompt will appear: `Pull latest base images (compiler/runtime)? (y/N)`
|
||||
Respond: 'N'. If you are curious then respond with 'y' (won't hurt).
|
||||
4. Build the ICU fuzzers:
|
||||
|
||||
```
|
||||
[sudo] python infra/helper.py build_fuzzers --sanitizer [address | memory | undefined] icu
|
||||
```
|
||||
Check that the fuzzer targets were built successfully: ```ls -l build/out/icu```
|
||||
|
||||
5. Reproduce the fuzzer finding.
|
||||
First, get the testdata the fuzzer used when finding the issue. In the fuzzer bug report look
|
||||
for 'Reproducer Testcase', a click on the link will download the testdata. Then execute
|
||||
|
||||
```
|
||||
[sudo] python infra/helper.py reproduce icu <icu_fuzzer> <testdata>
|
||||
```
|
||||
Concrete example:
|
||||
|
||||
```
|
||||
sudo python infra/helper.py reproduce icu uregex_open_fuzzer ~/Downloads/clusterfuzz-testcase-minimized-uregex_open_fuzzer-5732067058384896
|
||||
```
|
||||
|
||||
**Limitations:** When reproducing a fuzzer finding in the way outlined above the fuzzer environment
|
||||
will use the current ICU trunk from https://github.com/unicode-org/icu.git. Thus it is not possible
|
||||
to modify the code to try out a possible fix. What can be done is to redirect Docker to download ICU
|
||||
from a forked ICU repository. Open the file oss-fuzz/projects/icu/Dockerfile and adjust the line
|
||||
with `git clone --depth 1 https://github.com/unicode-org/icu.git icu` accordingly. Then modify
|
||||
the code in the forked repository and follow the steps above beginning with step 3, create a Docker
|
||||
image.
|
||||
|
||||
This of course is still a tedious way of reproducing and working on a fuzzer finding. Ticket
|
||||
[ICU-20734](https://unicode-org.atlassian.net/browse/ICU-20734) aims to introduce a fuzzer driver
|
||||
that can reproduce certain fuzzer findings in a local ICU workspace.
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 69 KiB |
@@ -1,143 +0,0 @@
|
||||
---
|
||||
layout: default
|
||||
title: Announcements
|
||||
parent: Release & Milestone Tasks
|
||||
grand_parent: Contributors
|
||||
nav_order: 10
|
||||
---
|
||||
|
||||
<!--
|
||||
© 2021 and later: Unicode, Inc. and others.
|
||||
License & terms of use: http://www.unicode.org/copyright.html
|
||||
-->
|
||||
|
||||
# Announcements
|
||||
{: .no_toc }
|
||||
|
||||
## Contents
|
||||
{: .no_toc .text-delta }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
|
||||
---
|
||||
|
||||
## First milestone/candidate for a release
|
||||
|
||||
We write drafts of announcements in the ["ICU announcements" Google
|
||||
doc](https://docs.google.com/document/d/1rSVKZaus2K3rF66bMfDjYZnA2KTSJZUBz4C58GQMHsQ/edit).
|
||||
|
||||
When there is no section yet for the upcoming release, then copy the whole
|
||||
section from one of the previous two full releases and adjust the version
|
||||
numbers everywhere in the copied headings and text.
|
||||
|
||||
### Announcement for a release candidate
|
||||
|
||||
Adjust the release candidate announcement subsection with a very brief summary
|
||||
gleaned from the download page.
|
||||
|
||||
Share the draft as a link to the doc heading with the ICU team, invite comments
|
||||
and edits.
|
||||
|
||||
When ready, send to the mailing lists as indicated in the doc. Make sure to use
|
||||
bcc as noted for the "announce" lists.
|
||||
|
||||
### Announcement for a new major version
|
||||
|
||||
Adjust the final-release announcement subsection with a very brief summary
|
||||
gleaned from the download page. Adjust the blog & tweet texts as well.
|
||||
|
||||
Share the draft as a link to the doc heading with the ICU team, invite comments
|
||||
and edits.
|
||||
|
||||
Share the draft as a link with the Unicode editorial committee, invite comments,
|
||||
discuss, adjust as agreed.
|
||||
|
||||
When ready, send to the mailing lists as indicated in the doc. Make sure to use
|
||||
bcc as noted for the "announce" lists.
|
||||
|
||||
Notify Rick McGowan for the Unicode blog posting and Ken Lunde for tweeting.
|
||||
|
||||
Notify Ken Whistler (Unicode) to update the ICU version & date on
|
||||
<https://www.unicode.org/releases/>
|
||||
|
||||
Put a News item on the ICU project site.
|
||||
|
||||
### Announcement for a maintenance update
|
||||
|
||||
Same as for a release candidate, but start by copying the section for the last
|
||||
maintenance update in the doc.
|
||||
|
||||
## Update the reference for ICU in the external sites
|
||||
|
||||
**For freshmeat.net**
|
||||
|
||||
1. <http://www.freshmeat.net/>
|
||||
2. Login under the correct account.
|
||||
3. I usually search for: icu unicode
|
||||
4. Click on the project returned
|
||||
5. Under the 'project' drop down, choose 'Add release'
|
||||
6. We have (thankfully) only one branch of development- choose Default
|
||||
7. Release info
|
||||
* release focus: Choose the one which best fits
|
||||
* Version
|
||||
* Changes: be VERY brief, and include links if needed.
|
||||
Please read the other ICU4C/ICU4J updates to get an idea for how we are
|
||||
wording these. DO NOT use bulleted lists. DO use short 'sentences'.
|
||||
Remember they can go to the release list and/or bug DB for the full
|
||||
scoop!
|
||||
Put the URL to the 'release notes' (i.e.
|
||||
http://oss.software.ibm.com/icu/download/(version)/ ) in the changes
|
||||
list.
|
||||
* tar/gz, zip: Put the FULL URLs to the actual files, here
|
||||
* Changelog: Leave BLANK. This is for a single page of ALL release changes
|
||||
(Actually: http://oss.software.ibm.com/icu/download/ fits this fairly
|
||||
well.)
|
||||
* License - leave at MIT/X
|
||||
* mailing list: I set this to the main mailing list page.
|
||||
(http://oss.software.ibm.com/icu/archives/)
|
||||
Click 'go on..'
|
||||
8. PREVIEW.
|
||||
Very important- Check:
|
||||
* SPELLING
|
||||
* links!!!!! Make sure they go where we want
|
||||
* accuracy - version number is OK?
|
||||
Remember that this is an important PR face for ICU4C and ICU4J.
|
||||
9. Submit!
|
||||
|
||||
That's it. Submissions take some time (<24 hrs) to go on. I guess they check the
|
||||
accuracy as well- not sure here. (Steven Loomis, 2001-06-14)
|
||||
|
||||
**Other sites**:
|
||||
|
||||
[slashdot](http://slashdot.org/), [newsforge](http://newsforge.com/),
|
||||
[developerWorks unicode zone](http://www.ibm.com/developerworks/unicode/),
|
||||
[developerWorks open-source
|
||||
zone](http://www.ibm.com/developerworks/opensource/), [developerWorks java
|
||||
zone](http://www.ibm.com/developerworks/java/) (send email to `dWnews <at>
|
||||
us.ibm.com`)
|
||||
|
||||
[ICU sourceforge News](http://sourceforge.net/projects/icu) - login here, go to
|
||||
News, Admin, and submit a news item as an administrator.
|
||||
|
||||
---
|
||||
|
||||
#### LanguageTech Net News (tm)
|
||||
|
||||
"LanguageTech Net News" features the latest news for language technology and
|
||||
multilingual computing. It is prepared from materials compiled and maintained by
|
||||
the research editors at "[MultiLingual Computing &
|
||||
Technology](http://www.multilingual.com/)" and the Language Technology Research
|
||||
Center.
|
||||
|
||||
Press releases and submissions are always welcomed and encouraged. Please send
|
||||
e-mail to [news@multilingual.com](mailto:news@multilingual.com), faxes to
|
||||
208-263-6310 and postal mail to MultiLingual Computing, Inc., 319 North First
|
||||
Avenue, Sandpoint, ID 83864 USA, or call 208-263-8178.
|
||||
|
||||
To view archived issues of "LanguageTech Net News" visit
|
||||
<http://www.multilingual.com/>.
|
||||
|
||||
To subscribe to "LanguageTech Net News" go to
|
||||
<http://www.multilingual.com/multilingualpress/> or send an e-mail to
|
||||
[news-l-subscribe@multilingual.com](mailto:news-l-subscribe@multilingual.com).
|
||||
@@ -1,115 +0,0 @@
|
||||
---
|
||||
layout: default
|
||||
title: Build Updates
|
||||
parent: Release & Milestone Tasks
|
||||
grand_parent: Contributors
|
||||
nav_order: 30
|
||||
---
|
||||
|
||||
<!--
|
||||
© 2021 and later: Unicode, Inc. and others.
|
||||
License & terms of use: http://www.unicode.org/copyright.html
|
||||
-->
|
||||
|
||||
# Build Updates
|
||||
{: .no_toc }
|
||||
|
||||
## Contents
|
||||
{: .no_toc .text-delta }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
|
||||
---
|
||||
|
||||
## Regenerate configure
|
||||
|
||||
Don't forget to re-run and check in the configure file along with configure.in.
|
||||
This is normally supposed to be done when configure.in or aclocal.m4 are
|
||||
modified. It also should be done whenever the version numbers change in
|
||||
**uvernum.h**
|
||||
|
||||
On a Linux system,
|
||||
|
||||
```sh
|
||||
cd icu4c/source
|
||||
autoconf
|
||||
```
|
||||
|
||||
Review the changes to configure, most commonly just an updated ICU version
|
||||
number. Sometimes there are differences due to different versions of the
|
||||
autoconf tool being used.
|
||||
|
||||
## Update urename.h
|
||||
|
||||
Update urename.h to catch all possible ICU4C library exports (especially on
|
||||
non-Windows systems that tend to ignore export qualifiers). See
|
||||
[icu4c/source/tools/genren/README](https://github.com/unicode-org/icu/blob/main/icu4c/source/tools/genren/README).
|
||||
|
||||
Diff the new one with the previous one; there are typically a few bad #defines
|
||||
in there.
|
||||
|
||||
You are looking to make sure that the defines all are correct exported symbols
|
||||
and that the Perl script hasn't inadvertently grabbed extraneous text. Each of
|
||||
the defines should be of the format "uxxxx_NamingConvention". If not then you
|
||||
need to determine if it's a script issue or a poorly named method.
|
||||
|
||||
Also, please look out for this type of message: "\***\*\* WARNING Bad namespace
|
||||
(not 'icu') on ShoeSize::ShoeSize()**" - it probably means that there is a class
|
||||
not defined inside the "icu" namespace. Consider adding **U_NAMESPACE_BEGIN**
|
||||
and **U_NAMESPACE_END** around the class and member definitions.
|
||||
|
||||
## Update the pool bundles
|
||||
|
||||
*Obsolete for ICU 64+*: The pool bundles are no longer checked in. Instead,
|
||||
they are built on the fly. (And smaller if the data is filtered.)
|
||||
|
||||
The locale data resource bundles use pool.res bundles (one per bundle tree) for
|
||||
sharing most of their resource table key strings. We should update the pool
|
||||
bundles once per release, or when we get new data from CLDR, or change the tree
|
||||
or key structure, to capture the changing set of key strings.
|
||||
|
||||
1. Build ICU4C
|
||||
2. Check the date and size of the old pool bundles, for comparison later
|
||||
1. `~/icu/mine/src$ find icu4c -name 'pool.res' -exec ls -l '{}' \;`
|
||||
3. Temporarily modify the data makefile: s/usePoolBundle/writePoolBundle/
|
||||
1. If you are not on Windows, you probably need to make other changes to
|
||||
the Makefile as well. See
|
||||
[ICU-8101](https://unicode-org.atlassian.net/browse/ICU-8101) (contains
|
||||
a data/Makefile patch; use the latest version there)
|
||||
2. I like to make a copy the Makefile, then change it, save that for
|
||||
possible reuse, and later copy the original back.
|
||||
3. Try to patch it in rather than redoing it manually:
|
||||
`~/icu/mine/bld/icu4c/data$ patch -p0 <
|
||||
~/Downloads/Makefile-writePoolBundle.patch`
|
||||
4. Rebuild the data (make sure it actually gets rebuilt: cd data, make clean,
|
||||
make)
|
||||
5. Copy all of the pool.res files like this
|
||||
1. Linux ICU 63, from inside the build output's data directory:
|
||||
2. ICUDT=icudt63l
|
||||
3. cp out/build/$ICUDT/pool.res ../../../src/icu4c/source/data/locales
|
||||
4. cp out/build/$ICUDT/lang/pool.res ../../../src/icu4c/source/data/lang
|
||||
5. cp out/build/$ICUDT/region/pool.res
|
||||
../../../src/icu4c/source/data/region
|
||||
6. cp out/build/$ICUDT/zone/pool.res ../../../src/icu4c/source/data/zone
|
||||
7. cp out/build/$ICUDT/curr/pool.res ../../../src/icu4c/source/data/curr
|
||||
8. cp out/build/$ICUDT/unit/pool.res ../../../src/icu4c/source/data/unit
|
||||
6. Double-check the date and size of the old pool bundles. Sizes are usually a
|
||||
few % higher than before, due to increased CLDR data.
|
||||
7. Revert the data makefile change (copy the original back, if you kept a copy)
|
||||
8. Rebuild the data (make sure it actually gets rebuilt: make clean, make)
|
||||
9. Build ICU4C & run tests
|
||||
10. Rebuild the ICU4J data .jar files
|
||||
1. See
|
||||
[icu4c/source/data/icu4j-readme.txt](https://github.com/unicode-org/icu/blob/main/icu4c/source/data/icu4j-readme.txt)
|
||||
2. And/or see "update Java data files" in
|
||||
[icu4c/source/data/unidata/changes.txt](https://github.com/unicode-org/icu/blob/main/icu4c/source/data/unidata/changes.txt)
|
||||
|
||||
If there are new bundle trees that should use pool.res files (like the "unit"
|
||||
tree in ICU 54 ticket
|
||||
[ICU-11092](https://unicode-org.atlassian.net/browse/ICU-11092)), then first
|
||||
modify the data makefiles (Windows and Linux) to add the new pool.res to the
|
||||
appropriate file lists and initially add --writePoolBundle which at the end also
|
||||
needs to be turned into --usePoolBundle. Or, simpler, initially copy (svn cp)
|
||||
the parent tree's pool bundle, and update it later (maybe during release task
|
||||
work).
|
||||
@@ -1,24 +0,0 @@
|
||||
---
|
||||
layout: default
|
||||
title: Copy Shared Tests
|
||||
parent: Release & Milestone Tasks
|
||||
grand_parent: Contributors
|
||||
nav_order: 40
|
||||
---
|
||||
|
||||
<!--
|
||||
© 2021 and later: Unicode, Inc. and others.
|
||||
License & terms of use: http://www.unicode.org/copyright.html
|
||||
-->
|
||||
|
||||
# Copy Shared Tests
|
||||
|
||||
Certain tests are structured so that they can be shared between CLDR and ICU.
|
||||
That allows the tests to be run in CLDR earlier in the development process. The
|
||||
tests include:
|
||||
|
||||
* LocaleMatcherTest
|
||||
* TransformTest (to be cleaned up)
|
||||
|
||||
Copy all of these from CLDR to ICU, and sanity check the differences, then check
|
||||
in.
|
||||
@@ -1,405 +0,0 @@
|
||||
---
|
||||
layout: default
|
||||
title: APIs & Docs
|
||||
parent: Release & Milestone Tasks
|
||||
grand_parent: Contributors
|
||||
nav_order: 20
|
||||
---
|
||||
|
||||
<!--
|
||||
© 2021 and later: Unicode, Inc. and others.
|
||||
License & terms of use: http://www.unicode.org/copyright.html
|
||||
-->
|
||||
|
||||
# APIs & Docs
|
||||
{: .no_toc }
|
||||
|
||||
## Contents
|
||||
{: .no_toc .text-delta }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
|
||||
---
|
||||
|
||||
## Update API status comments
|
||||
|
||||
### ICU4C
|
||||
|
||||
1. **Note: for ICU4C 49m2 or later, will require Doxygen 1.7.5.1 or later (
|
||||
see** [ICU-8862](https://unicode-org.atlassian.net/browse/ICU-8862) **).**
|
||||
On Linux with debian style package management,
|
||||
|
||||
`sudo apt-get install doxygen`
|
||||
|
||||
Or, to install manually
|
||||
1. To update Doxygen, first [download binary or source release
|
||||
here](http://doxygen.org/download.html#latestsrc)
|
||||
2. Unpack the release, and run something like<br>
|
||||
"`configure --prefix /usr/local`" (to install into /usr/local/bin etc)<br>
|
||||
(Note for the binary release, 'configure' just builds an installer.)
|
||||
3. Run "`make install`" with appropriate permission (perhaps
|
||||
"`sudo make install`".)
|
||||
4. Verify that '`doxygen --version`' gives the correct number.
|
||||
|
||||
2. Update the API documentation in all header files (.h file) to have correct
|
||||
@draft/@stable/@deprecated labels.
|
||||
3. Update docmain.h
|
||||
4. ./configure
|
||||
5. make doc
|
||||
6. Follow instructions in
|
||||
[tools/release/java/readme.txt](https://htmlpreview.github.io/?https://github.com/unicode-org/icu/blob/main/tools/release/java/readme.txt)
|
||||
to generate API status change report.
|
||||
7. Make sure that ICU headers work with U_HIDE_DRAFT_API and other such
|
||||
switches.
|
||||
8. Verify that U_DRAFT and U_STABLE match the @draft and @stable tags (same for
|
||||
other such pairs declaration macro vs. tag).
|
||||
1. For example, on Linux:
|
||||
2. grep --recursive --include="\*.h" --exclude-dir=".svn" -e "@deprecated"
|
||||
-A 5 . > at-deprecated.txt
|
||||
3. grep --recursive --include="\*.h" --exclude-dir=".svn" -e "U_DEPRECATED"
|
||||
-B 5 -A 1 . > u_deprecated.txt
|
||||
4. Do this for draft, deprecated, obsolete, internal.
|
||||
5. For @stable, we either verify by cleaning up all others or we need to
|
||||
write a script.
|
||||
|
||||
### ICU4J
|
||||
|
||||
Update the API documentation to have correct @draft/@stable/@deprecated labels.
|
||||
See the [User Guide, ICU Architectural Design, ICU API
|
||||
compatibility](https://unicode-org.github.io/icu/userguide/icu/design#icu-api-compatibility).
|
||||
|
||||
On ICU4J, run
|
||||
[com.ibm.icu.dev.tool.docs.CheckTags](https://github.com/unicode-org/icu/blob/main/icu4j/tools/build/src/com/ibm/icu/dev/tool/docs/CheckTags.java)
|
||||
(see file for instructions). This requires a JDK with javadoc available. The
|
||||
tool will need to change to reflect the release number to search for.
|
||||
|
||||
To check the API status changes, run the ant target "apireport" to generate the
|
||||
report since the previous official release.
|
||||
|
||||
Make sure **@internal APIs** are also marked as @deprecated:
|
||||
|
||||
```
|
||||
* @internal
|
||||
* @deprecated This API is ICU internal only.
|
||||
@Deprecated
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Promote draft APIs to stable
|
||||
|
||||
APIs previously introduced as @draft are reviewed for every new release. The
|
||||
current policy is to keep @draft status at least in one release cycle. For
|
||||
example, a new @draft API introduced in ICU 60 will be kept @draft in ICU 61.
|
||||
Then the API will be reviewed by ICU technical committee before ICU 62 release
|
||||
and the API can be promoted to @stable status.
|
||||
|
||||
### **ICU4C**
|
||||
|
||||
Andy's method (from email 2019-sep-05):
|
||||
|
||||
* Creation of the promotion spreadsheet was done by hand. It's something of an
|
||||
iterative process.
|
||||
* Start by making a copy of the previous release spread sheet, to get the
|
||||
format, and probable entries for older @draft items that are perpetually
|
||||
being held back.
|
||||
* Do the API promotion - search the sources for @draft, and make a best guess
|
||||
at what to promote. Add entries to the spread sheet as you go. Note any
|
||||
items you're not sure about.
|
||||
* Run the API change report tool, see what looks inconsistent, fix up the
|
||||
sources and rerun as needed. Deleted functions will first show up here, and
|
||||
are added to the spread sheet.
|
||||
* Cross-check the API change report and the spread sheet.
|
||||
* The whole thing takes a bit of time to do.
|
||||
|
||||
### **ICU4J**
|
||||
|
||||
1. Create a new GitHub branch.
|
||||
2. Run next ant target: `$ ant draftAPIsTSV`
|
||||
3. This **ant** target generates a tab-separated values file at
|
||||
icu4j/out/draftAPIs.tsv.
|
||||
4. Import the TSV file to Google spread sheet - for example, [ICU4J 61 Draft
|
||||
API
|
||||
Promotion](https://docs.google.com/spreadsheets/d/135RDyY6cWKBBvNuVE9ne8HfsItZu_CvLh47hKjr3UqM/edit#gid=1384666589).
|
||||
* Create the spreadsheet in the shared ***ICU Projects*** folder.
|
||||
5. Review all APIs introduced in 2 releases ago or older. For example, 59 or
|
||||
older for ICU 61 release.
|
||||
* Any API added 2 or more releases ago is a candidate for promotion.
|
||||
* Newer APIs should be left in @draft state.
|
||||
6. Check each API entry has a corresponding approved API proposal. For example,
|
||||
[ICU 59 API proposal
|
||||
status](https://docs.google.com/document/d/12dHTG3e9WXb7C1Xdc2bcXE0BK8f6IoufoUAPcz4ZT2I/edit).
|
||||
Note: An API proposal might have been created before the API proposal doc
|
||||
version, so you may need to look at older version of API proposal status
|
||||
doc.
|
||||
7. Mark APIs proposed for promotion, and add note in the spread sheet.
|
||||
8. Update API comments in ICU4J source code. In this case, @draft is replaced
|
||||
with @stable (do not change ICU version, for example, "@draft ICU 59" will
|
||||
be changed to "@stable ICU 59").
|
||||
9. Run next ant target to generate an API change report html: `$ ant apireport`
|
||||
10. Review the report, sent the report file and the link of the spread sheet to
|
||||
ICU technical committee members for review.
|
||||
11. Once ICU TC approves the proposed change, create a pull request (PR), wait
|
||||
for review, and merge the branch into the main branch.
|
||||
|
||||
---
|
||||
|
||||
## Update the API Change Report
|
||||
|
||||
### ICU4C
|
||||
|
||||
This work is done in the root of icu4c:
|
||||
|
||||
1. Update the API documentation in all header files (.h file) to have correct
|
||||
@draft/@stable/@deprecated/@obsolete labels.
|
||||
2. Update docmain.h with additions or removals as needed.
|
||||
3. `source/configure`
|
||||
4. `make doc`
|
||||
5. Follow instructions in
|
||||
[tools/release/java/readme.txt](https://github.com/unicode-org/icu/blob/main/tools/release/java/readme.txt).
|
||||
|
||||
### ICU4J
|
||||
|
||||
This work is done in the root of icu4j:
|
||||
|
||||
1. Make sure JAVA_HOME is set to JDK 8. This report creation fails with JDK 11.
|
||||
For example, in Linux:
|
||||
* `export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64`
|
||||
2. Then run ant task "clean" and "apireport" at <icu4j_root>:
|
||||
* `ant clean apireport`
|
||||
3. Above will produce API change report file
|
||||
<code><icu4j_root>/<b>out</b>/icu4j_compare_xxx_yyy.html</code>
|
||||
4. Make sure there are any new doc tag errors are reported. (As of ICU 4.4,
|
||||
ArabicShaping constants do not have proper tags - otherwise, clean)
|
||||
5. Copy generated report file to `<icu4j_root>/APIChangeReport.html` and check
|
||||
it in.
|
||||
|
||||
Once official release version is shipped, we need to keep API signature
|
||||
information file for next iteration. This is not done for milestone releases,
|
||||
only after the final official release.
|
||||
|
||||
1. Run ant task "gatherapi" at <icu4j_root>
|
||||
2. Above will produce API signature information file
|
||||
<icu4j_root>/out/icu4jxx.api2.gz
|
||||
3. Copy icu4jxxapi2.gz to <icu4j_root>/tools/build and add it to the repository
|
||||
|
||||
---
|
||||
|
||||
## Check in API signature data file (ICU4J)
|
||||
|
||||
Once APIs are frozen for a reference release, we should check in the API
|
||||
signature data file into the repository. The data file will be used for future
|
||||
API change report.
|
||||
|
||||
1. Run ant task "gatherapi" at <icu4j_root>
|
||||
2. The output file icu4j<ver>.api3.gz is created in <icu4j_root>/out directory.
|
||||
3. Copy the output .gz file to <icu4j_root>/tools directory and check in the
|
||||
file to the repository.
|
||||
4. You may delete very old versions of the API signature files. Keeping 10
|
||||
versions to the latest should be good enough.
|
||||
|
||||
Note: This task is only necessary for reference releases, because we won't
|
||||
change public APIs in maintenance releases. The API signature file for an ICU4J
|
||||
version is generated and checked into trunk just before ***maint-xx*** branch is
|
||||
created for the major version, so we can keep track of API changes in
|
||||
maintenance releases of the major version (A maintenance release should have no
|
||||
API changes although).
|
||||
|
||||
---
|
||||
|
||||
## Verify that @draft is surrounded by #ifndef U_HIDE_DRAFT_API etc
|
||||
|
||||
In ICU4C, we want every (consecutive group of) `@draft` API to be surrounded by
|
||||
`#ifndef U_HIDE_DRAFT_API`. This allows users to `-DU_HIDE_DRAFT_API` to make
|
||||
sure they don't use unstable API.
|
||||
|
||||
```
|
||||
#ifndef U_HIDE_DRAFT_API
|
||||
/** @draft ICU 51 */
|
||||
U_CAPI u_newFunction1();
|
||||
/** @draft ICU 51 */
|
||||
U_CAPI u_newFunction2();
|
||||
#endif // U_HIDE_DRAFT_API
|
||||
```
|
||||
|
||||
Same for `@deprecated` & <code>#ifndef U_HIDE_<b>DEPRECATED</b>_API</code> ..
|
||||
`#endif // U_HIDE_DEPRECATED_API`
|
||||
|
||||
Same for `@internal` & <code>#ifndef U_HIDE_<b>INTERNAL</b>_API</code> ..
|
||||
`#endif // U_HIDE_INTERNAL_API`
|
||||
|
||||
Same for `@system` & <code>#ifndef U_HIDE_<b>SYSTEM</b>_API</code> ..
|
||||
`#endif // U_HIDE_SYSTEM_API`
|
||||
|
||||
Same for `@obsolete` & <code>#ifndef U_HIDE_<b>OBSOLETE</b>_API</code> ..
|
||||
`#endif // U_HIDE_OBSOLETE_API`
|
||||
|
||||
### Caution
|
||||
|
||||
* We cannot #ifndef-guard virtual methods because that makes the vtable
|
||||
incompatible.
|
||||
* // Cannot use #ifndef U_HIDE_DRAFT_API for the following draft method
|
||||
since it is virtual.
|
||||
* When you #ifndef-guard enum constants, normally the following unguarded ones
|
||||
(e.g., a _COUNT or _LIMIT) should retain the same numeric values as if the
|
||||
guard was absent.
|
||||
|
||||
<span style="background-color:yellow">For more details (and cautions) see the
|
||||
Coding Guidelines section [C/C++ Hiding Un-@stable
|
||||
APIs](https://unicode-org.github.io/icu/userguide/dev/codingguidelines#cc-hiding-un-stable-apis).</span>
|
||||
|
||||
### Task
|
||||
|
||||
a) For each of these API status tags, for each API that is tagged with it,
|
||||
verify that the API is surrounded by the appropriate #ifndef..#endif.
|
||||
|
||||
Note: It is best to not use one single guard for APIs with different ICU
|
||||
versions, since they will become stable and thus lose their guards at different
|
||||
times. Use one #ifndef..#endif guard per API status and ICU version.
|
||||
|
||||
b) For each of these U_HIDE_..._API guards, verify that it only and exactly
|
||||
surrounds APIs with the corresponding status tag. In particular, make sure that
|
||||
U_HIDE_DRAFT_API does not surround (newly) @stable API.
|
||||
|
||||
We don't have tools for this. <span style="background-color:yellow">One approach
|
||||
is to</span> use "grep" or similar on the public *common*, *i18n*, *io* header
|
||||
files. Use grep options like -A 3, -B 2 and -C 3 for context After, Before, and
|
||||
Around the matching line. <span style="background-color:yellow">A better
|
||||
approach if you have the tools available is to use a programming-oriented text
|
||||
editor that can do (a) powerful regex search across (b) multiple files in
|
||||
several specified directories, and (c) display the matched lines in context such
|
||||
that (d) they or their context can be edited in place; an example of such a tool
|
||||
on macOS is BBEdit. This permits a comprehensive search using an expression such
|
||||
as
|
||||
"(U_HIDE_\[A-Z_\]+)|(@draft)|(@deprecated)|(@obsolete)|(@system)|(@internal)|(@preview)"
|
||||
which permits relatively easy checking for correct enclosure of status tags in
|
||||
conditionals.</span>
|
||||
|
||||
<span style="background-color:yellow">As part of this, you may need to run side
|
||||
searches, for example to verify that no conditionalized type, function or value
|
||||
is used by an unconditionalized portion of a header file.</span>
|
||||
|
||||
<span style="background-color:yellow">There is no magic bullet; however you
|
||||
carry out this step, it will require several hours of going through the
|
||||
grep/regex results and manually checking for correct enclosure, checking that
|
||||
conditionalized values are not needed elsewhere, etc.</span>
|
||||
|
||||
---
|
||||
|
||||
## Update udraft.h, usystem.h, uintrnl.h, uobslete.h and udeprctd.h
|
||||
|
||||
*Ignore this step for ICU49 and later. In ICU 49 and above, these header files
|
||||
and the gendraft/genheaders.pl tool are gone. (Ticket
|
||||
[ICU-8571](https://unicode-org.atlassian.net/browse/ICU-8571))*
|
||||
|
||||
Instructions ***for ICU4C 4.8.x and earlier***:
|
||||
|
||||
1. make doc
|
||||
2. `cd source/tools/gendraft ; make install-headers`
|
||||
3. Double check the modified files in <icu>/source/common/unicode folder and
|
||||
commit.
|
||||
|
||||
---
|
||||
|
||||
## Compare ICU4J APIs with JDK
|
||||
|
||||
Run the ICU4J versus JDK API comparison tool against the target JDK (anything
|
||||
that will come out before our next release, basically) with the tool
|
||||
[com.ibm.icu.dev.tool.docs.ICUJDKCompare](https://github.com/unicode-org/icu/blob/main/icu4j/tools/build/src/com/ibm/icu/dev/tool/docs/ICUJDKCompare.java)
|
||||
and make sure ICU4J adequately covers the JDK API for the classes we replicate.
|
||||
|
||||
---
|
||||
|
||||
## Build API documentations
|
||||
|
||||
### **ICU4C**
|
||||
|
||||
Build the API documentation pages for the new release. Run Doxygen to create the
|
||||
javadoc files. Create icu4c-X_X_X-docs.zip
|
||||
|
||||
**Note: for ICU4C 49m2 or later, requires Doxygen 1.7.5.1 or later ( see
|
||||
[ICU-8862](https://unicode-org.atlassian.net/browse/ICU-8862) )**
|
||||
|
||||
#### Create the ICU4C docs zip file:
|
||||
|
||||
1. Go to .../icu4c/source
|
||||
2. Generate the API document pages:<br>
|
||||
`make doc-searchengine`
|
||||
3. The generated API docs are in <path>/icu4c/source/doc/html/<br>
|
||||
`cd <path>/icu4c/source/doc/html/`
|
||||
4. Create a zip file, e.g.,<br>
|
||||
`tar cvfz /tmp/icu4c641.tar.gz * # '641' needs to be replaced by the respective release label.`
|
||||
5. ~~Upload this zip file to the GitHub release page. For the live API docs, see below.~~
|
||||
|
||||
#### Create a PR for ICU4C docs using the docs zip file
|
||||
|
||||
1. Follow directions in [How to update ICU docs](https://unicode-org.github.io/icu-docs/HOWTO-Update.html)
|
||||
|
||||
a. First, bring the `main` branch of your icu-docs local copy up to date.
|
||||
|
||||
b. Create and switch to a feature branch based of the latest `main`. Ex: `git checkout -b ICU-<TICKET-NUMBER>`.
|
||||
|
||||
c. Copy the zip file into your local copy of icu-docs into `apidoc/released/icu4c`
|
||||
(or `apidoc/dev/icu4c` if not a release)
|
||||
|
||||
d. Unzip the file, replacing all documentation. Ex: `tar xvfz /tmp/icu4c641.tar.gz`
|
||||
|
||||
e. Remove the zip file
|
||||
|
||||
f. `git add .`
|
||||
|
||||
g. `git commit -m "ICU-<TICKET-NUMBER> Update ICU4C API reference docs for
|
||||
XX.Y"`<br>
|
||||
Example: [ICU-21546 Update ICU4C API reference docs for 69.1](https://github.com/unicode-org/icu-docs/pull/25)
|
||||
|
||||
h. `git push origin ICU-<TICKET-NUMBER>`
|
||||
|
||||
i. Create Pull Request from your personal github fork of icu-docs from your newly-pushed branch `ICU-<TICKET-NUMBER>` with a destination of
|
||||
the `unicode-org/icu-docs` repo's `main` branch
|
||||
|
||||
j. Request review
|
||||
|
||||
Note: This is also referenced below [Upload API documentations](docs.md#upload-api-documentations) for how to make the API docs public.
|
||||
|
||||
### ICU4J
|
||||
|
||||
**Note:** JCite must be installed for building ICU4J documentation:
|
||||
<https://icu.unicode.org/setup/java/ant#TOC-Building-ICU4J-API-Reference-Document-with-JCite>
|
||||
|
||||
Build the API documentation pages for the new release:
|
||||
|
||||
```
|
||||
ant releaseDocs
|
||||
```
|
||||
|
||||
#### Alternative method:
|
||||
|
||||
**Note:** JCite must be installed for building ICU4J documentation:
|
||||
<https://icu.unicode.org/setup/java/ant#TOC-Building-ICU4J-API-Reference-Document-with-JCite>
|
||||
|
||||
Use the release target
|
||||
|
||||
```
|
||||
ant releaseVer
|
||||
```
|
||||
|
||||
which generate all release files.
|
||||
|
||||
* Upload the output files including icu4j-docs.jar to the release page first,
|
||||
* Then update the live API docs from the generated docs.jar.
|
||||
|
||||
See [Upload API documentations](docs.md#upload-api-documentations) below for how to make the API docs public.
|
||||
|
||||
### Upload API documentations
|
||||
|
||||
See <https://unicode-org.github.io/icu-docs/HOWTO-Update.html> for instructions
|
||||
to upload to <https://unicode-org.github.io/icu-docs/>
|
||||
|
||||
### Update the Readme.html for GA
|
||||
|
||||
If there are any last second changes:
|
||||
|
||||
* Make sure to document especially items that affect software that is
|
||||
currently compiled with the previous version of ICU.
|
||||
* Update build/installation instructions as necessary.
|
||||
@@ -1,557 +0,0 @@
|
||||
---
|
||||
layout: default
|
||||
title: Healthy Code
|
||||
parent: Release & Milestone Tasks
|
||||
grand_parent: Contributors
|
||||
nav_order: 50
|
||||
---
|
||||
|
||||
<!--
|
||||
© 2021 and later: Unicode, Inc. and others.
|
||||
License & terms of use: http://www.unicode.org/copyright.html
|
||||
-->
|
||||
|
||||
# Healthy Code
|
||||
{: .no_toc }
|
||||
|
||||
## Contents
|
||||
{: .no_toc .text-delta }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
|
||||
---
|
||||
|
||||
## Check for ClassID in new class hierarchies
|
||||
|
||||
Check that there are no "poor man's RTTI" methods in new class hierarchies.
|
||||
|
||||
After ICU 50: New class hierarchies should not declare getDynamicClassID() at
|
||||
all. UObject has a dummy implementation for it.
|
||||
|
||||
ICU 4.6..50: New class hierarchies used UOBJECT_DEFINE_NO_RTTI_IMPLEMENTATION.
|
||||
See Normalizer2 for an example declaration and implementation that satisfies the
|
||||
virtual-function override without adding new ClassID support.
|
||||
|
||||
We do need to keep and add "poor man's RTTI" in old classes, and in new classes
|
||||
extending existing class hierarchies (where parent classes have @stable RTTI
|
||||
functions). (For example, a new Format subclass.)
|
||||
|
||||
One easy way to check for this is to search through the API change report and
|
||||
look for "UClassID" or similar.
|
||||
|
||||
* Latest trunk report:
|
||||
[icu/icu4c/APIChangeReport.html](https://github.com/unicode-org/icu/blob/main/icu4c/APIChangeReport.html)
|
||||
* ~~The automated build system creates a new report:
|
||||
<https://cldr-build.unicode.org/jenkins/job/icu/job/icu-apidocs/>~~
|
||||
* Click "ICU4C API Change Report vs Latest"
|
||||
* Old: was http://bugs.icu-project.org/trac/build/icu4cdocs
|
||||
* Go into the latest successful build, the report is an attachment there.
|
||||
* Download the style sheet next to it:
|
||||
https://github.com/unicode-org/icu/blob/main/icu4c/icu4c.css
|
||||
|
||||
---
|
||||
|
||||
## ~~Check for non-ascii characters in ICU4C source files \[obsolete\]~~
|
||||
|
||||
~~Note: ICU4C and ICU4J source files are UTF-8. The ASCII check is no longer
|
||||
appropriate for them.~~
|
||||
|
||||
```sh
|
||||
cd icu4c/source
|
||||
find . \( -name "*.[ch]" -or -name "*.cpp" \) -exec grep -PHn [^[:ascii:]] {} \;
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Check source files for valid UTF-8 and correct text file line endings
|
||||
|
||||
Verify that all source and text files in the repository have plain LF line
|
||||
endings.
|
||||
|
||||
To do this on Linux, In an up-to-date git workspace,
|
||||
|
||||
```sh
|
||||
cd icu/icu4c/source
|
||||
tools/icu-file-utf8-check.py # reports problems
|
||||
```
|
||||
|
||||
The same python script from the icu4c tools will also check icu4j
|
||||
|
||||
```sh
|
||||
cd icu/icu4j
|
||||
../icu4c/source/tools/icu-file-utf8-check.py
|
||||
```
|
||||
|
||||
To double-check the line endings, the following grep will find all text files
|
||||
containing \\r characters. Do not run from Windows, where \\r\\n line endings
|
||||
are expected.
|
||||
|
||||
```sh
|
||||
cd icu
|
||||
grep -rPIl "\r" *
|
||||
```
|
||||
|
||||
Even when run from Mac or Linux, some WIndows specific files (.bat, etc) will be
|
||||
found by this check. This is OK.
|
||||
|
||||
## ~~Check UTF-8 file properties \[obsolete\]~~
|
||||
|
||||
*Note: As of ICU 63, the project moved from svn to GitHub. SVN file properties
|
||||
are no longer relevant.*
|
||||
|
||||
<span style="color:red">**Note: As of ICU 59, ICU4C source files are UTF-8
|
||||
encoded, and have the svn mime-type property "text/plain;charset=utf-8". They
|
||||
must not have a BOM.**</span>
|
||||
|
||||
This is checked by the above task, *Check svn properties, valid UTF-8 and text
|
||||
file line endings.*
|
||||
|
||||
The bomfix.py script, formerly used for this task, must *not* be run over the
|
||||
ICU4C sources
|
||||
|
||||
<span style="color:red">**~~Note: This task is only applicable to ICU4C. ICU4J
|
||||
.java source files are encoded by UTF-8, but must be without UTF-8
|
||||
BOM.~~**</span>
|
||||
|
||||
~~Check that the following match: Files marked as UTF-8 vs. Files beginning with
|
||||
the UTF-8 signature byte sequence ("BOM").~~
|
||||
|
||||
~~Run:~~
|
||||
|
||||
<pre><code><s>cd {icu}/icu4c</s>
|
||||
<s>python ../tools/release/c/bomfix.py</s></code></pre>
|
||||
|
||||
---
|
||||
|
||||
## Clean up import statements
|
||||
|
||||
The Eclipse IDE provides a feature which allow you to organize import statements
|
||||
for multiple files. Right click on projects/source folders/files, you can select
|
||||
\[Source\] - \[Organize Imports\] which resolve all wildcard imports and sort
|
||||
the import statements in a consistent order. (Note: You may experience OOM
|
||||
problem when your run this for projects/folders which contain many files. In
|
||||
this case, you may need to narrow a selection per iteration.)
|
||||
|
||||
---
|
||||
|
||||
## Check library dependencies
|
||||
|
||||
***ICU 64+ (2019+): Done automatically by a build bot, at least in one of the
|
||||
two modes (debug/release), ok to skip as BRS task.***
|
||||
|
||||
We want to keep dependencies between .c/.cpp/.o files reasonable, both between
|
||||
and inside ICU's libraries.
|
||||
|
||||
On Linux, run
|
||||
[source/test/depstest/depstest.py](https://github.com/unicode-org/icu/blob/main/icu4c/source/test/depstest/dependencies.py),
|
||||
for example:
|
||||
```sh
|
||||
~/icu/mine/src/icu4c/source/test/depstest$ ./depstest.py ~/icu/mine/dbg/icu4c
|
||||
```
|
||||
|
||||
Do this twice: Once for a release build (optimized) and once for a debug build
|
||||
(unoptimized). They pull in slightly different sets of standard library symbols
|
||||
(see comments in dependencies.txt).
|
||||
|
||||
If everything is fine, the test will print "OK: Specified and actual
|
||||
dependencies match." If not...
|
||||
|
||||
*Get changes reviewed by others, including Markus; including changes in
|
||||
dependencies.txt, .py scripts, or ICU4C code.*
|
||||
|
||||
At first, the test will likely complain about .o files not listed in its
|
||||
dependencies.txt or, if files were removed, the other way around. Try to add
|
||||
them to groups or create new groups as appropriate.
|
||||
|
||||
As a rule, smaller groups with fewer dependencies are preferable. If public API
|
||||
(e.g., constructing a UnicodeString via conversion) is not needed inside ICU
|
||||
(e.g., unistr_cnv), make its library depend on its new group.
|
||||
|
||||
If the test prints "Info: group icuplug does not need to depend on platform"
|
||||
then the plug-in code is disabled, as is the default since ICU 56. Consider
|
||||
enabling it for dependency checking, but make sure to revert that before you
|
||||
commit changes!
|
||||
|
||||
There are usually other "Info" messages where the tool thinks that dependencies
|
||||
on system symbols are not needed. These are harmless, that is, don't try to
|
||||
remove them: They are needed or not needed based on the compiler flags used. If
|
||||
you remove them, then you will likely cause an error for someone with different
|
||||
flags. Also, in an unoptimized build you only get half as many info messages.
|
||||
You get more in an optimized build because more system stuff gets inlined.
|
||||
|
||||
The test might complain "is this a new system symbol?" We should be careful
|
||||
about adding those. For example, we must not call printf() from library code,
|
||||
nor the global operator new.
|
||||
|
||||
The test might complain that some .o file "imports
|
||||
`icu_48::UnicodeString::UnicodeString(const char *)` but does not depend on
|
||||
unistr_cnv.o". This probably means that someone passes a simple `"string literal"`
|
||||
or a `char *` into a function that takes a UnicodeString, which invokes the
|
||||
default-conversion constructor. We do not want that! In most cases, such code
|
||||
should be fixed. Only implementations
|
||||
of API that require conversion should depend on it; for example, group
|
||||
formattable_cnv depends on group unistr_cnv, but then nothing inside ICU depends
|
||||
on that.
|
||||
|
||||
---
|
||||
|
||||
## Verify proper memory allocation functions
|
||||
|
||||
Verify the following for library code (common, i18n, layout, ustdio). The
|
||||
requirement is for ICU's memory management to be customizable by changing
|
||||
cmemory.h and the common base class.
|
||||
|
||||
**Note:** The requirement remains. The techniques to fix issues are valid.
|
||||
***For testing**, see the section "Check library dependencies" above.*
|
||||
|
||||
* No raw malloc/free/realloc but their uprv_ versions.
|
||||
* All C++ classes must inherit the common base class UObject or UMemory
|
||||
* But not for mixin/interface classes (pure virtual, no data members, no
|
||||
constructors) because that would break the single-inheritance model.
|
||||
* Also not for pure-static classes (used for name scoping) declare but
|
||||
don't implement a private default constructor to prevent instantiation.
|
||||
* Simple struct-like C++ classes (and structs) that do not have
|
||||
constructors, destructors, and virtual methods, need not inherit the
|
||||
base class but must then be allocated with uprv_malloc.
|
||||
* All simple types (UChar, int32_t, pointers(!), ...) must be allocated with
|
||||
uprv_malloc and released with uprv_free.
|
||||
* For Testing that this is the case, on Linux
|
||||
* run the Perl script ICUMemCheck.pl. Follow the instructions in the
|
||||
script. The script is in in the ICU4C sources at
|
||||
source/tools/memcheck/ICUMemCheck.pl.
|
||||
* also, depstest.py will show an error message if the global operator new
|
||||
is used (see section about checking dependencies)
|
||||
* For testing that this is the case, on Windows:
|
||||
* Don't bother, as of Nov, 2004. Failures appear in many dozens of files
|
||||
from the mixin class destructors. Do the check on Linux. But, for
|
||||
reference, here are the old instructions.
|
||||
* Make sure that in `uobject.h` `UObject::new` and `delete` are
|
||||
defined by default. Currently, this means to grep to see that
|
||||
`U_OVERRIDE_CXX_ALLOCATION` is defined to 1 (in `pwin32.h` for
|
||||
Windows).
|
||||
* Check the \*.obj's for linkage to the global (non-UObject::)
|
||||
operators new/delete; see uobject.cpp for details.
|
||||
* Global `new` must never be imported. Global `delete` will be
|
||||
imported and used by empty-virtual destructors in interface/mixin
|
||||
classes. However, they are not called because implementation classes
|
||||
always derive from UMemory. No other functions must use global
|
||||
delete.
|
||||
* There are now (2002-dec-17) definitions in `utypes.h` for global
|
||||
new/delete, with inline implementations that will always crash.
|
||||
These global new/delete operators are only defined for code inside
|
||||
the ICU4C libraries (but must be there for all of those). See ticket
|
||||
#2581.
|
||||
* If a global new/delete is used somewhere, then change the class
|
||||
inheritance and/or use uprv_malloc/free until no global new/delete
|
||||
are used in the libraries (and the tests still pass...). See the
|
||||
User Guide Coding Guidelines for details.
|
||||
|
||||
---
|
||||
|
||||
## Run static code analysis tools
|
||||
|
||||
(Purify, Boundary Checker, valgrind...)
|
||||
|
||||
Make sure we fixed all the memory leak problems that were discovered when
|
||||
running these tools.
|
||||
|
||||
Build ICU with debug information. On Linux,
|
||||
|
||||
```sh
|
||||
runConfigureICU --enable-debug --disable-release Linux
|
||||
```
|
||||
|
||||
Run all of the standard tests under valgrind. For intltest, for example,
|
||||
|
||||
```sh
|
||||
cd <where ever>/source/test/intltest
|
||||
LD_LIBRARY_PATH=../../lib:../../stubdata:../../tools/ctestfw:$LD_LIBRARY_PATH valgrind ./intltest
|
||||
```
|
||||
|
||||
You can grab the command line for running the tests from the output from "make
|
||||
check", and then just insert "valgrind" before the executable.
|
||||
|
||||
---
|
||||
|
||||
## Check the code coverage numbers
|
||||
|
||||
Our goal is that all releases go out to the public with 100% API test and at
|
||||
least 85% code coverage.
|
||||
|
||||
---
|
||||
|
||||
## Test ICU4C headers
|
||||
|
||||
### Test ICU4C public headers
|
||||
|
||||
Testing external dependencies in header files:
|
||||
|
||||
(on Unixes) Prerequisite: Configure with --prefix
|
||||
(../icu4c/source/runConfigureICU Linux --prefix=/some/temp/folder) and do 'make
|
||||
install'. Then set the PATH so that the installed icu-config script can be
|
||||
found. (export PATH=/some/temp/folder/**bin**:$PATH)
|
||||
|
||||
Then go to the 'icu4c/test/hdrtst' directory (note: not 'source/test/hdrtst')
|
||||
and do 'make check'. This will attempt to compile against each header file
|
||||
individually to make sure there aren't any problems. Output looks like this, if
|
||||
no error springs up all is in order.
|
||||
|
||||
~~If a C++ file fails to compile as a C file, add it to the 'cxxfiles.txt'
|
||||
located in the hdrtst directory.~~
|
||||
|
||||
<span style="color:red">**As of ICU 65, the hdrtst is now run as part of the
|
||||
regular CI builds, and the C++ headers are now guarded with the macro
|
||||
"U_SHOW_CPLUSPLUS_API".**</span>
|
||||
|
||||
There is no longer any "cxxfiles.txt" file. Instead the public C++ headers are
|
||||
all guarded with the macro "U_SHOW_CPLUSPLUS_API" which is set to 1 by default
|
||||
if __cplusplus is defined. Users of ICU can explicitly set the macro before
|
||||
including any ICU headers if they wish to only use the C APIs. Any new public
|
||||
C++ header needs to be similarly guarded with the macro, though this should be
|
||||
caught in the CI builds for a pull-request before it is merged.
|
||||
|
||||
Run this test with all the uconfig.h variations (see below).
|
||||
|
||||
```sh
|
||||
ctest unicode/docmain.h
|
||||
ctest unicode/icudataver.h
|
||||
ctest unicode/icuplug.h
|
||||
```
|
||||
|
||||
### Test ICU4C internal headers
|
||||
|
||||
Run the following script straight from the source tree (from inside the "source"
|
||||
folder, not on the top level), no need to build nor install.
|
||||
|
||||
For a new release, also look for new tools and tests and add their folders to
|
||||
the script. You can ignore messages stating that no '\*.h' files were found in a
|
||||
particular directory.
|
||||
|
||||
The command line is simply
|
||||
|
||||
```sh
|
||||
~/git.icu/icu4c/source$ test/hdrtst/testinternalheaders.sh
|
||||
```
|
||||
|
||||
See https://unicode-org.atlassian.net/browse/ICU-12141 "every header file should
|
||||
include all other headers if it depends on definitions from them"
|
||||
|
||||
<span style="color:red">**As of ICU 68, the internal header test is now
|
||||
automated as part of Travis CI.**</span>
|
||||
|
||||
---
|
||||
|
||||
## Test uconfig.h variations
|
||||
|
||||
Test ICU completely, and run the header test (above) with:
|
||||
|
||||
1. **none** of the 'UCONFIG_NO_XXX' switches turned on (i.e., the normal case)
|
||||
2. **all** of the 'UCONFIG_NO_XXX' switches turned on (set to 1)
|
||||
3. For each switch, test once with **just** that one switch on. But note the
|
||||
exception regarding [UCONFIG_NO_CONVERSION test](healthy-code.md) below.
|
||||
|
||||
(See
|
||||
[common/unicode/uconfig.h](https://github.com/unicode-org/icu/blob/main/icu4c/source/common/unicode/uconfig.h)
|
||||
for more documentation.)
|
||||
|
||||
There is a script available which will automatically test ICU in this way on
|
||||
UNIXes, it lives in:
|
||||
[tools/release/c/uconfigtest.sh](https://github.com/unicode-org/icu/blob/main/tools/release/c/uconfigtest.sh).
|
||||
See docs at top of script for information.
|
||||
|
||||
<span style="background-color:yellow">When guard conditionals (e.g. #ifndef
|
||||
U_HIDE_INTERNAL_API) are removed because they cause header test failures, please
|
||||
note in the header file the reason that guard conditionals cannot be used in
|
||||
that location, or they will lkeiely be re-added in the future.
|
||||
|
||||
---
|
||||
|
||||
## Test C++ Namespace Use
|
||||
|
||||
Verify that ICU builds without enabling the default use of the ICU namespace. To
|
||||
test on Linux,
|
||||
|
||||
```sh
|
||||
./runConfigureICU Linux CXXFLAGS="-DU_USING_ICU_NAMESPACE=0"
|
||||
make check
|
||||
```
|
||||
|
||||
Any problems will show up as compilation errors.
|
||||
|
||||
When definitions outside the ICU C++ namespace refer to ICU C++ classes, those
|
||||
need to be qualified with "`icu::`", as in "`icu::UnicodeString`". In rare
|
||||
cases, a C++ type is also visible in C code (e.g., ucol_imp.h has definitions
|
||||
that are visible to cintltst) and then we use `U_NAMESPACE_QUALIFIER` which is
|
||||
defined to be empty when compiling for C.
|
||||
|
||||
The automated build system should have a machine that sets both
|
||||
`-DU_USING_ICU_NAMESPACE=0` and `-DU_CHARSET_IS_UTF8=1`.
|
||||
|
||||
---
|
||||
|
||||
## Test UCONFIG_NO_CONVERSION
|
||||
|
||||
Make sure that the ICU4C common and i18n libraries build with
|
||||
UCONFIG_NO_CONVERSION set to 1. We cannot do this as part of "Test uconfig.h
|
||||
variations" because the test suites cannot be built like this, but the library
|
||||
code must support it.
|
||||
|
||||
The simplest is to take an ICU4C workspace, modify uconfig.h *==temporarily==*
|
||||
by changing the value of UCONFIG_NO_CONVERSION to 1, and do "make -j 6" (not
|
||||
"make check" or "make tests"). Verify that the stubdata, common & i18n libraries
|
||||
build fine; layout should build too but toolutil will fail, that's expected.
|
||||
|
||||
Fix any stubdata/common/i18n issues, revert the UCONFIG_NO_CONVERSION value, and
|
||||
verify that it still works with the normal setting.
|
||||
|
||||
If this breaks, someone probably inadvertently uses the `UnicodeString(const char *)` constructor.
|
||||
See the "Check library dependencies" section.
|
||||
|
||||
---
|
||||
|
||||
## Test U_CHARSET_IS_UTF8
|
||||
|
||||
Verify that ICU builds with default charset hardcoded to UTF-8. To test on
|
||||
Linux,
|
||||
|
||||
```sh
|
||||
./runConfigureICU Linux CPPFLAGS="-DU_CHARSET_IS_UTF8=1"
|
||||
make -j6 check
|
||||
```
|
||||
|
||||
Any problems will show up as compilation or test errors.
|
||||
|
||||
Rather than setting the CPPFLAGS, you can also temporarily add `#define
|
||||
U_CHARSET_IS_UTF8 1` in unicode/platform.h before it gets its default
|
||||
definition, or modify the default definition there. (In ICU 4.8 and earlier,
|
||||
this flag was in unicode/utypes.h.)
|
||||
|
||||
This works best on a machine that is set to use UTF-8 as its system charset,
|
||||
which is not possible on Windows.
|
||||
|
||||
The automated build system should have a machine that sets both
|
||||
`-DU_USING_ICU_NAMESPACE=0` and `-DU_CHARSET_IS_UTF8=1`.
|
||||
|
||||
---
|
||||
|
||||
## Test U_OVERRIDE_CXX_ALLOCATION=0
|
||||
|
||||
Verify that ICU builds with U_OVERRIDE_CXX_ALLOCATION=0 on Linux. Problems will
|
||||
show as build failures.
|
||||
|
||||
```sh
|
||||
CPPFLAGS="-DU_OVERRIDE_CXX_ALLOCATION=0" ./runConfigureICU Linux
|
||||
make clean
|
||||
make -j12 check
|
||||
```
|
||||
|
||||
## ~~Test ICU_USE_THREADS=0 \[Obsolete\]~~
|
||||
|
||||
***Only necessary up to ICU4C 49.***
|
||||
|
||||
* ICU 50m1 removes ICU_USE_THREADS from the runtime code (ticket
|
||||
[ICU-9010](https://unicode-org.atlassian.net/browse/ICU-9010)).
|
||||
* It is still possible to build and test intltest with ICU_USE_THREADS=0 but
|
||||
not nearly as important.
|
||||
* In ICU 50m1, the `--disable-threads` configure option is gone. If you want
|
||||
to test with ICU_USE_THREADS=0 then temporarily change this flag in
|
||||
intltest.h or in the intltest Makefile.
|
||||
|
||||
Verify that ICU builds and tests with threading disabled. To test on Linux,
|
||||
|
||||
```sh
|
||||
./runConfigureICU Linux --disable-threads
|
||||
make check
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Test ICU4C Samples and Demos
|
||||
|
||||
To build the ICU4C samples on Windows with Visual Studio, use the following
|
||||
steps:
|
||||
|
||||
* Open the "allinone" solution file located under
|
||||
"source\\allinone\\allinone.sln".
|
||||
* Build the Debug/Release + x86/x64 configurations (all 4 configurations).
|
||||
* Make sure the generated data file (ex: "icu4c\\bin\\icudt64.dll") is not
|
||||
stub data, it should be ~26MB.
|
||||
* Open the "all" Solution file located under "source\\samples\\all\\all.sln".
|
||||
* Build both x86 and x64 using the "Batch Build" option. This is located under
|
||||
the menu Build > Batch Build.
|
||||
* Click the "Select All" button.
|
||||
* Click the "Build" button.
|
||||
* If Visual Studio returns errors using the Batch Build option, build each
|
||||
configuration individually instead.
|
||||
* The samples should all build cleanly with no errors.
|
||||
|
||||
To test the sample programs, run the "source\\samples\\all\\samplecheck.bat"
|
||||
script for each configuration, and ensure that they are successful.
|
||||
|
||||
## **Test ICU4C Demos via Docker**
|
||||
|
||||
See <https://github.com/unicode-org/icu-demos/blob/main/icu-kube/README.md>
|
||||
|
||||
---
|
||||
|
||||
## **Test ICU4J Web Demos via Docker**
|
||||
|
||||
See: <https://github.com/unicode-org/icu-demos/blob/main/icu4jweb/README.md>
|
||||
|
||||
---
|
||||
|
||||
## Test ICU4J Demos
|
||||
|
||||
These are the demo applets, see above for the icu4jweb demos.
|
||||
|
||||
To test ICU4J demo applications, cd to ICU4J directory and build and run the
|
||||
demo.
|
||||
|
||||
```sh
|
||||
$ cd icu4j
|
||||
$ ant jarDemos
|
||||
$ java -jar icu4jdemos.jar
|
||||
```
|
||||
|
||||
Above command invokes GUI demo applications. As such it has to connect to a
|
||||
X-Server. The easiest way is to run via e.g. remote desktop on the machine on
|
||||
which it is executed instead of in a ssh shell.
|
||||
|
||||
The demos include calendar, charset detection, holidays, RBNF and
|
||||
transliterator. Check if each application is working OK.
|
||||
|
||||
To check ICU4J samples, open Eclipse workspace and import icu4j-samples project
|
||||
from directory <icu4j_root>/samples. Make sure these sample code has no build
|
||||
issues. Also run sample code with main and see if each sample code runs.
|
||||
|
||||
---
|
||||
|
||||
## Test, exhaustive mode, C & J
|
||||
|
||||
For ICU4J,
|
||||
|
||||
```sh
|
||||
$ ant exhaustiveCheck
|
||||
```
|
||||
|
||||
For ICU4C, testing with an optimized build will help reduce the elapsed time
|
||||
required for the tests to complete.
|
||||
|
||||
```sh
|
||||
$ make -j6 check-exhaustive
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Test ICU4C with the Thread Sanitizer
|
||||
|
||||
The build bots run the thread sanitizer on the most interesting multithreaded
|
||||
tests. These instructions run the sanitizer on the entire test suite. The clang
|
||||
compiler is required.
|
||||
|
||||
```sh
|
||||
$ CPPFLAGS=-fsanitize=thread LDFLAGS=-fsanitize=thread ./runConfigureICU --enable-debug --disable-release Linux --disable-renaming
|
||||
$ make clean
|
||||
$ make -j6 check
|
||||
```
|
||||
@@ -1,14 +0,0 @@
|
||||
---
|
||||
layout: default
|
||||
title: Release & Milestone Tasks
|
||||
parent: Contributors
|
||||
nav_order: 10
|
||||
has_children: true
|
||||
---
|
||||
|
||||
<!--
|
||||
© 2021 and later: Unicode, Inc. and others.
|
||||
License & terms of use: http://www.unicode.org/copyright.html
|
||||
-->
|
||||
|
||||
# Release & Milestone Tasks
|
||||
@@ -1,468 +0,0 @@
|
||||
---
|
||||
layout: default
|
||||
title: Integration Tests
|
||||
parent: Release & Milestone Tasks
|
||||
grand_parent: Contributors
|
||||
nav_order: 60
|
||||
---
|
||||
|
||||
<!--
|
||||
© 2021 and later: Unicode, Inc. and others.
|
||||
License & terms of use: http://www.unicode.org/copyright.html
|
||||
-->
|
||||
|
||||
# Integration Tests
|
||||
{: .no_toc }
|
||||
|
||||
## Contents
|
||||
{: .no_toc .text-delta }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
|
||||
---
|
||||
|
||||
## Run ICU4J Locale Service Provider tests
|
||||
|
||||
JDK 6 introduced the locale service provider interface. ICU4J release include a
|
||||
special jar file (icu4j-localespi.jar) implementing the locale service provider
|
||||
interface.
|
||||
|
||||
The test case for the ICU4J locale service provider requires JRE 6 or later
|
||||
version. Otherwise, there is no extra settings except for the standard ant set
|
||||
up ( necessary. To run the test cases, use ant with the top-level build.xml with
|
||||
target "localespiCheck"
|
||||
|
||||
```sh
|
||||
$ ant localespiCheck
|
||||
```
|
||||
|
||||
You should get the output like below -
|
||||
|
||||
```
|
||||
...
|
||||
build:
|
||||
|
||||
_runLocalespiCheck:
|
||||
[java] TestAll {
|
||||
[java] BreakIteratorTest {
|
||||
[java] TestGetInstance (7.581s) Passed
|
||||
[java] TestICUEquivalent (0.832s) Passed
|
||||
[java] } (8.669s) Passed
|
||||
[java] CollatorTest {
|
||||
[java] TestGetInstance (0.900s) Passed
|
||||
[java] TestICUEquivalent (0.040s) Passed
|
||||
[java] } (0.966s) Passed
|
||||
[java] CurrencyNameTest {
|
||||
[java] TestCurrencySymbols (2.682s) Passed
|
||||
[java] } (2.683s) Passed
|
||||
[java] DateFormatSymbolsTest {
|
||||
[java] TestGetInstance (0.348s) Passed
|
||||
[java] TestICUEquivalent (0.207s) Passed
|
||||
[java] TestNynorsk (0.000s) Passed
|
||||
[java] TestSetSymbols (0.071s) Passed
|
||||
[java] } (0.631s) Passed
|
||||
[java] DateFormatTest {
|
||||
[java] TestGetInstance (0.456s) Passed
|
||||
[java] TestICUEquivalent (0.113s) Passed
|
||||
[java] TestThaiDigit (0.000s) Passed
|
||||
[java] } (0.571s) Passed
|
||||
[java] DecimalFormatSymbolsTest {
|
||||
[java] TestGetInstance (0.098s) Passed
|
||||
[java] TestICUEquivalent (0.000s) Passed
|
||||
[java] TestSetSymbols (0.000s) Passed
|
||||
[java] } (0.099s) Passed
|
||||
[java] LocaleNameTest {
|
||||
[java] TestCountryNames (2.500s) Passed
|
||||
[java] TestLanguageNames (14.262s) Passed
|
||||
[java] TestVariantNames (8.638s) Passed
|
||||
[java] } (25.402s) Passed
|
||||
[java] NumberFormatTest {
|
||||
[java] TestGetInstance (0.266s) Passed
|
||||
[java] TestICUEquivalent (0.209s) Passed
|
||||
[java] } (0.475s) Passed
|
||||
[java] TimeZoneNameTest {
|
||||
[java] TestTimeZoneNames (17.766s) Passed
|
||||
[java] } (17.766s) Passed
|
||||
[java] } (57.268s) Passed
|
||||
[java]
|
||||
[java] Test cases taking excessive time (>10s):
|
||||
[java] TestAll/LocaleNameTest/TestLanguageNames (14.262s)
|
||||
[java] TestAll/TimeZoneNameTest/TestTimeZoneNames (17.766s)
|
||||
[java]
|
||||
|
||||
BUILD SUCCESSFUL
|
||||
Total time: 1 minute 46 seconds
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Run ICU4J Test cases with JDK TimeZone
|
||||
|
||||
ICU4J provides an option to alter TimeZone implementation to use the JRE's own
|
||||
implementation. The test cases used for this is the regular ICU4J unit test
|
||||
suite, except forcing ICU4J to use JRE TimeZone by special system property -
|
||||
com.ibm.icu.util.TimeZone.DefaultTimeZoneType=JDK.
|
||||
|
||||
To run the test, you have to apply the latest time zone patch from a JRE vendor,
|
||||
because some test cases are sensitive to actual time zone transitions. For
|
||||
Oracle JRE, you should go to the J2SE download page
|
||||
<http://www.oracle.com/technetwork/java/javase/downloads/index.html> and
|
||||
download the latest JDK DST Timezone Update Tool and apply the patch to your
|
||||
local JRE.
|
||||
|
||||
To run the test case, you just need to invoke the ant target "jdktzCheck".
|
||||
|
||||
```sh
|
||||
$ ant jdktzCheck
|
||||
```
|
||||
|
||||
**Note:** You might not be able to get the update tool matching the tzdata
|
||||
version used by ICU. In this case, some test cases may reports failures.
|
||||
Unfortunately, you have to walk though the failures to see if they are expected
|
||||
or not manually in this case.
|
||||
|
||||
---
|
||||
|
||||
## Verify the Eclipse ICU4J plug-in
|
||||
|
||||
1. Make sure the Eclipse ICU4J plug-in binaries are successfully produced.
|
||||
2. Run the ICU4J plug-in test cases.
|
||||
3. Update license files, build version strings for the new release.
|
||||
|
||||
---
|
||||
|
||||
## Run Tests Without ICU Data
|
||||
|
||||
### ICU4C
|
||||
|
||||
ICU data should be removed, so that tests cannot access it. Both cintltst and
|
||||
intltest should be run with -w option and they should not crash. Every crash
|
||||
should be investigated and fixed.
|
||||
|
||||
To do this, build and test normally, then replace the ICU data shared library
|
||||
with the stubdata library and run the tests again with the -w option.
|
||||
|
||||
Using an in-source build on Linux:
|
||||
|
||||
```sh
|
||||
cd icu4c/source
|
||||
./runConfigureICU Linux
|
||||
make -j2 check
|
||||
rm lib/libicudata.so*
|
||||
cp -P stubdata/libicudata.so* lib/
|
||||
cd test/intltest
|
||||
INTLTEST_OPTS=-w make check
|
||||
cd ../cintltst
|
||||
CINTLTST_OPTS=-w make check
|
||||
```
|
||||
|
||||
For debugging (for example using gdb) you cannot use `make check`. You need
|
||||
to set the `LD_LIBRARY_PATH` for the lib folder and then run
|
||||
`./cintltst -w` or `./intltest -w` etc. in the debugger.
|
||||
Example (in-source build on Linux):
|
||||
```sh
|
||||
cd test/cintltst
|
||||
export LD_LIBRARY_PATH=../../lib:../../tools/ctestfw
|
||||
./cintltst -w
|
||||
```
|
||||
|
||||
### ICU4J
|
||||
|
||||
ICU4J has the test target for this, but does not work as designed for now. For
|
||||
now, this task is not required for ICU4J.
|
||||
|
||||
---
|
||||
|
||||
## Verify that ICU4C tests pass without collation rule strings
|
||||
|
||||
***ICU4C 53 and later***
|
||||
|
||||
Background: [ICU-10636](https://unicode-org.atlassian.net/browse/ICU-10636)
|
||||
|
||||
### Rebuild ICU data without collation rule strings
|
||||
|
||||
#### ICU 64+
|
||||
|
||||
You need to configure ICU with a data filter config file that removes collation
|
||||
rule strings.
|
||||
|
||||
The file syntax is nicer with hjson rather than json.
|
||||
|
||||
```sh
|
||||
sudo apt install python3-pip
|
||||
pip3 install hjson
|
||||
```
|
||||
|
||||
Use an `~/icu/coll-norules.hjson` config file like this:
|
||||
|
||||
```
|
||||
{
|
||||
resourceFilters: [
|
||||
{
|
||||
categories: [
|
||||
coll_tree
|
||||
]
|
||||
rules: [
|
||||
-/UCARules
|
||||
-/collations/*/Sequence
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Configure ICU using this file:
|
||||
|
||||
```sh
|
||||
ICU_DATA_FILTER_FILE=~/icu/coll-norules.hjson ... runConfigureICU ...
|
||||
```
|
||||
|
||||
Run "make clean" (or delete just the collation .res files), then test as below.
|
||||
|
||||
#### ICU 54..63
|
||||
|
||||
This should work: `make GENRBOPTS='-k --omitCollationRules'` (-k is --strict)
|
||||
|
||||
For ICU 54..63, I went into the build output folder and did:
|
||||
|
||||
```sh
|
||||
cd data
|
||||
ICUDT=icudt63l
|
||||
rm out/build/$ICUDT/coll/*.res
|
||||
make GENRBOPTS='-k --omitCollationRules'
|
||||
```
|
||||
|
||||
If this does not work, then add this option to the configure'd data/Makefile,
|
||||
see ticket [ICU-10636](https://unicode-org.atlassian.net/browse/ICU-10636).
|
||||
|
||||
### Run the tests with data-errors-as-warnings
|
||||
|
||||
`INTLTEST_OPTS=-w CINTLTST_OPTS=-w make -j5 check`
|
||||
|
||||
See that they pass, or fix them to pass. See ticket #10636 test code changes for
|
||||
examples.
|
||||
|
||||
---
|
||||
|
||||
## Test ICU4J with only little-endian ICU4C data
|
||||
|
||||
*Only available since ICU 54.*
|
||||
|
||||
#### With ICU 64 and later:
|
||||
|
||||
* Reconfigure ICU4C with
|
||||
<code><b>ICU_DATA_BUILDTOOL_OPTS=--include_uni_core_data</b>
|
||||
./runConfigureICU Linux</code> or similar
|
||||
* Should be little-endian for coverage
|
||||
* Clean and build ICU4C: `make -j6 check`
|
||||
* Make a clean directory for testing
|
||||
* Find the .data file in the build output area, e.g.,
|
||||
`icu4c/source/data/out/tmp/icudt64l.dat`
|
||||
* Create a temporary directory such as `mkdir -p /tmp/icu4j_data_test`
|
||||
* Copy the .`dat` file to the new directory.
|
||||
* Build and test ICU4J without its own data: `ant clean && ant
|
||||
-Dicu4c.data.path=/tmp/icu4j_data_test check`
|
||||
* The configuration option sets the `ICUConfig.properties` data path
|
||||
* Verify that all tests pass.
|
||||
* If you get very many test failures, double-check that you enabled
|
||||
unicore data in the ICU4C build (see first step).
|
||||
|
||||
#### With ICU 55 through ICU 63:
|
||||
|
||||
* Rebuild ICU4C with <code>make <b>INCLUDE_UNI_CORE_DATA=1</b> check</code> or
|
||||
similar, and provide a path only for the .dat file.
|
||||
* Should be little-endian for coverage
|
||||
* Find the .data file in the build output area, e.g.,
|
||||
`data/out/tmp/icudt59l.dat`
|
||||
* Create a temporary directory such as `mkdir -p
|
||||
/tmp/icu/build/data/out/tmp`
|
||||
* Copy the .`dat` file to the new directory.
|
||||
* Build and test ICU4J without its own data: `ant clean && ant
|
||||
-Dicu4c.data.path=/tmp/icu/build/data/out/tmp check`
|
||||
* The configuration option sets the `ICUConfig.properties` data path
|
||||
* Verify that all tests pass.
|
||||
|
||||
#### ICU 54 method:
|
||||
|
||||
In `icu4j-core/src/com/ibm/icu/ICUConfig.properties` set
|
||||
`com.ibm.icu.impl.ICUBinary.dataPath` to a list of paths with all of the ICU4C
|
||||
data (should be little-endian for coverage), with a path to where the .dat file
|
||||
is and a path to the source/data/in files for data that is hardcoded in ICU4C
|
||||
and therefore not in the .dat file (e.g., uprops.icu).
|
||||
|
||||
Change `com.ibm.icu.impl.ICUData.logBinaryDataFromInputStream` to `true`, maybe
|
||||
set a breakpoint where such a message is logged.
|
||||
|
||||
Run all of the ICU4J tests, maybe in the debugger for the breakpoint, or look
|
||||
for logger output to the console.
|
||||
|
||||
Revert your config changes.
|
||||
|
||||
## Build and run testmap
|
||||
|
||||
Build and run the source/test/testmap project. (There is currently no Windows
|
||||
project defined for it.)
|
||||
|
||||
```sh
|
||||
$ cd <root of your ICU build tree>
|
||||
$ CONFIG_FILES=test/testmap/Makefile CONFIG_HEADERS= ./config.status
|
||||
$ cd test/testmap
|
||||
$ make check
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Verify XLIFF conversion
|
||||
|
||||
<span style="background-color:orange">**Note:** The following instruction does not work. Please read the comments with
|
||||
orange background. There are some issues in the current ICU XLIFF tools and the
|
||||
test case below. See the comments in
|
||||
[ticket ICU-6383](https://unicode-org.atlassian.net/browse/ICU-6383).</span>
|
||||
|
||||
Instructions for verifying the XLIFF conversion tools.
|
||||
|
||||
* Convert icu/source/test/testdata/ra.txt to XLIFF
|
||||
|
||||
`genrb -s icu/source/test/testdata -d icu/source/test/testdata/ -x -l en ra.txt`
|
||||
|
||||
<span style="background-color:orange">-d icu/source/test/testdata/ overwrite
|
||||
the existing ra.xlf. Specify another directory.</span>
|
||||
|
||||
<span style="background-color:orange">ra.txt has the top level item "ra",
|
||||
which is supposed to be the content language. Thus, with -l en, you'll get a
|
||||
warning - "The top level tag in the resource and language specified are not
|
||||
the same. Please check the input." We should use "-l ra" here.</span>
|
||||
|
||||
* Verify that the ra.xlf produced is identical to the one in CVS HEAD (except
|
||||
for generation date)
|
||||
|
||||
<span style="background-color:orange">If you use "-l ra" above, you'll get
|
||||
<file .... source-language = "ra" .... />, which is different from ra.xlf in
|
||||
the repository. Also, new line codes is broken for imported contents.</span>
|
||||
|
||||
* Convert icu/source/test/testdata/ra.xlf back to ICU format
|
||||
|
||||
`java -cp icu4j/classes com.ibm.icu.dev.tool.localeconverter.XLIFF2ICUConverter -d . -t ra ra.xlf`
|
||||
|
||||
<span style="background-color:orange">The option "-t ra" does not work,
|
||||
because ra.xlf does not contain target language data. Use "-c ra"
|
||||
instead.</span>
|
||||
|
||||
* Verify that the ra.txt produced is identical to the one in CVS HEAD (except
|
||||
for generation date)
|
||||
|
||||
<span style="background-color:orange">You cannot expect the generated ra.txt
|
||||
exactly matches the original one because of table item re-ordering, new line
|
||||
code changes, and explicit resource types (e.g. "ra {" vs. "ra:table
|
||||
{").</span>
|
||||
* Go through the steps given in
|
||||
<http://icu.sourceforge.net/docs/papers/localize_with_XLIFF_and_ICU.pdf>
|
||||
|
||||
---
|
||||
|
||||
## Test sample and demo programs
|
||||
|
||||
Build and run all of the sample and demo apps that are included with ICU, on
|
||||
each of the reference platforms. A list of them is in the
|
||||
[readme](http://icu.sanjose.ibm.com/internal/checklist/icu-progs-list.html).
|
||||
Also see the build system.
|
||||
|
||||
Another worthy test: Test suites and demos *from the previous release* should
|
||||
also compile and run with the libraries of the current release, at least when
|
||||
certain #defines are set (unless they test APIs that are deprecated and have
|
||||
been removed since)!
|
||||
|
||||
---
|
||||
|
||||
## Test data portability
|
||||
|
||||
Test if the data portability (under common endianness & charset family) is ok.
|
||||
On the ICU build server, you would use the "Build from source .dat archive"
|
||||
option. When it's not available, you would do the following:
|
||||
|
||||
1. Build ICU4C on Win32.
|
||||
2. Copy the icu/source/data/out/icudt<data_version>l.dat file into
|
||||
icu/source/data/in
|
||||
3. Delete non-essential directories from icu/source/data.
|
||||
4. Package up a clean copy of ICU for a non-Windows machine, like Linux on x86.
|
||||
5. Build ICU on the non-Windows machine from the newly created package.
|
||||
6. Run all tests on that non-Windows machine.
|
||||
|
||||
---
|
||||
|
||||
## Run the environment tests
|
||||
|
||||
Run
|
||||
[environmentTest.sh](https://github.com/unicode-org/icu/blob/main/tools/release/c/environmentTest.sh)
|
||||
on a Linux machine which has many (all possible?) POSIX locales installed. This
|
||||
test verifies that the ICU test suite will work regardless of a user's default
|
||||
locale and timezone. This test should be run on a fast machine with several CPU
|
||||
cores. This will take a long time to run. Here are the steps to run the test.
|
||||
|
||||
1. cd icu4c/source/
|
||||
2. ./runConfigureICU Linux
|
||||
3. make check
|
||||
4. cd ../../tools/release/c/
|
||||
5. ./environmentTest.sh
|
||||
6. Wait a while for the tests to finish. The logs will be written in each test
|
||||
directory. e.g. icu4c/source/test/intltest/intltest-\*.txt. A message will
|
||||
be printed from each spawned test script when it finishes.
|
||||
7. grep the logs for any test failures when the tests are done.
|
||||
|
||||
---
|
||||
|
||||
## Run Thread Sanitizer tests
|
||||
|
||||
Thread sanitizer testing is one of the standard Travis builds. If it is passing
|
||||
there, nothing further is required.
|
||||
|
||||
To run manually, on a Linux system with clang,
|
||||
|
||||
```sh
|
||||
cd icu4c/source
|
||||
CPPFLAGS=-fsanitize=thread LDFLAGS=-fsanitize=thread ./runConfigureICU --enable-debug --disable-release Linux
|
||||
make clean
|
||||
make -j6 check
|
||||
```
|
||||
|
||||
Errors are displayed at the point they occur, and stop further testing.
|
||||
|
||||
---
|
||||
|
||||
## Run Address Sanitizer tests
|
||||
|
||||
Address sanitizer testing is included in the standard Linux with Clang Travis
|
||||
builds. If it is passing there, nothing further is required.
|
||||
|
||||
To run manually, on a Linux system with clang,
|
||||
|
||||
```sh
|
||||
cd icu4c/source
|
||||
CPPFLAGS=-fsanitize=address LDFLAGS=-fsanitize=address ./runConfigureICU --enable-debug --disable-release Linux
|
||||
make clean
|
||||
make -j6 check
|
||||
```
|
||||
|
||||
Memory leaks are summarized at the end. Other errors are displayed at the point
|
||||
they occur, and stop further testing.
|
||||
|
||||
---
|
||||
|
||||
## ICU4J Serialization Compatibility Test Data
|
||||
|
||||
The regular ICU4J unit test includes serialization compatibility tests. The test
|
||||
case creates an ICU service object from serialized data created by a former
|
||||
version of ICU. When we prepare a new release, the serialization compatibility
|
||||
test data should be created and checked in for future testing. This task is
|
||||
usually done just before publishing release candidate.
|
||||
|
||||
1. Run regular ICU4J unit tests - `ant check`
|
||||
2. Make sure the unit tests pass successfully.
|
||||
3. Run - `ant serialTestData`
|
||||
4. Copy a folder with ICU version (e.g. ICU_61.1) generated under <icu4j
|
||||
root>/out/serialTestData to <icu4j
|
||||
root>/main/tests/core/src/com/ibm/icu/dev/test/serializable/data.
|
||||
5. You may delete older serialization test data from the directory (but keep
|
||||
the oldest one - ICU_3.6).
|
||||
6. Run `ant check` again before committing the changes.
|
||||
@@ -1,131 +0,0 @@
|
||||
#!/usr/bin/perl -w
|
||||
#
|
||||
# Copyright (C) 2012, 2017: Unicode, Inc. and others.
|
||||
# License & terms of use: http://www.unicode.org/copyright.html
|
||||
#
|
||||
#####################################################################
|
||||
# Name : find_textfiles #
|
||||
# Full path : /home/umesh/bin/find_textfiles #
|
||||
# Type : Perl Script. Needs Perl 5. #
|
||||
# Description : Finds the text files under a directory #
|
||||
# Prunes CVS directories if -nocvs is provided #
|
||||
# Prunes RCS directories if -norcs is provided #
|
||||
# Prunes SCCS directories if -nosccs is provided #
|
||||
# Author : Umesh Nair (umesh@google.com) #
|
||||
# #
|
||||
#-------------------------------------------------------------------#
|
||||
# Maintenance Block: #
|
||||
# Date Changed by Description #
|
||||
# ---- ---------- ----------- #
|
||||
# May 2000 Umesh Nair Initial version of the script #
|
||||
# Mar 2002 Umesh Nair Added -h option #
|
||||
# May 2008 Umesh Nair Added -nosvn and -novc #
|
||||
# #
|
||||
#####################################################################
|
||||
|
||||
use strict;
|
||||
use File::Find;
|
||||
|
||||
my ($skipCvs, $skipRcs, $skipSccs, $skipSvn, $skipVc, $skipEclipse) = (0, 0, 0, 0, 0);
|
||||
my $help = 0;
|
||||
|
||||
while ((defined $ARGV[0]) and ($ARGV[0] =~ /\A\-/)) {
|
||||
my $option = shift;
|
||||
chomp $option;
|
||||
($skipCvs = 1) if ($option =~ /nocvs/);
|
||||
($skipSccs = 1) if ($option =~ /nosccs/);
|
||||
($skipSvn = 1) if ($option =~ /nosvn/);
|
||||
($skipVc = 1) if ($option =~ /novc/);
|
||||
($skipEclipse = 1) if ($option =~ /noeclipse/);
|
||||
($help = 1) if ($option =~ /h/);
|
||||
}
|
||||
|
||||
if ($help) {
|
||||
exec "pod2man $0 | nroff -man |". ($ENV{"PAGER"} || 'more -s');
|
||||
}
|
||||
|
||||
if (@ARGV == 0) {
|
||||
find (\&wanted, '.');
|
||||
} else {
|
||||
find (\&wanted, @ARGV);
|
||||
}
|
||||
|
||||
sub wanted {
|
||||
unless (($skipVc and (/\.vcproj$/ or /\.vcxproj$/ or /\.sln$/ or /\.dsp$/ or /\.dsw$/ or /\.filters$/))
|
||||
or ($skipEclipse and (/\.project$/ or /\.classpath$/ or /\.prefs$/ or /\.launch$/))) {
|
||||
print "$File::Find::name\n" if (-f and -T );
|
||||
}
|
||||
($File::Find::prune = 1)
|
||||
if (
|
||||
($skipCvs and /^CVS$/) or
|
||||
($skipRcs and /^RCS$/) or
|
||||
($skipSccs and /^SCCS$/) or
|
||||
($skipSvn and /^\.svn$/)
|
||||
);
|
||||
}
|
||||
|
||||
__END__
|
||||
|
||||
=head1 NAME
|
||||
|
||||
find_textfiles - Finds the text files under a directory
|
||||
|
||||
=head1 SYNOPSIS
|
||||
|
||||
find_textfiles [C<-nocvs>] [C<-norcs>] [C<-nosccs>] [C<-nosvn>] [C<-novc>] [C<-noeclipse>] [ dir1 [ dir2 ] .... ]
|
||||
|
||||
find_textfiles [C<-h>]
|
||||
|
||||
=head1 DESCRIPTION
|
||||
|
||||
This script recursively searches the specified directories (current
|
||||
directory if no parameter is passed) for text files, and lists the
|
||||
filenames on the standard output, one file per line.
|
||||
|
||||
The directories named 'CVS', 'RCS' and 'SCCS' may be skipped by
|
||||
specifying the switches -nocvs, -norcs and -nosccs respectively.
|
||||
|
||||
=head1 OPTIONS
|
||||
|
||||
=over 4
|
||||
|
||||
=item C<-h>
|
||||
|
||||
Displays this help document.
|
||||
|
||||
=item C<-nocvs>
|
||||
|
||||
Prunes all directories named 'CVS' and everything under that.
|
||||
|
||||
=item C<-norcs>
|
||||
|
||||
Prunes all directories named 'RCS' and everything under that.
|
||||
|
||||
=item C<-nosccs>
|
||||
|
||||
Prunes all directories named 'SCCS' and everything under that.
|
||||
|
||||
=item C<-nosvn>
|
||||
|
||||
Prunes all directories named '.svn' and everything under that.
|
||||
|
||||
=item C<-novc>
|
||||
|
||||
Excludes all VC++ files, i.e., *.vcproj, *vcxproj, *.dsw, *.dsp, *.filters.
|
||||
|
||||
=item C<-noeclipse>
|
||||
|
||||
Excludes all Eclipse files, i.e., .project, .classpath, *.prefs, *.launch.
|
||||
|
||||
=back
|
||||
|
||||
=head1 SEE ALSO
|
||||
|
||||
find_binaries
|
||||
|
||||
=head1 AUTHOR
|
||||
|
||||
Umesh Nair, 2000.
|
||||
|
||||
=cut
|
||||
|
||||
@@ -1,54 +0,0 @@
|
||||
---
|
||||
layout: default
|
||||
title: Legalities
|
||||
parent: Release & Milestone Tasks
|
||||
grand_parent: Contributors
|
||||
nav_order: 70
|
||||
---
|
||||
|
||||
<!--
|
||||
© 2021 and later: Unicode, Inc. and others.
|
||||
License & terms of use: http://www.unicode.org/copyright.html
|
||||
-->
|
||||
|
||||
# Legalities
|
||||
|
||||
## 1. Scan for Copyright notices
|
||||
|
||||
Check source and data files, especially newly contributed ones, to make sure the
|
||||
proper copyright notice is in place. For example,
|
||||
|
||||
```
|
||||
© 2016 and later: Unicode, Inc. and others.
|
||||
License & terms of use: http://www.unicode.org/copyright.html#License
|
||||
```
|
||||
|
||||
Scan the source code to make sure that every file that was touched recently has
|
||||
the current year in the copyright statement. See the [ICU Copyright
|
||||
Scanner](../../../copyright-scan.md) page and follow the link to the scripts and
|
||||
the readme.
|
||||
|
||||
Scan the source code to include third-party company names in copyright notices
|
||||
if necessary.
|
||||
|
||||
~~Scan for text files that do not contain the word "Copyright": find_textfiles
|
||||
-nosvn -novc -noeclipse | xargs grep -i -L Copyright (See the find_textfiles
|
||||
Perl script attached to this page.) There are files without word "Copyright" in
|
||||
ICU source repository including some test data files (no comment syntax defined
|
||||
for these test data files), Unicode data files, tzcode source files and others.
|
||||
Review the output file list and determine if each of them should have ICU
|
||||
copyright statement or not.~~ ***The script [find_textfiles](find_textfiles)
|
||||
associated with this document is not maintained. Use the ICU Copyright Scanner
|
||||
above instead.***
|
||||
|
||||
## 2. Update license files
|
||||
|
||||
Check ICU, Unicode and other license terms. Make sure these files are up to
|
||||
date. The [Unicode data and software license
|
||||
term](http://www.unicode.org/copyright.html) is updated annually (usually year
|
||||
number only). The easiest way to get the updated license is to do View Source on
|
||||
unicode.org/copyright.html and scroll down to the plaintext version of the
|
||||
software license ("Exhibit A").
|
||||
|
||||
See [svn changeset r39632](https://github.com/unicode-org/icu/commit/0001f6c5e92f6f3a8d66c7dbc47cc24df7633a71)
|
||||
for an example; there should be only two files to update.
|
||||
@@ -1,110 +0,0 @@
|
||||
---
|
||||
layout: default
|
||||
title: Miscellaneous
|
||||
parent: Release & Milestone Tasks
|
||||
grand_parent: Contributors
|
||||
nav_order: 80
|
||||
---
|
||||
|
||||
<!--
|
||||
© 2021 and later: Unicode, Inc. and others.
|
||||
License & terms of use: http://www.unicode.org/copyright.html
|
||||
-->
|
||||
|
||||
# Miscellaneous
|
||||
{: .no_toc }
|
||||
|
||||
## Contents
|
||||
{: .no_toc .text-delta }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
|
||||
---
|
||||
|
||||
## Complete code reviews
|
||||
|
||||
Nag all the reviewers to finish reviewing the code and change the status of
|
||||
bugs.
|
||||
|
||||
---
|
||||
|
||||
## Check in serialization compatibility test data
|
||||
|
||||
ICU4J unit test contains serialization compatibility test cases. When a new
|
||||
reference version is released, we build serialized object data with the version
|
||||
and use it for future testing.
|
||||
|
||||
1. Run "ant serialTestData" at ICU4J root directory
|
||||
2. The target generates test data and runs some serialization test cases.
|
||||
3. Once you confirm the test runs clean, copy
|
||||
`out/serialTestData/ICU_<version>` to
|
||||
`main/tests/core/src/com/ibm/icu/dev/test/serializable/data/ICU_<version>`.
|
||||
|
||||
---
|
||||
|
||||
## Release ticket
|
||||
|
||||
After every milestone (GA / RC / Milestone-N) is completed, create a new release
|
||||
ticket in ICU Jira.
|
||||
The release ticket is used for BRS tasks, such as version update, tagging new version,
|
||||
merging post RC fixes from trunk and others.
|
||||
|
||||
---
|
||||
|
||||
## Check integrity of Jira issues in commit messages
|
||||
|
||||
Every commit being shipped in the next ICU release should be labeled with a Jira
|
||||
ticket that is marked as fixed with the correct fix version. Further, there
|
||||
should be no Jira tickets marked as fixed with the current fixVersion that do
|
||||
not have commits. To check this, run the following tool:
|
||||
|
||||
<https://github.com/unicode-org/icu/tree/main/tools/commit-checker>
|
||||
|
||||
Follow the instructions in the README file to generate the report and send it
|
||||
for review.
|
||||
|
||||
---
|
||||
|
||||
## Fix Mis-ticketted commits
|
||||
|
||||
If the commit checker tool above reports any malformed commit messages, it might
|
||||
mean that a bad commit made its way onto ICU main branch. To fix this, a rebase
|
||||
is required. Since rebases can be disruptive, confirm with ICU-TC before
|
||||
performing one. If ICU-TC agrees to perform a rebase, initiate the process as
|
||||
follows:
|
||||
|
||||
$ git checkout main; git pull --ff-only upstream main
|
||||
|
||||
$ git rebase -i --preserve-merges latest
|
||||
|
||||
Note: although tempting, please do not use --committer-date-is--author-date. It
|
||||
changes the order of commits in ways we don't want to do.
|
||||
|
||||
In the interactive rebase window, choose commit messages to rewrite using the
|
||||
"reword" option. Save and exit the interactive screen, and then edit each commit
|
||||
message.
|
||||
|
||||
When ready, force-push the main branch to your fork and give ICU-TC a day or two
|
||||
to review. Before force-pushing to upstream, create a new branch on upstream
|
||||
with the latest commit on the old main branch; name it something like
|
||||
"pre63-old-main". When ready, disable branch protection on main, force-push,
|
||||
and then reapply branch protection. Create a new branch named something like
|
||||
"pre63-new-main" to allow users to easily switch between the two heads.
|
||||
|
||||
Send an email to icu-support explaining the change and how to deal with it. You
|
||||
can use [this email](https://sourceforge.net/p/icu/mailman/message/36426642/) as
|
||||
a model.
|
||||
|
||||
---
|
||||
|
||||
## Scrub closed issues in code
|
||||
|
||||
(1) Search for "TODO(12345)" to detect TODO items for closed issues.
|
||||
|
||||
(2) Do the same for logKnownIssue. (the data related logKnownIssues are often
|
||||
addressed during CLDR data integration)
|
||||
|
||||
If the TODO or logKnownIssue references a closed issue, if the problem is fixed,
|
||||
remove the TODO/logKnownIssue, or if the problem is not fixed, open a new issue
|
||||
and update the reference in the code.
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 6.8 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 6.0 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 12 KiB |
@@ -1,525 +0,0 @@
|
||||
---
|
||||
layout: default
|
||||
title: Publish
|
||||
parent: Release & Milestone Tasks
|
||||
grand_parent: Contributors
|
||||
nav_order: 90
|
||||
---
|
||||
|
||||
<!--
|
||||
© 2021 and later: Unicode, Inc. and others.
|
||||
License & terms of use: http://www.unicode.org/copyright.html
|
||||
-->
|
||||
|
||||
# Publish
|
||||
{: .no_toc }
|
||||
|
||||
## Contents
|
||||
{: .no_toc .text-delta }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
|
||||
---
|
||||
|
||||
## Create a release branch in GitHub
|
||||
|
||||
Once the branch is created, only changes necessary for the target release are
|
||||
merged in from the trunk.
|
||||
|
||||
---
|
||||
|
||||
## Upgrade LocaleExplorer and other demos/samples
|
||||
|
||||
... to the ICU project site.
|
||||
|
||||
Build the icuapps module following the README's. Update code and/or docs as
|
||||
needed. "Reference" platforms for icuapps are: RedHat Linux and win32. On Linux,
|
||||
icuapps is built against the "make install "'ed ICU. So, run ICU4C's configure
|
||||
with --prefix=/some/where pointing to where ICU4C should be installed, and also
|
||||
follow icuapps's README.
|
||||
|
||||
Install the new locale explorer and other demos/samples onto the public demo
|
||||
hosting site.
|
||||
|
||||
---
|
||||
|
||||
## ICU Collation Demo
|
||||
|
||||
Update the ICU collation demo's `index.html` with the new ICU version’s
|
||||
available collators.
|
||||
|
||||
1. Do a clean build (configure, make clean, make install, make check).
|
||||
1. Otherwise, the data build may not pick up a new locale into the
|
||||
coll/res_index.txt file.
|
||||
2. Run [icu-demos >
|
||||
webdemo/collation/build.sh](https://github.com/unicode-org/icu-demos/blob/main/webdemo/collation/build.sh)
|
||||
(after modifying it for your system).
|
||||
3. Copy-paste the output `available-collators.txt` into `index.html`.
|
||||
1. Or, easier: Use a GUI difftool (e.g., meld) to compare the two and move
|
||||
the changes into index.html.
|
||||
2. `meld webdemo/collation/index.html ../available-collators.txt`
|
||||
4. See for example the changes for
|
||||
[ICU-11355](https://unicode-org.atlassian.net/browse/ICU-11355)
|
||||
|
||||
For details see the comments at the start of the build.sh file.
|
||||
|
||||
---
|
||||
|
||||
## Repository Branch and Tags
|
||||
|
||||
⚠ Careful! The following examples contain specific version, revision and ticket
|
||||
numbers. Adjust them for the current release! Easiest: Paste into an editor,
|
||||
fix, then paste into the terminal.
|
||||
|
||||
### Creating Maintenance Branch.
|
||||
|
||||
Sanity check: Update to the latest repository revision. (Main branch if you
|
||||
copy from main, maintenance branch if you copy from there.)
|
||||
|
||||
```sh
|
||||
git checkout main
|
||||
git pull upstream main
|
||||
git log -n1
|
||||
commit bcd0... (HEAD -> main, upstream/main, ...)
|
||||
```
|
||||
|
||||
Ensure that your local branch is in sync with the upstream branch. Make sure you
|
||||
are checking the upstream remote, and not your fork!
|
||||
|
||||
Build & test ICU4C & ICU4J on your machine.
|
||||
|
||||
Create the maintenance branch from the current known good main ref.
|
||||
|
||||
```sh
|
||||
git checkout -b maint/maint-63
|
||||
git push -u upstream maint/maint-63
|
||||
```
|
||||
|
||||
#### Tagging
|
||||
|
||||
Use the GitHub GUI to create both the "release" and the "tag" at the same time:
|
||||
|
||||
<https://github.com/unicode-org/icu/releases/new>
|
||||
|
||||
Fill in the tag name, such as "release-63-rc" or "release-63-1", and make the
|
||||
target the "maint/maint-xx" branch (such as maint/maint-63).
|
||||
|
||||
Set the title to "ICU 63 RC" or "ICU 63.1". Fill in the description using the
|
||||
text from the announcement email. (You can also look at previous releases and
|
||||
possibly re-use some of the generic text, such as links to the API docs, etc.)
|
||||
|
||||
Your screen should look like this:
|
||||
|
||||
|
||||
|
||||
For the release candidate tag, please check the box that says "Pre-Release".
|
||||
|
||||
We no longer need to add the note about Git LFS files, as GitHub now includes
|
||||
them in the auto-generated .zip downloads.
|
||||
|
||||
Click the "Publish Release" button to make the tag.
|
||||
|
||||
Note: The "latest" tag is no longer updated. It was agreed by the ICU-TC to be
|
||||
deleted around the 64.2 time-frame, as it doesn't work well with with Git. (You
|
||||
need to force-push the new tag, and if somebody has already cloned the
|
||||
repository, they might have something different for the "latest" tag).
|
||||
A possible future alternative might be a sym-link folder, or HTTP redirect that
|
||||
points to the latest release.
|
||||
|
||||
#### Maintenance release
|
||||
|
||||
Create the maintenance release tag from the maintenance branch, as above.
|
||||
|
||||
Update the "latest" tag.
|
||||
|
||||
### ~~ICU 58 and earlier~~
|
||||
|
||||
~~Tag related svn files, for **icu**, **icu4j** and (for final releases)
|
||||
**tools** file trees. We tag the tools tree so that we can reproduce the Unicode
|
||||
tools that were used for the Unicode data files in this release.~~
|
||||
|
||||
~~For a Release Candidate, just tag, don't branch, and only tag icu & icu4j.~~
|
||||
|
||||
~~For the final release, branch then tag. Copy the trunk to maint/maint-4-8 and
|
||||
copy that to tags/release-4-8. Specify the source revision explicitly via -r so
|
||||
that you don't inadvertently pick up an unexpected changeset. Make sure that the
|
||||
trunk at the source revision is good.~~
|
||||
|
||||
~~We do not tag the data & icuapps trees. Steven Loomis writes on 2011-may-23:~~
|
||||
|
||||
> ~~My thought had been (in the CVS days) to take a 'snapshot' of these items.
|
||||
> However, in SVN all you need is a date or a revision number (such as
|
||||
> r30140).~~
|
||||
|
||||
> ~~So, probably, we don't need to tag these two (icuapps or data).~~
|
||||
|
||||
> ~~Tools are more important because those tools are actually used in the
|
||||
> release.~~
|
||||
|
||||
### Create ICU download page
|
||||
|
||||
Create the download page before the first milestone, if we have one, or before
|
||||
the release candidate.
|
||||
|
||||
Log into the ICU homepage Google Site. (If you don't have write access, ask one
|
||||
of the editors to grant you access.)
|
||||
|
||||
Verify that there is not already a page for the upcoming release: Either via the
|
||||
sitemap, or on the bottom of the main download page where it says something like
|
||||
"Subpages (27): View All".
|
||||
|
||||
Copy the download page for the last release: Go to that page, gear menu, Copy
|
||||
page; "Name your page" using just the major version number for now (e.g., "61")
|
||||
so that the page URL is just that number, select "Put page under Downloading
|
||||
ICU", click the red COPY button.
|
||||
|
||||
Adjust the new page as needed: Change the title to "Download ICU 61 Release
|
||||
Candidate" (with the right version number...), remove contents specific to the
|
||||
previous release, update all version numbers, update all links.
|
||||
|
||||
Put a big, bold+italics, red warning at the top like "This version has not been
|
||||
released yet. Use it for testing but not in production!"
|
||||
|
||||
Save the page. It is generally a good idea to save frequently. For further
|
||||
edits, simply go back into edit mode. (Keyboard shortcut 'e' for edit, ctrl+s
|
||||
for save.)
|
||||
|
||||
Add new contents for the upcoming release: Grab some text from the sibling
|
||||
Unicode and CLDR release notes, look at the proposal status doc for this
|
||||
release, make a pass through the api/enhancement tickets fixed in this release
|
||||
or under reviewing/reviewfeedback.
|
||||
|
||||
Look at the download pages of the last two releases for templates for things
|
||||
like a Migration Issues section etc.
|
||||
|
||||
Ask everyone on the team to add stuff & details.
|
||||
|
||||
**Once the page has been created and various people are invited to edit it,
|
||||
everyone should keep editing short so that the page is not locked for others for
|
||||
long periods of time.**
|
||||
|
||||
### Maintenance release
|
||||
|
||||
For a maintenance release, look at the ICU 60 page which includes 60.2.
|
||||
|
||||
---
|
||||
|
||||
## Milestone on the main download page
|
||||
|
||||
We had the following HTML on the main download page for ICU 4.8M1 = 4.7.1:
|
||||
|
||||
```html
|
||||
<h3 style="background-color:rgb(102, 102, 102);color:white;margin-bottom:0pt;margin-top:12pt;padding-left:0.75em;font-size:1em;font-family:Arial,Helvetica,sans-serif">Development Milestones</h3>
|
||||
<table border="0"><p style="font-size:10pt;font-family:Arial,Helvetica,sans-serif">Development milestone versions of ICU can be downloaded below. A development milestone is a stable snapshot build for next ICU major version. These binaries and source code are provided for evaluation purpose and should be not be used in production environments. New APIs or features in a milestone release might be changed or removed without notice. </p>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td style="width:105px;height:16px"> <b>Release</b></td>
|
||||
<td style="width:792px;height:16px"> <b>Major Changes<br>
|
||||
</b></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td style="width:105px;height:29px"> <a href="https://sites.google.com/site/icusite/download/471">4.8M1 (4.7.1)</a><br>
|
||||
</td>
|
||||
<td style="width:792px;height:29px"> CLDR 1.9.1+, Parent locale override, Dictionary type trie, Alphabetic index (C), Compound text encoding (C), JDK7 Locale conversion (J)<br>
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</span><br>
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Upload Release Source / Binaries
|
||||
|
||||
Download Directories are located at, for example,
|
||||
`icu-project.org:/home/htdocs/ex/files/icu4c/4.4.2`
|
||||
corresponding to <http://download.icu-project.org/ex/files/icu4c/4.4.2/>
|
||||
Look at previous releases for an example.
|
||||
|
||||
### Java Source/Bin:
|
||||
|
||||
Follow instructions here: [Building ICU4J Release Files](../release-build.md)
|
||||
|
||||
### C source/binary:
|
||||
|
||||
<span style="background:yellow">***WORK IN PROGRESS***</a>
|
||||
|
||||
#### Source and Linux Binaries:
|
||||
|
||||
Important: this step works with Unix make + docker.
|
||||
|
||||
First, install *docker* and *docker-compose. D*o not proceed until *docker run
|
||||
hello-world* works!
|
||||
|
||||
```sh
|
||||
$ git clone https://github.com/unicode-org/icu-docker.git
|
||||
$ cd icu-docker/src
|
||||
$ git clone --branch release-64-rc --depth 1 https://github.com/unicode-org/icu.git
|
||||
$ cd icu
|
||||
$ git lfs fetch
|
||||
$ git lfs checkout
|
||||
$ cd ../..
|
||||
$ less [README.md](https://github.com/unicode-org/icu-docker/blob/main/README.md) # Follow these instructions.
|
||||
```
|
||||
|
||||
* Source and binaries are created in ./dist/.
|
||||
* The names [don't match what's needed on
|
||||
output](https://github.com/unicode-org/icu-docker/issues/1) so be sure to
|
||||
rename.
|
||||
|
||||
**Note:** If you only want to make a source tarball (.tgz/.zip), then you can
|
||||
run \`make dist\`.
|
||||
|
||||
* This will produce a source tarball and will include a pre-compiled .dat file
|
||||
under icu4c/source/data/in/.
|
||||
* Note: This tarball will also omit all of the data sub-directories containing
|
||||
locale data.
|
||||
* Note that the source is taken from the git repository itself, and not your
|
||||
local checkout. (Thus it will exclude any local uncommitted changes).
|
||||
|
||||
#### Windows Binary:
|
||||
|
||||
* Manual process:
|
||||
* Build with MSVC x64 Release. (See the ICU
|
||||
[readme.html](https://github.com/unicode-org/icu/blob/main/icu4c/readme.html)
|
||||
file for details).
|
||||
* Open a command prompt.
|
||||
```
|
||||
cd C:\icu\icu4c\ (or wherever you have ICU located).
|
||||
powershell
|
||||
Set-ExecutionPolicy -Scope Process Unrestricted
|
||||
.\packaging\distrelease.ps1 -arch x64
|
||||
```
|
||||
This will produce the file "source\dist\icu-windows.zip", which will
|
||||
need to be renamed before uploading.
|
||||
* For example, the binaries for ICU4C v61.1 generated with VS2017 were
|
||||
named "icu4c-61_1-Win64-MSVC2017.zip".
|
||||
* Note: As of ICU 68, the pre-built binaries use MSVC2019 instead of
|
||||
MSVC2017.
|
||||
* Using the output from the build bots:
|
||||
* Navigate to the GitHub page for the commits on the
|
||||
`maint/maint-<version>` branch.
|
||||
* Ex: https://github.com/unicode-org/icu/commits/maint/maint-64
|
||||
* Click on the green check mark (✔) on the most recent/last commit. (It
|
||||
might be a red X if the builds failed, hopefully not).
|
||||
* This will open up a pop-up with links to various CI builds.
|
||||
* Click on one of the various links that says "Details" for the Azure CI
|
||||
builds.
|
||||
* This will open up the GitHub overview of the build status.<br>
|
||||
<br>
|
||||
* Click on the link "View more details on Azure Pipelines" link.
|
||||
* This will take you to the actual Azure CI build page.
|
||||
* On the top right of the page there should be a button titled
|
||||
"Artifacts". Click this and it should show a drop-down with various ZIP
|
||||
files that you can download.<br>
|
||||
<br>
|
||||
* The ZIP may automatically download for you.
|
||||
* However, if you are signed-in to visualstudio.com then you might see a
|
||||
dialog titled "Artifacts explorer".
|
||||
* In this case click on the name, then the "..." button to download the
|
||||
zip file.<br>
|
||||
<br>
|
||||
* Download both the x64 (64-bit) and x86 (32-bit) ZIP files.
|
||||
* For each architecture:
|
||||
* Extract the Zip file. (It will have a name like
|
||||
"20190829.6_ICU4C_MSVC_x64_Release.zip").
|
||||
* Navigate into the folder with the same name.
|
||||
* Rename the file "icu-windows.zip" to the appropriate name:
|
||||
* Ex: The x64 zip for version 64.1 was named
|
||||
"icu4c-64_1-Win64-MSVC2017.zip"
|
||||
* Ex: The x86 zip for version 64.1 was named
|
||||
"icu4c-64_1-Win32-MSVC2017.zip"
|
||||
* Note: Note: As of ICU 68, the pre-built binaries use MSVC2019
|
||||
instead of MSVC2017.
|
||||
* Note: For RC releases the name looked like this:
|
||||
"icu4c-64rc-Win64-MSVC2017.zip"
|
||||
* ~~AIX Bin:~~ (AIX is broken and ignored for now.)
|
||||
* ~~login to gcc119.fsffrance.org and copy the ICU4C source archive
|
||||
created above to there.~~
|
||||
* ~~$ gzip -dc icu4c-XXX-src.tgz | tar xf -~~
|
||||
* ~~$ cd icu~~
|
||||
* ~~$ PATH=/opt/IBM/xlC/13.1.3/bin:$PATH source/runConfigureICU AIX~~
|
||||
* ~~(The above command line doesn't actually work, see [ICU Ticket
|
||||
ICU-13639](https://unicode-org.atlassian.net/browse/ICU-13639) for a
|
||||
workaround.)~~
|
||||
* ~~$ gmake DESTDIR=/tmp/icu releaseDist~~
|
||||
* ~~That last step will create a directory in **/tmp/icu** - zip that up
|
||||
to make the release.~~
|
||||
* ~~In case /tmp happens to be full, see the [mailing list
|
||||
archive](https://sourceforge.net/p/icu/mailman/message/36275940/) for
|
||||
advice.~~
|
||||
|
||||
#### Output of icuexportdata:
|
||||
|
||||
This step publishes pre-processed Unicode property data, which may be ingested by downstream clients such as ICU4X.
|
||||
|
||||
* Using the output from the build bots:
|
||||
* Navigate to the Azure Pipelines artifacts as described in the previous section
|
||||
* Download the files named *icuexportdata_uprops_full.zip* and *icuexportdata_uprops_sample.zip*
|
||||
|
||||
### Signing archives and creating checksums:
|
||||
|
||||
#### Step 1. PGP files:
|
||||
|
||||
Sign all archives created above with your own personal PGP key. This creates a
|
||||
file with .asc as the suffix.
|
||||
|
||||
```sh
|
||||
$ gpg --armor --detach-sign icu4c-xxx-xxx.zip
|
||||
# To verify
|
||||
$ gpg --verify icu4c-xxx-xxx.zip.asc
|
||||
```
|
||||
|
||||
#### Step 2. MD5 files:
|
||||
|
||||
Use md5sum or [cfv](http://cfv.sf.net) to create [md5](https://en.wikipedia.org/wiki/MD5) hash sums for three groups of files:
|
||||
|
||||
* icu4j (all files),
|
||||
* icu4c (source),
|
||||
* icu4c (binaries).
|
||||
|
||||
Using md5sum to create and verify the checksum files:
|
||||
|
||||
<pre><code><b><b>md5sum source1 source2 ... sourceN > icu4c_sources.md5</b></b> # To verifymd5sum -c icu4c_sources.md5
|
||||
</code></pre>
|
||||
|
||||
Alternatively, use cfv to create and verify md5 files:
|
||||
|
||||
```sh
|
||||
cfv -t md5 -C -f icu-……-src.md5 somefile.zip somefile.tgz …
|
||||
# To verify
|
||||
cfv -f icu-……-src.md5
|
||||
```
|
||||
|
||||
#### Step 3. SHASUM512.txt
|
||||
|
||||
Create an additional hash sum file SHASUM512.txt file with:
|
||||
|
||||
```sh
|
||||
shasum -a 512 *.zip *.tgz | tee SHASUM512.txt
|
||||
```
|
||||
|
||||
This file should also be GPG signed. Check the .asc with \`gpg verify\`.
|
||||
|
||||
### Update the Download Page Gadgets
|
||||
|
||||
Update the gadgets on the download page to point at the new URL for the
|
||||
binaries.
|
||||
|
||||
1. Edit the download page.
|
||||
2. Click on the Gadget area.
|
||||
3. Click on the "gear" icon.
|
||||
4. Update the URL field with the new URL.
|
||||
1. For example: The ICU4C 63.1 Binaries URL was:
|
||||
<http://apps.icu-project.org/icu-jsp/downloadSection.jsp?ver=63.1&base=c&svn=release-63-1>
|
||||
|
||||
#### Check the ICU public site for the new release
|
||||
|
||||
Make sure that, aside from download pages, homepages, news items, feature lists
|
||||
and feature comparisons, etc. are updated. Upload the new API references. Update
|
||||
the User Guide.
|
||||
|
||||
#### Update the Trac release number list for ICU4C and ICU4J. <<?? STILL VALID ??>>
|
||||
|
||||
Update the ICU release number list by going to "Admin>Versions" in Trac, and add
|
||||
the new ICU version.
|
||||
|
||||
#### Post-release cleanup
|
||||
|
||||
* Cleanup the milestone in the ICU Trac. Move left over items to future
|
||||
milestones. Close the milestone.
|
||||
* Look for TODO comments in the source code and file new tickets as required.
|
||||
* Delete and retag
|
||||
[latest](http://source.icu-project.org/repos/icu/tags/latest/) (**ONLY**
|
||||
after GA release, including maintenance!) << IS THIS STILL VALID WITH GIT?
|
||||
>>
|
||||
|
||||
---
|
||||
|
||||
## Update online demos
|
||||
|
||||
Update online demos/tools to the latest version:
|
||||
|
||||
* ICU demos (eg <https://icu4c-demos.unicode.org/icu-bin/idnbrowser>)
|
||||
* CLDR Survey tool (eg <http://unicode.org/cldr/apps/about.jsp>)
|
||||
* Unicode Utilities (eg <http://unicode.org/cldr/utility/character.jsp>)
|
||||
|
||||
### Online information update
|
||||
|
||||
Collation and [comparison](../../../../charts/comparison/index.md) charts need
|
||||
to be updated. See [charts/Performance & Size](../../../../charts/index.md).
|
||||
|
||||
### Old sensitive tickets
|
||||
|
||||
Unset the "sensitive" flag on old tickets. For example, on tickets that were
|
||||
fixed two or more releases ago.
|
||||
|
||||
[Sample ticket query for ICU 65, for tickets fixed in 63 or
|
||||
earlier](https://unicode-org.atlassian.net/issues/?jql=project%20%3D%20ICU%20AND%20Level%3DSensitive%20AND%20fixVersion%20not%20in%20(65.1%2C%2064.2%2C%2064.1)%20AND%20status%3DDone).
|
||||
Adjust the fixVersion selection as appropriate. Check the list in the ICU
|
||||
meeting.
|
||||
|
||||
Check duplicates and fixedbyotherticket! Keep the "sensitive" flag on tickets
|
||||
that were closed as duplicates of other tickets that are not yet fixed or have
|
||||
been fixed only very recently.
|
||||
|
||||
For removing the flag:
|
||||
|
||||
* Enter bulk edit. Select all query results.
|
||||
* Uncheck duplicates of unfixed or too-recent tickets.
|
||||
* Edit fields:
|
||||
* Security Level = None
|
||||
* Add label "was_sensitive"
|
||||
* No notification emails
|
||||
* Confirm bulk edit.
|
||||
|
||||
---
|
||||
|
||||
## Punt tickets
|
||||
|
||||
Double-check that tickets with commits in this release are closed/fixed. Close
|
||||
as needed. (Hopefully none misticketed at this point...)
|
||||
|
||||
Then punt remaining tickets marked for this release:
|
||||
|
||||
1. In Jira, search for all tickets with Project=ICU, Fix Version=<this
|
||||
release>, Status≠Done.
|
||||
2. Go to Bulk Edit ("..." menu in the upper right corner)
|
||||
3. Select all
|
||||
4. Edit fields:
|
||||
1. Fix Version: replace all with "future"
|
||||
2. Labels: add "punt<this release>" (e.g., "punt63")
|
||||
3. No email notifications
|
||||
5. Confirm bulk edit
|
||||
6. Send a courtesy email to the team with a Jira query URL for the
|
||||
Label="punt<this release>" tickets.
|
||||
|
||||
After punting, please also update the "To-Do for Next ICU Release" widget on
|
||||
Jira.
|
||||
|
||||
1. Open <https://unicode-org.atlassian.net/issues/?filter=10007>
|
||||
2. Use the drop-down to change the fix version to the next ICU version
|
||||
3. Click "Save" next to the filter title
|
||||
|
||||
## Update readme
|
||||
|
||||
Update [ICU4C
|
||||
readme.html](https://github.com/unicode-org/icu/blob/main/icu4c/readme.html)
|
||||
and [ICU4J
|
||||
readme.html](https://github.com/unicode-org/icu/blob/main/icu4j/readme.html)
|
||||
before every milestone (GA / RC / Milestone-N). Make sure the following items
|
||||
are up to date.
|
||||
|
||||
* Release version
|
||||
* Last update date
|
||||
* Description - descriptions for GA, RC and Milesone-N are already included in
|
||||
the readme file.
|
||||
* Comment/uncomment parts as appropriate.
|
||||
* If the readme should remain the same between milestones, we can skip
|
||||
directly to the GA description. Otherwise, pick the right one for the
|
||||
release type.
|
||||
* **Since ICU 67, we have skipped from GA to GA**, without marking &
|
||||
unmarking the readme specifically for the release candidate.
|
||||
* Build steps - make sure supported compiler versions are up to date
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 74 KiB |
@@ -1,213 +0,0 @@
|
||||
---
|
||||
layout: default
|
||||
title: Release Build
|
||||
parent: Release & Milestone Tasks
|
||||
grand_parent: Contributors
|
||||
nav_order: 100
|
||||
---
|
||||
|
||||
<!--
|
||||
© 2021 and later: Unicode, Inc. and others.
|
||||
License & terms of use: http://www.unicode.org/copyright.html
|
||||
-->
|
||||
|
||||
# Release Build
|
||||
{: .no_toc }
|
||||
|
||||
## Contents
|
||||
{: .no_toc .text-delta }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
|
||||
---
|
||||
|
||||
## Building ICU4J Release Files
|
||||
|
||||
<span style="color:red">*Note: This instruction is reviewed for ICU4J 69.1. Some
|
||||
description may not apply to previous ICU releases. ICU4J library (as of ICU 69)
|
||||
supports JRE 7 or later Java runtime environment. To avoid the Java's system
|
||||
library class's backward compatibility problem, ICU4J binary files must be
|
||||
compiled with Java system libraries from JRE 7.*</span>
|
||||
|
||||
## Prerequisites
|
||||
|
||||
* Java SE Development Kit 8 (to run ant, javac, javadoc..)
|
||||
* Java Runtime Environment 7 (used as bootclasspath for cross-compilation)
|
||||
* Apache Ant 1.10.x (the latest available version)
|
||||
\[<http://ant.apache.org/>\]
|
||||
* JCite 1.13.0 \[<https://arrenbrecht.ch/jcite/>\]
|
||||
|
||||
### Other Prerequisites only for GA
|
||||
|
||||
* Maven Ant Tasks 2.1.3 (GA only) \[<http://maven.apache.org/ant-tasks>\]
|
||||
(Note: This is component is no longer maintained, and ICU project should
|
||||
update the process in near future)
|
||||
* gpg (GNU Privacy Guard)
|
||||
* ICU Project PGP key (icuintl@us.ibm.com - [published to MIT key
|
||||
server](https://pgp.mit.edu/pks/lookup?op=vindex&search=0x44CE7BF2825EA2CD))
|
||||
* Sonatype Nexus Repository Manager account for managing artifacts
|
||||
com.ibm.icu.\* on OSSRH (Reference: [OSSRH
|
||||
Guide](http://central.sonatype.org/pages/ossrh-guide.html))
|
||||
|
||||
<span style="color:red">*Note: Contact Yoshito for the PGP key and Sonatype
|
||||
account.*</span>
|
||||
|
||||
### Configuration
|
||||
|
||||
* Set environment variable `JAVA_HOME` to point to the JDK directory.
|
||||
* Set environment variable `JCITE_DIR` to specify JCite binary location (see[
|
||||
Building ICU4J API Reference Document with
|
||||
JCite](../../../setup/java/ant/index.md))
|
||||
* See `build.properties` to check version strings are properly set.
|
||||
* `api.report.version` / `api.report.prev.version` are used for locating
|
||||
API signature files and should be integer numbers.
|
||||
* `release.file.ver` is used as a part of build output file name.
|
||||
* `api.doc.version` is used for the API document title
|
||||
* `mavne.pom.ver` is used in pom.xml when release files are uploaded to
|
||||
OSSRH
|
||||
|
||||
For ICU 69.1 GA, the contents of the properties file should be:
|
||||
```
|
||||
api.report.version = 69
|
||||
api.report.prev.version = 68
|
||||
release.file.ver = 69_1
|
||||
api.doc.version = 69.1
|
||||
maven.pom.ver = 69.1
|
||||
```
|
||||
|
||||
* Create `build-local.properties` in the ICU4J root directory
|
||||
|
||||
* Edit `build-local.properties` to add a property - `java7.bootclasspath
|
||||
(java6.bootclasspath for ICU57 to ICU62)`
|
||||
|
||||
The value of javaX.bootclasspath should include JRE's system library path. A
|
||||
set of jar files included in the system library path may vary depending on
|
||||
JRE vendor (Oracle, OpenJDK and IBM use different set of jars) and version.
|
||||
The easiest way to get the system library path is to run a simple Java
|
||||
program on the target JRE.
|
||||
|
||||
System.out.println(System.getProperty("sun.boot.class.path"));
|
||||
|
||||
For example, the values on my Linux system (Ubuntu) look like below:
|
||||
|
||||
java7.bootclasspath=/home/yoshito/jdk/oracle/jdk1.7.0_80/jre/lib/resources.jar:/home/yoshito/jdk/oracle/jdk1.7.0_80/jre/lib/rt.jar:/home/yoshito/jdk/oracle/jdk1.7.0_80/jre/lib/sunrsasign.jar:/home/yoshito/jdk/oracle/jdk1.7.0_80/jre/lib/jsse.jar:/home/yoshito/jdk/oracle/jdk1.7.0_80/jre/lib/jce.jar:/home/yoshito/jdk/oracle/jdk1.7.0_80/jre/lib/charsets.jar:/home/yoshito/jdk/oracle/jdk1.7.0_80/jre/lib/jfr.jar:/home/yoshito/jdk/oracle/jdk1.7.0_80/jre/classes
|
||||
|
||||
One Windows system, path separator back slash '\\' should be replaced with
|
||||
forward slash '/'. For example:
|
||||
|
||||
java7.bootclasspath=C:/jdk/oracle/7u80/jre/lib/resources.jar;C:/jdk/oracle/7u80/jre/lib/rt.jar;C:/jdk/oracle/7u80/jre/lib/sunrsasign.jar;C:/jdk/oracle/7u80/jre/lib/jsse.jar;C:/jdk/oracle/7u80/jre/lib/jce.jar;C:/jdk/oracle/7u80/jre/lib/charsets.jar;C:/jdk/oracle/7u80/jre/lib/jfr.jar;C:/jdk/oracle/7u80/jre/classes
|
||||
|
||||
### Additional Configuration for GA
|
||||
|
||||
* Add 2 more properties to `build-local.properties`
|
||||
* `maven-ant-tasks.jar` - Specify the path to the Maven Ant Tasks jar
|
||||
file.
|
||||
* `gpg.user` - Specify the ICU Project PGP key user name
|
||||
|
||||
For example, build-local.properties look like below:
|
||||
|
||||
```
|
||||
java7.bootclasspath=/home/yoshito/jdk/oracle/jdk1.7.0_80/jre/lib/resources.jar:/home/yoshito/jdk/oracle/jdk1.7.0_80/jre/lib/rt.jar:/home/yoshito/jdk/oracle/jdk1.7.0_80/jre/lib/sunrsasign.jar:/home/yoshito/jdk/oracle/jdk1.7.0_80/jre/lib/jsse.jar:/home/yoshito/jdk/oracle/jdk1.7.0_80/jre/lib/jce.jar:/home/yoshito/jdk/oracle/jdk1.7.0_80/jre/lib/charsets.jar:/home/yoshito/jdk/oracle/jdk1.7.0_80/jre/lib/jfr.jar:/home/yoshito/jdk/oracle/jdk1.7.0_80/jre/classes
|
||||
maven-ant-tasks.jar = /home/yoshito/java-libs/maven-ant-tasks/maven-ant-tasks-2.1.3.jar
|
||||
gpg.user = ICU Project
|
||||
```
|
||||
|
||||
<span style="color:red">*Note: Maven Ant Tasks is no longer maintained. ICU team should find out another method to publish ICU artifacts to the repository (ticket [ICU-12811](https://unicode-org.atlassian.net/browse/ICU-12811)).*</span>
|
||||
|
||||
* Create maven local setting file `$HOME/.m2/settings.xml` if absent. Then add
|
||||
the Sonatype Nexus Repository Manager account information in the file. For
|
||||
example:
|
||||
```xml
|
||||
<settings>
|
||||
<servers>
|
||||
<server>
|
||||
<id>icu4j-releases</id>
|
||||
<username>yumaoka</username>
|
||||
<password>OSSRH-SonaType-Password</password>
|
||||
</server>
|
||||
<server>
|
||||
<id>icu4j-snapshots</id>
|
||||
<username>yumaoka</username>
|
||||
<password>OSSRH-SonaType-Password</password>
|
||||
</server>
|
||||
</servers>
|
||||
</settings>
|
||||
```
|
||||
|
||||
### Building Release Files for Milestones/Release Candidates
|
||||
|
||||
1. Checkout tagged version - Use clean checkout of Git (SVN up to ICU62) tag
|
||||
(e.g. release-61-0-1, release-61-rc..) for building release files.
|
||||
2. Run `$ ant releaseVer` - This build target generates ICU release files with
|
||||
version number in a folder `release` under the ICU4J root directory.
|
||||
|
||||
This ant target prints out warning messages as below if the build system is
|
||||
not properly configured.
|
||||
```
|
||||
_verify_config_for_release:
|
||||
[echo] ################################################################
|
||||
[echo] [WARNING] Insufficient Build Configuration for ICU4J Release
|
||||
[echo] JDK Tools Version: 1.8.0_152
|
||||
[echo] JRE 7 System Library Path: Not Defined!
|
||||
[echo] JCite Library Path: /home/yoshito/java-libs/jcite-1.13.0/build/jcite.jar:/home/yoshito/java-libs/jcite-1.13.0/lib/commons-discovery-0.4.jar:/home/yoshito/java-libs/jcite-1.13.0/lib/commons-logging-1.1.jar:/home/yoshito/java-libs/jcite-1.13.0/lib/java2html.jar`
|
||||
[echo] ################################################################
|
||||
|
||||
BUILD SUCCESSFUL
|
||||
Total time: 5 minutes 31 seconds
|
||||
```
|
||||
If you see such warnings at the end of build log, fix the issue and run the
|
||||
target again.
|
||||
3. Upload all files generated under the `release` directory to the ICU site.
|
||||
|
||||
## Building Release Files for Final Release (GA)
|
||||
|
||||
A final (official) release files will be posted to the Maven Central Repository
|
||||
as well as the ICU project site. The build target `releaseFinal` generates all
|
||||
necessary files for the Maven repository and upload them after regular release
|
||||
build target is executed.
|
||||
1. Checkout tagged version - Use clean checkout of Git (SVN up to ICU62) tag
|
||||
(e.g. release-61-1, release-61-2) for building release files.
|
||||
2. Run `$ ant releaseVer` once to make sure there are no build configuration
|
||||
issues. Run `$ ant clean` to clean up build.
|
||||
3. \[Optional\] In the next step, the build target invokes gpg to generate
|
||||
signature of build output files. gpg will request you to type in the
|
||||
passphrase for the ICU Project (icuintl@us.ibm.com) PGP key once. gpg
|
||||
pinentry is used for this purpose, but it may not work well on some systems
|
||||
when pgp was invoked from ant (such as Mac OS X). If this is the case, you
|
||||
would want to run gpg-agent to cache passphrase. More specifically, once you
|
||||
set up gpg-agent, run `$ gpg -ab -u "ICU Project" foo.txt` to sign arbitrary
|
||||
file. This operation will prompt you to type in the ICU Project key's
|
||||
passphrase and cache it. So it does not need to ask you to type passphrase
|
||||
again while running the ant release target.
|
||||
4. Run `$ ant releaseFinal` . This target generates release files for ICU site,
|
||||
as well as files for Maven Central Repository, then upload the files to OSSRH
|
||||
staging repository. You should see build log as below at the end.
|
||||
```
|
||||
[deploy] Transferring 37K from icu4j-releases
|
||||
[deploy] Uploaded 37K
|
||||
[deploy] Uploading: com/ibm/icu/icu4j-localespi/61.1/icu4j-localespi-61.1-sources.jar.asc to repository icu4j-releases at https://oss.sonatype.org/service/local/staging/deploy/maven2
|
||||
[deploy] Transferring 0K from icu4j-releases
|
||||
[deploy] Uploaded 0K
|
||||
[deploy] Uploading: com/ibm/icu/icu4j-localespi/61.1/icu4j-localespi-61.1-javadoc.jar to repository icu4j-releases at https://oss.sonatype.org/service/local/staging/deploy/maven2
|
||||
[deploy] Transferring 22K from icu4j-releases
|
||||
[deploy] Uploaded 22K
|
||||
[deploy] Uploading: com/ibm/icu/icu4j-localespi/61.1/icu4j-localespi-61.1-javadoc.jar.asc to repository icu4j-releases at https://oss.sonatype.org/service/local/staging/deploy/maven2
|
||||
[deploy] Transferring 0K from icu4j-releases
|
||||
[deploy] Uploaded 0K
|
||||
|
||||
BUILD SUCCESSFUL
|
||||
Total time: 6 minutes 19 seconds
|
||||
```
|
||||
After the successful target execution, copy files generated under `release`
|
||||
directory and upload them to ICU site.
|
||||
5. Log on to [OSSRH Nexus Repository Manager](https://oss.sonatype.org/). Select
|
||||
"Staging Repositories" under "Build Promotion" on the left navigation pane,
|
||||
then type "com.ibm.icu" on the search field on the upper right corner. You
|
||||
should find a repository entry corresponding to the one just uploaded. Select
|
||||
the entry, and sanity check content. Once you confirm the contents, click
|
||||
"Close" on the menu. If the repository is closed successfully, you can
|
||||
"Release" after that. The release operation will promote the staging
|
||||
repository to production. It usually takes 20 to 30 minutes.
|
||||
|
||||
|
||||
@@ -1,99 +0,0 @@
|
||||
---
|
||||
layout: default
|
||||
title: Standards Update
|
||||
parent: Release & Milestone Tasks
|
||||
grand_parent: Contributors
|
||||
nav_order: 110
|
||||
---
|
||||
|
||||
<!--
|
||||
© 2021 and later: Unicode, Inc. and others.
|
||||
License & terms of use: http://www.unicode.org/copyright.html
|
||||
-->
|
||||
|
||||
# Standards Updates
|
||||
{: .no_toc }
|
||||
|
||||
## Contents
|
||||
{: .no_toc .text-delta }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
|
||||
---
|
||||
|
||||
## Time Zones
|
||||
|
||||
Make sure the timezone (Olson) data is up to date.
|
||||
|
||||
Olson timezone data should be regularly updated in the trunk. If the data is out
|
||||
of date, follow the instruction described in the
|
||||
[readme](https://github.com/unicode-org/icu-data/blob/main/tzdata/readme.txt)
|
||||
file.
|
||||
|
||||
Once the timezone resource file is generated by the process above, the file
|
||||
should be checked into
|
||||
[zoneinfo64.txt](https://github.com/unicode-org/icu/blob/main/icu4c/source/data/misc/zoneinfo64.txt)
|
||||
in the trunk.
|
||||
|
||||
## Unicode
|
||||
|
||||
### Update Unicode Data
|
||||
|
||||
See the [Unicode Update page](../../unicode-update.md) for an overview, and the
|
||||
[change log for Unicode updates](https://github.com/unicode-org/icu/blob/main/icu4c/source/data/unidata/changes.txt)
|
||||
for details.
|
||||
|
||||
### Update script code enums
|
||||
|
||||
Starting with ICU 55, we do not add UScriptCode constants for new scripts any more
|
||||
until they are encoded in Unicode,
|
||||
or can be assumed to be encoded in the next Unicode version.
|
||||
|
||||
Script enum constant names want to follow the Unicode script property value aliases,
|
||||
which are assigned only when the scripts are encoded.
|
||||
When we encode scripts early and guess wrong, then we have confusing enum constants
|
||||
and have sometimes added aliases.
|
||||
|
||||
Normally, we add script codes during the ICU update for a Unicode beta,
|
||||
at the same time as we add new API constants for new Unicode blocks and
|
||||
for new values of other supported properties.
|
||||
|
||||
Variant script codes like Latf and Aran that are not subject to separate encoding
|
||||
can be added at any time.
|
||||
(For example, Aran could be added as USCRIPT_ARABIC_NASTALIQ.)
|
||||
|
||||
We add script codes used in CLDR or in the spoof checker.
|
||||
This includes combination/alias codes like Hanb and Jamo.
|
||||
See https://unicode.org/reports/tr35/#unicode_script_subtag_validity
|
||||
and look for "alias" on https://unicode.org/iso15924/iso15924-codes.html
|
||||
|
||||
We add special Z* script codes like Zsye.
|
||||
|
||||
For new script codes see https://www.unicode.org/iso15924/codechanges.html
|
||||
Check for recent changes with the code highlighted in the left column and
|
||||
"Add" in the right column.
|
||||
|
||||
Update script code enums with new codes from ISO 15924, send a notice for them
|
||||
as public API (born `@stable`) to the icu-design list.
|
||||
|
||||
See details in the
|
||||
[change log for Unicode updates](https://github.com/unicode-org/icu/blob/main/icu4c/source/data/unidata/changes.txt).
|
||||
|
||||
## CLDR
|
||||
|
||||
### Sync ICU with CLDR and tag CLDR.
|
||||
|
||||
Update ICU data from CLDR, tag CLDR. Instructions
|
||||
[here](https://github.com/unicode-org/icu/blob/main/icu4c/source/data/cldr-icu-readme.txt).
|
||||
|
||||
### Update MeasureUnit Code
|
||||
|
||||
<https://github.com/unicode-org/icu/blob/main/docs/processes/release/tasks/updating-measure-unit.md>
|
||||
|
||||
## ISO 4217 numeric codes
|
||||
|
||||
Make sure the ISO 4217 numeric code mapping data is up to date.
|
||||
|
||||
See the instruction in
|
||||
<https://github.com/unicode-org/icu/blob/main/tools/currency/readme.txt>
|
||||
@@ -1,150 +0,0 @@
|
||||
---
|
||||
layout: default
|
||||
title: Updating MeasureUnit with new CLDR data
|
||||
parent: Release & Milestone Tasks
|
||||
grand_parent: Contributors
|
||||
nav_order: 120
|
||||
---
|
||||
|
||||
<!--
|
||||
© 2020 and later: Unicode, Inc. and others.
|
||||
License & terms of use: http://www.unicode.org/copyright.html
|
||||
-->
|
||||
|
||||
# Updating MeasureUnit with new CLDR data
|
||||
{: .no_toc }
|
||||
|
||||
## Contents
|
||||
{: .no_toc .text-delta }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
|
||||
---
|
||||
|
||||
This document explains how to update the C++ and Java version of the MeasureUnit
|
||||
class with new CLDR data.
|
||||
|
||||
Code is generated by running MeasureUnitTest.java unit tests, which writes
|
||||
generated code to System.out. Two ways to access this:
|
||||
|
||||
1. Within **eclipse**:
|
||||
- Open MeasureUnitTest.java, run it by clicking on the green play button on
|
||||
menu bar.
|
||||
- Copy the generated code from the eclipse console to the clipboard.
|
||||
|
||||
2. With **ant**:
|
||||
- Run: `ant checkTest
|
||||
-Dtestclass='com.ibm.icu.dev.test.format.MeasureUnitTest'`
|
||||
- Open the checkTest output: `out/junit-results/checkTest/html/index.html`
|
||||
- Navigate to "System.out" at the bottom of the MeasureUnitTest page to find
|
||||
the generated code, and copy to the clipboard.
|
||||
|
||||
After syncing CLDR data with ICU do the following. This documentation assumes
|
||||
that you are updating the MeasureUnit clases for ICU 68.
|
||||
|
||||
* Check out
|
||||
$GIT_ROOT/icu4j/main/tests/core/src/com/ibm/icu/dev/test/format/MeasureUnitTest.java
|
||||
* Open MeasureUnitTest.java.
|
||||
* Find the `testZZZ` test, its code should all be commented out. This test will
|
||||
execute last and will run the desired code.
|
||||
|
||||
Make sure DRAFT_VERSIONS at top of MeasureUnitTest.java is set correctly.
|
||||
These are the ICU versions that have draft methods.
|
||||
|
||||
## Update MeasureUnit.java
|
||||
|
||||
* Change `testZZZ` to run `generateConstants(“68”); // ICU 68.`
|
||||
* Run MeasureUnitTest.java, copy the generated code (see instructions above).
|
||||
* Open MeasureUnit.java:
|
||||
$GIT_ROOT/icu4j/main/classes/core/src/com/ibm/icu/util/MeasureUnit.java
|
||||
* Look for line containing:
|
||||
|
||||
`// Start generated MeasureUnit constants`
|
||||
* Look for line containing:
|
||||
|
||||
`// End generated MeasureUnit constants`
|
||||
* Replace all the generated code in between with the contents of the clipboard
|
||||
* Run the MeasureUnitTest.java to ensure that the new code is backward
|
||||
compatible. These compatibility tests are called something like
|
||||
`TestCompatible65`, which tests backward compatibility with ICU 65.
|
||||
* Create a compatibility test for ICU 68. Change `testZZZ` to run
|
||||
`generateBackwardCompatibilityTest(“68”)`
|
||||
* Run tests.
|
||||
* Copy generated test (see instructions above) into MeasureUnitTest.java
|
||||
* Run tests again to ensure that new code is backward compatible with itself
|
||||
|
||||
## Update ICU4C
|
||||
|
||||
* checkout ICU4C
|
||||
|
||||
### Update measunit.h
|
||||
|
||||
* Change testZZZ to run `generateCXXHConstants(“68”); // ICU 68`.
|
||||
* Run MeasureUnitTest.java, copy the generated code (see instructions above).
|
||||
* Open $GIT_ROOT/icu4c/source/i18n/unicode/measunit.h. Look for line containing:
|
||||
|
||||
`// Start generated createXXX methods`
|
||||
* Look for line:
|
||||
|
||||
`// End generated createXXX methods`
|
||||
* Replace all the generated code in between with the contents of the clipboard
|
||||
|
||||
### Update measunit.cpp
|
||||
|
||||
* Change testZZZ to run generateCXXConstants();
|
||||
* Run MeasureUnitTest.java, copy the generated code (see instructions above).
|
||||
* Open $GIT_ROOT/icu4c/source/i18n/measunit.cpp. Look for line containing:
|
||||
|
||||
`// Start generated code for measunit.cpp`
|
||||
* Look for lines
|
||||
|
||||
`// End generated code for measunit.cpp`
|
||||
* Replace all the generated code in between with the contents of the clipboard
|
||||
|
||||
### Run C++ tests
|
||||
|
||||
* Run `./intltest format/MeasureFormatTest` from `test/intltest` to ensure new
|
||||
code is backward compatible.
|
||||
* Create a compatibility test for ICU 68. Change `testZZZ` in eclipse to run
|
||||
`generateCXXBackwardCompatibilityTest(“68”)`
|
||||
* Run tests.
|
||||
* Copy generated test (see instructions above) into
|
||||
$GIT_ROOT/icu4c/source/test/intltest/measfmttest.cpp. Make other necessary
|
||||
changes to make test compile. You can find these changes by searching for
|
||||
`TestCompatible65()`
|
||||
* Run tests again to ensure that new code is backward compatible with itself
|
||||
|
||||
## Finishing changes
|
||||
|
||||
These last changes are necessary to permanently record the ICU version number of
|
||||
any new measure units. Without these changes any new functions for this release
|
||||
will be considered new for the next release too.
|
||||
|
||||
* Change `testZZZ` to run `updateJAVAVersions(“68”);`
|
||||
* Run MeasureUnitTest.java, copy the generated code (see instructions above).
|
||||
* Append the clipboard contents to the values of the JAVA_VERSIONS variable
|
||||
near the top of MeasureUnitTest.java.
|
||||
|
||||
**Important:** what you are copying are just the new functions for the current
|
||||
ICU version, in this case 68. Therefore append, do not replace.
|
||||
|
||||
## Updating units.txt and unitConstants
|
||||
|
||||
The standard ldml2icu process is used to update ICU's resource files (see
|
||||
[cldr-icu-readme.txt](https://github.com/unicode-org/icu/blob/main/icu4c/source/data/cldr-icu-readme.txt)).
|
||||
CLDR's units.xml defines conversion rates in terms of some constants defined in
|
||||
`unitConstants`.
|
||||
|
||||
For efficiency and simplicity, ICU does not read `unitConstants` from the
|
||||
resource file. If any new constants are added, some code changes would be
|
||||
needed. This would be caught by `testUnitConstantFreshness` unit test in
|
||||
`units_test.cpp`.
|
||||
|
||||
They are hard-coded:
|
||||
* Java: `UnitConverter.java` has the constant names in
|
||||
`UnitConverter.Factor.addEntity()` and constant values in
|
||||
`UnitConverter.Factor.getConversionRate()`.
|
||||
* C++: `units_converter.cpp` has the constant names in
|
||||
`addSingleFactorConstant()`, with the constant values in `double
|
||||
constantsValues[]` in the `units_converter.h` header file.
|
||||
@@ -1,324 +0,0 @@
|
||||
---
|
||||
layout: default
|
||||
title: Version Numbers
|
||||
parent: Release & Milestone Tasks
|
||||
grand_parent: Contributors
|
||||
nav_order: 140
|
||||
---
|
||||
|
||||
<!--
|
||||
© 2021 and later: Unicode, Inc. and others.
|
||||
License & terms of use: http://www.unicode.org/copyright.html
|
||||
-->
|
||||
|
||||
# Version Numbers
|
||||
{: .no_toc }
|
||||
|
||||
## Contents
|
||||
{: .no_toc .text-delta }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
|
||||
---
|
||||
|
||||
## ICU Version Number
|
||||
|
||||
For reference, see the section in the User Guide about Version numbers here:
|
||||
|
||||
<https://unicode-org.github.io/icu/userguide/icu/design#version-numbers-in-icu>
|
||||
|
||||
### ICU Version Number for Front load, RC and GA tasks
|
||||
|
||||
The process of releasing a new ICU version (E.G. ICU 70.1) is divided in three phases:
|
||||
* Front loading tasks
|
||||
* Release Candidate (RC) tasks
|
||||
* General Availability (GA) tasks
|
||||
|
||||
As of ICU 70, the ICU Version number changes for each of these tasks.
|
||||
|
||||
When "front loading" tasks, the version number will consist of a Major number, Minor number, and a Patch number.
|
||||
For example: ICU version 70.0.1
|
||||
|
||||
For the RC and GA tasks, the ICU version number will consist of a Major number and a Minor number.
|
||||
For example: ICU version 70.1
|
||||
|
||||
This means that when updating from the front load tasks to the RC tasks, files such as
|
||||
[icu4c/source/common/unicode/uvernum.h](https://github.com/unicode-org/icu/blob/main/icu4c/source/common/unicode/uvernum.h)
|
||||
need to be correspondingly updated. See below for more files to be updated and steps to be followed.
|
||||
|
||||
|
||||
### ICU Data
|
||||
|
||||
[icu4c/source/data/misc/icuver.txt](https://github.com/unicode-org/icu/blob/main/icu4c/source/data/misc/icuver.txt)
|
||||
needs to be updated with the correct version number for ICU and its data.
|
||||
|
||||
#### Since ICU 68
|
||||
|
||||
In
|
||||
[tools/cldr/cldr-to-icu/build-icu-data.xml](https://github.com/unicode-org/icu/blob/main/tools/cldr/cldr-to-icu/build-icu-data.xml)
|
||||
edit `<property name="icuVersion" value="67.1.0.0"/>` and `<property
|
||||
name="icuDataVersion" value="67.1.0.0"/>`.
|
||||
|
||||
The CLDR-to-ICU converter will then generate icuver.txt with the new version
|
||||
numbers.
|
||||
|
||||
Run the converter, or coordinate with the person who will run it.
|
||||
|
||||
#### Until ICU 67 (inclusive)
|
||||
|
||||
Edit icuver.txt directly.
|
||||
|
||||
### ICU4C
|
||||
|
||||
1. The instructions for updating the version number are in
|
||||
[icu4c/source/common/unicode/uvernum.h](https://github.com/unicode-org/icu/blob/main/icu4c/source/common/unicode/uvernum.h).
|
||||
2. [icu4c/source/stubdata/stubdata.vcxproj](https://github.com/unicode-org/icu/blob/main/icu4c/source/stubdata/stubdata.vcxproj)
|
||||
and
|
||||
[icu4c/source/data/makedata.mak](https://github.com/unicode-org/icu/blob/main/icu4c/source/data/makedata.mak)
|
||||
may also need to updated with the correct version for icudt.
|
||||
3. After uvernum.h is updated, the Windows project files should be updated by
|
||||
running the UNIX makefile target '**update-windows-makefiles**' in
|
||||
icu/source.
|
||||
* You will need to rerun "./configure" first though before you can run the
|
||||
command "make update-windows-makefiles".
|
||||
* Note: You can use MSYS+MinGW to run the UNIX makefile on Windows
|
||||
platforms as well.
|
||||
4. As well, the ICU4C "configure" script should be updated so that it reflects
|
||||
the latest version number.
|
||||
* Note: When updating the version number in the configure files be careful
|
||||
not to just blindly search-and-replace the version number.
|
||||
For example, there are lines like this: "As of ICU 62, both C and C++
|
||||
files require them" which need to have the exact version number
|
||||
retained.
|
||||
|
||||
### ICU4J
|
||||
|
||||
Since ICU4J 4.6, you can quickly check the current version information by
|
||||
running jar main in icu4j.jar. For example,
|
||||
|
||||
```sh
|
||||
$ java -jar icu4j.jar
|
||||
```
|
||||
|
||||
prints out -
|
||||
|
||||
<pre>
|
||||
International Component for Unicode for Java 4.8
|
||||
Implementation Version: 4.8
|
||||
Unicode Data Version: 6.0
|
||||
CLDR Data Version: 2.0
|
||||
Time Zone Data Version: 2011g
|
||||
</pre>
|
||||
|
||||
For updating ICU version numbers, follow the steps below.
|
||||
|
||||
1. [icu4j/main/shared/build/common.properties](https://github.com/unicode-org/icu/blob/main/icu4j/main/shared/build/common.properties)
|
||||
|
||||
* icu4j.spec.version: This is API spec version, therefore, 2-digit major
|
||||
version only. The version number won't be changed for maintenance
|
||||
releases. (e.g. "55")
|
||||
* icu4j.impl.version: This version is used for the actual release version,
|
||||
including maintenance release numbers. (e.g. "55.1"). Note: We do not
|
||||
use .0 - For example, ICU4J 54.1 uses "54.1", instead of "54.1.0" For
|
||||
milestone 1, use <major>.0.1 (e.g. 60.0.1).
|
||||
* icu4j.data.version: This version number is corresponding to the data
|
||||
structure, therefore, won't be changed in maintenance releases. Use
|
||||
2-digit number used by the data path. (e.g. "55")
|
||||
* \[*Not applicable for ICU4J 60+*\] current.year: Some build script embed
|
||||
the year taken from this into a copyright template. Make sure this value
|
||||
is updated to the current year for a new release (also applicable to a
|
||||
maintenance release).
|
||||
|
||||
2. icu4j/build.properties (For API change report and release target)
|
||||
|
||||
* api.report.version: 2 digit release number. Note: If necessary, we may
|
||||
include maintenance release number. (e.g "54", "481")
|
||||
* api.report.prev.version: The previous version compared against the current
|
||||
version. (e.g. "46")
|
||||
* release.file.ver: This string is used for files names. For milestone
|
||||
releases, use <2-digit major version> + "m" + <milestone number>, e.g.
|
||||
"55m1". For release candidate, use <2-digit major version> + "rc", e.g.
|
||||
54rc. For official releases, use full version numbers using under bar as the
|
||||
separator, e.g. "54_1", "54_1_1".
|
||||
* api.doc.version: The version displayed in API reference doc - use full
|
||||
version number such as "60.1" for official and RC releases, "60 Milestone 1" for
|
||||
milestone 1.
|
||||
* maven.pom.ver: The version used in ICU pom.xml files. Use full version
|
||||
number such as "60.1" for official releases, "61.1-SNAPSHOT" until 61.1
|
||||
release, after 60.1.
|
||||
|
||||
3. [icu4j/main/classes/core/src/com/ibm/icu/util/VersionInfo.java](https://github.com/unicode-org/icu/blob/main/icu4j/main/classes/core/src/com/ibm/icu/util/VersionInfo.java)
|
||||
|
||||
There is a static block starting at line 501 (as of 54.1) in the source file -
|
||||
|
||||
<pre>
|
||||
/**
|
||||
* Initialize versions only after MAP_ has been created
|
||||
*/
|
||||
static {
|
||||
UNICODE_1_0 = getInstance(1, 0, 0, 0);
|
||||
UNICODE_1_0_1 = getInstance(1, 0, 1, 0);
|
||||
UNICODE_1_1_0 = getInstance(1, 1, 0, 0);
|
||||
UNICODE_1_1_5 = getInstance(1, 1, 5, 0);
|
||||
UNICODE_2_0 = getInstance(2, 0, 0, 0);
|
||||
UNICODE_2_1_2 = getInstance(2, 1, 2, 0);
|
||||
UNICODE_2_1_5 = getInstance(2, 1, 5, 0);
|
||||
UNICODE_2_1_8 = getInstance(2, 1, 8, 0);
|
||||
UNICODE_2_1_9 = getInstance(2, 1, 9, 0);
|
||||
UNICODE_3_0 = getInstance(3, 0, 0, 0);
|
||||
UNICODE_3_0_1 = getInstance(3, 0, 1, 0);
|
||||
UNICODE_3_1_0 = getInstance(3, 1, 0, 0);
|
||||
UNICODE_3_1_1 = getInstance(3, 1, 1, 0);
|
||||
UNICODE_3_2 = getInstance(3, 2, 0, 0);
|
||||
UNICODE_4_0 = getInstance(4, 0, 0, 0);
|
||||
UNICODE_4_0_1 = getInstance(4, 0, 1, 0);
|
||||
UNICODE_4_1 = getInstance(4, 1, 0, 0);
|
||||
UNICODE_5_0 = getInstance(5, 0, 0, 0);
|
||||
UNICODE_5_1 = getInstance(5, 1, 0, 0);
|
||||
UNICODE_5_2 = getInstance(5, 2, 0, 0);
|
||||
UNICODE_6_0 = getInstance(6, 0, 0, 0);
|
||||
UNICODE_6_1 = getInstance(6, 1, 0, 0);
|
||||
UNICODE_6_2 = getInstance(6, 2, 0, 0);
|
||||
UNICODE_6_3 = getInstance(6, 3, 0, 0);
|
||||
UNICODE_7_0 = getInstance(7, 0, 0, 0);
|
||||
<b>
|
||||
ICU_VERSION = getInstance(54, 1, 0, 0);
|
||||
ICU_DATA_VERSION = ICU_VERSION;</b>
|
||||
UNICODE_VERSION = UNICODE_7_0;
|
||||
|
||||
UCOL_RUNTIME_VERSION = getInstance(8);
|
||||
UCOL_BUILDER_VERSION = getInstance(9);
|
||||
UCOL_TAILORINGS_VERSION = getInstance(1);
|
||||
}
|
||||
</pre>
|
||||
|
||||
In the same file, starting at line 164 (as of 54.1) -
|
||||
|
||||
<pre>
|
||||
/**
|
||||
* Data version string for ICU's internal data.
|
||||
* Used for appending to data path (e.g. icudt43b)
|
||||
* @internal
|
||||
* @deprecated This API is ICU internal only.
|
||||
*/
|
||||
@Deprecated
|
||||
public static final String <b>ICU_DATA_VERSION_PATH = "54b";</b>
|
||||
</pre>
|
||||
|
||||
4. [icu4j/pom.xml](http://source.icu-project.org/repos/icu/trunk/icu4j/pom.xml)
|
||||
(before ICU4J 60 only)
|
||||
|
||||
<pre>
|
||||
<groupId>com.ibm.icu</groupId>
|
||||
<artifactId>icu4j</artifactId>
|
||||
<version><b>55-SNAPSHOT</b></version>
|
||||
<name>ICU4J</name>
|
||||
</pre>
|
||||
|
||||
Only for the final release (including maintenance release), update the <version>
|
||||
item to the actual release version (e.g. "54.1", "4.8.1") Otherwise, use (next
|
||||
ver)-SNAPSHOT. (e.g. "55-SNAPSHOT").
|
||||
|
||||
5. Time Bombs (before ICU4J 52)
|
||||
|
||||
***Note: We no longer use time bombs since ICU4J 52. In the trunk,
|
||||
logKnownIssue() is used for skipping known test failures. The new scheme no
|
||||
longer depends on the current ICU4J version, so there are no updates
|
||||
necessary in test codes when version number is updated. See [Skipping Known
|
||||
Test Failures](../../../setup/eclipse/time.md) for more details.***
|
||||
|
||||
There might be some test cases intentionally skipped for the current ICU4J
|
||||
version. When ICU4J version is updated, these time bombed test cases may
|
||||
show up. In this case, you should:
|
||||
|
||||
* Inform the list of failing test cases because of the version change to
|
||||
icu-core ML - ask if someone has plan to fix them.
|
||||
* File a ticket if a time bomb does not have any corresponding Jira ticket
|
||||
and put the ticket number as comment
|
||||
* Move the time bomb version to the next milestone.
|
||||
|
||||
The time bomb looks like below -
|
||||
|
||||
<pre>
|
||||
if (<b>isICUVersionBefore(49, 1)</b>) { // ICU-6806
|
||||
logln(showOrderComparison(uLocale, style1, style2, order1, order2));
|
||||
} else {
|
||||
errln(showOrderComparison(uLocale, style1, style2, order1, order2));
|
||||
}
|
||||
</pre>
|
||||
|
||||
Note: ICU4J time bomb - Before
|
||||
[ICU-7973](https://unicode-org.atlassian.net/browse/ICU-7973), we used to
|
||||
use skipIfBeforeICU(int,int,int).
|
||||
|
||||
When a test case with time bomb still fails before a major release, the time
|
||||
bomb may be moved to the version before the first milestone of the next
|
||||
major release stream. For example, the time bomb (49,1) is not yet resolved
|
||||
before ICU4J 49 release, it should be updated to (50,0,1). This will prevent
|
||||
the error test case showing up during 49 maintenance releases, and appear
|
||||
before the first milestone of 50.0.1.
|
||||
|
||||
6. ICU4J Eclipse plug-in version (Eclipse release only)
|
||||
|
||||
ICU4J Eclipse plug-in use the standard Eclipse versioning scheme -
|
||||
X.Y.Z.v<build date>, for example, com.ibm.icu_4.2.1.v20100412.jar. By
|
||||
default, the build script compose the version string from
|
||||
icu4j.plugin.impl.version.string property in
|
||||
[eclipse-build/build.properties](https://github.com/unicode-org/icu/blob/main/icu4j/eclipse-build/build.properties)
|
||||
with current date at the build time. However, when we tag a version, we want
|
||||
to freeze the build date part. To force a fixed version string, we add a
|
||||
property - icu4j.eclipse.build.version.string in the build.properties. For
|
||||
example, see
|
||||
[tags/release-4-4-2-eclipse37-20110208/eclipse-build/build.properties](http://source.icu-project.org/repos/icu/tags/icu4j/release-4-4-2-eclipse37-20110208/eclipse-build/build.properties).
|
||||
|
||||
7. [DebugUtilitiesData.java](https://github.com/unicode-org/icu/blob/main/icu4j/main/tests/core/src/com/ibm/icu/dev/test/util/DebugUtilities.java)
|
||||
|
||||
This file is automatically generated when data is generated for ICU4J. The
|
||||
ICU4C version number string should be check to ensure that the correct
|
||||
version number is being used. public static final String
|
||||
ICU4C_VERSION="50.0.2"; Note: The ICU4C version number string in this JAVA
|
||||
file is not really used for anything except for a few logln method calls.
|
||||
Perhaps this member is not really needed.
|
||||
|
||||
## Data Versions
|
||||
|
||||
Make sure data file versions (for **data contents**) are properly assigned.
|
||||
|
||||
If any of the data files in the
|
||||
[/icu/source/data/](https://github.com/unicode-org/icu/tree/main/icu4c/source/data)
|
||||
directory has changed **MANUALLY**, upgrade the version number accordingly as
|
||||
well. If the contents of a resource bundle has changed, then increase the
|
||||
version number (at least the minor version field). The CLDR generated data
|
||||
should have the correct number. **Note** from Markus (20090514, ICU 4.2
|
||||
timeframe): Most data files automatically get their version numbers set by the
|
||||
LDML2ICUConverter, from CLDR version numbers. It is not clear what, if anything,
|
||||
needs to be done for this task.
|
||||
|
||||
The ICU4J data files depend on the ICU4C data files. Thus, in order to
|
||||
regenerate the updated ICU4J data files, you will first need to update the ICU4C
|
||||
data files version.
|
||||
|
||||
Once the ICU4C data files are updated, follow the instructions in the Readme
|
||||
file here:
|
||||
<https://github.com/unicode-org/icu/blob/main/icu4c/source/data/icu4j-readme.txt>
|
||||
|
||||
## Data File *Format* Versions
|
||||
|
||||
Make sure data file **format** versions are updated. See
|
||||
<https://unicode-org.github.io/icu/userguide/icu_data/#icu-data-file-formats>
|
||||
|
||||
For Unicode data files, it is also useful to look at recent tools-tree changes:
|
||||
[icu/commits/main/tools/unicode](https://github.com/unicode-org/icu/commits/main/tools/unicode)
|
||||
|
||||
If the format of a binary data file has changed, upgrade the format version in
|
||||
the UDataInfo header for that file. Better: Change the format version
|
||||
immediately when the format is changed. The change must be made in both the
|
||||
producer/generator and consumer/runtime code.
|
||||
|
||||
It is desirable to maintain backward compatibility, but sometimes impractical.
|
||||
Update the major version number for incompatible changes. Runtime code should
|
||||
permit higher minor version numbers for supported major versions.
|
||||
|
||||
We rarely use the third and fourth version number fields, except for UTrie (only
|
||||
version 1, not UTrie2) parameters that don't really change.
|
||||
@@ -1,286 +0,0 @@
|
||||
<!--
|
||||
Copyright (C) 2016 and later: Unicode, Inc. and others.
|
||||
License & terms of use: http://www.unicode.org/copyright.html
|
||||
-->
|
||||
|
||||
Updating ICU's built-in Break Iterator rules
|
||||
============================================
|
||||
|
||||
Here are instructions for updating ICU's built-in break iterator rules, for Grapheme, Word, Line and Sentence breaks.
|
||||
|
||||
The ICU rules implement the boundary behavior from Unicode [UAX-14](https://www.unicode.org/reports/tr14/) and [UAX-29](https://www.unicode.org/reports/tr29/), with tailorings from CLDR and some ICU-specific enhancements. ICU rules updates are needed in response to changes from Unicode or CLDR, or for bug fixes. Often ideas for CLDR or UAX updates are prototyped in ICU first, before becoming official.
|
||||
|
||||
This is not a cook book process. Familiarity with ICU break iterator behavior and rules is needed. Sets of break rules often interact in subtle and difficult to understand ways. Expect some bumps.
|
||||
|
||||
### Have clear specifications for the change.
|
||||
|
||||
The changes will typically come from a proposed update to Unicode UAX 29 or UAX 14,
|
||||
or from CLDR based tailorings to these specifications.
|
||||
|
||||
As an example, see [CLDR proposal for Extended Indic Grapheme Clusters](https://github.com/unicode-org/cldr/tree/main/common/properties/segments).
|
||||
|
||||
Often ICU will implement draft versions of proposed specification updates, to check that they are complete and consistent, and to identify any issues before they are released.
|
||||
|
||||
### Files that typically will need to be updated:
|
||||
|
||||
|
||||
| File | Contents |
|
||||
|-------------------------------------|--------------------------
|
||||
| icu/icu4c/source/...
|
||||
| .../test/testdata/rbbitst.txt | Data driven test file. Typically the first to be updated.
|
||||
| .../data/brkitr/rules/*.txt | Main break rule files.
|
||||
| .../test/intltest/rbbitst.cpp | Monkey Test Rules (as code; also called the 'original' or 'old' monkey test).
|
||||
| .../test/testdata/break_rules/*.txt | Monkey Test Rules (as data; also called the 'new' or 'rule-based' monkey test).
|
||||
| .../test/testdata/*BreakTest.txt | Unicode Supplied Test Files.
|
||||
|||
|
||||
| icu/icu4j/...
|
||||
| .../main/shared/data/icudata.jar | Data jar, includes break rules. Derived from ICU4C.
|
||||
| .../main/tests/core/src/com/ibm/icu/dev/test/rbbi/rbbitst.txt | Test data, copied from ICU4C.
|
||||
| .../main/tests/core/src/com/ibm/icu/dev/test/rbbi/break_rules/* | Monkey test rules, copied from ICU4C.
|
||||
| .../main/tests/core/src/com/ibm/icu/dev/test/rbbi/RBBITestMonkey.java | Monkey test w rules as code. Port from ICU4C.
|
||||
|
||||
|
||||
### ICU4C
|
||||
|
||||
The rule updates are done first for ICU4C, and then ported (code changes) or moved (data changes) to ICU4J. This order is easiest because the the break rule source files are part of the ICU4C project, as is the rule builder.
|
||||
|
||||
1. **Add basic tests**` to icu4c/source/test/testdata/rbbitst.txt`
|
||||
|
||||
This file contains data driven tests, which are basically text strings marked up with their expected break positions. The test syntax is documented in the file itself.
|
||||
|
||||
Add tests to to spot check the basics of the changes, to verify that some simple, straight forward cases work as expected. There is no need to thoroughly check corner cases; the goal at this step is a quick sanity check that will fail before the rule update and pass afterwards.
|
||||
|
||||
The [Unicode Utilities](http://www.unicode.org/cldr/utility/) can be very helpful at this point, for showing what characters
|
||||
match a UnicodeSet expression, and for listing the properties of a particular character.
|
||||
|
||||
Tests added for the above example:
|
||||
|
||||
#
|
||||
# ICU-13637 and CLDR-10994 - Indic Grapheme Cluster Boundary changes to support aksaras
|
||||
# New rule: LinkingConsonant ExtCccZwj* Virama ExtCccZwj* × LinkingConsonant
|
||||
# Sample Chars: LinkingConsonant: \u0915
|
||||
# Virama: \u094d [also Extend]
|
||||
# ExtCccZWJ: \u0308
|
||||
# Extend but not ExtCCCZWJ \u093A
|
||||
<char>
|
||||
<data>•\u0915\u094d\u0915•</data>
|
||||
<data>•\u0915\u0308\u0308\u094d\u0308\u0308\u0915•</data>
|
||||
<data>•\u0915\u0308\u0308\u094d\u0308\u0308•\u0041•</data>
|
||||
<data>•\u0915\u0308\u0308\u094d\u093A\u093A•\u0915•</data>
|
||||
|
||||
Two copies of the test file exist in the ICU repository, one for C++ and one for Java. There are two because
|
||||
there is no common place that will always be present and that the two build systems can both access.
|
||||
|
||||
The two should be identical. Verify this before starting to make changes. If they differ, it
|
||||
probably means that some earlier change to ICU4C was not fully ported to ICU4J, and this
|
||||
needs to be resolved before proceeding.
|
||||
|
||||
diff icu4c/source/test/testdata/rbbitst.txt icu4j/main/tests/core/src/com/ibm/icu/dev/test/rbbi/rbbitst.txt
|
||||
|
||||
Should show no differrence.
|
||||
|
||||
|
||||
2. **Run the ICU4C break iterator tests**, and verify that the newly added tests fail as expected.
|
||||
(We haven't updated the rules yet)
|
||||
|
||||
cd icu/icu4c/source
|
||||
make -j6 check
|
||||
|
||||
To run just the RBBI Tests (you will be doing this a lot)
|
||||
|
||||
cd test/intltest
|
||||
LD_LIBRARY_PATH=../../lib:../../stubdata:../../tools/ctestfw:$LD_LIBRARY_PATH ./intltest rbbi
|
||||
|
||||
A snippet from the test output, showing one of many expected failures:
|
||||
|
||||
=== Handling test: rbbi: ===
|
||||
rbbi {
|
||||
TestExtended {
|
||||
code alpha extend alphanum type word sent line name
|
||||
------------------------------------------------ 0
|
||||
915 1 0 1 Lo LE LE AL DEVANAGARI LETTER KA
|
||||
94d 0 1 0 Mn Extend EX CM DEVANAGARI SIGN VIRAMA
|
||||
------------------------------------------------ 2
|
||||
915 1 0 1 Lo LE LE AL DEVANAGARI LETTER KA
|
||||
Forward Iteration, break found, but not expected. Pos= 2 File line,col= 175, 21
|
||||
Reverse Itertion, break found, but not expected. Pos= 2 File line,col= 175, 21
|
||||
|
||||
3. **Update the main rule file for the break iterator type in question.**
|
||||
|
||||
For this example, the rule file is `icu4c/source/data/brkitr/rules/char.txt`.
|
||||
(If the change is for word or line break, which have multiple rule files for tailorings, only update the root file at this time.)
|
||||
|
||||
Start by looking at how existing similar rules are being handled, and also refer to the ICU user guide section on [Break Rules](../userguide/boundaryanalysis/break-rules.md) for an explanation of rule syntax and behavior.
|
||||
|
||||
The transformation from UAX or CLDR style rules to ICU rules can be non-trivial. Sources of difficulties include:
|
||||
|
||||
- All ICU rules run in parallel, while UAX/CLDR rules are applied sequentially, stopping after the first match. The ICU rules sometimes require extra logic to prevent a later rule from preempting an earlier rule. This can be quite tricky to express.
|
||||
|
||||
- ICU rules match a run of text that does not have boundaries in its interior (unless the rule contains a "hard break", represented by a '/'. UAX and CLDR rules, on the other hand, tell whether a single text position is or is not a break, with the rule expressing pre and post context around that position. This transformation is generally not hard, and the ICU form of the rules is often simpler.
|
||||
|
||||
|
||||
4. **Rebuild the ICU data with the updated rules.**
|
||||
|
||||
cd icu4c/source/data
|
||||
make
|
||||
|
||||
5. **Rerun the data-driven test**, `rbbi/TestExtended`. With luck, it may pass. Failures fall into two classes:
|
||||
|
||||
- The newly added test failed. Either something is wrong with the test cases, or something is wrong with the rule updates.
|
||||
|
||||
- A previously existing test started failing. Examine the test case; it probably conflicts with the new rules. Either change the expected boundaries, or change the test string to something that generates the previous boundaries. Or remove the test. Whatever seems most sensible.
|
||||
|
||||
Fix any failures before proceeding. Don't try to run other tests yet; a large number of failures are very likely.
|
||||
|
||||
6. **Run any relevant Unicode test data.** ICU's copies of the test files are here:
|
||||
|
||||
icu4c/source/test/testdata/GraphemeBreakTest.txt
|
||||
icu4c/source/test/testdata/LineBreakTest.txt
|
||||
icu4c/source/test/testdata/SentenceBreakTest.txt
|
||||
icu4c/source/test/testdata/WordBreakTest.txt
|
||||
|
||||
If the update includes new versions of any of these files, copy them to the above locations.
|
||||
|
||||
To run the test:
|
||||
|
||||
cd icu4c/source/test/intltest
|
||||
LD_LIBRARY_PATH=../../lib:../../stubdata:../../tools/ctestfw:$LD_LIBRARY_PATH ./intltest rbbi/RBBITest/TestUnicodeFiles
|
||||
|
||||
The test files are from the Unicode Consortium. The official, released versions are at https://www.unicode.org/Public/UCD/latest/ucd/auxiliary/. The files are copied, unmodified, into the ICU source tree to make them accessible to the ICU tests.
|
||||
|
||||
If the update is for a new Unicode version, or for a new CLDR tailoring of the root Unicode rules, it should include updated test data files. If they're missing, ask whoever is requesting or providing the updated rules for help. The test data is generated by CLDR tooling.
|
||||
|
||||
Copy any new Unicode test data files to their location in icu, and rerun the test.
|
||||
|
||||
Historically, failures have roughly equal chance of being problems with the test data or problems with the ICU rules. In either event, track down and fix any problems before proceeding.
|
||||
|
||||
*Note:* Known issues with the test data file are accounted for in the test code, in the function `RBBITest::testCaseIsKnownIssue()` in the file `rbbitst.cpp`. Test cases are skipped when ICU behavior has been patched or enhanced for some reason, relative to standard Unicode behavior.
|
||||
|
||||
7. **Other Break Iterator Tests, except for Monkey Tests**
|
||||
|
||||
cd icu4c/source/test/intltest
|
||||
LD_LIBRARY_PATH=../../lib:../../stubdata:../../tools/ctestfw:$LD_LIBRARY_PATH ./intltest rbbi
|
||||
|
||||
This runs all of the RBBI tests, including the Monkey tests. For this step, ignore Monkey failures, and track down and fix any others.
|
||||
|
||||
There is a real mish-mash of old tests, checking random bits of hard coded data.
|
||||
|
||||
8. **Monkey Tests**
|
||||
|
||||
Monkey testing compares the breaking behavior of the main ICU RBBI implementation with that of a reference implementation, using random data.
|
||||
|
||||
Monkey testing has proved to be by far the most effective way to check obscure edge and corner case behavior, to the point that it no longer seems worth while to hand-write more than fairly basic test cases.
|
||||
|
||||
ICU has two independent RBBI monkey tests. The original one implements the break rules directly in code. The algorithm sticks pretty close to that of the Unicode UAX specifications. The original monkey test checks only the root break iterator behavior, not any tailorings.
|
||||
|
||||
The newer, data-driven monkey test takes its reference rules from test data files, instead of hard coding them. It covers tailorings. Its algorithm differs somewhat from that of the UAX specifications, in that the rules match runs of text rather than testing pre or post context around a potential break. (This was intended as a first step in driving a revised algorithm back to the specifications, a task that hasn't yet happened.)
|
||||
|
||||
The original plan was to retire the old monkey test in favor of the newer data-driven one, but each tends to uncover problems that the other misses, so they both remain.
|
||||
|
||||
9. **Original Monkey Test**
|
||||
|
||||
To run the test:
|
||||
|
||||
LD_LIBRARY_PATH=../../lib:../../stubdata:../../tools/ctestfw:$LD_LIBRARY_PATH ./intltest rbbi/RBBITest/TestMonkey@"type=char loop=-1"
|
||||
|
||||
Test parameters (following the '@'
|
||||
seed=nnnnn Random number starting seed.
|
||||
Setting the seed allows errors to be reproduced.
|
||||
loop=nnn Looping count. Controls running time.
|
||||
-1: run forever.
|
||||
0 or greater: run length.
|
||||
|
||||
type = char | word | line | sent | title
|
||||
|
||||
Updating the test with new or revised rules requires changing the test source code, in `icu4c/source/test/intltest/rbbitst.cpp`. Look for the classes RBBICharMonkey, RBBIWordMonkey, RBBISentMonkey and RBBILineMonkey. The body of each class tracks the corresponding UAX-14 or UAX-29 specifications in defining the character classes and break rules.
|
||||
|
||||
After making changes, as a final check, let the test run for an extended period of time, on the order of several hours.
|
||||
Run it from a terminal, and just interrupt it (Ctrl-C) when it's gone long enough.
|
||||
|
||||
10. **New Monkey Test**
|
||||
|
||||
To run the test:
|
||||
|
||||
intltest rbbi/RBBIMonkeyTest/testMonkey@rules=grapheme.txt,loop=-1
|
||||
|
||||
The @rules parameter is the test rules file to run; test rules files are located in the directory `icu4c/source/test/testdata/break_rules`
|
||||
|
||||
The test should initially fail, because ICU's library rules have been updated (steps 3 and 4), but the reference rules used
|
||||
by this test have not yet been.
|
||||
|
||||
Make the updates to the test rules and re-run. The rule syntax is described in
|
||||
[icu4c/source/test/testdata/break_rules/README.md](https://github.com/unicode-org/icu/blob/main/icu4c/source/test/testdata/break_rules/README.md)
|
||||
The test reference rules are in this same directory.
|
||||
|
||||
Again, after everything appears to be working, let the test run for an extended length of time. Long runs are especially important with the more complex break rule sets, such as line break.
|
||||
|
||||
11. **Tailorings**
|
||||
|
||||
If this is an update to word or line break root behavior, the rule changes must be propagated from from the root rule files to the tailored files, for both the main rules (source/data/brkitr/rules/*) and the monkey test rules (source/test/testdata/break_rules/*).
|
||||
|
||||
The easiest and safest way to do this is to create a patch file of the diffs to the root rule file, typically using `git diff`.
|
||||
Apply it to the various tailorings of the break type being updated.
|
||||
|
||||
Merge conflicts when applying the patch would indicate that the same rules modified by the new change were also modified in the tailoring. When this happens, you just have to dig in and understand the intent of the rules and the tailoring were, and figure out what makes sense. Fortunately, conflicts are not common.
|
||||
|
||||
As with the main rules, after everything appears to be working, run the rule based monkey test for an extended period of time (with loop=-1).
|
||||
|
||||
### ICU4J
|
||||
|
||||
1. **Copy the Data Driven Test File to ICU4J**
|
||||
|
||||
Copy the file `rbbitst.txt` from ICU4C to ICU4J, and run the Java test. It should fail until the rules are updated.
|
||||
|
||||
cd <top level icu directory>
|
||||
cp icu4c/source/test/testdata/rbbitst.txt icu4j/main/tests/core/src/com/ibm/icu/dev/test/rbbi/rbbitst.txt
|
||||
|
||||
Run the test from Eclipse.
|
||||
|
||||
Navigate to `/icu4j-core-tests/src/com/ibm/icu/dev/test/rbbi/RBBITestExtended.java`.
|
||||
|
||||
Select the `TestExtended()` function in the source code, right-click it and choose "Run As Junit Test".
|
||||
|
||||
Errors (expected because the break rules have not yet been updated) should show the failing line in `rbbitst.txt`. For example:
|
||||
|
||||
java.lang.AssertionError: Forward Iteration, break found, but not expected. Pos=2 File line,col= 175, 21
|
||||
|
||||
2. **Refresh ICU4J data from ICU4C**.
|
||||
|
||||
This will bring over the updated break rules, refreshing the file `main/shared/data/icudata.jar`. Follow the instructions from `icu4c/source/data/icu4j-readme.txt`.
|
||||
|
||||
Rerun the ICU4J tests. `TestExtended` should now pass. Others may start failing.
|
||||
|
||||
3. **Port the code-based Monkey Test changes from ICU4C**
|
||||
|
||||
ICU4C file to port from: `source/test/intltest/rbbitst.cpp`
|
||||
|
||||
ICU4J file to port to: `main/tests/core/src/com/ibm/icu/dev/test/rbbi/RBBITestMonkey.java`
|
||||
|
||||
To conveniently run the individual tests, look for the test functions `TestCharMonkey()`, `TestWordMonkey()`, etc. in `RBBITestMonkey.java`.
|
||||
|
||||
Test parameters are passed via the Eclipse Run Configuration settings, arguments tab, VM parameters. For example,
|
||||
|
||||
-ea -Dseed=554654 -Dloop=1
|
||||
|
||||
When the test appears to be working, run for an extended time (with -Dloop=-1).
|
||||
|
||||
4. **New (rule driven) Monkey Test**
|
||||
|
||||
Copy the updated monkey test rules from ICU4C and run the test.**
|
||||
|
||||
ICU4C directory, to copy from: `source/test/testdata/break_rules/`
|
||||
|
||||
ICU4J directory, to copy to: `main/tests/core/src/com/ibm/icu/dev/test/rbbi/break_rules/`
|
||||
|
||||
Then rerun the rule based monkey test, in the file `main/tests/core/src/com/ibm/icu/dev/test/rbbi/RBBIMonkeyTest.java`. Find the test function `TestMonkey()`; it include comments describing how to run it with parameters from Eclipse.
|
||||
|
||||
Run the test(s) for the changed rules for an extended amount of time (with Dloop=-1).
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
@@ -1,157 +0,0 @@
|
||||
---
|
||||
layout: default
|
||||
title: Unicode Update
|
||||
parent: Release & Milestone Tasks
|
||||
grand_parent: Contributors
|
||||
nav_order: 130
|
||||
---
|
||||
|
||||
<!--
|
||||
© 2021 and later: Unicode, Inc. and others.
|
||||
License & terms of use: http://www.unicode.org/copyright.html
|
||||
-->
|
||||
|
||||
# Unicode Update
|
||||
{: .no_toc }
|
||||
|
||||
## Contents
|
||||
{: .no_toc .text-delta }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
|
||||
---
|
||||
|
||||
The International Components for Unicode (ICU) implement the Unicode Standard
|
||||
and many of its Standard Annexes and Technical Standards,
|
||||
and are updated to each new Unicode version.
|
||||
Usually, the ICU team participates in the Unicode beta process by
|
||||
updating to a beta snapshot of the new Unicode version and testing it thoroughly.
|
||||
In the past, this has sometimes uncovered problems that could be
|
||||
fixed before the release of the new Unicode version.
|
||||
|
||||
(Note that ICU does not provide any access to Unihan data,
|
||||
mostly because of low demand and the large size of the Unihan data.)
|
||||
|
||||
## Update process
|
||||
|
||||
For the last several updates, there is a
|
||||
[change log for Unicode updates](https://github.com/unicode-org/icu/blob/main/icu4c/source/data/unidata/changes.txt).
|
||||
|
||||
For each new Unicode version, during the beta period,
|
||||
* Copy the change log for the previous version to the top of this file.
|
||||
* Adjust the versions, tickets, URLs, and paths.
|
||||
* Work through the steps listed in the log, top to bottom, adjusting the log as necessary.
|
||||
* Report problems to the UTC and/or CLDR and/or ICU.
|
||||
|
||||
Before the data is final, “turn the crank” several more times,
|
||||
using appropriate subsets of the steps.
|
||||
|
||||
At the start of the process, most of the Unicode data files are copied into the ICU repository, either
|
||||
without modification or, for some files, with comments removed and lines merged
|
||||
to reduce their size.
|
||||
|
||||
Some of the data files are not part of the Unicode release but are output from
|
||||
various Unicode Tools, as noted in the change log.
|
||||
(See also https://github.com/unicode-org/unicodetools)
|
||||
|
||||
Note: We have looked at using the [UCD XML](https://www.unicode.org/ucd/#UCDinXML) files,
|
||||
but decided against it and instead developed a simpler format for a combined Unicode data file.
|
||||
See https://icu.unicode.org/design/props/ppucd#TOC-Why-not-UCD-XML-files-
|
||||
(There was an outdated, experimental, partial UCD XML parser here:
|
||||
<https://github.com/unicode-org/icu-docs/tree/main/design/properties/genudata>)
|
||||
|
||||
The ICU Unicode tools parse the text files, process the data somewhat, and write
|
||||
binary data for runtime use. Most of these tools live in a
|
||||
[source tree](https://github.com/unicode-org/icu/tree/main/tools/unicode) separate
|
||||
from the ICU4C/ICU4J sources, and link with ICU4C.
|
||||
|
||||
The following steps are necessarily manual:
|
||||
|
||||
* New property values and properties need to be reviewed.
|
||||
* For new property values, enum constants are added to the API.
|
||||
* For new properties, APIs are added and the tools are modified to write the
|
||||
additional data into new fields in the data structures; sometimes new data
|
||||
structures need to be developed for new properties.
|
||||
* Some properties are not exposed via simple, direct data access APIs but
|
||||
in more high-level APIs (like case mapping and normalization functions).
|
||||
* Sometimes changes in which property aliases are canonical vs. actual aliases
|
||||
require manual changes to helper files or tools.
|
||||
|
||||
New properties (whether they are supported via dedicated API or not) should be added to the
|
||||
[Properties User Guide chapter](https://unicode-org.github.io/icu/userguide/strings/properties).
|
||||
|
||||
### Bazel build process
|
||||
|
||||
The tools for building ICU data for Unicode properties are in a separate subtree of the ICU repo.
|
||||
They depend on parts of the ICU libraries and generate files that go back into the source tree
|
||||
in order to make updated properties available to higher-level parts of the library and tools.
|
||||
|
||||
In the past, we boot-strapped this by doing a `make install` on ICU with the old data,
|
||||
using cmake to build the tools, running some of the tools with their output
|
||||
going back into the source tree, rebuilding ICU and the tools, running more tools, etc.
|
||||
|
||||
This was very manual and cumbersome.
|
||||
|
||||
Instead, starting with ICU 70 (2021),
|
||||
we now use the [Bazel build system](https://bazel.build/) to build only small parts of the libraries,
|
||||
just enough to build and run the initial tools.
|
||||
We still need a layer outside of Bazel in order to copy the tool output into the source tree,
|
||||
because Bazel on its own does not allow modifying the source tree.
|
||||
We use a shell script to automate alternately building tools and copying files.
|
||||
|
||||
This simplifies the process.
|
||||
|
||||
It should also make it much easier to customize Unicode properties,
|
||||
for example by patching ppucd.txt with real properties for PUA (private use) characters.
|
||||
|
||||
Finally, it should make it easier to modify the binary data file format for a property
|
||||
because we build the library code that depends on the data only after generating that data.
|
||||
|
||||
For the initial setup of this Bazel build system for ICU see
|
||||
https://unicode-org.atlassian.net/browse/ICU-21117 “sane build system for Unicode data”
|
||||
|
||||
This was completed while working on
|
||||
https://unicode-org.atlassian.net/browse/ICU-21635 “Unicode 14”
|
||||
|
||||
#### Bazel setup
|
||||
|
||||
It should be possible to run the `bazel` command directly,
|
||||
but the Bazel team recommends using the `bazelisk` wrapper.
|
||||
It downloads and runs the latest version of Bazel, or,
|
||||
if the root folder contains a .bazelisk file with an entry like
|
||||
```
|
||||
USE_BAZEL_VERSION=3.7.1
|
||||
```
|
||||
then it downloads that specific version. If there are any incompatible changes in Bazel behavior,
|
||||
then this insulates us from those.
|
||||
|
||||
We do have an $ICU_SRC/.bazeliskrc file with such a line.
|
||||
Consider running `bazelisk --version` outside of the $ICU_SRC folder
|
||||
to find out the latest `bazel` version, and copying that version number into the config file.
|
||||
(Revert if you find incompatibilities, or, better, update our build & config files.)
|
||||
|
||||
Right in $ICU_SRC we also have a file called WORKSPACE which tells Bazel that
|
||||
our repo root is also the root of its build system.
|
||||
We build library “targets” relative to that. For example,
|
||||
`//icu4c/source/common:normalizer2` refers to the cc_library named `normalizer2` in
|
||||
$ICU_SRC/icu4c/source/common/BUILD .
|
||||
|
||||
## Testing
|
||||
|
||||
The ICU test suites include some tests for Unicode data. Some just check the
|
||||
data from the API against the original .txt files. Some tests simply check for
|
||||
certain hardcoded values, which have to be updated when those values change
|
||||
deliberately. Other tests perform consistency checks between some properties, or
|
||||
between different implementations.
|
||||
|
||||
There is a program as a part of CLDR that uses regular expressions to test the
|
||||
segmentation rules and properties (LineBreak, WordBreak, etc). That is, there is
|
||||
a regular expression corresponding to each of the rules, and a brute force
|
||||
evaluation of them. That is used to generate the tables and test data. The
|
||||
segmentation rules in ICU are later modified by hand to match the
|
||||
specifications. That has to be done by hand, because there are some areas where
|
||||
the rules don't correspond 1:1 with the spec. There are a series of ICU
|
||||
consistency tests for those rules. ICU also includes regression tests with
|
||||
"golden files" that are used to detect unanticipated side effects of revisions
|
||||
to the rules.
|
||||
File diff suppressed because one or more lines are too long
|
Before Width: | Height: | Size: 53 KiB |
@@ -1,473 +0,0 @@
|
||||
---
|
||||
layout: default
|
||||
title: Break Rules
|
||||
nav_order: 1
|
||||
parent: Boundary Analysis
|
||||
---
|
||||
<!--
|
||||
© 2020 and later: Unicode, Inc. and others.
|
||||
License & terms of use: http://www.unicode.org/copyright.html
|
||||
-->
|
||||
|
||||
# Break Rules
|
||||
{: .no_toc }
|
||||
|
||||
## Contents
|
||||
{: .no_toc .text-delta }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
|
||||
---
|
||||
|
||||
## Introduction
|
||||
|
||||
ICU locates boundary positions within text by means of rules, which are a form
|
||||
of regular expressions. The form of the rules is similar, but not identical,
|
||||
to the boundary rules from the Unicode specifications
|
||||
[[UAX-14](https://www.unicode.org/reports/tr14/),
|
||||
[UAX-29](https://www.unicode.org/reports/tr29/)], and there is a reasonably close
|
||||
correspondence between the two.
|
||||
|
||||
Taken as a set, the ICU rules describe how to move forward to the next boundary,
|
||||
starting from a known boundary.
|
||||
ICU includes rules for the standard boundary types (word, line, etc.).
|
||||
Applications may also create customized break iterators from their own rules.
|
||||
|
||||
ICU's built-in rules are located at
|
||||
[icu/icu4c/source/data/brkitr/rules/](https://github.com/unicode-org/icu/tree/main/icu4c/source/data/brkitr/rules).
|
||||
These can serve as examples when writing your own, and as starting point for
|
||||
customizations.
|
||||
|
||||
### Rule Tutorial
|
||||
|
||||
Rules most commonly describe a range of text that should remain together,
|
||||
unbroken. For example, this rule
|
||||
|
||||
```
|
||||
[\p{Letter}]+;
|
||||
```
|
||||
|
||||
matches a run of one or more letters, and would cause them to remain unbroken.
|
||||
|
||||
The part within `[`brackets`]` follows normal ICU [UnicodeSet pattern syntax](../strings/unicodeset.md).
|
||||
|
||||
The qualifier, '`+`' in this case, can be one of
|
||||
|
||||
| Qualifier | Meaning |
|
||||
| --------- | ------------------------ |
|
||||
| empty | Match exactly once |
|
||||
| `?` | Match zero or one time |
|
||||
| `+` | Match one or more times |
|
||||
| `*` | Match zero or more times |
|
||||
|
||||
#### Variables
|
||||
|
||||
A variable names a set or rule sub-expression. They are useful for documenting
|
||||
what something represents, and for simplifying complex expressions by breaking
|
||||
them up.
|
||||
|
||||
"Variable" is something if a misnomer; they cannot be reassigned, but are more
|
||||
of a constant expression.
|
||||
|
||||
They start with a '`$`', both in the definition and use.
|
||||
|
||||
```
|
||||
# Variable Definition
|
||||
$ASCIILetNum = [A-Za-z0-9];
|
||||
# Variable Use
|
||||
$ASCIILetNum+;
|
||||
```
|
||||
|
||||
#### Comments and Semicolons
|
||||
|
||||
'`#`' begins a comment, which extends to the end of a line.
|
||||
|
||||
Comments may stand alone, or appear after another statement on a line.
|
||||
|
||||
All rule statements or expressions are terminated by semicolons.
|
||||
|
||||
#### Chained Matching
|
||||
|
||||
Most ICU rule sets use the concept of "chained matching". The idea is that
|
||||
complete match can be composed from multiple pieces, with each piece coming from
|
||||
an individual rule of a rule set.
|
||||
|
||||
This idea is unique to ICU break rules, it is not a concept found in other
|
||||
regular expression based matchers. Some of the Unicode standard break rules
|
||||
would be difficult to implement without it.
|
||||
|
||||
Starting with an example,
|
||||
|
||||
```
|
||||
!!chain;
|
||||
word_char = [\p{Letter}];
|
||||
word_joiner = [_-];
|
||||
$word_char+;
|
||||
$word_char $word_joiner $word_char;
|
||||
```
|
||||
|
||||
These rules will match "`abc`", "`hello_world`", `"hi-there"`,
|
||||
"`a-bunch_of-joiners-here`".
|
||||
|
||||
They will not match "`-abc`", "`multiple__joiners`", "`tail-`"
|
||||
|
||||
A full match is composed of pieces or submatches, possibly from different rules,
|
||||
with adjacent submatches linked by at least one overlapping character.
|
||||
|
||||
In the example below, matching "`hello_world`",
|
||||
|
||||
* '`1`' shows matches of the first rule, `word_char+`
|
||||
|
||||
* '`2`' shows matches of the second rule, `$word_char $word_joiner $word_char`
|
||||
|
||||
```
|
||||
hello_world
|
||||
11111 11111
|
||||
222
|
||||
```
|
||||
|
||||
There is an overlap of the matched regions, which causes the chaining mechanism
|
||||
to join them into a single overall match.
|
||||
|
||||
The mechanism is a good match to, for example, [Unicode's word break
|
||||
rules](http://www.unicode.org/reports/tr29/#Word_Boundary_Rules), where rules
|
||||
WB5 through WB13 combine to piece together longer words from multiple short
|
||||
segments.
|
||||
|
||||
`!!chain;` enables chaining in a rule set. It is disabled by default for back
|
||||
compatibility—very old versions of ICU did not support it, and it was
|
||||
originally introduced as an option.
|
||||
|
||||
#### Parentheses and Alternation
|
||||
|
||||
Rule expressions can contain parentheses and '`|`' operators, representing
|
||||
alternation or "or" operations. This follows conventional regular expression
|
||||
behavior.
|
||||
|
||||
For example, the following would match a simplified identifier:
|
||||
|
||||
```
|
||||
$Letter ($Letter | $Digit)*;
|
||||
```
|
||||
|
||||
#### String and Character Literals
|
||||
|
||||
Similarly to common regular expressions, literal characters that do not have
|
||||
other special meaning represent themselves. So the rule
|
||||
|
||||
```
|
||||
Hello;
|
||||
```
|
||||
|
||||
would match the literal input "`Hello`".
|
||||
|
||||
In practice, nearly all break rules are composed from `[`sets`]` based on Unicode
|
||||
character properties; literal characters in rules are very rare.
|
||||
|
||||
To prevent random typos in rules from being treated as literals, use this
|
||||
option:
|
||||
|
||||
```
|
||||
!!quoted_literals_only;
|
||||
```
|
||||
|
||||
With the option, the naked `Hello` becomes a rule syntax error while a quoted
|
||||
`"hello"` still matches a literal hello.
|
||||
|
||||
`!!quoted_literals_only` is strongly recommended for all rule sets. The random
|
||||
typo problem is very real, and surprisingly hard to recognize and debug.
|
||||
|
||||
#### Explicit Break Rules
|
||||
|
||||
A rule containing a slash (`/`) will force a boundary when it matches, even when
|
||||
other rules or chaining would otherwise lead to a longer match. Also called Hard
|
||||
Break Rules, these have the form
|
||||
|
||||
```
|
||||
pre-context / post-context;
|
||||
```
|
||||
|
||||
where the pre and post-context look like normal break rules. Both the pre and
|
||||
post context are required, and must not allow a zero-length match. There should
|
||||
be no overlap between characters that end a match of the pre-context and those
|
||||
that begin a match of the post-context.
|
||||
|
||||
Chaining into a hard break rule operates normally. There is no chaining out of a
|
||||
hard break rule; when the post-context matches a break is forced immediately.
|
||||
|
||||
Note: future versions of ICU may loosen the restrictions on explicit break
|
||||
rules. The behavior of rules with missing or overlapping contexts is subject to
|
||||
change.
|
||||
|
||||
#### Chaining Control
|
||||
|
||||
Chaining into a rule can be dis-allowed by beginning that rule with a '`^`'. Rules
|
||||
so marked can begin a match after a preceding boundary or at the start of text,
|
||||
but cannot extend a match via chaining from another rule.
|
||||
|
||||
~~The !!LBCMNoChain; statement modifies chaining behavior by preventing chaining
|
||||
from one rule to another from occurring on any character whose Line Break
|
||||
property is Combining Mark. This option is subject to change or removal, and
|
||||
should not be used in general. Within ICU, it is used only with the line break
|
||||
rules. We hope to replace it with something more general.~~
|
||||
|
||||
> :point_right: **Note**: `!!LBCMNoChain` is deprecated, and will be removed
|
||||
> completely from a future version of ICU.
|
||||
|
||||
## Rule Status Values
|
||||
|
||||
Break rules can be tagged with a number, which is called the *rule status*.
|
||||
After a boundary has been located, the status number of the specific rule that
|
||||
determined the boundary position is available to the application through the
|
||||
function `getRuleStatus()`.
|
||||
|
||||
For the predefined word boundary rules, status values are available to
|
||||
distinguish between boundaries associated with words, numbers, and those around
|
||||
spaces or punctuation. Similarly for line break boundaries, status values
|
||||
distinguish between mandatory line endings (new line characters) and break
|
||||
opportunities that are appropriate points for line wrapping. Refer to the ICU
|
||||
API documentation for the C header file `ubrk.h` or to Java class
|
||||
`RuleBasedBreakIterator` for a complete list of the predefined boundary
|
||||
classifications.
|
||||
|
||||
When creating custom sets of break rules, integer status values can be
|
||||
associated with boundary rules in whatever way will be convenient for the
|
||||
application. There is no need to remain restricted to the predefined values and
|
||||
classifications from the standard rules.
|
||||
|
||||
It is possible for a set of break rules to contain more than a single rule that
|
||||
produces some boundary in an input text. In this event, `getRuleStatus()` will
|
||||
return the numerically largest status value from the matching rules, and the
|
||||
alternate function `getRuleStatusVec()` will return a vector of the values from
|
||||
all of the matching rules.
|
||||
|
||||
In the source form of the break rules, status numbers appear at end of a rule,
|
||||
and are enclosed in `{`braces`}`.
|
||||
|
||||
Hard break rules that also have a status value place the status at the end, for
|
||||
example
|
||||
|
||||
```
|
||||
pre-context / post-context {1234};
|
||||
```
|
||||
|
||||
### Word Dictionaries
|
||||
|
||||
For some languages that don't normally use spaces between words, break iterators
|
||||
are able to supplement the rules with dictionary based breaking. Some languages,
|
||||
Thai or Lao, for example, use a dictionary for both word and line breaking.
|
||||
Others, such as Japanese, use a dictionary for word breaking, but not for line
|
||||
breaking.
|
||||
|
||||
To enable dictionary use,
|
||||
|
||||
1. The break rules must select, as unbroken chunks, ranges of text to be passed
|
||||
off to the word dictionary for further subdivision.
|
||||
2. The break rules must define a character class named `$dictionary` that
|
||||
contains the characters (letters) to be handled by the dictionary.
|
||||
|
||||
The dictionary implementation, on receiving a range of text, will map it to a
|
||||
specific dictionary based on script, and then delegate to that dictionary for
|
||||
subdividing the range into words.
|
||||
|
||||
See, for example, this snippet from the [line break rules](https://github.com/unicode-org/icu/blob/main/icu4c/source/data/brkitr/rules/line.txt):
|
||||
|
||||
```
|
||||
# Dictionary character set, for triggering language-based break engines. Currently
|
||||
# limited to LineBreak=Complex_Context (SA).
|
||||
$dictionary = [$SA];
|
||||
```
|
||||
|
||||
## Rule Options
|
||||
|
||||
| Option | Description |
|
||||
| --------------- | ----------- |
|
||||
| `!!chain` | Enable rule chaining. Default is no chaining. |
|
||||
| `!!forward` | The rules that follow are for forward iteration. Forward rules are now the only type of rules needed or used. |
|
||||
|
||||
### Deprecated Rule Options
|
||||
|
||||
| Deprecated Option | Description |
|
||||
| --------------- | ----------- |
|
||||
| ~~`!!reverse`~~ | ~~*[deprecated]* The rules that follow are for reverse iteration. No longer needed; any rules in a Reverse rule section are ignored.~~ |
|
||||
| ~~`!!safe_forward`~~ | ~~*[deprecated]* The rules that follow are for safe forward iteration. No longer needed; any rules in such a section are ignored.~~ |
|
||||
| ~~`!!safe_reverse`~~ | ~~*[deprecated]* The rules that follow are for safe reverse iteration. No longer needed; any rules in such a section are ignored.~~ |
|
||||
| ~~`!!LBCMNoChain`~~ | ~~*[deprecated]* Disable chaining when the overlap character matches `\p{Line_Break=Combining_Mark}`~~ |
|
||||
|
||||
## Rule Syntax
|
||||
|
||||
Here is the syntax for the boundary rules. (The EBNF Syntax is given below.)
|
||||
|
||||
| Rule Name | Rule Values | Notes |
|
||||
| ---------- | ----------- | ----- |
|
||||
| rules | statement+ | |
|
||||
| statement | assignment \| rule \| control |
|
||||
| control | (`!!forward` \| `!!reverse` \| `!!safe_forward` \| `!!safe_reverse` \| `!!chain`) `;`
|
||||
| assignment | variable `=` expr `;` | 5 |
|
||||
| rule | `^`? expr (`{`number`}`)? `;` | 8,9 |
|
||||
| number | [0-9]+ | 1 |
|
||||
| break-point | `/` | 10 |
|
||||
| expr | expr-q \| expr `\|` expr \| expr expr | 3 |
|
||||
| expr-q | term \| term `*` \| term `?` \| term `+` |
|
||||
| term | rule-char \| unicode-set \| variable \| quoted-sequence \| `(` expr `)` \| break-point |
|
||||
| rule-special | *any printing ascii character except letters or numbers* \| white-space |
|
||||
| rule-char | *any non-escaped character that is not rule-special* \| `.` \| *any escaped character except* `\p` *or* `\P` |
|
||||
| variable | `$` name-start-char name-char* | 7 |
|
||||
| name-start-char | `_` \| \p{L} |
|
||||
| name-char | name-start-char \| \\p{N} |
|
||||
| quoted-sequence | `'` *(any char except single quote or line terminator or two adjacent single quotes)*+ `'` |
|
||||
| escaped-char | *See “Character Quoting and Escaping” in the [UnicodeSet](../strings/unicodeset.md) chapter* |
|
||||
| unicode-set | See [UnicodeSet](../strings/unicodeset.md) | 4 |
|
||||
| comment | unescaped `#` *(any char except new-line)** new-line | 2 |
|
||||
| s | unescaped \p{Z}, tab, LF, FF, CR, NEL | 6 |
|
||||
| new-line | LF, CR, NEL | 2 |
|
||||
|
||||
### Rule Syntax Notes
|
||||
|
||||
1. The number associated with a rule that actually determined a break position
|
||||
is available to the application after the break has been returned. These
|
||||
numbers are *not* Perl regular expression repeat counts.
|
||||
|
||||
2. Comments are recognized and removed separately from otherwise parsing the
|
||||
rules. They may appear wherever a space would be allowed (and ignored.)
|
||||
|
||||
3. The implicit concatenation of adjacent terms has higher precedence than the
|
||||
`|` operation. "`ab|cd`" is interpreted as "`(ab)|(cd)`", not as "`a(b|c)d`" or
|
||||
"`(((ab)|c)d)`"
|
||||
|
||||
4. The syntax for [unicode-set](../strings/unicodeset.md) is defined (and parsed) by the `UnicodeSet` class.
|
||||
It is not repeated here.
|
||||
|
||||
5. For `$`variables that will be referenced from inside of a `UnicodeSet`, the
|
||||
definition must consist only of a Unicode Set. For example, when variable `$a`
|
||||
is used in a rule like `[$a$b$c]`, then this definition of `$a` is ok:
|
||||
“`$a=[:Lu:];`” while this one “`$a=abcd;`” would cause an error when `$a` was
|
||||
used.
|
||||
|
||||
6. Spaces are allowed nearly anywhere, and are not significant unless escaped.
|
||||
Exceptions to this are noted.
|
||||
|
||||
7. No spaces are allowed within a variable name. The variable name `$dictionary`
|
||||
is special. If defined, it must be a Unicode Set, the characters of which
|
||||
will trigger the use of word dictionary based boundaries.
|
||||
|
||||
8. A leading `^` on a rule prevents chaining into that rule. It can only match
|
||||
immediately after a preceding boundary, or at the start of text.
|
||||
|
||||
9. `{`nnn`}` appearing at the end of a rule is a Rule Status number, not a repeat
|
||||
count as it would be with conventional regular expression syntax.
|
||||
|
||||
10. A `/` in a rule specifies a hard break point. If the rule matches, a
|
||||
boundary will be forced at the position of the `/` within the match.
|
||||
|
||||
### EBNF Syntax used for the RBBI rules syntax description
|
||||
|
||||
| syntax | description |
|
||||
| -- | ------------------------- |
|
||||
| a? | zero or one instance of a |
|
||||
| a+ | one or more instances of a |
|
||||
| a* | zero or more instances of a |
|
||||
| a \| b | either a or b, but not both |
|
||||
| `a` "`a`" | the literal string between the quotes or displayed as `monospace` |
|
||||
|
||||
## Planned Changes and Removed or Deprecated Rule Features
|
||||
|
||||
1. Reverse rules could formerly be indicated by beginning them with an
|
||||
exclamation `!`. This syntax is deprecated, and will be removed from a
|
||||
future version of ICU.
|
||||
|
||||
2. `!!LBCMNoChain` was a global option that specified that characters with the
|
||||
line break property of "Combining Character" would not participate in rule
|
||||
chaining. This option was always considered internal, is deprecated and will
|
||||
be removed from a future version of ICU.
|
||||
|
||||
3. Naked rule characters. Plain text, in the context of a rule, is treated as
|
||||
literal text to be matched, much like normal regular expressions. This turns
|
||||
out to be very error prone, has been the source of bugs in released versions
|
||||
of ICU, and is not useful in implementing normal text boundary rules. A
|
||||
future version will reject literal text that is not escaped.
|
||||
|
||||
4. Exact reverse rules and safe forward rules: planned changes to the break
|
||||
engine implementation will remove the need for exact reverse rules and safe
|
||||
forward rules.
|
||||
|
||||
5. `{bof}` and `{eof}`, appearing within `[`sets`]`, match the beginning or ending of
|
||||
the input text, respectively. This is an internal (not documented) feature
|
||||
that will probably be removed in a future version of ICU. They are currently
|
||||
used by the standard rules for word, line and sentence breaking. An
|
||||
alternative is probably needed. The existing implementation is incomplete.
|
||||
|
||||
## Additional Sample Code
|
||||
|
||||
**C/C++**
|
||||
See [icu/source/samples/break/](https://github.com/unicode-org/icu/tree/main/icu4c/source/samples/break/)
|
||||
in the ICU source distribution for code samples showing the use of ICU boundary analysis.
|
||||
|
||||
## Details about Dictionary-Based Break Iteration
|
||||
|
||||
> :point_right: **Note**: This section below is originally from August 2012.
|
||||
> It is probably out of date, for example `brkfiles.mk` does not exist anymore.
|
||||
|
||||
Certain Unicode characters have a "dictionary" bit set in the break iteration
|
||||
rules, and text made up of these characters cannot be handled by the rules-based
|
||||
break iteration code for lines or words. Rather, they must be handled by a
|
||||
dictionary-based approach. The ICU approach is as follows:
|
||||
|
||||
Once the Dictionary bit is detected, the set of characters with that bit is
|
||||
handed off to "dictionary code." This code then inspects the characters more
|
||||
carefully, and splits them by script (Thai, Khmer, Chinese, Japanese, Korean).
|
||||
If text in this script has not yet been handled, it loads the appropriate
|
||||
dictionary from disk, and initializes a specialized "BreakEngine" class for that
|
||||
script.
|
||||
|
||||
There are three such specialized classes: Thai, Khmer and CJK.
|
||||
|
||||
Thai and Khmer use very similar approaches. They look through a dictionary that
|
||||
is not weighted by word frequency, and attempt to find the longest total "match"
|
||||
that can be made in the text.
|
||||
|
||||
For Chinese and Japanese text, on the other hand, we have a unified dictionary
|
||||
(due to the fact that both use some of the same characters, it is difficult to
|
||||
distinguish them) that contains information about word frequencies. The
|
||||
algorithm to match text then uses dynamic programming to find the set of breaks
|
||||
it considers "most likely" based on the frequency of the words created by the
|
||||
breaks. This algorithm could also be used for Thai and Khmer, but we do not have
|
||||
sufficient data to do so. This algorithm could also be used for Korean, but once
|
||||
again we do not have the data to do so.
|
||||
|
||||
Code of interest is in `source/common/dictbe.{h, cpp}`, `source/common/brkeng.{h,
|
||||
cpp}`, `source/common/dictionarydata.{h, cpp}`. The dictionaries use the `BytesTrie`
|
||||
and `UCharsTrie` as their data store. The binary form of these dictionaries is
|
||||
produced by the `gendict` tool, which has source in `source/tools/gendict`.
|
||||
|
||||
In order to add new dictionary implementations, a few changes have to be made.
|
||||
First, you should create a new subclass of `DictionaryBreakEngine` or
|
||||
`LanguageBreakEngine` in `dictbe.cpp` that implements your algorithm. Then, in
|
||||
`brkeng.cpp`, you should add logic to create this dictionary break engine if we
|
||||
strike the appropriate script - which should only be 3 or so lines of code at
|
||||
the most. Lastly, you should add the correct data file. If your data is to be
|
||||
represented as a `.dict` file - as is recommended, and in fact required if you
|
||||
don't want to make substantial code changes to the engine loader - you need to
|
||||
simply add a file in the correct format for gendict to the `source/data/brkitr`
|
||||
directory, and add its name to the list of `BRK_DICT_SOURCE` in
|
||||
`source/data/brkitr/brkfiles.mk`. This will cause your dictionary (say, `foo.txt`)
|
||||
to be added as a `UCharsTrie` dictionary with the name foo.dict. If you want your
|
||||
dictionary to be a `BytesTrie` dictionary, you will need to specify a transform
|
||||
within the `Makefile`. To do so, find the part of `source/data/Makefile.in` and
|
||||
`source/data/makedata.mak` that deals with `thaidict.dict` and `khmerdict.dict` and
|
||||
add a similar set of lines for your script. Lastly, in
|
||||
`source/data/brkitr/root.txt`, add a line to the dictionaries `{}` section of the
|
||||
form:
|
||||
|
||||
```
|
||||
shortscriptname:process(dependency){"dictionaryname.dict"}
|
||||
```
|
||||
|
||||
For example, for Katakana:
|
||||
|
||||
```
|
||||
Kata:process(dependency){"cjdict.dict"}
|
||||
```
|
||||
|
||||
Make sure to add appropriate tests for the new implementation.
|
||||
@@ -1,544 +0,0 @@
|
||||
---
|
||||
layout: default
|
||||
title: Boundary Analysis
|
||||
nav_order: 1300
|
||||
has_children: true
|
||||
---
|
||||
<!--
|
||||
© 2020 and later: Unicode, Inc. and others.
|
||||
License & terms of use: http://www.unicode.org/copyright.html
|
||||
-->
|
||||
|
||||
# Boundary Analysis
|
||||
{: .no_toc }
|
||||
|
||||
## Contents
|
||||
{: .no_toc .text-delta }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
|
||||
---
|
||||
|
||||
## Overview of Text Boundary Analysis
|
||||
|
||||
Text boundary analysis is the process of locating linguistic boundaries while
|
||||
formatting and handling text. Examples of this process include:
|
||||
|
||||
1. Locating appropriate points to word-wrap text to fit within specific margins
|
||||
while displaying or printing.
|
||||
|
||||
2. Locating the beginning of a word that the user has selected.
|
||||
|
||||
3. Counting characters, words, sentences, or paragraphs.
|
||||
|
||||
4. Determining how far to move the text cursor when the user hits an arrow key
|
||||
(Some characters require more than one position in the text store and some
|
||||
characters in the text store do not display at all).
|
||||
|
||||
5. Making a list of the unique words in a document.
|
||||
|
||||
6. Figuring out if a given range of text contains only whole words.
|
||||
|
||||
7. Capitalizing the first letter of each word.
|
||||
|
||||
8. Locating a particular unit of the text (For example, finding the third word
|
||||
in the document).
|
||||
|
||||
The `BreakIterator` classes were designed to support these kinds of tasks. The
|
||||
BreakIterator objects maintain a location between two characters in the text.
|
||||
This location will always be a text boundary. Clients can move the location
|
||||
forward to the next boundary or backward to the previous boundary. Clients can
|
||||
also check if a particular location within a source text is on a boundary or
|
||||
find the boundary which is before or after a particular location.
|
||||
|
||||
## Four Types of BreakIterator
|
||||
|
||||
ICU `BreakIterator`s can be used to locate the following kinds of text boundaries:
|
||||
|
||||
1. Character Boundary
|
||||
|
||||
2. Word Boundary
|
||||
|
||||
3. Line-break Boundary
|
||||
|
||||
4. Sentence Boundary
|
||||
|
||||
Each type of boundary is found in accordance with the rules specified by Unicode
|
||||
Standard Annex #29, *Unicode Text Segmentation*
|
||||
(<https://www.unicode.org/reports/tr29/> ) or Unicode Standard Annex #14, *Unicode
|
||||
Line Breaking Algorithm* (<https://www.unicode.org/reports/tr14/>)
|
||||
|
||||
### Character Boundary
|
||||
|
||||
The character-boundary iterator locates the boundaries according to the rules
|
||||
defined in <http://www.unicode.org/reports/tr29/#Grapheme_Cluster_Boundaries>.
|
||||
These boundaries try to match what a user would think of as a "character"—a
|
||||
basic unit of a writing system for a language—which may be more than just a
|
||||
single Unicode code point.
|
||||
|
||||
The letter `Ä`, for example, can be represented in Unicode either with a single
|
||||
code-point value or with two code-point values (one representing the `A` and
|
||||
another representing the umlaut `¨`). The character-boundary iterator will treat
|
||||
either representation as a single character.
|
||||
|
||||
End-user characters, as described above, are also called grapheme clusters, in
|
||||
an attempt to limit the confusion caused by multiple meanings for the word
|
||||
"character".
|
||||
|
||||
### Word Boundary
|
||||
|
||||
The word-boundary iterator locates the boundaries of words, for purposes such as
|
||||
double click selection or "Find whole words" operations.
|
||||
|
||||
Words boundaries are identified according to the rules in
|
||||
<https://www.unicode.org/reports/tr29/#Word_Boundaries>, supplemented by a word
|
||||
dictionary for text in Chinese, Japanese, Thai or Khmer. The rules used for
|
||||
locating word breaks take into account the alphabets and conventions used by
|
||||
different languages.
|
||||
|
||||
Here's an example of a sentence, showing the boundary locations that will be
|
||||
identified by a word break iterator:
|
||||
|
||||
> :point_right: **Note**: TODO: An example needs to be added here.
|
||||
|
||||
### Line-break Boundary
|
||||
|
||||
The line-break iterator locates positions that would be appropriate points to
|
||||
wrap lines when displaying the text. The boundary rules are define here:
|
||||
<https://www.unicode.org/reports/tr14/>
|
||||
|
||||
This example shows the differences in the break locations produced by word and
|
||||
line break iterators:
|
||||
|
||||
> :point_right: **Note**: TODO: An example needs to be added here.
|
||||
|
||||
### Sentence Boundary
|
||||
|
||||
A sentence-break iterator locates sentence boundaries according to the rules
|
||||
defined here: <https://www.unicode.org/reports/tr29/#Sentence_Boundaries>
|
||||
|
||||
## Dictionary-Based BreakIterator
|
||||
|
||||
Some languages are written without spaces, and word and line breaking requires
|
||||
more than rules over character sequences. ICU provides dictionary support for
|
||||
word boundaries in Chinese, Japanese, Thai, Lao, Khmer and Burmese.
|
||||
|
||||
Use of the dictionaries is automatic when text in one of the dictionary
|
||||
languages is encountered. There is no separate API, and no extra programming
|
||||
steps required by applications making use of the dictionaries.
|
||||
|
||||
## Usage
|
||||
|
||||
To locate boundaries in a document, create a BreakIterator using the
|
||||
`BreakIterator::create***Instance` family of methods in C++, or the `ubrk_open()`
|
||||
function (C), where "`***`" is `Character`, `Word`, `Line` or `Sentence`,
|
||||
depending on the type of iterator wanted. These factory methods also take a
|
||||
parameter that specifies the locale for the language of the text to be processed.
|
||||
|
||||
When creating a `BreakIterator`, a locale is also specified, and the behavior of
|
||||
the BreakIterator obtained may be specialized in some way for that locale. For
|
||||
most locales the default break iterator behavior is used.
|
||||
|
||||
Applications also may register customized BreakIterators for use in specific
|
||||
locales. Once such a break iterator has been registered, any requests for break
|
||||
iterators for that locale will return copies of the registered break iterator.
|
||||
|
||||
ICU may cache service instances. Therefore, registration should be done during
|
||||
startup, before opening services by locale ID.
|
||||
|
||||
In the general-usage-model, applications will use the following basic steps to
|
||||
analyze a piece of text for boundaries:
|
||||
|
||||
1. Create a `BreakIterator` with the desired behavior
|
||||
|
||||
2. Use the `setText()` method to set the iterator to analyze a particular piece
|
||||
of text.
|
||||
|
||||
3. Locate the desired boundaries using the appropriate combination of `first()`,
|
||||
`last()`, `next()`, `previous()`, `preceding()`, and `following()` methods.
|
||||
|
||||
The `setText()` method can be called more than once, allowing reuse of a
|
||||
BreakIterator on new pieces of text. Because the creation of a `BreakIterator` can
|
||||
be relatively time-consuming, it makes good sense to reuse them when practical.
|
||||
|
||||
The iterator always points to a boundary position between two characters. The
|
||||
numerical value of the position, as returned by `current()` is the zero-based
|
||||
index of the character following the boundary. Thus a position of zero
|
||||
represents a boundary preceding the first character of the text, and a position
|
||||
of one represents a boundary between the first and second characters.
|
||||
|
||||
The `first()` and `last()` methods reset the iterator's current position to the
|
||||
beginning or end of the text (the beginning and the end are always considered
|
||||
boundaries). The `next()` and `previous()` methods advance the iterator one boundary
|
||||
forward or backward from the current position. If the `next()` or `previous()`
|
||||
methods run off the beginning or end of the text, it returns DONE. The `current()`
|
||||
method returns the current position.
|
||||
|
||||
The `following()` and `preceding()` methods are used for random access, to move the
|
||||
iterator to an arbitrary position within the text. Since a BreakIterator always
|
||||
points to a boundary position, the `following()` and `preceding()` methods will
|
||||
never set the iterator to point to the position specified by the caller (even if
|
||||
it is, in fact, a boundary position). `BreakIterator` will, however, set the
|
||||
iterator to the nearest boundary position before or after the specified
|
||||
position.
|
||||
|
||||
`isBoundary()` returns true if the specified position is a boundary.
|
||||
|
||||
### Thread Safety
|
||||
|
||||
`BreakIterator`s are not thread safe. This is inherit in their design—break
|
||||
iterators are stateful, holding a reference to and position in the text, meaning
|
||||
that a single instance cannot operate in parallel on multiple texts.
|
||||
|
||||
For concurrent break iteration, each thread must use its own break iterator.
|
||||
These can be obtained by creating separate break iterators of the desired type,
|
||||
or by initially creating a main break iterator and then creating a clone for
|
||||
each thread.
|
||||
|
||||
### Line Breaking Strictness, a CSS Property
|
||||
|
||||
CSS has the concept of "[Line Breaking
|
||||
Strictness](https://www.w3.org/TR/css-text-3/#line-break-property)". This
|
||||
property specifies the strictness of line-breaking rules applied within an
|
||||
element: especially how wrapping interacts with punctuation and symbols. ICU
|
||||
line break iterators can choose a strictness using locale tags:
|
||||
|
||||
| Locale | Behavior |
|
||||
| ------------ | ----------- |
|
||||
| `en@lb=strict` <br/> `ja@lb=strict` | Breaks text using the most stringent set of line-breaking rules |
|
||||
| `en@lb=normal` <br/> `ja@lb=normal` | Breaks text using the most common set of line-breaking rules. |
|
||||
| `en@lb=loose` <br/> `ja@lb=loose` | Breaks text using the least restrictive set of line-breaking rules. Typically used for short lines, such as in newspapers. |
|
||||
|
||||
### Sentence Break Filters
|
||||
|
||||
Sentence breaking can return false positives - an indication that sentence ends
|
||||
in an incorrect position - in the presence of abbreviations. For example,
|
||||
consider the sentence
|
||||
|
||||
> In the meantime Mr. Weston arrived with his small ship.
|
||||
|
||||
Default sentence break shows a false boundary following the "Mr."
|
||||
|
||||
ICU includes lists of common abbreviations that can be used to filter, to
|
||||
ignore, these false sentence boundaries. Filtering is enabled by the presence of
|
||||
the `ss` locale tag when creating the break iterator.
|
||||
|
||||
| Locale | Behavior |
|
||||
| ---------------- | ------------------------------------------------------- |
|
||||
| `en` | no filtering |
|
||||
| `en@ss=standard` | Filter based on common English language abbreviations. |
|
||||
| `es@ss=standard` | Filter with common Spanish abbreviations. |
|
||||
|
||||
Abbreviation lists are available (as of ICU 64) for English, German, Spanish,
|
||||
French, Italian and Portuguese.
|
||||
|
||||
## Accuracy
|
||||
|
||||
ICU's break iterators are based on the default boundary rules described in the
|
||||
Unicode Standard Annexes [14](https://www.unicode.org/reports/tr14/) and
|
||||
[29](https://www.unicode.org/reports/tr29/). These are relatively
|
||||
simple boundary rules that can be implemented efficiently, and are sufficient
|
||||
for many purposes and languages. However, some languages and applications will
|
||||
require a more sophisticated linguistic analysis of the text in order to find
|
||||
boundaries with good accuracy. Such an analysis is not directly available from
|
||||
ICU at this time.
|
||||
|
||||
Break Iterators based on custom, user-supplied boundary rules can be created and
|
||||
used by applications with requirements that are not met by the standard default
|
||||
boundary rules.
|
||||
|
||||
## BreakIterator Boundary Analysis Examples
|
||||
|
||||
### Print out all the word-boundary positions in a UnicodeString
|
||||
|
||||
**In C++:**
|
||||
|
||||
```c++
|
||||
void listWordBoundaries(const UnicodeString& s) {
|
||||
UErrorCode status = U_ZERO_ERROR;
|
||||
BreakIterator* bi = BreakIterator::createWordInstance(Locale::getUS(), status);
|
||||
bi->setText(s);
|
||||
int32_t p = bi->first();
|
||||
while (p != BreakIterator::DONE) {
|
||||
printf("Boundary at position %d\n", p);
|
||||
p = bi->next();
|
||||
}
|
||||
delete bi;
|
||||
}
|
||||
```
|
||||
|
||||
**In C:**
|
||||
|
||||
```c
|
||||
void listWordBoundaries(const UChar* s, int32_t len) {
|
||||
UBreakIterator* bi;
|
||||
int32_t p;
|
||||
UErrorCode err = U_ZERO_ERROR;
|
||||
bi = ubrk_open(UBRK_WORD, 0, s, len, &err);
|
||||
if (U_FAILURE(err)) return;
|
||||
p = ubrk_first(bi);
|
||||
while (p != UBRK_DONE) {
|
||||
printf("Boundary at position %d\n", p);
|
||||
p = ubrk_next(bi);
|
||||
}
|
||||
ubrk_close(bi);
|
||||
}
|
||||
```
|
||||
|
||||
### Get the boundaries of the word that contains a double-click position
|
||||
|
||||
**In C++:**
|
||||
|
||||
```c++
|
||||
void wordContaining(BreakIterator& wordBrk,
|
||||
int32_t idx,
|
||||
const UnicodeString& s,
|
||||
int32_t& start,
|
||||
int32_t& end) {
|
||||
// this function is written to assume that we have an
|
||||
// appropriate BreakIterator stored in an object or a
|
||||
// global variable somewhere-- When possible, programmers
|
||||
// should avoid having the create() and delete calls in
|
||||
// a function of this nature.
|
||||
if (s.isEmpty())
|
||||
return;
|
||||
wordBrk.setText(s);
|
||||
start = wordBrk.preceding(idx + 1);
|
||||
end = wordBrk.next();
|
||||
// NOTE: for this and similar operations, use preceding() and next()
|
||||
// as shown here, not following() and previous(). preceding() is
|
||||
// faster than following() and next() is faster than previous()
|
||||
// NOTE: By using preceding(idx + 1) above, we're adopting the convention
|
||||
// that if the double-click comes right on top of a word boundary, it
|
||||
// selects the word that _begins_ on that boundary (preceding(idx) would
|
||||
// instead select the word that _ends_ on that boundary).
|
||||
}
|
||||
```
|
||||
|
||||
**In C:**
|
||||
|
||||
```c
|
||||
void wordContaining(UBreakIterator* wordBrk,
|
||||
int32_t idx,
|
||||
const UChar* s,
|
||||
int32_t sLen,
|
||||
int32_t* start,
|
||||
int32_t* end,
|
||||
UErrorCode* err) {
|
||||
if (wordBrk == NULL || s == NULL || start == NULL || end == NULL) {
|
||||
*err = U_ILLEGAL_ARGUMENT_ERROR;
|
||||
return;
|
||||
}
|
||||
ubrk_setText(wordBrk, s, sLen, err);
|
||||
if (U_SUCCESS(*err)) {
|
||||
*start = ubrk_preceding(wordBrk, idx + 1);
|
||||
*end = ubrk_next(wordBrk);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Check for Whole Words
|
||||
|
||||
Use the following to check if a range of text is a "whole word":
|
||||
|
||||
**In C++:**
|
||||
|
||||
```c++
|
||||
UBool isWholeWord(BreakIterator& wordBrk,
|
||||
const UnicodeString& s,
|
||||
int32_t start,
|
||||
int32_t end) {
|
||||
if (s.isEmpty())
|
||||
return FALSE;
|
||||
wordBrk.setText(s);
|
||||
if (!wordBrk.isBoundary(start))
|
||||
return FALSE;
|
||||
return wordBrk.isBoundary(end);
|
||||
}
|
||||
```
|
||||
|
||||
**In C:**
|
||||
|
||||
```c
|
||||
UBool isWholeWord(UBreakIterator* wordBrk,
|
||||
const UChar* s,
|
||||
int32_t sLen,
|
||||
int32_t start,
|
||||
int32_t end,
|
||||
UErrorCode* err) {
|
||||
UBool result = FALSE;
|
||||
if (wordBrk == NULL || s == NULL) {
|
||||
*err = U_ILLEGAL_ARGUMENT_ERROR;
|
||||
return FALSE;
|
||||
}
|
||||
ubrk_setText(wordBrk, s, sLen, err);
|
||||
if (U_SUCCESS(*err)) {
|
||||
result = ubrk_isBoundary(wordBrk, start) && ubrk_isBoundary(wordBrk, end);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
```
|
||||
|
||||
Count the words in a document (C++ only):
|
||||
|
||||
```c++
|
||||
int32_t containsLetters(RuleBasedBreakIterator& bi, const UnicodeString& s, int32_t start) {
|
||||
bi.setText(s);
|
||||
int32_t count = 0;
|
||||
while (start != BreakIterator::DONE) {
|
||||
int breakType = bi.getRuleStatus();
|
||||
if (breakType != UBRK_WORD_NONE) {
|
||||
// Exclude spaces, punctuation, and the like.
|
||||
// A status value UBRK_WORD_NONE indicates that the boundary does
|
||||
// not start a word or number.
|
||||
//
|
||||
++count;
|
||||
}
|
||||
start = bi.next();
|
||||
}
|
||||
return count;
|
||||
}
|
||||
```
|
||||
|
||||
The function `getRuleStatus()` returns an enum giving additional information on
|
||||
the text preceding the last break position found. Using this value, it is
|
||||
possible to distinguish between numbers, words, words containing kana
|
||||
characters, words containing ideographic characters, and non-word characters,
|
||||
such as spaces or punctuation. The sample uses the break status value to filter
|
||||
out, and not count, boundaries associated with non-word characters.
|
||||
|
||||
### Word-wrap a document (C++ only)
|
||||
|
||||
The sample function below wraps a paragraph so that each line is less than or
|
||||
equal to 72 characters. The function fills in an array passed in by the caller
|
||||
with the starting offsets of
|
||||
each line in the document. Also, it fills in a second array to track how many
|
||||
trailing white space characters there are in the line. For simplicity, it is
|
||||
assumed that an outside process has already broken the document into paragraphs.
|
||||
For example, it is assumed that every string the function is passed has a single
|
||||
newline at the end only.
|
||||
|
||||
```c++
|
||||
int32_t wrapParagraph(const UnicodeString& s,
|
||||
const Locale& locale,
|
||||
int32_t lineStarts[],
|
||||
int32_t trailingwhitespace[],
|
||||
int32_t maxLines,
|
||||
UErrorCode &status) {
|
||||
|
||||
int32_t numLines = 0;
|
||||
int32_t p, q;
|
||||
const int32_t MAX_CHARS_PER_LINE = 72;
|
||||
UChar c;
|
||||
|
||||
BreakIterator *bi = BreakIterator::createLineInstance(locale, status);
|
||||
if (U_FAILURE(status)) {
|
||||
delete bi;
|
||||
return 0;
|
||||
}
|
||||
bi->setText(s);
|
||||
|
||||
|
||||
p = 0;
|
||||
while (p < s.length()) {
|
||||
// jump ahead in the paragraph by the maximum number of
|
||||
// characters that will fit
|
||||
q = p + MAX_CHARS_PER_LINE;
|
||||
|
||||
// if this puts us on a white space character, a control character
|
||||
// (which includes newlines), or a non-spacing mark, seek forward
|
||||
// and stop on the next character that is not any of these things
|
||||
// since none of these characters will be visible at the end of a
|
||||
// line, we can ignore them for the purposes of figuring out how
|
||||
// many characters will fit on the line)
|
||||
if (q < s.length()) {
|
||||
c = s[q];
|
||||
while (q < s.length()
|
||||
&& (u_isspace(c)
|
||||
|| u_charType(c) == U_CONTROL_CHAR
|
||||
|| u_charType(c) == U_NON_SPACING_MARK
|
||||
)) {
|
||||
++q;
|
||||
c = s[q];
|
||||
}
|
||||
}
|
||||
|
||||
// then locate the last legal line-break decision at or before
|
||||
// the current position ("at or before" is what causes the "+ 1")
|
||||
q = bi->preceding(q + 1);
|
||||
|
||||
// if this causes us to wind back to where we started, then the
|
||||
// line has no legal line-break positions. Break the line at
|
||||
// the maximum number of characters
|
||||
if (q == p) {
|
||||
p += MAX_CHARS_PER_LINE;
|
||||
lineStarts[numLines] = p;
|
||||
trailingwhitespace[numLines] = 0;
|
||||
++numLines;
|
||||
}
|
||||
// otherwise, we got a good line-break position. Record the start of this
|
||||
// line (p) and then seek back from the end of this line (q) until you find
|
||||
// a non-white space character (same criteria as above) and
|
||||
// record the number of white space characters at the end of the
|
||||
// line in the other results array
|
||||
else {
|
||||
lineStarts[numLines] = p;
|
||||
int32_t nextLineStart = q;
|
||||
|
||||
for (q--; q > p; q--) {
|
||||
c = s[q];
|
||||
if (!(u_isspace(c)
|
||||
|| u_charType(c) == U_CONTROL_CHAR
|
||||
|| u_charType(c) == U_NON_SPACING_MARK)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
trailingwhitespace[numLines] = nextLineStart - q -1;
|
||||
p = nextLineStart;
|
||||
++numLines;
|
||||
}
|
||||
if (numLines >= maxLines) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
delete bi;
|
||||
return numLines;
|
||||
}
|
||||
```
|
||||
|
||||
Most text editors would not break lines based on the number of characters on a
|
||||
line. Even with a monospaced font, there are still many Unicode characters that
|
||||
are not displayed and therefore should be filtered out of the calculation. With
|
||||
a proportional font, character widths are added up until a maximum line width is
|
||||
exceeded or an end of the paragraph marker is reached.
|
||||
|
||||
Trailing white space does not need to be counted in the line-width measurement
|
||||
because it does not need to be displayed at the end of a line. The sample code
|
||||
above returns an array of trailing white space values because an external
|
||||
rendering process needs to be able to measure the length of the line (without
|
||||
the trailing white space) to justify the lines. For example, if the text is
|
||||
right-justified, the invisible white space would be drawn outside the margin.
|
||||
The line would actually end with the last visible character.
|
||||
|
||||
In either case, the basic principle is to jump ahead in the text to the location
|
||||
where the line would break (without taking word breaks into account). Then, move
|
||||
backwards using the preceding() method to find the last legal breaking position
|
||||
before that location. Iterating straight through the text with next() method
|
||||
will generally be slower.
|
||||
|
||||
## ICU BreakIterator Data Files
|
||||
|
||||
The source code for the ICU break rules for the standard boundary types is
|
||||
located in the directory
|
||||
[icu4c/source/data/brkitr/rules](https://github.com/unicode-org/icu/tree/main/icu4c/source/data/brkitr/rules).
|
||||
These files will be built, and the corresponding binary state tables
|
||||
incorporated into ICU's data, by the standard ICU4C build process.
|
||||
|
||||
The dictionary word lists used by word break, and for some languages, line break
|
||||
are in
|
||||
[icu4c/source/data/brkitr/dictionaries](https://github.com/unicode-org/icu/tree/main/icu4c/source/data/brkitr/dictionaries).
|
||||
|
||||
The same data is used by both ICU4C and ICU4J. In the normal ICU build process,
|
||||
the source data is processed into a binary form using ICU4C, and the resulting
|
||||
binary tables are incorporated into ICU4J.
|
||||
@@ -1,711 +0,0 @@
|
||||
---
|
||||
layout: default
|
||||
title: API Details
|
||||
nav_order: 6
|
||||
parent: Collation
|
||||
---
|
||||
<!--
|
||||
© 2020 and later: Unicode, Inc. and others.
|
||||
License & terms of use: http://www.unicode.org/copyright.html
|
||||
-->
|
||||
|
||||
# Collation API Details
|
||||
{: .no_toc }
|
||||
|
||||
## Contents
|
||||
{: .no_toc .text-delta }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
This section describes some of the usage conventions for the ICU Collation
|
||||
Service API.
|
||||
|
||||
## Collator Instantiation
|
||||
|
||||
To use the Collation Service, you must instantiate a `Collator`. The
|
||||
Collator defines the properties and behavior of the sort ordering. The Collator
|
||||
can be repeatedly referenced until all collation activities have been performed.
|
||||
The Collator can then be closed and removed.
|
||||
|
||||
### Instantiating the Predefined Collators
|
||||
|
||||
ICU comes with a large set of already predefined collators that are suited for
|
||||
specific locales. Most of the ICU locales have a predefined collator. In the worst
|
||||
case, the CLDR default set of rules,
|
||||
which is mostly equivalent to the UCA default ordering (DUCET), is used.
|
||||
The default sort order itself is designed to work well for many languages.
|
||||
(For example, there are no tailorings for the standard sort orders for
|
||||
English, German, French, etc.)
|
||||
|
||||
To instantiate a predefined collator, use the APIs `ucol_open`, `createInstance` and
|
||||
`getInstance` for C, C++ and Java codes respectively. The C API takes a locale ID
|
||||
(or language tag) string argument, C++ takes a Locale object, and Java takes a
|
||||
Locale or ULocale.
|
||||
|
||||
For some languages, multiple collation types are available; for example,
|
||||
"de-u-co-phonebk" / "de@collation=phonebook". They can be enumerated via
|
||||
`Collator::getKeywordValuesForLocale()`. See also the list of available collation
|
||||
tailorings in the online [ICU Collation
|
||||
Demo](https://icu4c-demos.unicode.org/icu-bin/collation.html).
|
||||
|
||||
Starting with ICU 54, collation attributes can be specified via locale keywords
|
||||
as well, in the old locale extension syntax ("el@colCaseFirst=upper") or in
|
||||
language tag syntax ("el-u-kf-upper"). Keywords and values are case-insensitive.
|
||||
|
||||
See the [LDML Collation spec, Collation
|
||||
Settings](http://www.unicode.org/reports/tr35/tr35-collation.html#Collation_Settings),
|
||||
and the [data
|
||||
file](https://github.com/unicode-org/cldr/blob/main/common/bcp47/collation.xml) listing
|
||||
the valid collation keywords and their values. (The deprecated attributes
|
||||
kh/colHiraganaQuaternary and vt/variableTop are not supported.)
|
||||
|
||||
For the [old locale extension
|
||||
syntax](http://www.unicode.org/reports/tr35/tr35.html#Old_Locale_Extension_Syntax),
|
||||
the data file's alias names are used (first alias, if defined, otherwise the
|
||||
name): "de@collation=phonebook;colCaseLevel=yes;kv=space"
|
||||
|
||||
For the language tag syntax, the non-alias names are used, and "true" values can
|
||||
be omitted: "de-u-co-phonebk-kc-kv-space"
|
||||
|
||||
This example demonstrates the instantiation of a collator.
|
||||
|
||||
**C:**
|
||||
|
||||
```c
|
||||
UErrorCode status = U_ZERO_ERROR;
|
||||
UCollator *coll = ucol_open("en_US", &status);
|
||||
if(U_SUCCESS(status)) {
|
||||
/* close the collator*/
|
||||
ucol_close(coll);
|
||||
}
|
||||
```
|
||||
|
||||
**C++:**
|
||||
|
||||
```c++
|
||||
UErrorCode status = U_ZERO_ERROR;
|
||||
Collator *coll = Collator::createInstance(Locale("en", "US"), status);
|
||||
if(U_SUCCESS(status)) {
|
||||
//close the collator
|
||||
delete coll;
|
||||
}
|
||||
```
|
||||
|
||||
**Java:**
|
||||
|
||||
```java
|
||||
Collator col = null;
|
||||
try {
|
||||
col = Collator.getInstance(Locale.US);
|
||||
} catch (Exception e) {
|
||||
System.err.println("English collation creation failed.");
|
||||
e.printStackTrace();
|
||||
}
|
||||
```
|
||||
|
||||
### Instantiating Collators Using Custom Rules
|
||||
|
||||
If the ICU predefined collators are not appropriate for your intended usage, you
|
||||
can define your own set of rules and instantiate a collator that uses them. For more
|
||||
details, please see [the section on collation customization](customization/index).
|
||||
|
||||
This example demonstrates the instantiation of a collator.
|
||||
|
||||
**C:**
|
||||
|
||||
```c
|
||||
UErrorCode status = U_ZERO_ERROR;
|
||||
U_STRING_DECL(rules, "&9 < a, A < b, B < c, C; ch, cH, Ch, CH < d, D, e, E", 52);
|
||||
UCollator *coll;
|
||||
|
||||
U_STRING_INIT(rules, "&9 < a, A < b, B < c, C; ch, cH, Ch, CH < d, D, e, E", 52);
|
||||
coll = ucol_openRules(rules, -1, UCOL_ON, UCOL_DEFAULT_STRENGTH, NULL, &status);
|
||||
if(U_SUCCESS(status)) {
|
||||
/* close the collator*/
|
||||
ucol_close(coll);
|
||||
}
|
||||
```
|
||||
|
||||
**C++:**
|
||||
|
||||
```c++
|
||||
UErrorCode status = U_ZERO_ERROR;
|
||||
UnicodeString rules(u"&9 < a, A < b, B < c, C; ch, cH, Ch, CH < d, D, e, E");
|
||||
Collator *coll = new RuleBasedCollator(rules, status);
|
||||
if(U_SUCCESS(status)) {
|
||||
//close the collator
|
||||
delete coll;
|
||||
}
|
||||
```
|
||||
|
||||
**Java:**
|
||||
|
||||
```java
|
||||
RuleBasedCollator coll = null;
|
||||
String ruleset = "&9 < a, A < b, B < c, C; ch, cH, Ch, CH < d, D, e, E";
|
||||
try {
|
||||
coll = new RuleBasedCollator(ruleset);
|
||||
} catch (Exception e) {
|
||||
System.err.println("Customized collation creation failed.");
|
||||
e.printStackTrace();
|
||||
}
|
||||
```
|
||||
|
||||
## Compare
|
||||
|
||||
Two of the most used functions in ICU collation API, `ucol_strcoll` and `ucol_getSortKey`, have their counterparts in both Win32 and ANSI APIs:
|
||||
|
||||
ICU C | ICU C++ | ICU Java | ANSI/POSIX | WIN32
|
||||
----------------- | --------------------------- | -------------------------- | ---------- | -----
|
||||
`ucol_strcoll` | `Collator::compare` | `Collator.compare` | `strcoll` | `CompareString`
|
||||
`ucol_getSortKey` | `Collator::getSortKey` | `Collator.getCollationKey` | `strxfrm` | `LCMapString`
|
||||
| `Collator::getCollationKey` | | |
|
||||
|
||||
For more sophisticated usage, such as user-controlled language-sensitive text
|
||||
searching, an iterating interface to collation is provided. Please refer to the
|
||||
section below on `CollationElementIterator` for more details.
|
||||
|
||||
The `ucol_compare` function compares one pair of strings at a time. Comparing two
|
||||
strings is much faster than calculating sort keys for both of them. However, if
|
||||
comparisons should be done repeatedly on a very large number of strings, generating
|
||||
and storing sort keys can improve performance. In all other cases (such as quick
|
||||
sort or bubble sort of a
|
||||
moderately-sized list of strings), comparing strings works very well.
|
||||
|
||||
The C API used for comparing two strings is `ucol_strcoll`. It requires two
|
||||
`UChar *` strings and their lengths as parameters, as well as a pointer to a valid
|
||||
`UCollator` instance. The result is a `UCollationResult` constant, which can be one
|
||||
of `UCOL_LESS`, `UCOL_EQUAL` or `UCOL_GREATER`.
|
||||
|
||||
The C++ API offers the method `Collator::compare` with several overloads.
|
||||
Acceptable input arguments are `UChar *` with length of strings, or `UnicodeString`
|
||||
instances. The result is a member of the `UCollationResult` or `EComparisonResult` enums.
|
||||
|
||||
The Java API provides the method `Collator.compare` with one overload. Acceptable
|
||||
input arguments are Strings or Objects. The result is an int value, which is
|
||||
less than zero if source is less than target, zero if source and target are
|
||||
equal, or greater than zero if source is greater than target.
|
||||
|
||||
There are also several convenience functions and methods returning a boolean
|
||||
value, such as `ucol_greater`, `ucol_greaterOrEqual`, `ucol_equal` (in C)
|
||||
`Collator::greater`, `Collator::greaterOrEqual`, `Collator::equal` (in C++) and
|
||||
`Collator.equals` (in Java).
|
||||
|
||||
### Examples
|
||||
|
||||
**C:**
|
||||
|
||||
```c
|
||||
UChar *s [] = { /* list of Unicode strings */ };
|
||||
uint32_t listSize = sizeof(s)/sizeof(s[0]);
|
||||
UErrorCode status = U_ZERO_ERROR;
|
||||
UCollator *coll = ucol_open("en_US", &status);
|
||||
uint32_t i, j;
|
||||
if(U_SUCCESS(status)) {
|
||||
for(i=listSize-1; i>=1; i--) {
|
||||
for(j=0; j<i; j++) {
|
||||
if(ucol_strcoll(s[j], -1, s[j+1], -1) == UCOL_LESS) {
|
||||
swap(s[j], s[j+1]);
|
||||
}
|
||||
}
|
||||
}
|
||||
ucol_close(coll);
|
||||
}
|
||||
```
|
||||
|
||||
**C++:**
|
||||
|
||||
```c++
|
||||
UnicodeString s [] = { /* list of Unicode strings */ };
|
||||
uint32_t listSize = sizeof(s)/sizeof(s[0]);
|
||||
UErrorCode status = U_ZERO_ERROR;
|
||||
Collator *coll = Collator::createInstance(Locale("en", "US"), status);
|
||||
uint32_t i, j;
|
||||
if(U_SUCCESS(status)) {
|
||||
for(i=listSize-1; i>=1; i--) {
|
||||
for(j=0; j<i; j++) {
|
||||
if(coll->compare(s[j], s[j+1]) == UCOL_LESS) {
|
||||
swap(s[j], s[j+1]);
|
||||
}
|
||||
}
|
||||
}
|
||||
delete coll;
|
||||
}
|
||||
```
|
||||
|
||||
**Java:**
|
||||
|
||||
```java
|
||||
String s [] = { /* list of Unicode strings */ };
|
||||
try {
|
||||
Collator coll = Collator.getInstance(Locale.US);
|
||||
for (int i = s.length - 1; i > = 1; i --) {
|
||||
for (j=0; j<i; j++) {
|
||||
if (coll.compare(s[j], s[j+1]) == -1) {
|
||||
swap(s[j], s[j+1]);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
System.err.println("English collation creation failed.");
|
||||
e.printStackTrace();
|
||||
}
|
||||
```
|
||||
|
||||
## GetSortKey
|
||||
|
||||
The C API provides the `ucol_getSortKey` function, which requires (apart from a
|
||||
pointer to a valid `UCollator` instance), an original `UChar` pointer, together with
|
||||
its length. It also requires a pointer to a receiving buffer and its length.
|
||||
|
||||
The C++ API provides the `Collator::getSortKey` method with similar parameters as
|
||||
the C version. It also provides `Collator::getCollationKey`, which produces a
|
||||
`CollationKey` object instance (a wrapper around a sort key).
|
||||
|
||||
The Java API provides only the `Collator.getCollationKey` method, which produces a
|
||||
`CollationKey` object instance (a wrapper around a sort key).
|
||||
|
||||
Sort keys are generally only useful in databases or other circumstances where
|
||||
function calls are extremely expensive. See [Sortkeys vs
|
||||
Comparison](concepts#sortkeys-vs-comparison).
|
||||
|
||||
### Sort Key Features
|
||||
|
||||
ICU writes sort keys as sequences of bytes.
|
||||
|
||||
Each sort key ends with one 00 byte and does not contain any other 00 byte. The
|
||||
terminating 00 byte is included in the length of the sort key as returned by the
|
||||
API (unlike any other ICU API where terminating NUL bytes or characters are not
|
||||
counted as part of the length).
|
||||
|
||||
Sort key byte sequences must be compared with an unsigned-byte comparison, as
|
||||
with `strcmp()`.
|
||||
|
||||
Comparing the sort keys of two strings from the same collator yields the same
|
||||
ordering as using the collator to compare the two strings directly. That is:
|
||||
`strcmp(coll.getSortKey(str1), coll.getSortKey(str2))` is equivalent to
|
||||
`coll.compare(str1, str2)`.
|
||||
|
||||
Sort keys from different collators (different locale or strength or any other
|
||||
attributes/settings) are not comparable.
|
||||
|
||||
Sort keys can be "merged" as described in [UTS #10 Merging Sort
|
||||
Keys](http://www.unicode.org/reports/tr10/#Merging_Sort_Keys), via
|
||||
`ucol_mergeSortkeys()` or Java `CollationKey.merge()`.
|
||||
|
||||
* Since CLDR 1.9/ICU 4.6, the same effect can be achieved by concatenating
|
||||
strings with U+FFFE between them. The concatenation has the same sort order
|
||||
as the merged sort keys.
|
||||
* However, it is not guaranteed that the sort key of the concatenated strings
|
||||
is the same as the merged result of the individual sort keys. (That is,
|
||||
merge(getSortKey(str1), getSortKey(str2)) may differ from getSortKey(str1 +
|
||||
'\\uFFFE' + str2).)
|
||||
* In particular, a future version of ICU is likely to generate shorter sort
|
||||
keys when concatenating strings with U+FFFE between them (by using
|
||||
compression across the U+FFFE weights).
|
||||
* *The recommended way to achieve "merged" sorting is via strings with
|
||||
U+FFFE.*
|
||||
|
||||
Any further analysis or parsing of sort keys is not supported.
|
||||
|
||||
Sort keys will change from one ICU version to another; therefore, if sort keys
|
||||
are stored in a database or other persistent storage, then each upgrade requires
|
||||
their regeneration.
|
||||
|
||||
* The details of the underlying data change with every Unicode and CLDR
|
||||
version.
|
||||
* Sort keys are also subject to enhancements and bug fixes in the builder and
|
||||
implementation code.
|
||||
* On the other hand, the sort *order* is much more stable. It is subject to
|
||||
deliberate changes to the default Unicode collation order, which is kept
|
||||
quite stable, and subject to deliberate changes in CLDR data as new data is
|
||||
added and feedback on existing data is taken into account.
|
||||
|
||||
Implementation notes: (Not supported as permanent constraints on sort keys)
|
||||
|
||||
Byte 02 was unique as a merge separator for some versions of ICU before version
|
||||
ICU 53. Since ICU 53, 02 is also used in regular collation weights where there
|
||||
is no conflict (to expand the number of available short weights).
|
||||
|
||||
Byte 01 has been unique as a level separator. This is not strictly necessary for
|
||||
non-primary levels. (A level's compressible "common" weight as its level
|
||||
separator would yield shorter sort keys.) However, the current implementation of
|
||||
`ucol_mergeSortkeys()` relies on it. (Also, test code currently examines sort keys
|
||||
for finding the strength of a comparison difference.) This may change in the
|
||||
future, especially if `ucol_mergeSortkeys()` were to become deprecated.
|
||||
|
||||
Level separators are likely to be equivalent to single-byte weights (possibly
|
||||
compressible): Multi-byte level separators would noticeably lengthen sort keys
|
||||
for short strings.
|
||||
|
||||
The byte values used in several ICU versions for sort keys and collation
|
||||
elements are documented in the [“Special Byte Values” design
|
||||
doc](https://icu.unicode.org/design/collation/bytes) on the ICU site.
|
||||
|
||||
### Sort Key Output Buffer
|
||||
|
||||
`ucol_getSortKey()` can operate in 'preflighting' mode, which returns the amount
|
||||
of memory needed to store the resulting sort key. This mode is automatically
|
||||
activated if the output buffer size passed is set to zero. Should the sort key
|
||||
become longer than the buffer provided, function again slips into preflighting
|
||||
mode. The overall performance is poorer than if the function is called with a
|
||||
zero output buffer. If the size of the sort key returned is greater than the
|
||||
size of the buffer provided, the content of the result buffer is undefined. In
|
||||
that case, the result buffer could be reallocated to its proper size and the
|
||||
sort key generator function can be used again.
|
||||
|
||||
The best way to generate a series of sort keys is to do the following:
|
||||
|
||||
1. Create a big temporary buffer on the stack. Typically, this buffer is
|
||||
allocated only once, and reused with every sort key generated. There is no
|
||||
need to keep it as small as possible. A recommended size for the temporary
|
||||
buffer is four times the length of the longest string processed.
|
||||
|
||||
2. Start the loop. Call `ucol_getSortKey()` to find out how big the sort key
|
||||
buffer should be, and fill in the temporary buffer at the same time.
|
||||
|
||||
3. If the temporary buffer is too small, allocate or reallocate more space.
|
||||
Fill in the sort key values in the overflow buffer.
|
||||
|
||||
4. Allocate the sort key buffer with the size returned by `ucol_getSortKey()` and
|
||||
call `memcpy` to copy the sort key content from the temp buffer to the sort
|
||||
key buffer.
|
||||
|
||||
5. Loop back to step 1 until you are done.
|
||||
|
||||
6. Delete the overflow buffer if you created one.
|
||||
|
||||
### Example
|
||||
|
||||
```c
|
||||
void GetSortKeys(const Ucollator* coll, const UChar*
|
||||
const *source, uint32_t arrayLength)
|
||||
{
|
||||
char[1000] buffer; // allocate stack buffer
|
||||
char* currBuffer = buffer;
|
||||
int32_t bufferLen = sizeof(buffer);
|
||||
int32_t expectedLen = 0;
|
||||
UErrorCode err = U_ZERO_ERROR;
|
||||
|
||||
for (int i = 0; i < arrayLength; ++i) {
|
||||
expectedLen = ucol_getSortKey(coll, source[i], -1, currBuffer, bufferLen);
|
||||
if (expectedLen > bufferLen) {
|
||||
if (currBuffer == buffer) {
|
||||
currBuffer = (char*)malloc(expectedLen);
|
||||
} else {
|
||||
currBuffer = (char*)realloc(currBuffer, expectedLen);
|
||||
}
|
||||
}
|
||||
bufferLen = ucol_getSortKey(coll, source[i], -1, currBuffer, expectedLen);
|
||||
}
|
||||
processSortKey(i, currBuffer, bufferLen);
|
||||
|
||||
|
||||
if (currBuffer != buffer && currBuffer != NULL) {
|
||||
free(currBuffer);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
> :point_right: **Note** Although the API allows you to call
|
||||
> `ucol_getSortKey` with `NULL` to see what the
|
||||
> sort key length is, it is strongly recommended that you NOT determine the length
|
||||
> first, then allocate and fill the sort key buffer. If you do, it requires twice
|
||||
> the processing since computing the length has to do the same calculation as
|
||||
> actually getting the sort key. Instead, the example shown above uses a stack buffer.
|
||||
|
||||
### Using Iterators for String Comparison
|
||||
|
||||
ICU4C's `ucol_strcollIter` API allows for comparing two strings that are supplied
|
||||
as character iterators (`UCharIterator`). This is useful when you need to compare
|
||||
differently encoded strings using `strcoll`. In that case, converting the strings
|
||||
first would probably be wasteful, since `strcoll` usually gives the result
|
||||
before whole strings are processed. This API is implemented only as a C function
|
||||
in ICU4C. There are no equivalent C++ or ICU4J functions.
|
||||
|
||||
```c
|
||||
...
|
||||
/* we are arriving with two char*: utf8Source and utf8Target, with their
|
||||
* lengths in utf8SourceLen and utf8TargetLen
|
||||
*/
|
||||
UCharIterator sIter, tIter;
|
||||
uiter_setUTF8(&sIter, utf8Source, utf8SourceLen);
|
||||
uiter_setUTF8(&tIter, utf8Target, utf8TargetLen);
|
||||
compareResultUTF8 = ucol_strcollIter(myCollation, &sIter, &tIter, &status);
|
||||
...
|
||||
```
|
||||
|
||||
### Obtaining Partial Sort Keys
|
||||
|
||||
When using different sort algorithms, such as radix sort, sometimes it is useful
|
||||
to process strings only as much as needed to feed into the sorting algorithm.
|
||||
For that purpose, ICU provides the `ucol_nextSortKeyPart` API, which also takes
|
||||
character iterators. This API allows for iterating over subsequent pieces of an
|
||||
uncompressed sort key. Between calls to the API you need to save a 64-bit state.
|
||||
Following is an example of simulating a string compare function using the partial
|
||||
sort key API. Your usage model is bound to look much different.
|
||||
|
||||
```c
|
||||
static UCollationResult compareUsingPartials(UCollator *coll,
|
||||
const UChar source[], int32_t sLen,
|
||||
const UChar target[], int32_t tLen,
|
||||
int32_t pieceSize, UErrorCode *status) {
|
||||
int32_t partialSKResult = 0;
|
||||
UCharIterator sIter, tIter;
|
||||
uint32_t sState[2], tState[2];
|
||||
int32_t sSize = pieceSize, tSize = pieceSize;
|
||||
int32_t i = 0;
|
||||
uint8_t sBuf[16384], tBuf[16384];
|
||||
if(pieceSize > 16384) {
|
||||
*status = U_BUFFER_OVERFLOW_ERROR;
|
||||
return UCOL_EQUAL;
|
||||
}
|
||||
*status = U_ZERO_ERROR;
|
||||
sState[0] = 0; sState[1] = 0;
|
||||
tState[0] = 0; tState[1] = 0;
|
||||
while(sSize == pieceSize && tSize == pieceSize && partialSKResult == 0) {
|
||||
uiter_setString(&sIter, source, sLen);
|
||||
uiter_setString(&tIter, target, tLen);
|
||||
sSize = ucol_nextSortKeyPart(coll, &sIter, sState, sBuf, pieceSize, status);
|
||||
tSize = ucol_nextSortKeyPart(coll, &tIter, tState, tBuf, pieceSize, status);
|
||||
partialSKResult = memcmp(sBuf, tBuf, pieceSize);
|
||||
}
|
||||
|
||||
if(partialSKResult < 0) {
|
||||
return UCOL_LESS;
|
||||
} else if(partialSKResult > 0) {
|
||||
return UCOL_GREATER;
|
||||
} else {
|
||||
return UCOL_EQUAL;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Other Examples
|
||||
|
||||
A longer example is presented in the 'Examples' section. Here is an illustration
|
||||
of the usage model.
|
||||
|
||||
**C:**
|
||||
|
||||
```c
|
||||
#define MAX_KEY_SIZE 100
|
||||
#define MAX_BUFFER_SIZE 10000
|
||||
#define MAX_LIST_LENGTH 5
|
||||
const char text[] = {
|
||||
"Quick",
|
||||
"fox",
|
||||
"Moving",
|
||||
"trucks",
|
||||
"riddle"
|
||||
};
|
||||
const UChar s [5][20];
|
||||
int i;
|
||||
int32_t length, expectedLen;
|
||||
uint8_t temp[MAX_BUFFER _SIZE];
|
||||
|
||||
|
||||
uint8_t *temp2 = NULL;
|
||||
uint8_t keys [MAX_LIST_LENGTH][MAX_KEY_SIZE];
|
||||
UErrorCode status = U_ZERO_ERROR;
|
||||
|
||||
temp2 = temp;
|
||||
|
||||
length = MAX_BUFFER_SIZE;
|
||||
for( i = 0; i < 5; i++)
|
||||
{
|
||||
u_uastrcpy(s[i], text[i]);
|
||||
}
|
||||
UCollator *coll = ucol_open("en_US",&status);
|
||||
uint32_t length;
|
||||
if(U_SUCCESS(status)) {
|
||||
for(i=0; i<MAX_LIST_LENGTH; i++) {
|
||||
expectedLen = ucol_getSortKey(coll, s[i], -1,temp2,length );
|
||||
if (expectedLen > length) {
|
||||
if (temp2 == temp) {
|
||||
temp2 =(char*)malloc(expectedLen);
|
||||
} else {
|
||||
temp2 =(char*)realloc(temp2, expectedLen);
|
||||
}
|
||||
length =ucol_getSortKey(coll, s[i], -1, temp2, expectedLen);
|
||||
}
|
||||
memcpy(key[i], temp2, length);
|
||||
}
|
||||
}
|
||||
qsort(keys, MAX_LIST_LENGTH,MAX_KEY_SIZE*sizeof(uint8_t), strcmp);
|
||||
for (i = 0; i < MAX_LIST_LENGTH; i++) {
|
||||
free(key[i]);
|
||||
}
|
||||
ucol_close(coll);
|
||||
```
|
||||
|
||||
**C++:**
|
||||
|
||||
```c++
|
||||
#define MAX_LIST_LENGTH 5
|
||||
const UnicodeString s [] = {
|
||||
"Quick",
|
||||
"fox",
|
||||
"Moving",
|
||||
"trucks",
|
||||
"riddle"
|
||||
};
|
||||
CollationKey *keys[MAX_LIST_LENGTH];
|
||||
UErrorCode status = U_ZERO_ERROR;
|
||||
Collator *coll = Collator::createInstance(Locale("en_US"), status);
|
||||
uint32_t i;
|
||||
if(U_SUCCESS(status)) {
|
||||
for(i=0; i<listSize; i++) {
|
||||
keys[i] = coll->getCollationKey(s[i], -1);
|
||||
}
|
||||
qsort(keys, MAX_LIST_LENGTH, sizeof(CollationKey),compareKeys);
|
||||
delete[] keys;
|
||||
delete coll;
|
||||
}
|
||||
```
|
||||
|
||||
**Java:**
|
||||
|
||||
```java
|
||||
String s [] = {
|
||||
"Quick",
|
||||
"fox",
|
||||
"Moving",
|
||||
"trucks",
|
||||
"riddle"
|
||||
};
|
||||
CollationKey keys[] = new CollationKey[s.length];
|
||||
try {
|
||||
Collator coll = Collator.getInstance(Locale.US);
|
||||
for (int i = 0; i < s.length; i ++) {
|
||||
keys[i] = coll.getCollationKey(s[i]);
|
||||
}
|
||||
|
||||
Arrays.sort(keys);
|
||||
}
|
||||
catch (Exception e) {
|
||||
System.err.println("Error creating English collator");
|
||||
e.printStackTrace();
|
||||
}
|
||||
```
|
||||
|
||||
## Collation ElementIterator
|
||||
|
||||
A collation element iterator can only be used in one direction. This is
|
||||
established at the time of the first call to retrieve a collation element. Once
|
||||
`ucol_next` (C), `CollationElementIterator::next` (C++) or
|
||||
`CollationElementIterator.next` (Java) are invoked,
|
||||
`ucol_previous` (C),
|
||||
`CollationElementIterator::previous` (C++) or `CollationElementIterator.previous`
|
||||
(Java) should not be used (and vice versa). The direction can be changed
|
||||
immediately after `ucol_first`, `ucol_last`, `ucol_reset` (in C),
|
||||
`CollationElementIterator::first`, `CollationElementIterator::last`,
|
||||
`CollationElementIterator::reset` (in C++) or `CollationElementIterator.first`,
|
||||
`CollationElementIterator.last`, `CollationElementIterator.reset` (in Java) is
|
||||
called, or when it reaches the end of string while traversing the string.
|
||||
|
||||
When `ucol_next` is called at the end of the string buffer, `UCOL_NULLORDER` is
|
||||
always returned with any subsequent calls to `ucol_next`. The same applies to
|
||||
`ucol_previous`.
|
||||
|
||||
An example of how iterators are used is the Boyer-Moore search implementation,
|
||||
which can be found in the samples section.
|
||||
|
||||
### API Example
|
||||
|
||||
**C:**
|
||||
|
||||
```c
|
||||
UCollator *coll = ucol_open("en_US",status);
|
||||
UErrorCode status = U_ZERO_ERROR;
|
||||
UChar text[20];
|
||||
UCollationElements *collelemitr;
|
||||
uint32_t collelem;
|
||||
|
||||
u_uastrcpy(text, "text");
|
||||
collelemitr = ucol_openElements(coll, text, -1, &status);
|
||||
collelem = 0;
|
||||
do {
|
||||
collelem = ucol_next(collelemitr, &status);
|
||||
} while (collelem != UCOL_NULLORDER);
|
||||
|
||||
ucol_closeElements(collelemitr);
|
||||
ucol_close(coll);
|
||||
```
|
||||
|
||||
**C++:**
|
||||
|
||||
```c++
|
||||
UErrorCode status = U_ZERO_ERROR;
|
||||
Collator *coll = Collator::createInstance(Locale::getUS(), status);
|
||||
UnicodeString text("text");
|
||||
CollationElementIterator *collelemitr = coll->createCollationElementIterator(text);
|
||||
uint32_t collelem = 0;
|
||||
do {
|
||||
collelem = collelemitr->next(status);
|
||||
} while (collelem != CollationElementIterator::NULLORDER);
|
||||
|
||||
delete collelemitr;
|
||||
delete coll;
|
||||
```
|
||||
|
||||
**Java:**
|
||||
|
||||
```java
|
||||
try {
|
||||
RuleBasedCollator coll = (RuleBasedCollator)Collator.getInstance(Locale.US);
|
||||
String text = "text";
|
||||
CollationElementIterator collelemitr = coll.getCollationElementIterator(text);
|
||||
int collelem = 0;
|
||||
do {
|
||||
collelem = collelemitr.next();
|
||||
} while (collelem != CollationElementIterator.NULLORDER);
|
||||
} catch (Exception e) {
|
||||
System.err.println("Error in collation iteration");
|
||||
e.printStackTrace();
|
||||
}
|
||||
```
|
||||
|
||||
## Setting and Getting Attributes
|
||||
|
||||
The general attribute setting APIs are `ucol_setAttribute` (in C) and
|
||||
`Collator::setAttribute` (in C++). These APIs take an attribute name and an
|
||||
attribute value. If the name and the value pass a syntax and range check, the
|
||||
property of the collator is changed. If the name and value do not pass a syntax
|
||||
and range check, however, the state is not changed and the error code variable
|
||||
is set to an error condition. The Java version does not provide general
|
||||
attribute setting APIs; instead, each attribute has its own setter API of
|
||||
the form `RuleBasedCollator.setATTRIBUTE_NAME(arguments)`.
|
||||
|
||||
The attribute getting APIs are `ucol_getAttribute` (C) and `Collator::getAttribute`
|
||||
(C++). Both APIs require an attribute name as an argument and return an
|
||||
attribute value if a valid attribute name was supplied. If a valid attribute
|
||||
name was not supplied, however, they return an undefined result and set the
|
||||
error code. Similarly to the setter APIs for the Java version, no generic getter
|
||||
API is provided. Each attribute has its own setter API of the form
|
||||
`RuleBasedCollator.getATTRIBUTE_NAME()` in the Java version.
|
||||
|
||||
## References
|
||||
|
||||
1. Ken Whistler, Markus Scherer: "Unicode Technical Standard #10, Unicode Collation
|
||||
Algorithm" (<http://www.unicode.org/reports/tr10/>)
|
||||
|
||||
2. ICU Design doc: "Collation v2" (<https://icu.unicode.org/design/collation/v2>)
|
||||
|
||||
3. Mark Davis: "ICU Collation Design Document"
|
||||
(<https://htmlpreview.github.io/?https://github.com/unicode-org/icu-docs/blob/main/design/collation/ICU_collation_design.htm>)
|
||||
|
||||
3. The Unicode Standard, chapter 5, "Implementation guidelines"
|
||||
(<http://www.unicode.org/uni2book/ch05.pdf>)
|
||||
|
||||
4. Laura Werner: "Efficient text searching in Java: Finding the right string in
|
||||
any language"
|
||||
(<http://icu-project.org/docs/papers/efficient_text_searching_in_java.html>)
|
||||
|
||||
5. Mark Davis, Martin Dürst: "Unicode Standard Annex #15: Unicode Normalization
|
||||
Forms" (<http://www.unicode.org/reports/tr15/>).
|
||||
@@ -1,579 +0,0 @@
|
||||
---
|
||||
layout: default
|
||||
title: Architecture
|
||||
nav_order: 2
|
||||
parent: Collation
|
||||
---
|
||||
<!--
|
||||
© 2020 and later: Unicode, Inc. and others.
|
||||
License & terms of use: http://www.unicode.org/copyright.html
|
||||
-->
|
||||
|
||||
# Collation Service Architecture
|
||||
{: .no_toc }
|
||||
|
||||
## Contents
|
||||
{: .no_toc .text-delta }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
This section describes the design principles, architecture and coding
|
||||
conventions of the ICU Collation Service.
|
||||
|
||||
## Collator
|
||||
|
||||
To use the Collation Service, a Collator must first be instantiated. An
|
||||
Collator is a data structure or object that maintains all of the property
|
||||
and state information necessary to define and support the specific collation
|
||||
behavior provided. Examples of properties described in the Collator are the
|
||||
locale, whether normalization is to be performed, and how many levels of
|
||||
collation are to be evaluated. Examples of the state information described in
|
||||
the Collator include the direction of a Collation Element Iterator (forward
|
||||
or backward) and the status of the last API executed.
|
||||
|
||||
The Collator is instantiated either by referencing a locale or by defining a
|
||||
custom set of rules (a tailoring).
|
||||
|
||||
The Collation Service uses the paradigm:
|
||||
|
||||
1. Open a Collator,
|
||||
|
||||
2. Use while necessary,
|
||||
|
||||
3. Close the Collator.
|
||||
|
||||
Collator instances cannot be shared among threads. You should open them
|
||||
instead, and use a different collator for each separate thread. The safe clone
|
||||
function is supported for cloning collators in a thread-safe fashion.
|
||||
|
||||
The Collation Service follows the ICU conventions for locale designation
|
||||
when opening collators:
|
||||
|
||||
1. NULL means the default locale.
|
||||
|
||||
2. The empty locale name ("") means the root locale.
|
||||
The Collation Service adheres to the ICU conventions described in the
|
||||
"[ICU Architectural Design](../design.md) " section of the users guide.
|
||||
In particular:
|
||||
|
||||
3. The standard error code convention is usually followed. (Functions that do
|
||||
not take an error code parameter do so for backward compatibility.)
|
||||
|
||||
4. The string length convention is followed: when passing a `UChar *`, the
|
||||
length is required in a separate argument. If -1 is passed for the length,
|
||||
it is assumed that the string is zero terminated.
|
||||
|
||||
### Collation locale and keyword handling
|
||||
|
||||
When a collator is created from a locale, the collation service (like all ICU
|
||||
services) must map the requested locale to the localized collation data
|
||||
available to ICU at the time. It does so using the standard ICU locale fallback
|
||||
mechanism. See the fallback section of the [locale
|
||||
chapter](../locale/index.md) for more details.
|
||||
|
||||
If you pass a regular locale in, like "en_US", the collation service first
|
||||
searches with fallback for "collations/default" key. The first such key it finds
|
||||
will have an associated string value; this is the keyword name for the collation
|
||||
that is default for this locale. If the search falls all the way back to the
|
||||
root locale, the collation service will us the "collations/default" key there,
|
||||
which has the value "standard".
|
||||
|
||||
If there is a locale with a keyword, like "de-u-co-phonebk" or "de@collation=phonebook", the
|
||||
collation service searches with fallback for "collations/phonebook". If the
|
||||
search is successful, the collation service uses the string value it finds to
|
||||
instantiate a Collator. If the search fails because no such key is present in
|
||||
any of ICU's locale data (e.g., "de@collation=funky"), the service returns a
|
||||
collator implementing the default tailoring of the locale.
|
||||
If the fallback is all the way to the root locale, then
|
||||
the return `UErrorCode` is `U_USING_DEFAULT_WARNING`.
|
||||
|
||||
## Input values for collation
|
||||
|
||||
Collation deals with processing strings. ICU generally requires that all the
|
||||
strings should be in UTF-16 format, and that all the required conversion should
|
||||
done before ICU functions are used. In the case of collation, there are APIs
|
||||
that can also take instances of character iterators (`UCharIterator`)
|
||||
or UTF-8 directly.
|
||||
|
||||
Theoretically, character iterators can iterate strings
|
||||
in any encoding. ICU currently provides character iterator implementations for
|
||||
UTF-8 and UTF-16BE (useful when processing data from a big endian platform on an
|
||||
little endian machine). It should be noted, however, that using iterators for
|
||||
collation APIs has a performance impact. It should be used in situations when it
|
||||
is not desirable to convert whole strings before the operation - such as when
|
||||
using a string compare function.
|
||||
|
||||
## Collation Elements
|
||||
|
||||
As discussed in the introduction, there are many possible orderings for sorted
|
||||
text, depending on language and other factors. Ideally, there is a way to
|
||||
describe each ordering as a set of rules for calculating numeric values for each
|
||||
string of text. The collation process then becomes one of simply comparing these
|
||||
numeric values.
|
||||
|
||||
This essentially describes the way the Collation Service works. To implement
|
||||
a particular sort ordering, first the relationship between each character or
|
||||
character sequence is derived. For example, a Spanish ordering defines the
|
||||
letter sequence "CH" to be between the letters "C" and "D". As also discussed in
|
||||
the introduction, to order strings properly requires that comparison of base
|
||||
letters must be considered separately from comparison of accents. Letter case
|
||||
must also be considered separately from either base letters or accents. Any
|
||||
ordering specification language must provide a way to define the relationships
|
||||
between characters or character sequences on multiple levels. ICU supports this
|
||||
by using "<" to describe a relationship at the primary level, using "<<" to
|
||||
describe a relationship at the secondary level, and using "<<<" to describe a
|
||||
relationship at the tertiary level. Here are some example usages:
|
||||
|
||||
Symbol | Example | Description
|
||||
------ | -------- | -----------
|
||||
`<` | `c < ch` | Make a primary (base letter) difference between "c" and the character sequence "ch"
|
||||
`<<` | `a << ä` | Make a secondary (accent) difference between "a" and "ä"
|
||||
`<<<` | `a<<<A` | Make a tertiary difference between "a" and "A"
|
||||
|
||||
A more complete description of the ordering specification symbols and their
|
||||
meanings is provided in the section on Collation Tailoring.
|
||||
|
||||
Once a sort ordering is defined by specifying the desired relationships between
|
||||
characters and character sequences, ICU can convert these relationships to a
|
||||
series of numerical values (one for each level) that satisfy these same
|
||||
relationships.
|
||||
|
||||
This series of numeric values, representing the relative weighting of a
|
||||
character or character sequence, is called a Collation Element (CE).
|
||||
One possible encoding of a Collation Element is a 32-bit value consisting of
|
||||
a 16-bit primary weight, a 8-bit secondary weight,
|
||||
2 case bits, and a 6-bit tertiary weight.
|
||||
|
||||
The sort weight of a string is represented by the collation elements of its
|
||||
component characters and character sequences. For example, the sort weight of
|
||||
the string "apple" would consist of its component Collation Elements, as shown
|
||||
here:
|
||||
|
||||
"Apple" | "Apple" Collation Elements
|
||||
------- | --------------------------
|
||||
a | `[1900.05.05]`
|
||||
p | `[3700.05.05]`
|
||||
p | `[3700.05.05]`
|
||||
l | `[2F00.05.05]`
|
||||
e | `[2100.05.05]`
|
||||
|
||||
In this example, the letter "a" has a 16-bit primary weight of 1900 (hex), an
|
||||
8-bit secondary weight of 05 (hex), and a combined 8-bit case-tertiary weight of
|
||||
05 (hex).
|
||||
|
||||
String comparison is performed by comparing the collation elements of each
|
||||
string. Each of the primary weights are compared. If a difference is found, that
|
||||
difference determines the relationship between the two strings. If no
|
||||
differences are found, the secondary weights are compared and so forth.
|
||||
|
||||
With ICU it is possible to specify how many levels should be compared. For some
|
||||
applications, it can be desirable to compare only primary levels or to compare
|
||||
only primary and secondary levels.
|
||||
|
||||
## Sort Keys
|
||||
|
||||
If a string is to be compared thousands or millions of times,
|
||||
it can be more efficient to use sort keys.
|
||||
Sort keys are useful in situations where a large amount of data is indexed
|
||||
and frequently searched. The sort key is generated once and used in subsequent
|
||||
comparisons, rather than repeatedly generating the string's Collation Elements.
|
||||
The comparison of sort keys is a very efficient and simple binary compare of strings of
|
||||
unsigned bytes.
|
||||
|
||||
An important property of ICU sort keys is that you can obtain the same results
|
||||
by comparing 2 strings as you do by comparing the sort keys of the 2 strings
|
||||
(provided that the same ordering and related collation attributes are used).
|
||||
|
||||
An ICU sort key is a pre-processed sequence of bytes generated from a Unicode
|
||||
string. The weights for each comparison level are concatenated, separated by a
|
||||
"0x01" byte between levels.
|
||||
The entire sequence is terminated with a 0x00 byte for convenience in C APIs.
|
||||
(This 0x00 terminator is counted in the sort key length —
|
||||
unlike regular strings where the NUL terminator is excluded from the string length.)
|
||||
|
||||
ICU actually compresses the sort keys so that they take the
|
||||
minimum storage in memory and in databases.
|
||||
|
||||
<!-- TODO: (diagram was missing in Google Sites already)
|
||||
The diagram below represents an uncompressed sort key in ICU for ease of understanding. -->
|
||||
|
||||
### Sort key size
|
||||
|
||||
One of the more important issues when considering using sort keys is the sort
|
||||
key size. Unfortunately, it is very hard to give a fast exact answer to the
|
||||
following question: "What is the maximum size for sort keys generated for
|
||||
strings of size X". This problem is twofold:
|
||||
|
||||
1. The maximum size of the sort key depends on the size of the collation
|
||||
elements that are used to build it. Size of collation elements vary greatly
|
||||
and depends both on the alphabet in question and on the locale used.
|
||||
|
||||
2. Compression is used in building sort keys. Most 'regular' sequences of
|
||||
characters produce very compact sort keys.
|
||||
|
||||
If one is to assume the worst case and use too-big buffers, a lot of space will
|
||||
be wasted. However, if you use too-small buffers, you will lose performance if
|
||||
generated sort keys are longer than supplied buffers too often
|
||||
(and you have to reallocate for each of those).
|
||||
A good strategy
|
||||
for this problem would be to manually manage a large buffer for storing sortkeys
|
||||
and keep a list of indices to sort keys in this buffer (see the "large buffers"
|
||||
[Collation Example](examples#using-large-buffers-to-manage-sort-keys)
|
||||
for more details).
|
||||
|
||||
Here are some rules of a thumb, please do not rely on them. If you are looking
|
||||
at the East Asian locales, you probably want to go with 5 bytes per code point.
|
||||
For Thai, 3 bytes per code point should be sufficient. For all the other locales
|
||||
(mostly Latin and Cyrillic), you should be fine with 2 bytes per code point.
|
||||
These values are based on average lengths of sort keys generated with tertiary
|
||||
strength. If you need quaternary and identical strength (you should not), add 3
|
||||
bytes per code point to each of these.
|
||||
|
||||
### Partial sort keys
|
||||
|
||||
In some cases, most notably when implementing [radix
|
||||
sorting](http://en.wikipedia.org/wiki/Radix_sort), it is useful to produce only
|
||||
parts of sort keys at a time. ICU4C 2.6+ provides an API that allows producing
|
||||
parts of sort keys (`ucol_nextSortKeyPart` API). These sort keys may or may not be
|
||||
compressed; that is, they may or may not be compatible with regular sort keys.
|
||||
|
||||
### Merging sort keys
|
||||
|
||||
Sometimes, it is useful to be able to merge sort keys. One example is having
|
||||
separate sort keys for first and last names. If you need to perform an operation
|
||||
that requires a sort key generated on the whole name, instead of concatenating
|
||||
strings and regenerating sort keys, you should merge the sort keys. The merging
|
||||
is done by merging the corresponding levels while inserting a terminator between
|
||||
merged parts. The reserved sort key byte value for the merge terminator is 0x02.
|
||||
For more details see [UCA section 1.6, Merging Sort
|
||||
Keys](http://www.unicode.org/reports/tr10/#Interleaved_Levels).
|
||||
|
||||
* C API: unicode/ucol.h `ucol_mergeSortkeys()`
|
||||
* Java API: `com.ibm.icu.text.CollationKey merge(CollationKey source)`
|
||||
|
||||
CLDR 1.9/ICU 4.6 and later map U+FFFE to a special collation element that is
|
||||
intended to allow concatenating strings like firstName+\\uFFFE+lastName to yield
|
||||
the same results as merging their individual sort keys.
|
||||
This has been fully implemented in ICU since version 53.
|
||||
|
||||
### Generating bounds for a sort key (prefix matching)
|
||||
|
||||
Having sort keys for strings allows for easy creation of bounds - sort keys that
|
||||
are guaranteed to be smaller or larger than any sort key from a give range. For
|
||||
example, if bounds are produced for a sortkey of string "smith", strings between
|
||||
upper and lower bounds with one level would include "Smith", "SMITH", "sMiTh".
|
||||
Two kinds of upper bounds can be generated - the first one will match only
|
||||
strings of equal length, while the second one will match all the strings with
|
||||
the same initial prefix.
|
||||
|
||||
CLDR 1.9/ICU 4.6 and later map U+FFFF to a collation element with the maximum
|
||||
primary weight, so that for example the string "smith\\uFFFF" can be used as the
|
||||
upper bound rather than modifying the sort key for "smith".
|
||||
|
||||
## Collation Element Iterator
|
||||
|
||||
The collation element iterator is used for traversing Unicode string collation
|
||||
elements one at a time. It can be used to implement language-sensitive text
|
||||
search algorithms like Boyer-Moore.
|
||||
|
||||
For most applications, the two API categories, compare and sort key, are
|
||||
sufficient. Most people do not need to manipulate collation elements directly.
|
||||
|
||||
Example:
|
||||
|
||||
Consider iterating over "apple" and "äpple". Here are sequences of collation
|
||||
elements:
|
||||
|
||||
String 1 | String 1 Collation Elements
|
||||
-------- | ---------------------------
|
||||
a | `[1900.05.05]`
|
||||
p | `[3700.05.05]`
|
||||
p | `[3700.05.05]`
|
||||
l | `[2F00.05.05]`
|
||||
e | `[2100.05.05]`
|
||||
|
||||
String 2 | String 2 Collation Elements
|
||||
-------- | ---------------------------
|
||||
a | `[1900.05.05]`
|
||||
\\u0308 | `[0000.9D.05]`
|
||||
p | `[3700.05.05]`
|
||||
p | `[3700.05.05]`
|
||||
l | `[2F00.05.05]`
|
||||
e | `[2100.05.05]`
|
||||
|
||||
The resulting CEs are typically masked according to the desired strength, and
|
||||
zero CEs are discarded. In the above example, masking with 0xFFFF0000 (for primary strength)
|
||||
produces the results of NULL secondary and tertiary differences. The collator then
|
||||
ignores the NULL differences and declares a match. For more details see the
|
||||
paper "Efficient text searching in Java™: Finding the right string in any
|
||||
language" by Laura Werner (
|
||||
<http://icu-project.org/docs/papers/efficient_text_searching_in_java.html>).
|
||||
|
||||
## Collation Attributes
|
||||
|
||||
The Collation Service has a number of attributes whose values can be changed
|
||||
during run time. These attributes affect both the functionality and the
|
||||
performance of the Collation Service. This section describes these
|
||||
attributes and, where possible, their performance impact. Performance
|
||||
indications are only approximate and timings may vary significantly depending on
|
||||
the CPU, compiler, etc.
|
||||
|
||||
Although string comparison by ICU and comparison of each string's sort key give
|
||||
the same results, attribute settings can impact the execution time of each
|
||||
method differently. To be precise in the discussion of performance, this section
|
||||
refers to the API employed in the measurement. The `ucol_strcoll` function is the
|
||||
API for string comparison. The `ucol_getSortKey` function is used to create sort
|
||||
keys.
|
||||
|
||||
> :point_right: **Note** There is a special attribute value, `UCOL_DEFAULT`,
|
||||
> that can be used to set any attribute to its default value
|
||||
> (which is inherited from the UCA and the tailoring).
|
||||
|
||||
### Attribute Types
|
||||
|
||||
#### Strength level
|
||||
|
||||
Collation strength, or the maximum collation level used for comparison, is set
|
||||
by using the `UCOL_STRENGTH` attribute. Valid values are:
|
||||
|
||||
1. `UCOL_PRIMARY`
|
||||
|
||||
2. `UCOL_SECONDARY`
|
||||
|
||||
3. `UCOL_TERTIARY` (default)
|
||||
|
||||
4. `UCOL_QUATERNARY`
|
||||
|
||||
5. `UCOL_IDENTICAL`
|
||||
|
||||
#### French collation
|
||||
|
||||
The `UCOL_FRENCH_COLLATION` attribute determines whether to sort the secondary
|
||||
differences in reverse order. Valid values are:
|
||||
|
||||
1. `UCOL_OFF` (default): compares secondary differences in the order they appear
|
||||
in the string.
|
||||
|
||||
2. `UCOL_ON`: causes secondary differences to be considered in reverse order, as
|
||||
it is done in the French language.
|
||||
|
||||
#### Normalization mode
|
||||
|
||||
The `UCOL_NORMALIZATION_MODE` attribute, or its alias `UCOL_DECOMPOSITION_MODE`,
|
||||
controls whether text normalization is performed on the input strings. Valid
|
||||
values are:
|
||||
|
||||
1. `UCOL_OFF` (default): turns off normalization check
|
||||
|
||||
2. `UCOL_ON` : normalization is checked and the collator performs normalization
|
||||
if it is needed.
|
||||
|
||||
X | FCD | NFC | NFD
|
||||
--------------------- | --- | --- | ---
|
||||
A-ring | Y | Y |
|
||||
Angstrom | Y | |
|
||||
A + ring | Y | | Y
|
||||
A + grave | Y | Y |
|
||||
A-ring + grave | Y | |
|
||||
A + cedilla + ring | Y | | Y
|
||||
A + ring + cedilla | | |
|
||||
A-ring + cedilla | | Y |
|
||||
|
||||
With normalization mode turned on, the `ucol_strcoll` function slows down by 10%.
|
||||
In addition, the time to generate a sort key also increases by about 25%.
|
||||
|
||||
#### Alternate handling
|
||||
|
||||
This attribute allows shifting of the variable characters (usually spaces and
|
||||
punctuation, in the UCA also most symbols) from the primary to the quaternary
|
||||
strength level. This is set by using the `UCOL_ALTERNATE_HANDLING` attribute. For
|
||||
details see [UCA: Variable
|
||||
Weighting](http://www.unicode.org/reports/tr10/#Variable_Weighting), [LDML:
|
||||
Collation
|
||||
Settings](http://www.unicode.org/reports/tr35/tr35-collation.html#Collation_Settings),
|
||||
and [“Ignore Punctuation” Options](customization/ignorepunct.md).
|
||||
|
||||
1. `UCOL_NON_IGNORABLE` (CLDR/ICU default): variable characters are treated as
|
||||
all the other characters
|
||||
|
||||
2. `UCOL_SHIFTED` (UCA default): all the variable characters will be ignored at
|
||||
the primary, secondary and tertiary levels and their primary strengths will
|
||||
be shifted to the quaternary level.
|
||||
|
||||
#### Case Ordering
|
||||
|
||||
Some conventions require uppercase letters to sort before lowercase ones, while
|
||||
others require the opposite. This attribute is controlled by the value of the
|
||||
`UCOL_CASE_FIRST`. The case difference in the UCA is contained in the tertiary
|
||||
weights along with other appearance characteristics (like circling of letters).
|
||||
The case-first attribute allows for emphasizing of the case property of the
|
||||
letters by reordering the tertiary weights with either upper-first, and/or
|
||||
lowercase-first. This difference gets the most significant bit in the weight.
|
||||
Valid values for this attribute are:
|
||||
|
||||
1. `UCOL_OFF` (default): leave tertiary weights unaffected
|
||||
|
||||
2. `UCOL_LOWER_FIRST`: causes lowercase letters and uncased characters to sort
|
||||
before uppercase
|
||||
|
||||
3. `UCOL_UPPER_FIRST` : causes uppercase letters to sort first
|
||||
|
||||
The case-first attribute does not affect the performance substantially.
|
||||
|
||||
#### Case level
|
||||
|
||||
When this attribute is set, an additional level is formed between the secondary
|
||||
and tertiary levels, known as the Case Level. The case level is used to
|
||||
distinguish large and small Japanese Kana characters. Case level could also be
|
||||
used in other situations. for example to distinguish certain Pinyin characters.
|
||||
Case level is controlled by `UCOL_CASE_LEVEL` attribute. Valid values for this
|
||||
attribute are
|
||||
|
||||
1. `UCOL_OFF` (default): no additional case level
|
||||
|
||||
2. `UCOL_ON` : adds a case level
|
||||
|
||||
#### Hiragana Quaternary
|
||||
|
||||
*This setting is deprecated and ignored in recent versions of ICU.*
|
||||
|
||||
Hiragana Quaternary can be set to `UCOL_ON`, in which case Hiragana code points
|
||||
will sort before everything else on the quaternary level. If set to `UCOL_OFF`
|
||||
Hiragana letters are treated the same as all the other code points. This setting
|
||||
can be changed on run-time using the `UCOL_HIRAGANA_QUATERNARY_MODE` attribute.
|
||||
You probably won't need to use it.
|
||||
|
||||
#### Variable Top
|
||||
|
||||
Variable Top is a boundary which decides whether the code points will be treated
|
||||
as variable (shifted to quaternary level in the **shifted** mode) or
|
||||
non-ignorable. Special APIs are used for setting of variable top. It can
|
||||
basically be set either to a codepoint or a primary strength value.
|
||||
|
||||
## Performance
|
||||
|
||||
ICU collation is designed to be fast, small and customizable. Several techniques
|
||||
are used to enhance the performance:
|
||||
|
||||
1. Providing optimized processing for Latin characters.
|
||||
|
||||
2. Comparing strings incrementally and stopping at the first significant
|
||||
difference.
|
||||
|
||||
3. Tuning to eliminate unnecessary file access or memory allocation.
|
||||
|
||||
4. Providing efficient preflight functions that allows fast sort key size
|
||||
generation.
|
||||
|
||||
5. Using a single, shared copy of UCA in memory for the read-only default sort
|
||||
order. Only small tailoring tables are kept in memory for locale-specific
|
||||
customization.
|
||||
|
||||
6. Compressing sort keys efficiently.
|
||||
|
||||
7. Making the sort order be data-driven.
|
||||
|
||||
In general, the best performance from the Collation Service is expected by
|
||||
doing the following:
|
||||
|
||||
1. After opening a collator, keep and reuse it until done. Do not open new
|
||||
collators for the same sort order. (Note the restriction on
|
||||
multi-threading.)
|
||||
|
||||
2. Use `ucol_strcoll` etc. when comparing strings. If it is necessary to
|
||||
compare strings thousands or millions of times,
|
||||
create the sort keys first and compare the sort keys instead.
|
||||
Generating the sort keys of two strings is about 5-10
|
||||
times slower than just comparing them directly.
|
||||
|
||||
3. Follow the best practice guidelines for generating sort keys. Do not call
|
||||
`ucol_getSortKey` twice to first size the key and then allocate the sort key
|
||||
buffer and repeat the call to the function to fill in the buffer.
|
||||
|
||||
### Performance and Storage Implications of Attributes
|
||||
|
||||
Most people use the default attributes when comparing strings or when creating
|
||||
sort keys. When they do want to customize the ordering, the most common options
|
||||
are the following :
|
||||
|
||||
`UCOL_ALTERNATE_HANDLING == UCOL_SHIFTED`\
|
||||
Used to ignore space and punctuation characters
|
||||
|
||||
`UCOL_ALTERNATE_HANDLING == UCOL_SHIFTED` **and** `UCOL_STRENGTH == UCOL_QUATERNARY`\
|
||||
Used to ignore the space and punctuation characters except when there are no previous letter, accent, or case/variable differences.
|
||||
|
||||
`UCOL_CASE_FIRST == UCOL_LOWER_FIRST` **or** `UCOL_CASE_FIRST == UCOL_UPPER_FIRST`\
|
||||
Used to change the ordering of upper vs. lower case letters (as
|
||||
well as small vs. large kana)
|
||||
|
||||
`UCOL_CASE_LEVEL == UCOL_ON` **and** `UCOL_STRENGTH == UCOL_PRIMARY`\
|
||||
Used to ignore only the accent differences.
|
||||
|
||||
`UCOL_NORMALIZATION_MODE == UCOL_ON`\
|
||||
Force to always check for normalization. This
|
||||
is used if the input text may not be in FCD form.
|
||||
|
||||
`UCOL_FRENCH_COLLATION == UCOL_OFF`\
|
||||
This is only useful for languages like French and Catalan that may turn this attribute on.
|
||||
(It is the default only for Canadian French ("fr-CA").)
|
||||
|
||||
In String Comparison, most of these options have little or no effect on
|
||||
performance. The only noticeable one is normalization, which can cost 10%-40% in
|
||||
performance.
|
||||
|
||||
For Sort Keys, most of these options either leave the storage alone or reduce
|
||||
it. Shifting can reduce the storage by about 10%-20%; case level + primary-only
|
||||
can decrease it about 20% to 40%. Using no French accents can reduce the storage
|
||||
by about 38% , but only for languages like French and Catalan that turn it on by
|
||||
default. On the other hand, using Shifted + Quaternary can increase the storage by
|
||||
10%-15%. (The Identical Level also increases the length, but this option is not
|
||||
recommended).
|
||||
|
||||
> :point_right: **Note** All of the above numbers are based on
|
||||
> tests run on a particular machine, with a particular set of data.
|
||||
> (The data for each language is a large number of names
|
||||
> in that language in the format <first_name>, <last name>.)
|
||||
> The performance and storage may vary, depending on the particular computer,
|
||||
> operating system, and data.
|
||||
|
||||
## Versioning
|
||||
|
||||
Sort keys are often stored on disk for later reuse. A common example is the use
|
||||
of keys to build indexes in databases. When comparing keys, it is important to
|
||||
know that both keys were generated by the same algorithms and weightings.
|
||||
Otherwise, identical strings with keys generated on two different dates, for
|
||||
example, might compare as unequal. Sort keys can be affected by new versions of
|
||||
ICU or its data tables, new sort key formats, or changes to the Collator.
|
||||
Starting with release 1.8.1, ICU provides a versioning mechanism to identify the
|
||||
version information of the following (but not limited to),
|
||||
|
||||
1. The run-time executable
|
||||
|
||||
2. The collation element content
|
||||
|
||||
3. The Unicode/UCA database
|
||||
|
||||
4. The tailoring table
|
||||
|
||||
The version information of Collator is a 32-bit integer. If a new version of ICU
|
||||
has changes affecting the content of collation elements, the version information
|
||||
will be changed. In that case, to use the new version of ICU collator will
|
||||
require regenerating any saved or stored sort keys.
|
||||
|
||||
However, it is possible to modify ICU code or data without changing relevant version numbers,
|
||||
so it is safer to regenerate sort keys any time after any part of ICU has been updated.
|
||||
|
||||
Since ICU4C 1.8.1.
|
||||
it is possible to build your program so that it uses more than one version of
|
||||
ICU (only in C/C++, not in Java). Therefore, you could use the current version
|
||||
for the features you need and use the older version for collation.
|
||||
|
||||
## Programming Examples
|
||||
|
||||
See the [Collation Examples](examples.md) chapter for an example of how to
|
||||
compare and create sort keys with the default locale in C, C++ and Java.
|
||||
@@ -1,831 +0,0 @@
|
||||
---
|
||||
layout: default
|
||||
title: Concepts
|
||||
nav_order: 1
|
||||
parent: Collation
|
||||
---
|
||||
<!--
|
||||
© 2020 and later: Unicode, Inc. and others.
|
||||
License & terms of use: http://www.unicode.org/copyright.html
|
||||
-->
|
||||
|
||||
# Collation Concepts
|
||||
{: .no_toc }
|
||||
|
||||
## Contents
|
||||
{: .no_toc .text-delta }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
The previous section demonstrated many of the requirements imposed on string
|
||||
comparison routines that try to correctly collate strings according to
|
||||
conventions of more than a hundred different languages, written in many
|
||||
different scripts. This section describes the principles and architecture behind
|
||||
the ICU Collation Service.
|
||||
|
||||
## Sortkeys vs Comparison
|
||||
|
||||
Sort keys are most useful in databases, where the overhead of calling a function
|
||||
for each comparison is very large.
|
||||
|
||||
Generating a sort key from a Collator is many times more expensive than doing a
|
||||
compare with the Collator (for common use cases). That's if the two functions
|
||||
are called from Java or C. So for those languages, unless there is a very large
|
||||
number of comparisons, it is better to call the compare function.
|
||||
|
||||
Here is an example, with a little back-of-the-envelope calculation. Let's
|
||||
suppose that with a given language on a given platform, the compare performance
|
||||
(CP) is 100 faster than sortKey performance (SP), and that you are doing a
|
||||
binary search of a list with 1,000 elements. The binary comparison performance
|
||||
is BP. We'd do about 10 comparisons, getting:
|
||||
|
||||
compare: 10 \* CP
|
||||
|
||||
sortkey: 1 \* SP + 10 \* BP
|
||||
|
||||
Even if BP is free, compare would be better. One has to get up to where log2(n)
|
||||
= 100 before they break even.
|
||||
|
||||
But even this calculation is only a rough guide. First, the binary comparison is
|
||||
not completely free. Secondly, the performance of compare function varies
|
||||
radically with the source data. We optimized for maximizing performance of
|
||||
collation in sorting and binary search, so comparing strings that are "close" is
|
||||
optimized to be much faster than comparing strings that are "far away". That
|
||||
optimization is important because normal sort/lookup operations compare close
|
||||
strings far more often -- think of binary search, where the last few comparisons
|
||||
are always with the closest strings. So even the above calculation is not very
|
||||
accurate.
|
||||
|
||||
## Comparison Levels
|
||||
|
||||
In general, when comparing and sorting objects, some properties can take
|
||||
precedence over others. For example, in geometry, you might consider first the
|
||||
number of sides a shape has, followed by the number of sides of equal length.
|
||||
This causes triangles to be sorted together, then rectangles, then pentagons,
|
||||
etc. Within each category, the shapes would be ordered according to whether they
|
||||
had 0, 2, 3 or more sides of the same length. However, this is not the only way
|
||||
the shapes can be sorted. For example, it might be preferable to sort shapes by
|
||||
color first, so that all red shapes are grouped together, then blue, etc.
|
||||
Another approach would be to sort the shapes by the amount of area they enclose.
|
||||
|
||||
Similarly, character strings have properties, some of which can take precedence
|
||||
over others. There is more than one way to prioritize the properties.
|
||||
|
||||
For example, a common approach is to distinguish characters first by their
|
||||
unadorned base letter (for example, without accents, vowels or tone marks), then
|
||||
by accents, and then by the case of the letter (upper vs. lower). Ideographic
|
||||
characters might be sorted by their component radicals and then by the number of
|
||||
strokes it takes to draw the character.
|
||||
An alternative ordering would be to sort these characters by strokes first and
|
||||
then by their radicals.
|
||||
|
||||
The ICU Collation Service supports many levels of comparison (named "Levels",
|
||||
but also known as "Strengths"). Having these categories enables ICU to sort
|
||||
strings precisely according to local conventions. However, by allowing the
|
||||
levels to be selectively employed, searching for a string in text can be
|
||||
performed with various matching conditions.
|
||||
|
||||
Performance optimizations have been made for ICU collation with the default
|
||||
level settings. Performance specific impacts are discussed in the Performance
|
||||
section below.
|
||||
|
||||
Following is a list of the names for each level and an example usage:
|
||||
|
||||
1. Primary Level: Typically, this is used to denote differences between base
|
||||
characters (for example, "a" < "b"). It is the strongest difference. For
|
||||
example, dictionaries are divided into different sections by base character.
|
||||
This is also called the level-1 strength.
|
||||
|
||||
2. Secondary Level: Accents in the characters are considered secondary
|
||||
differences (for example, "as" < "às" < "at"). Other differences between
|
||||
letters can also be considered secondary differences, depending on the
|
||||
language. A secondary difference is ignored when there is a primary
|
||||
difference anywhere in the strings. This is also called the level-2
|
||||
strength.
|
||||
Note: In some languages (such as Danish), certain accented letters are
|
||||
considered to be separate base characters. In most languages, however, an
|
||||
accented letter only has a secondary difference from the unaccented version
|
||||
of that letter.
|
||||
|
||||
3. Tertiary Level: Upper and lower case differences in characters are
|
||||
distinguished at the tertiary level (for example, "ao" < "Ao" < "aò"). In
|
||||
addition, a variant of a letter differs from the base form on the tertiary
|
||||
level (such as "A" and "Ⓐ"). Another example is the difference between large
|
||||
and small Kana. A tertiary difference is ignored when there is a primary or
|
||||
secondary difference anywhere in the strings. This is also called the
|
||||
level-3 strength.
|
||||
|
||||
4. Quaternary Level: When punctuation is ignored (see Ignoring Punctuations
|
||||
(§)) at level 1-3, an additional level can be used to distinguish words with
|
||||
and without punctuation (for example, "ab" < "a-b" < "aB"). This difference
|
||||
is ignored when there is a primary, secondary or tertiary difference. This
|
||||
is also known as the level-4 strength. The quaternary level should only be
|
||||
used if ignoring punctuation is required or when processing Japanese text
|
||||
(see Hiragana processing (§)).
|
||||
|
||||
5. Identical Level: When all other levels are equal, the identical level is
|
||||
used as a tiebreaker. The Unicode code point values of the NFD form of each
|
||||
string are compared at this level, just in case there is no difference at
|
||||
levels 1-4. For example, Hebrew cantillation marks are only distinguished
|
||||
at this level. This level should be used sparingly, as only code point
|
||||
value differences between two strings is an extremely rare occurrence.
|
||||
Using this level substantially decreases the performance for
|
||||
both incremental comparison and sort key generation (as well as increasing
|
||||
the sort key length). It is also known as level 5 strength.
|
||||
|
||||
## Backward Secondary Sorting
|
||||
|
||||
Some languages require words to be ordered on the secondary level according to
|
||||
the *last* accent difference, as opposed to the *first* accent difference. This
|
||||
was previously the default for all French locales, based on some French
|
||||
dictionary ordering traditions, but is currently only applicable to Canadian
|
||||
French (locale **fr_CA**), for conformance with the [Canadian sorting
|
||||
standard](http://www.unicode.org/reports/tr10/#CanStd). The difference in
|
||||
ordering is only noticeable for a small number of pairs of real words. For more
|
||||
information see [UCA: Contextual
|
||||
Sensitivity](http://www.unicode.org/reports/tr10/#Contextual_Sensitivity).
|
||||
|
||||
Example:
|
||||
|
||||
Forward secondary | Backward secondary
|
||||
----------------- | ------------------
|
||||
cote | cote
|
||||
coté | côte
|
||||
côte | coté
|
||||
côté | côté
|
||||
|
||||
## Contractions
|
||||
|
||||
A contraction is a sequence consisting of two or more letters. It is considered
|
||||
a single letter in sorting.
|
||||
|
||||
For example, in the traditional Spanish sorting order, "ch" is considered a
|
||||
single letter. All words that begin with "ch" sort after all other words
|
||||
beginning with "c", but before words starting with "d".
|
||||
|
||||
Other examples of contractions are "ch" in Czech, which sorts after "h", and
|
||||
"lj" and "nj" in Croatian and Latin Serbian, which sort after "l" and "n"
|
||||
respectively.
|
||||
|
||||
Example:
|
||||
|
||||
Order without contraction | Order with contraction "lj" sorting after letter "l"
|
||||
------------------------- | ----------------------------------------------------
|
||||
la | la
|
||||
li | li
|
||||
lj | lk
|
||||
lja | lz
|
||||
ljz | lj
|
||||
lk | lja
|
||||
lz | ljz
|
||||
ma | ma
|
||||
|
||||
Contracting sequences such as the above are not very common in most languages.
|
||||
|
||||
> :point_right: **Note** Since ICU 2.2, and as required by the UCA,
|
||||
> if a completely ignorable code point
|
||||
> appears in text in the middle of contraction, it will not break the contraction.
|
||||
> For example, in Czech sorting, cU+0000h will sort as it were ch.
|
||||
|
||||
## Expansions
|
||||
|
||||
If a letter sorts as if it were a sequence of more than one letter, it is called
|
||||
an expansion.
|
||||
|
||||
For example, in German phonebook sorting (de@collation=phonebook or BCP 47
|
||||
de-u-co-phonebk), "ä" sorts as though it were equivalent to the sequence "ae."
|
||||
All words starting with "ä" will sort between words starting with "ad" and words
|
||||
starting with "af".
|
||||
|
||||
In the case of Unicode encoding, characters can often be represented either as
|
||||
pre-composed characters or in decomposed form. For example, the letter "à" can
|
||||
be represented in its decomposed (a+\`) and pre-composed (à) form. Most
|
||||
applications do not want to distinguish text by the way it is encoded. A search
|
||||
for "à" should find all instances of the letter, regardless of whether the
|
||||
instance is in pre-composed or decomposed form. Therefore, either form of the
|
||||
letter must result in the same sort ordering. The architecture of the ICU
|
||||
Collation Service supports this.
|
||||
|
||||
## Contractions Producing Expansions
|
||||
|
||||
It is possible to have contractions that produce expansions.
|
||||
|
||||
One example occurs in Japanese, where the vowel with a prolonged sound mark is
|
||||
treated to be equivalent to the long vowel version:
|
||||
|
||||
カアー<<< カイー and\
|
||||
キイー<<< キイー
|
||||
|
||||
> :point_right: **Note** Since ICU 2.0 Japanese tailoring uses
|
||||
> [prefix analysis](http://www.unicode.org/reports/tr35/tr35-collation.html#Context_Sensitive_Mappings)
|
||||
> instead of contraction producing expansions.
|
||||
|
||||
## Normalization
|
||||
|
||||
In the section on expansions, we discussed that text in Unicode can often be
|
||||
represented in either pre-composed or decomposed forms. There are other types of
|
||||
equivalences possible with Unicode, including Canonical and Compatibility. The
|
||||
process of
|
||||
Normalization ensures that text is written in a predictable way so that searches
|
||||
are not made unnecessarily complicated by having to match on equivalences. Not
|
||||
all text is normalized, however, so it is useful to have a collation service
|
||||
that can address text that is not normalized, but do so with efficiency.
|
||||
|
||||
The ICU Collation Service handles un-normalized text properly, producing the
|
||||
same results as if the text were normalized.
|
||||
|
||||
In practice, most data that is encountered is in normalized or semi-normalized
|
||||
form already. The ICU Collation Service is designed so that it can process a
|
||||
wide range of normalized or un-normalized text without a need for normalization
|
||||
processing. When a case is encountered that requires normalization, the ICU
|
||||
Collation Service drops into code specific to this purpose. This maximizes
|
||||
performance for the majority of text that does not require normalization.
|
||||
|
||||
In addition, if the text is known with certainty not to contain un-normalized
|
||||
text, then even the overhead of checking for normalization can be eliminated.
|
||||
The ICU Collation Service has the ability to turn Normalization Checking either
|
||||
on or off. If Normalization Checking is turned off, it is the user's
|
||||
responsibility to insure that all text is already in the appropriate form. This
|
||||
is true in a great majority of the world languages, so normalization checking is
|
||||
turned off by default for most locales.
|
||||
|
||||
If the text requires normalization processing, Normalization Checking should be
|
||||
on. Any language that uses multiple combining characters such as Arabic, ancient
|
||||
Greek, Hebrew, Hindi, Thai or Vietnamese either requires Normalization Checking
|
||||
to be on, or the text to go through a normalization process before collation.
|
||||
|
||||
For more information about Normalization related reordering please see
|
||||
[Unicode Technical Note #5](http://www.unicode.org/notes/tn5/) and
|
||||
[UAX #15.](http://www.unicode.org/reports/tr15/)
|
||||
|
||||
> :point_right: **Note** ICU supports two modes of normalization: on and off.
|
||||
> Java.text.\* classes offer compatibility decomposition mode, which is not supported in ICU.
|
||||
|
||||
## Ignoring Punctuation
|
||||
|
||||
In some cases, punctuation can be ignored while searching or sorting data. For
|
||||
example, this enables a search for "biweekly" to also return instances of
|
||||
"bi-weekly". In other cases, it is desirable for punctuated text to be
|
||||
distinguished from text without punctuation, but to have the text sort close
|
||||
together.
|
||||
|
||||
These two behaviors can be accomplished if there is a way for a character to be
|
||||
ignored on all levels except for the quaternary level. If this is the case, then
|
||||
two strings which compare as identical on the first three levels (base letter,
|
||||
accents, and case) are then distinguished at the fourth level based on their
|
||||
punctuation (if any). If the comparison function ignores differences at the
|
||||
fourth level, then strings that differ by punctuation only are compared as
|
||||
equal.
|
||||
|
||||
The following table shows the results of sorting a list of terms in 3 different
|
||||
ways. In the first column, punctuation characters (space " ", and hyphen "-")
|
||||
are not ignored (" " < "-" < "b"). In the second column, punctuation characters
|
||||
are ignored in the first 3 levels and compared only in the fourth level. In the
|
||||
third column, punctuation characters are ignored in the first 3 levels and the
|
||||
fourth level is not considered. In the last column, punctuated terms are
|
||||
equivalent to the identical terms without punctuation.
|
||||
|
||||
For more options and details see the [“Ignore Punctuation”
|
||||
Options](customization/ignorepunct.md) page.
|
||||
|
||||
Non-ignorable | Ignorable and Quaternary strength | Ignorable and Tertiary strength
|
||||
------------- | --------------------------------- | -------------------------------
|
||||
black bird | black bird | **black bird**
|
||||
black Bird | black-bird | **black-bird**
|
||||
black birds | blackbird | **blackbird**
|
||||
black-bird | black Bird | black Bird
|
||||
black-Bird | black-Bird | black-Bird
|
||||
black-birds | blackBird | blackBird
|
||||
blackbird | black birds | black birds
|
||||
blackBird | black-birds | black-birds
|
||||
blackbirds | blackbirds | blackbirds
|
||||
|
||||
> :point_right: **Note** The strings with the same font format in the last column are
|
||||
compared as equal by ICU Collator.\
|
||||
> Since ICU 2.2 and as prescribed by the UCA, primary ignorable code points that
|
||||
> follow shifted code points will be completely ignored. This means that an accent
|
||||
> following a space will compare as if it was a space alone.
|
||||
|
||||
## Case Ordering
|
||||
|
||||
The tertiary level is used to distinguish text by case, by small versus large
|
||||
Kana, and other letter variants as noted above.
|
||||
|
||||
Some applications prefer to emphasize case differences so that words starting
|
||||
with the same case sort together. Some Japanese applications require the
|
||||
difference between small and large Kana be emphasized over other tertiary
|
||||
differences.
|
||||
|
||||
The UCA does not provide means to separate out either case or Kana differences
|
||||
from the remaining tertiary differences. However, the ICU Collation Service has
|
||||
two options that help in customize case and/or Kana differences. Both options
|
||||
are turned off by default.
|
||||
|
||||
### CaseFirst
|
||||
|
||||
The Case-first option makes case the most significant part of the tertiary
|
||||
level. Primary and secondary levels are unaffected. With this option, words
|
||||
starting with the same case sort together. The Case-first option can be set to
|
||||
make either lowercase sort before
|
||||
uppercase or uppercase sort before lowercase.
|
||||
|
||||
Note: The case-first option does not constitute a separate level; it is simply a
|
||||
reordering of the tertiary level.
|
||||
|
||||
ICU makes use of the following three case categories for sorting
|
||||
|
||||
1. uppercase: "ABC"
|
||||
|
||||
2. mixed case: "Abc", "aBc"
|
||||
|
||||
3. normal (lowercase or no case): "abc", "123"
|
||||
|
||||
Mixed case is always sorted between uppercase and normal case when the
|
||||
"case-first" option is set.
|
||||
|
||||
### CaseLevel
|
||||
|
||||
The Case Level option makes a separate level for case differences. This is an
|
||||
extra level positioned between secondary and tertiary. The case level is used in
|
||||
Japanese to make the difference between small and large Kana more important than
|
||||
the other tertiary differences. It also can be used to ignore other tertiary
|
||||
differences, or even secondary differences. This is especially useful in
|
||||
matching. For example, if the strength is set to primary only (level-1) and the
|
||||
case level is turned on, the comparison ignores accents and tertiary differences
|
||||
except for case. The contents of the case level are affected by the case-first
|
||||
option.
|
||||
|
||||
The case level is independent from the strength of comparison. It is possible to
|
||||
have a collator set to primary strength with the case level turned on. This
|
||||
provides for comparison that takes into account the case differences, while at
|
||||
the same time ignoring accents and tertiary differences other than case. This
|
||||
may be used in searching.
|
||||
|
||||
Example:
|
||||
|
||||
**Case-first off, Case level off**
|
||||
|
||||
apple\
|
||||
ⓐⓟⓟⓛⓔ\
|
||||
Abernathy\
|
||||
ⒶⒷⒺⓇⓃⒶⓉⒽⓎ\
|
||||
ähnlich\
|
||||
Ähnlichkeit
|
||||
|
||||
**Lowercase-first, Case level off**
|
||||
|
||||
apple\
|
||||
ⓐⓟⓟⓛⓔ\
|
||||
ähnlich\
|
||||
Abernathy\
|
||||
ⒶⒷⒺⓇⓃⒶⓉⒽⓎ\
|
||||
Ähnlichkeit
|
||||
|
||||
**Uppercase-first, Case level off**
|
||||
|
||||
Abernathy\
|
||||
ⒶⒷⒺⓇⓃⒶⓉⒽⓎ\
|
||||
Ähnlichkeit\
|
||||
apple\
|
||||
ⓐⓟⓟⓛⓔ\
|
||||
ähnlich
|
||||
|
||||
**Lowercase-first, Case level on**
|
||||
|
||||
apple\
|
||||
Abernathy\
|
||||
ⓐⓟⓟⓛⓔ\
|
||||
ⒶⒷⒺⓇⓃⒶⓉⒽⓎ\
|
||||
ähnlich\
|
||||
Ähnlichkeit
|
||||
|
||||
**Uppercase-first, Case level on**
|
||||
|
||||
Abernathy\
|
||||
apple\
|
||||
ⒶⒷⒺⓇⓃⒶⓉⒽⓎ\
|
||||
ⓐⓟⓟⓛⓔ\
|
||||
Ähnlichkeit\
|
||||
ähnlich
|
||||
|
||||
## Script Reordering
|
||||
|
||||
Script reordering allows scripts and some other groups of characters to be moved
|
||||
relative to each other. This reordering is done on top of the DUCET/CLDR
|
||||
standard collation order. Reordering can specify groups to be placed at the
|
||||
start and/or the end of the collation order.
|
||||
|
||||
By default, reordering codes specified for the start of the order are placed in
|
||||
the order given after several special non-script blocks. These special groups of
|
||||
characters are space, punctuation, symbol, currency, and digit. Script groups
|
||||
can be intermingled with these special non-script groups if those special groups
|
||||
are explicitly specified in the reordering.
|
||||
|
||||
The special code `others` stands for any script that is not explicitly mentioned
|
||||
in the list. Anything that is after others will go at the very end of the list
|
||||
in the order given. For example, `[Grek, others, Latn]` will result in an
|
||||
ordering that puts all scripts other than Greek and Latin between them.
|
||||
|
||||
### Examples:
|
||||
|
||||
Note: All examples below use the string equivalents for the scripts and reorder
|
||||
codes that would be used in collator rules. The script and reorder code
|
||||
constants that would be used in API calls will be different.
|
||||
|
||||
**Example 1:**\
|
||||
set reorder code - `[Grek]`\
|
||||
result - `[space, punctuation, symbol, currency, digit, Grek, others]`
|
||||
|
||||
**Example 2:**\
|
||||
set reorder code - `[Grek]`\
|
||||
result - `[space, punctuation, symbol, currency, digit, Grek, others]`
|
||||
|
||||
followed by: set reorder code - `[Hani]`\
|
||||
result -` [space, punctuation, symbol, currency, digit, Hani, others]`
|
||||
|
||||
That is, setting a reordering always modifies
|
||||
the DUCET/CLDR order, replacing whatever was previously set, rather than adding
|
||||
on to it. In order to cumulatively modify an ordering, you have to retrieve the
|
||||
existing ordering, modify it, and then set it.
|
||||
|
||||
**Example 3:**\
|
||||
set reorder code - `[others, digit]`\
|
||||
result - `[space, punctuation, symbol, currency, others, digit]`
|
||||
|
||||
**Example 4:**\
|
||||
set reorder code - `[space, Grek, punctuation]`\
|
||||
result - `[symbol, currency, digit, space, Grek, punctuation, others]`
|
||||
|
||||
**Example 5:**\
|
||||
set reorder code - `[Grek, others, Hani]`\
|
||||
result - `[space, punctuation, symbol, currency, digit, Grek, others, Hani]`
|
||||
|
||||
**Example 6:**\
|
||||
set reorder code - `[Grek, others, Hani, symbol, Tglg]`\
|
||||
result - `[space, punctuation, currency, digit, Grek, others, Hani, symbol, Tglg]`
|
||||
|
||||
followed by:\
|
||||
set reorder code - `[NONE]`\
|
||||
result - DUCET/CLDR
|
||||
|
||||
**Example 7:**\
|
||||
set reorder code - `[Grek, others, Hani, symbol, Tglg]`\
|
||||
result - `[space, punctuation, currency, digit, Grek, others, Hani, symbol, Tglg]`
|
||||
|
||||
followed by:\
|
||||
set reorder code - `[DEFAULT]`\
|
||||
result - original reordering for the locale which may or may not be DUCET/CLDR
|
||||
|
||||
**Example 8:**\
|
||||
set reorder code - `[Grek, others, Hani, symbol, Tglg]`\
|
||||
result - `[space, punctuation, currency, digit, Grek, others, Hani, symbol, Tglg]`
|
||||
|
||||
followed by:\
|
||||
set reorder code - `[]`\
|
||||
result - original reordering for the locale which may or may not be DUCET/CLDR
|
||||
|
||||
**Example 9:**\
|
||||
set reorder code - `[Hebr, Phnx]`\
|
||||
result - error
|
||||
|
||||
Beginning with ICU 55, scripts only reorder together if they are primary-equal,
|
||||
for example Hiragana and Katakana.
|
||||
|
||||
ICU 4.8-54:
|
||||
|
||||
* Scripts were reordered in groups, each normally starting with a [Recommended
|
||||
Script](http://www.unicode.org/reports/tr31/#Table_Recommended_Scripts).
|
||||
* Reorder codes moved as a group (were “equivalent”) if their scripts shared a
|
||||
primary-weight lead byte.
|
||||
* For example, Hebr and Phnx were “equivalent” reordering codes and were
|
||||
reordered together. Their order relative to each other could not be changed.
|
||||
* Only any one code out of any group could be reordered, not multiple of the
|
||||
same group.
|
||||
|
||||
## Sorting of Japanese Text (JIS X 4061)
|
||||
|
||||
Japanese standard JIS X 4061 requires two changes to the collation procedures:
|
||||
special processing of Hiragana characters and (for performance reasons) prefix
|
||||
analysis of text.
|
||||
|
||||
### Hiragana Processing
|
||||
|
||||
JIS X 4061 standard requires more levels than provided by the UCA. To offer
|
||||
conformant sorting order, ICU uses the quaternary level to distinguish between
|
||||
Hiragana and Katakana. Hiragana symbols are given smaller values than Katakana
|
||||
symbols on quaternary level, thus causing Hiragana sequences to sort before
|
||||
corresponding Katakana sequences.
|
||||
|
||||
### Prefix Analysis
|
||||
|
||||
Another characteristics of sorting according to the JIS X 4061 is a large number
|
||||
of contractions followed by expansions (see
|
||||
[Contractions Producing Expansions](#contractions-producing-expansions)).
|
||||
This causes all the Hiragana and Katakana codepoints to be treated as
|
||||
contractions, which reduces performance. The solution we adopted introduces the
|
||||
prefix concept which allows us to improve the performance of Japanese sorting.
|
||||
More about this can be found in the [customization
|
||||
chapter](customization/index.md) .
|
||||
|
||||
## Thai/Lao reordering
|
||||
|
||||
UCA requires that certain Thai and Lao prevowels be reordered with a code point
|
||||
following them. This option is always on in the ICU implementation, as
|
||||
prescribed by the UCA.
|
||||
|
||||
This rule takes effect when:
|
||||
|
||||
1. A Thai vowel of the range \\U0E40-\\U0E44 precedes a Thai consonant of the
|
||||
range \\U0E01-\\U0E2E
|
||||
or
|
||||
|
||||
2. A Lao vowel of the range \\U0EC0-\\U0EC4 precedes a Lao consonant of the
|
||||
range \\U0E81-\\U0EAE. In these cases the vowel is placed after the
|
||||
consonant for collation purposes.
|
||||
|
||||
> :point_right: **Note** There is a difference between java.text.\* classes and ICU in regard to Thai
|
||||
> reordering. Java.text.\* classes allow tailorings to turn off reordering by
|
||||
> using the '!' modifier. ICU ignores the '!' modifier and always reorders Thai
|
||||
> prevowels.
|
||||
|
||||
## Space Padding
|
||||
|
||||
In many database products, fields are padded with null. To get correct results,
|
||||
the input to a Collator should omit any superfluous trailing padding spaces. The
|
||||
problem arises with contractions, expansions, or normalization. Suppose that
|
||||
there are two fields, one containing "aed" and the other with "äd". German
|
||||
phonebook sorting (de@collation=phonebook or BCP 47 de-u-co-phonebk) will
|
||||
compare "ä" as if it were "ae" (on a primary level), so the order will be "äd" <
|
||||
"aed". But if both fields are padded with spaces to a length of 3, then this
|
||||
will reverse the order, since the first will compare as if it were one character
|
||||
longer. In other words, when you start with strings 1 and 2
|
||||
|
||||
1 | a | e | d | \<space\>
|
||||
-- | -- | -- | --------- | ---------
|
||||
2 | ä | d | \<space\> | \<space\>
|
||||
|
||||
they end up being compared on a primary level as if they were 1' and 2'
|
||||
|
||||
1' | a | e | d | \<space\> |
|
||||
-- | -- | -- | -- | --------- | ---------
|
||||
2' | a | e | d | \<space\> | \<space\>
|
||||
|
||||
Since 2' has an extra character (the extra space), it counts as having a primary
|
||||
difference when it shouldn't. The correct result occurs when the trailing
|
||||
padding spaces are removed, as in 1" and 2"
|
||||
|
||||
1" | a | e | d
|
||||
-- | -- | -- | --
|
||||
2" | a | e | d
|
||||
|
||||
## Collator naming scheme
|
||||
|
||||
***Starting with ICU 54, the following naming scheme and its API functions are deprecated.***
|
||||
Use `ucol_open()` with language tag collation keywords instead
|
||||
(see [Collation API Details](api.md)). For example,
|
||||
`ucol_open("de-u-co-phonebk-ka-shifted", &errorCode)` for German Phonebook order
|
||||
with "ignore punctuation" mode.
|
||||
|
||||
When collating or matching text, a number of attributes can be used to affect
|
||||
the desired result. The following describes the attributes, their values, their
|
||||
effects, their normal usage, and the string comparison performance and sort key
|
||||
length implications. It also includes single-letter abbreviations for both the
|
||||
attributes and their values. These abbreviations allow a 'short-form'
|
||||
specification of a set of collation options, such as "UCA4.0.0_AS_LSV_S", which
|
||||
can be used to specific that the desired options are: UCA version 4.0.0; ignore
|
||||
spaces, punctuation and symbols; use Swedish linguistic conventions; compare
|
||||
case-insensitively.
|
||||
|
||||
A number of attribute values are common across different attributes; these
|
||||
include **Default** (abbreviated as D), **On** (O), and **Off** (X). Unless
|
||||
otherwise stated, the examples use the UCA alone with default settings.
|
||||
|
||||
> :point_right: **Note** In order to achieve uniqueness, a collator name always
|
||||
> has the attribute abbreviations sorted.
|
||||
|
||||
### Main References
|
||||
|
||||
1. For a full list of supported locales in ICU, see [Locale
|
||||
Explorer](https://icu4c-demos.unicode.org/icu-bin/locexp) , which also contains
|
||||
an on-line demo showing sorting for each locale. The demo allows you to try
|
||||
different attribute values, to see how they affect sorting.
|
||||
|
||||
2. To see tabular results for the UCA table itself, see the [Unicode Collation
|
||||
Charts](http://www.unicode.org/charts/collation/) .
|
||||
|
||||
3. For the UCA specification, see [UTS #10: Unicode Collation
|
||||
Algorithm](http://www.unicode.org/reports/tr10/) .
|
||||
|
||||
4. For more detail on the precise effects of these options, see [Collation
|
||||
Customization](customization/index.md) .
|
||||
|
||||
#### Collator Naming Attributes
|
||||
|
||||
Attribute | Abbreviation | Possible Values
|
||||
---------------------- | ------------ | ---------------
|
||||
Locale | L | \<language\>
|
||||
Script | Z | \<script\>
|
||||
Region | R | \<region\>
|
||||
Variant | V | \<variant\>
|
||||
Keyword | K | \<keyword\>
|
||||
| |
|
||||
Strength | S | 1, 2, 3, 4, I, D
|
||||
Case_Level | E | X, O, D
|
||||
Case_First | C | X, L, U, D
|
||||
Alternate | A | N, S, D
|
||||
Variable_Top | T | \<hex digits\>
|
||||
Normalization Checking | N | X, O, D
|
||||
French | F | X, O, D
|
||||
Hiragana | H | X, O, D
|
||||
|
||||
#### Collator Naming Attribute Descriptions
|
||||
|
||||
The **Locale** attribute is typically the most
|
||||
important attribute for correct sorting and matching, according to the user
|
||||
expectations in different countries and regions. The default UCA ordering will
|
||||
only sort a few languages such as Dutch and Portuguese correctly ("correctly"
|
||||
meaning according to the normal expectations for users of the languages).
|
||||
Otherwise, you need to supply the locale to UCA in order to properly collate
|
||||
text for a given language. Thus a locale needs to be supplied so as to choose a
|
||||
collator that is correctly **tailored** for that locale. The choice of a locale
|
||||
will automatically preset the values for all of the attributes to something that
|
||||
is reasonable for that locale. Thus most of the time the other attributes do not
|
||||
need to be explicitly set. In some cases, the choice of locale will make a
|
||||
difference in string comparison performance and/or sort key length.
|
||||
|
||||
In short attribute names,
|
||||
`<language>_<script>_<region>_<variant>@collation=<keyword>` is
|
||||
represented by: `L<language>_Z<script>_R<region>_V<variant>_K<keyword>`. Not
|
||||
all the elements are required. Valid values for locale elements are general
|
||||
valid values for RFC 3066 locale naming.
|
||||
|
||||
**Example:**\
|
||||
**Locale="sv" (Swedish)** "Kypper" < "Köpfe"\
|
||||
**Locale="de" (German)** "Köpfe" < "Kypper"
|
||||
|
||||
The **Strength** attribute determines whether accents or
|
||||
case are taken into account when collating or matching text. ( (In writing
|
||||
systems without case or accents, it controls similarly important features). The
|
||||
default strength setting usually does not need to be changed for collating
|
||||
(sorting), but often needs to be changed when **matching** (e.g. SELECT). The
|
||||
possible values include Default (D), Primary (1), Secondary (2), Tertiary (3),
|
||||
Quaternary (4), and Identical (I).
|
||||
|
||||
For example, people may choose to ignore accents or ignore accents and case when
|
||||
searching for text.
|
||||
|
||||
Almost all characters are distinguished by the first three levels, and in most
|
||||
locales the default value is thus Tertiary. However, if Alternate is set to be
|
||||
Shifted, then the Quaternary strength (4) can be used to break ties among
|
||||
whitespace, punctuation, and symbols that would otherwise be ignored. If very
|
||||
fine distinctions among characters are required, then the Identical strength (I)
|
||||
can be used (for example, Identical Strength distinguishes between the
|
||||
**Mathematical Bold Small A** and the **Mathematical Italic Small A.** For more
|
||||
examples, look at the cells with white backgrounds in the collation charts).
|
||||
However, using levels higher than Tertiary - the Identical strength - result in
|
||||
significantly longer sort keys, and slower string comparison performance for
|
||||
equal strings.
|
||||
|
||||
**Example:**\
|
||||
**S=1** role = Role = rôle\
|
||||
**S=2** role = Role < rôle\
|
||||
**S=3** role < Role < rôle
|
||||
|
||||
The **Case_Level** attribute is used when ignoring accents
|
||||
**but not** case. In such a situation, set Strength to be Primary, and
|
||||
Case_Level to be On. In most locales, this setting is Off by default. There is a
|
||||
small string comparison performance and sort key impact if this attribute is set
|
||||
to be On.
|
||||
|
||||
**Example:**\
|
||||
**S=1, E=X** role = Role = rôle\
|
||||
**S=1, E=O** role = rôle < Role
|
||||
|
||||
The **Case_First** attribute is used to control whether
|
||||
uppercase letters come before lowercase letters or vice versa, in the absence of
|
||||
other differences in the strings. The possible values are Uppercase_First (U)
|
||||
and Lowercase_First (L), plus the standard Default and Off. There is almost no
|
||||
difference between the Off and Lowercase_First options in terms of results, so
|
||||
typically users will not use Lowercase_First: only Off or Uppercase_First.
|
||||
(People interested in the detailed differences between X and L should consult
|
||||
the [Collation Customization](customization/index.md) ).
|
||||
Specifying either L or U won't affect string comparison performance, but will
|
||||
affect the sort key length.
|
||||
|
||||
**Example:**\
|
||||
**C=X or C=L** "china" < "China" < "denmark" < "Denmark"\
|
||||
**C=U** "China" < "china" < "Denmark" < "denmark"
|
||||
|
||||
The **Alternate** attribute is used to control the handling of
|
||||
the so-called **variable **characters in the UCA: whitespace, punctuation and
|
||||
symbols. If Alternate is set to Non-Ignorable (N), then differences among these
|
||||
characters are of the same importance as differences among letters. If Alternate
|
||||
is set to Shifted (S), then these characters are of only minor importance. The
|
||||
Shifted value is often used in combination with Strength set to Quaternary. In
|
||||
such a case, white-space, punctuation, and symbols are considered when comparing
|
||||
strings, but only if all other aspects of the strings (base letters, accents,
|
||||
and case) are identical. If Alternate is not set to Shifted, then there is no
|
||||
difference between a Strength of 3 and a Strength of 4.
|
||||
|
||||
For more information and examples, see
|
||||
[Variable_Weighting](http://www.unicode.org/reports/tr10/#Variable_Weighting) in
|
||||
the UCA.
|
||||
|
||||
The reason the Alternate values are not simply On and Off is that
|
||||
additional Alternate values may be added in the future.
|
||||
|
||||
The UCA option
|
||||
**Blanked** is expressed with Strength set to 3, and Alternate set to Shifted.
|
||||
|
||||
The default for most locales is Non-Ignorable. If Shifted is selected, it may be
|
||||
slower if there are many strings that are the same except for punctuation; sort
|
||||
key length will not be affected unless the strength level is also increased.
|
||||
|
||||
**Example:**\
|
||||
**S=3, A=N** di Silva < Di Silva < diSilva < U.S.A. < USA\
|
||||
**S=3, A=S** di Silva = diSilva < Di Silva < U.S.A. = USA\
|
||||
**S=4, A=S** di Silva < diSilva < Di Silva < U.S.A. < USA
|
||||
|
||||
The **Variable_Top** attribute is only meaningful if the
|
||||
Alternate attribute is not set to Non-Ignorable. In such a case, it controls
|
||||
which characters count as ignorable. The \<hex\> value specifies the "highest"
|
||||
character sequence (in UCA order) weight that is to be considered ignorable.
|
||||
|
||||
Thus, for example, if a user wanted white-space to be ignorable, but not any
|
||||
visible characters, then s/he would use the value Variable_Top=0020 (space). The
|
||||
digits should only be a single character. All characters of the same primary
|
||||
weight are equivalent, so Variable_Top=3000 (ideographic space) has the same
|
||||
effect as Variable_Top=0020.
|
||||
|
||||
This setting (alone) has little impact on string comparison performance; setting
|
||||
it lower or higher will make sort keys slightly shorter or longer respectively.
|
||||
|
||||
**Example:**\
|
||||
**S=3, A=S** di Silva = diSilva < U.S.A. = USA\
|
||||
**S=3, A=S, T=0020** di Silva = diSilva < U.S.A. < USA
|
||||
|
||||
The **Normalization** setting determines whether
|
||||
text is thoroughly normalized or not in comparison. Even if the setting is off
|
||||
(which is the default for many locales), text as represented in common usage
|
||||
will compare correctly (for details, see [UTN
|
||||
#5](http://www.unicode.org/notes/tn5/)). Only if the accent marks are in
|
||||
non-canonical order will there be a problem. If the setting is On, then the best
|
||||
results are guaranteed for all possible text input.There is a medium string
|
||||
comparison performance cost if this attribute is On, depending on the frequency
|
||||
of sequences that require normalization. There is no significant effect on sort
|
||||
key length.If the input text is known to be in NFD or NFKD normalization forms,
|
||||
there is no need to enable this Normalization option.
|
||||
|
||||
**Example:**\
|
||||
**N=X** ä = a + ◌̈ < ä + ◌̣ < ạ + ◌̈\
|
||||
**N=O** ä = a + ◌̈ < ä + ◌̣ = ạ + ◌̈
|
||||
|
||||
Some **French** dictionary ordering traditions sort strings with
|
||||
different accents from the back of the string. This attribute is automatically
|
||||
set to On for the Canadian French locale (fr_CA). Users normally would not need
|
||||
to explicitly set this attribute. There is a string comparison performance cost
|
||||
when it is set On, but sort key length is unaffected.
|
||||
|
||||
**Example:**\
|
||||
**F=X** cote < coté < côte < côté\
|
||||
**F=O** cote < côte < coté < côté
|
||||
|
||||
Compatibility with JIS x 4061 requires the introduction of an
|
||||
additional level to distinguish **Hiragana** and Katakana characters. If
|
||||
compatibility with that standard is required, then this attribute is set On, and
|
||||
the strength should be set to at least Quaternary.
|
||||
|
||||
This attribute is an implementation detail of the CLDR Japanese tailoring. The
|
||||
implementation might change to use a different mechanism to achieve the same
|
||||
Japanese sort order. Since ICU 50, this attribute is not settable any more.
|
||||
|
||||
**Example:**\
|
||||
**H=X, S=4** きゅう = キュウ < きゆう = キユウ\
|
||||
**H=O, S=4** きゅう < キュウ < きゆう < キユウ
|
||||
|
||||
> :point_right: **Note** If attributes in collator name are not overridden,
|
||||
> it is assumed that they are the same as for the given locale.
|
||||
> For example, a collator opened with an empty
|
||||
> string has the same attribute settings as **AN_CX_EX_FX_HX_KX_NX_S3_T0000**.*
|
||||
|
||||
### Summary of Value Abbreviations
|
||||
|
||||
Value | Abbreviation
|
||||
------------- | ------------
|
||||
Default | D
|
||||
On | O
|
||||
Off | X
|
||||
Primary | 1
|
||||
Secondary | 2
|
||||
Tertiary | 3
|
||||
Quaternary | 4
|
||||
Identical | I
|
||||
Shifted | S
|
||||
Non-Ignorable | N
|
||||
Lower-First | L
|
||||
Upper-First | U
|
||||
@@ -1,178 +0,0 @@
|
||||
---
|
||||
layout: default
|
||||
title: Ignore Punctuation Options
|
||||
nav_order: 8
|
||||
parent: Collation
|
||||
---
|
||||
<!--
|
||||
© 2020 and later: Unicode, Inc. and others.
|
||||
License & terms of use: http://www.unicode.org/copyright.html
|
||||
-->
|
||||
|
||||
# “Ignore Punctuation” Options
|
||||
{: .no_toc }
|
||||
|
||||
## Contents
|
||||
{: .no_toc .text-delta }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
By default, spaces and punctuation characters add primary (base character)
|
||||
differences. Such characters sort less-than digits and letters. For example, the
|
||||
default collation yields “De Anza” < “de-luge” < “deanza”.
|
||||
|
||||
UCA/CLDR/ICU provide several options for “ignore punctuation” collation
|
||||
settings, also known as Variable Weighting or Alternate Handling. These options
|
||||
change the sorting behavior of “variable” characters algorithmically. “Variable”
|
||||
characters are those with low (but non-zero) primary weights up to a threshold,
|
||||
the “variable top”. By default, CLDR and ICU treat spaces and punctuation as
|
||||
variable. (This can be changed via API.) The DUCET also includes most symbols.
|
||||
|
||||
## Non-Ignorable
|
||||
|
||||
The default behavior in CLDR & ICU, shown above, is to not ignore punctuation
|
||||
(alternate=non-ignorable) but to map variable characters to their normal primary
|
||||
collation elements.
|
||||
|
||||
All of the following options cause variable characters to be ignored on levels
|
||||
1..3. Only when strings compare equal up to the tertiary level may variable
|
||||
characters make a difference, depending on the options.
|
||||
|
||||
See also
|
||||
|
||||
* [UCA: Variable
|
||||
Weighting](http://www.unicode.org/reports/tr10/#Variable_Weighting)
|
||||
* [LDML: Setting
|
||||
Options](https://htmlpreview.github.io/?https://github.com/unicode-org/cldr/blob/main/docs/ldml/tr35-collation.html#Setting_Options)
|
||||
|
||||
Here is an overview of the sorting results with these options.
|
||||
|
||||
Non-ignorable | Blanked | Shifted | Shift-Trimmed | Variable-After
|
||||
------------- | ------------ | ------- | ------------- | --------------
|
||||
delug | delug | delug | delug | delug
|
||||
de-luge | de-luge | de-luge | *deluge* | *deluge*
|
||||
delu-ge | delu-ge (*) | delu-ge | de-luge | deluge-
|
||||
*deluge* | *deluge* (*) | *deluge* | delu-ge | delu-ge
|
||||
Deluge | deluge- (*) | deluge- | deluge- | de-luge
|
||||
deluge- | Deluge | Deluge | Deluge | Deluge
|
||||
|
||||
Items with (*) compare equal to the preceding ones, and their relative order
|
||||
is arbitrary. These only occur in the Blanked column. This table shows the
|
||||
results of a stable sort algorithm with the non-ignorable column as input.
|
||||
|
||||
## Blanked
|
||||
|
||||
The simplest option is to “ignore punctuation” completely, as if all variable
|
||||
characters (and following combining marks) had been removed from the input
|
||||
strings before comparing them.
|
||||
|
||||
For example: “De Anza” = “De-Anza” = “DeAnza”.
|
||||
|
||||
In ICU, this option is selected with alternate=shifted and
|
||||
strength=primary|secondary|tertiary. (ICU does not support Blanked combined with
|
||||
strength=identical.)
|
||||
|
||||
The implementation “blanks” out all weights of the variable characters’
|
||||
collation elements.
|
||||
|
||||
*With all of the following options, variable characters are ignored on levels
|
||||
1..3 but add distinctions on level 4 (quaternary level).*
|
||||
|
||||
## Shifted
|
||||
|
||||
Among strings that compare tertiary-equal, that is, they contain the same
|
||||
letters, accents and casing:
|
||||
|
||||
* Sorts all variable characters less-than (before) regular characters.
|
||||
* Appending a variable character makes a string sort *greater-than* the string
|
||||
without it.
|
||||
* *Inserting* a variable character makes a string sort *less-than* the string
|
||||
without it.
|
||||
* Inserting a variable character *earlier* in a string makes it sort
|
||||
*less-than* inserting the variable character *later* in the string.
|
||||
|
||||
The result is similar to [Merging Sort
|
||||
Keys](http://www.unicode.org/reports/tr10/#Merging_Sort_Keys) (with shorter
|
||||
prefixes sorting less-than longer ones), like in last-name+first-name sorting,
|
||||
except only among tertiary-equal strings.
|
||||
|
||||
For example: “de-luge” < “delu-ge” < “deluge” < “deluge-”.
|
||||
|
||||
In ICU, this option is selected with alternate=shifted and
|
||||
strength=quaternary|identical.
|
||||
|
||||
The implementation “shifts” the primary weight p of the collation element \[p,
|
||||
s, t, q\] of each variable characters down three levels: \[0, 0, 0, p\]. Regular
|
||||
characters with primary collation elements get a high quaternary weight, higher
|
||||
than that of any variable character.
|
||||
|
||||
Note that this behavior is different from collation on secondary and tertiary
|
||||
level, because normal collation elements get low secondary & tertiary weights
|
||||
but high quaternary weights. Adding an accent difference anywhere makes a string
|
||||
sort greater-than the string without it, and adding an accent difference earlier
|
||||
makes it sort greater-than adding it later. For example, “deanza” < “deanzä” <
|
||||
“deänza” < “dëanza”. (Compare the ‘ä’/‘ë’ positions here with the ‘-’ positions
|
||||
above.)
|
||||
|
||||
## Shift-Trimmed
|
||||
|
||||
*Note: This method is not currently implemented in ICU.*
|
||||
|
||||
Among strings that compare tertiary-equal:
|
||||
|
||||
* Sorts variable characters sometimes less-than, sometimes greater-than
|
||||
regular characters.
|
||||
* Inserting a variable character anywhere makes a string sort *greater-than*
|
||||
the string without it. (The string without variable characters gets an empty
|
||||
quaternary level.)
|
||||
* Inserting a variable character *earlier* in a string makes it sort
|
||||
*less-than* inserting the variable character *later* in the string.
|
||||
|
||||
For example: “deluge” < “de-luge” < “delu-ge” < “deluge-”.
|
||||
|
||||
The Shift-Trimmed method works like Shifted, except that *trailing*
|
||||
high-quaternary weights (from regular characters) are removed (trimmed).
|
||||
Compared with Shifted, the Shift-Trimmed method sorts strings without variable
|
||||
characters before ones with variable characters added, rather than producing the
|
||||
equivalent of [Merging Sort
|
||||
Keys](http://www.unicode.org/reports/tr10/#Merging_Sort_Keys).
|
||||
|
||||
Shift-Trimmed is more complicated to implement than all of the other options:
|
||||
When comparing strings, a lookahead (or equivalent) is needed to determine
|
||||
whether a non-variable character gets a zero quaternary weight (if no variables
|
||||
follow) or a high quaternary weight (if at least one variable follows). When
|
||||
building sort keys, trailing high/common quaternary weights are trimmed (backed
|
||||
out) at the end of the quaternary level.
|
||||
|
||||
## Variable-After
|
||||
|
||||
*Note: This method is not currently implemented in ICU.*
|
||||
|
||||
Among strings that compare tertiary-equal:
|
||||
|
||||
* Sorts all variable characters greater-than (after) regular characters.
|
||||
* Inserting a variable character anywhere makes a string sort *greater-than*
|
||||
the string without it. (Like Shift-Trimmed.)
|
||||
* Inserting a variable character *earlier* in a string makes it sort
|
||||
*greater-than* inserting the variable character *later* in the string. (Like
|
||||
accent differences.)
|
||||
|
||||
For example: “deluge” < “deluge-” < “delu-ge” < “de-luge”.
|
||||
|
||||
The implementation “shifts” the primary weight p of the collation element \[p,
|
||||
s, t, q\] of each variable characters down three levels: \[0, 0, 0, p\]. Regular
|
||||
characters with primary collation elements get a *low* quaternary weight,
|
||||
*lower* than that of any variable character. This is consistent with collation
|
||||
on secondary and tertiary levels but unlike [Merging Sort
|
||||
Keys](http://www.unicode.org/reports/tr10/#Merging_Sort_Keys).
|
||||
|
||||
This method extends the [UCA well-formedness condition
|
||||
2](http://www.unicode.org/reports/tr10/#WF2) to apply to quaternary weights.
|
||||
(UCA versions before UCA 6.2 did not limit WF2 to secondary & tertiary weights,
|
||||
which meant that several of the Variable Weighting options technically created
|
||||
ill-formed quaternary weights.)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,333 +0,0 @@
|
||||
---
|
||||
layout: default
|
||||
title: Collation Examples
|
||||
nav_order: 7
|
||||
parent: Collation
|
||||
---
|
||||
<!--
|
||||
© 2020 and later: Unicode, Inc. and others.
|
||||
License & terms of use: http://www.unicode.org/copyright.html
|
||||
-->
|
||||
|
||||
# Collation Examples
|
||||
{: .no_toc }
|
||||
|
||||
## Contents
|
||||
{: .no_toc .text-delta }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
|
||||
---
|
||||
|
||||
## Simple Collation Sample Customization
|
||||
|
||||
The following program demonstrates how to compare and create sort keys with
|
||||
default locale.
|
||||
|
||||
In **C:**
|
||||
|
||||
```c
|
||||
#include <stdio.h>
|
||||
#include <memory.h>
|
||||
#include <string.h>
|
||||
#include "unicode/ustring.h"
|
||||
#include "unicode/utypes.h"
|
||||
#include "unicode/uloc.h"
|
||||
#include "unicode/ucol.h"
|
||||
|
||||
#define MAXBUFFERSIZE 100
|
||||
#define BIGBUFFERSIZE 5000
|
||||
|
||||
UBool collateWithLocaleInC(const char* locale, UErrorCode *status)
|
||||
{
|
||||
UChar dispName [MAXBUFFERSIZE];
|
||||
int32_t bufferLen = 0;
|
||||
UChar source [MAXBUFFERSIZE];
|
||||
UChar target [MAXBUFFERSIZE];
|
||||
UCollationResult result = UCOL_EQUAL;
|
||||
uint8_t sourceKeyArray [MAXBUFFERSIZE];
|
||||
uint8_t targetKeyArray [MAXBUFFERSIZE];
|
||||
int32_t sourceKeyOut = 0,
|
||||
targetKeyOut = 0;
|
||||
UCollator *myCollator = 0;
|
||||
if (U_FAILURE(*status))
|
||||
{
|
||||
return FALSE;
|
||||
}
|
||||
u_uastrcpy(source, "This is a test.");
|
||||
u_uastrcpy(target, "THIS IS A TEST.");
|
||||
myCollator = ucol_open(locale, status);
|
||||
if (U_FAILURE(*status)){
|
||||
bufferLen = uloc_getDisplayName(locale, 0, dispName, MAXBUFFERSIZE, status);
|
||||
/*Report the error with display name... */
|
||||
fprintf(stderr,
|
||||
"Failed to create the collator for : \"%s\"\n", dispName);
|
||||
return FALSE;
|
||||
}
|
||||
result = ucol_strcoll(myCollator, source, u_strlen(source), target, u_strlen(target));
|
||||
/* result is 1, secondary differences only for ignorable space characters*/
|
||||
if (result != UCOL_LESS)
|
||||
{
|
||||
fprintf(stderr,
|
||||
"Comparing two strings with only secondary differences in C failed.\n");
|
||||
return FALSE;
|
||||
}
|
||||
/* To compare them with just primary differences */
|
||||
ucol_setStrength(myCollator, UCOL_PRIMARY);
|
||||
result = ucol_strcoll(myCollator, source, u_strlen(source), target, u_strlen(target));
|
||||
/* result is 0 */
|
||||
if (result != 0)
|
||||
{
|
||||
fprintf(stderr,
|
||||
"Comparing two strings with no differences in C failed.\n");
|
||||
return FALSE;
|
||||
}
|
||||
|
||||
/* Now, do the same comparison with keys */
|
||||
sourceKeyOut = ucol_getSortKey(myCollator, source, -1, sourceKeyArray, MAXBUFFERSIZE);
|
||||
targetKeyOut = ucol_getSortKey(myCollator, target, -1, targetKeyArray, MAXBUFFERSIZE);
|
||||
result = 0;
|
||||
result = strcmp(sourceKeyArray, targetKeyArray);
|
||||
if (result != 0)
|
||||
{
|
||||
fprintf(stderr,
|
||||
"Comparing two strings with sort keys in C failed.\n");
|
||||
return FALSE;
|
||||
}
|
||||
ucol_close(myCollator);
|
||||
return TRUE;
|
||||
}
|
||||
```
|
||||
|
||||
In **C++:**
|
||||
|
||||
```c++
|
||||
#include <stdio.h>
|
||||
#include "unicode/unistr.h"
|
||||
#include "unicode/utypes.h"
|
||||
#include "unicode/locid.h"
|
||||
#include "unicode/coll.h"
|
||||
#include "unicode/tblcoll.h"
|
||||
#include "unicode/coleitr.h"
|
||||
#include "unicode/sortkey.h"
|
||||
UBool collateWithLocaleInCPP(const Locale& locale, UErrorCode& status)
|
||||
{
|
||||
UnicodeString dispName;
|
||||
UnicodeString source("This is a test.");
|
||||
UnicodeString target("THIS IS A TEST.");
|
||||
Collator::EComparisonResult result = Collator::EQUAL;
|
||||
CollationKey sourceKey;
|
||||
CollationKey targetKey;
|
||||
Collator *myCollator = 0;
|
||||
if (U_FAILURE(status))
|
||||
{
|
||||
return FALSE;
|
||||
}
|
||||
myCollator = Collator::createInstance(locale, status);
|
||||
if (U_FAILURE(status)){
|
||||
locale.getDisplayName(dispName);
|
||||
/*Report the error with display name... */
|
||||
fprintf(stderr,
|
||||
"%s: Failed to create the collator for : \"%s\"\n", dispName);
|
||||
return FALSE;
|
||||
}
|
||||
result = myCollator->compare(source, target);
|
||||
/* result is 1, secondary differences only for ignorable space characters*/
|
||||
if (result != UCOL_LESS)
|
||||
{
|
||||
fprintf(stderr,
|
||||
"Comparing two strings with only secondary differences in C failed.\n");
|
||||
return FALSE;
|
||||
}
|
||||
/* To compare them with just primary differences */
|
||||
myCollator->setStrength(Collator::PRIMARY);
|
||||
result = myCollator->compare(source, target);
|
||||
/* result is 0 */
|
||||
if (result != 0)
|
||||
{
|
||||
fprintf(stderr,
|
||||
"Comparing two strings with no differences in C failed.\n");
|
||||
return FALSE;
|
||||
}
|
||||
/* Now, do the same comparison with keys */
|
||||
myCollator->getCollationKey(source, sourceKey, status);
|
||||
myCollator->getCollationKey(target, targetKey, status);
|
||||
result = Collator::EQUAL;
|
||||
|
||||
result = sourceKey.compareTo(targetKey);
|
||||
if (result != 0)
|
||||
{
|
||||
fprintf(stderr,
|
||||
"%s: Comparing two strings with sort keys in C failed.\n");
|
||||
return FALSE;
|
||||
}
|
||||
delete myCollator;
|
||||
return TRUE;
|
||||
}
|
||||
```
|
||||
|
||||
### Main Function
|
||||
|
||||
```c++
|
||||
extern "C" UBool collateWithLocaleInC(const char* locale, UErrorCode *status);
|
||||
int main()
|
||||
{
|
||||
UErrorCode status = U_ZERO_ERROR;
|
||||
fprintf(stdout, "\n");
|
||||
if (collateWithLocaleInCPP(Locale("en", "US"), status) != TRUE)
|
||||
{
|
||||
fprintf(stderr,
|
||||
"Collate with locale in C++ failed.\n");
|
||||
} else
|
||||
{
|
||||
fprintf(stdout, "Collate with Locale C++ example worked!!\n");
|
||||
}
|
||||
status = U_ZERO_ERROR;
|
||||
fprintf(stdout, "\n");
|
||||
if (collateWithLocaleInC("en_US", &status) != TRUE)
|
||||
{
|
||||
fprintf(stderr,
|
||||
"%s: Collate with locale in C failed.\n");
|
||||
} else
|
||||
{
|
||||
fprintf(stdout, "Collate with Locale C example worked!!\n");
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
```
|
||||
|
||||
In **Java:**
|
||||
|
||||
```java
|
||||
import com.ibm.icu.text.Collator;
|
||||
import com.ibm.icu.text.CollationElementIterator;
|
||||
import com.ibm.icu.text.CollationKey;
|
||||
import java.util.Locale;
|
||||
|
||||
public class CollateExample
|
||||
{
|
||||
|
||||
public static void main(String arg[])
|
||||
{
|
||||
CollateExample example = new CollateExample();
|
||||
try {
|
||||
if (!example.collateWithLocale(Locale.US)) {
|
||||
System.err.println("Collate with locale example failed.");
|
||||
}
|
||||
else {
|
||||
System.out.println("Collate with Locale example worked!!");
|
||||
}
|
||||
} catch (Exception e) {
|
||||
System.err.println("Collating with locale failed");
|
||||
e.printStackTrace();
|
||||
}
|
||||
}
|
||||
|
||||
public boolean collateWithLocale(Locale locale) throws Exception
|
||||
{
|
||||
String source = "This is a test.";
|
||||
String target = "THIS IS A TEST.";
|
||||
Collator myCollator = Collator.getInstance(locale);
|
||||
|
||||
int result = myCollator.compare(source, target);
|
||||
// result is 1, secondary differences only for ignorable space characters
|
||||
if (result >= 0) {
|
||||
System.err.println(
|
||||
"Comparing two strings with only secondary differences failed.");
|
||||
return false;
|
||||
}
|
||||
// To compare them with just primary differences
|
||||
myCollator.setStrength(Collator.PRIMARY);
|
||||
result = myCollator.compare(source, target);
|
||||
// result is 0
|
||||
if (result != 0) {
|
||||
System.err.println(
|
||||
"Comparing two strings with no differences failed.");
|
||||
return false;
|
||||
}
|
||||
// Now, do the same comparison with keys
|
||||
CollationKey sourceKey = myCollator.getCollationKey(source);
|
||||
CollationKey targetKey = myCollator.getCollationKey(target);
|
||||
result = sourceKey.compareTo(targetKey);
|
||||
if (result != 0) {
|
||||
System.err.println("Comparing two strings with sort keys failed.");
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Language-sensitive searching
|
||||
|
||||
String searching is a well-researched area, and there are algorithms that can
|
||||
optimize the searching process. Perhaps the best is the Boyer-Moore method. For a
|
||||
full description of this concept, please see Laura
|
||||
Werner's text searching article for more details
|
||||
(<http://icu-project.org/docs/papers/efficient_text_searching_in_java.html>).
|
||||
|
||||
However, implementing collation-based search with the Boyer-Moore method
|
||||
while getting correct results is very tricky, and ICU no longer uses this method
|
||||
(as of ICU4C 4.0 and ICU4J 53).
|
||||
|
||||
Please see the [String Search Service](./string-search) chapter.
|
||||
|
||||
## Using large buffers to manage sort keys
|
||||
|
||||
A good solution for the problem of not knowing the sort key size in advance is
|
||||
to allocate a large buffer and store all the sort keys there, while keeping a
|
||||
list of indexes or pointers to that buffer.
|
||||
|
||||
Following is sample code that will take a pointer to an array of UChar pointer,
|
||||
an array of key indexes. It will allocate and fill a buffer with sort keys and
|
||||
return the maximum size for a sort key. Once you have done this to your string,
|
||||
you just need to allocate a field of maximum size and copy your sortkeys from
|
||||
the buffer to fields.
|
||||
|
||||
```c++
|
||||
uint32_t fillBufferWithKeys(UCollator *coll, UChar **source, uint32_t *keys,
|
||||
uint32_t sourceSize, uint8_t **buffer,
|
||||
uint32_t *maxSize, UErrorCode *status)
|
||||
{
|
||||
if(status == NULL || U_FAILURE(*status)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
uint32_t bufferSize = 16384;
|
||||
uint32_t increment = 16384;
|
||||
uint32_t currentOffset = 0;
|
||||
uint32_t keySize = 0;
|
||||
uint32_t i = 0;
|
||||
*maxSize = 0;
|
||||
|
||||
*buffer = (uint8_t *)malloc(bufferSize * sizeof(uint8_t));
|
||||
if(buffer == NULL) {
|
||||
*status = U_MEMORY_ALLOCATION_ERROR;
|
||||
return 0;
|
||||
}
|
||||
|
||||
for(i = 0; i < sourceSize; i++) {
|
||||
keys[i] = currentOffset;
|
||||
keySize = ucol_getSortKey(coll, source[i], -1, *buffer+currentOffset, bufferSize-currentOffset);
|
||||
if(keySize > bufferSize-currentOffset) {
|
||||
*buffer = (uint8_t *)realloc(*buffer, bufferSize+increment);
|
||||
if(buffer == NULL) {
|
||||
*status = U_MEMORY_ALLOCATION_ERROR;
|
||||
return 0;
|
||||
}
|
||||
bufferSize += increment;
|
||||
keySize = ucol_getSortKey(coll, source[i], -1, *buffer+currentOffset, bufferSize-currentOffset);
|
||||
}
|
||||
/* here you can hook code that does something interesting with the keySize -
|
||||
* remembers the maximum or similar...
|
||||
*/
|
||||
if(keySize > *maxSize) {
|
||||
*maxSize = keySize;
|
||||
}
|
||||
currentOffset += keySize;
|
||||
}
|
||||
|
||||
return currentOffset;
|
||||
}
|
||||
```
|
||||
@@ -1,70 +0,0 @@
|
||||
---
|
||||
layout: default
|
||||
title: Collation FAQ
|
||||
nav_order: 5
|
||||
parent: Collation
|
||||
---
|
||||
<!--
|
||||
© 2020 and later: Unicode, Inc. and others.
|
||||
License & terms of use: http://www.unicode.org/copyright.html
|
||||
-->
|
||||
|
||||
# Collation FAQ
|
||||
{: .no_toc }
|
||||
|
||||
## Contents
|
||||
{: .no_toc .text-delta }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
|
||||
---
|
||||
|
||||
## Q. Should I turn Full Normalization on all the time?
|
||||
|
||||
**A.** You can if you want, but you don't typically need to. The key is that
|
||||
normalization for most characters is already built into ICU's collation by
|
||||
default. Everything that can be done without affecting performance is already
|
||||
there, and will work with most languages. So the normalization parameter in ICU
|
||||
really only changes whether full normalization is invoked.
|
||||
|
||||
The outlying cases are situations where a language uses multiple accents
|
||||
(non-spacing marks) on the same base letter, such as Vietnamese or Arabic. In
|
||||
those cases, full normalization needs to be turned on. If you use the right
|
||||
locale (or language) when creating a collation in ICU, then full normalization
|
||||
will be turned on or off according to what the language typically requires.
|
||||
|
||||
## Q. Are there any cases where I would want to override the Full Normalization setting?
|
||||
|
||||
**A.** The only case where you really need to worry about that parameter is for
|
||||
very unusual cases, such as sorting an list containing of names according to
|
||||
English conventions, but where the list contains, for example, some Vietnamese
|
||||
names. One way to check for such a situation is to open a collator for each of
|
||||
the languages you expect to find, and see if any of them have the full
|
||||
normalization flags set.
|
||||
|
||||
## Q. How can collation rules mimic word sorting?
|
||||
|
||||
Word sort is a way of sorting where certain interpunction characters are
|
||||
completely ignored, while other are considered. An example of word sort below
|
||||
ignores hyphens and apostrophes:
|
||||
|
||||
Word Sort | String Sort
|
||||
--------- | -----------
|
||||
billet | bill's
|
||||
bills | billet
|
||||
bill's | bills
|
||||
cannot | can't
|
||||
cant | cannot
|
||||
can't | cant
|
||||
con | co-op
|
||||
coop | con
|
||||
co-op | coop
|
||||
|
||||
This specific behavior can be mimicked using a tailoring that makes these
|
||||
characters completely ignorable. In this case, an appropriate rule would be
|
||||
`"&\\u0000 = '' = '-'"`.
|
||||
|
||||
Please note that we don't think that such solution is correct, since different
|
||||
languages have different word elements. Instead one should use shifted mode for
|
||||
comparison.
|
||||
@@ -1,148 +0,0 @@
|
||||
---
|
||||
layout: default
|
||||
title: Collation
|
||||
nav_order: 1200
|
||||
has_children: true
|
||||
---
|
||||
<!--
|
||||
© 2020 and later: Unicode, Inc. and others.
|
||||
License & terms of use: http://www.unicode.org/copyright.html
|
||||
-->
|
||||
|
||||
# Collation
|
||||
|
||||
## Overview
|
||||
|
||||
Information is displayed in sorted order to enable users to easily find the
|
||||
items they are looking for. However, users of different languages might have
|
||||
very different expectations of what a "sorted" list should look like. Not only
|
||||
does the alphabetical order vary from one language to another, but it also can
|
||||
vary from document to document within the same language. For example, phonebook
|
||||
ordering might be different than dictionary ordering. String comparison is one
|
||||
of the basic functions most applications require, and yet implementations often
|
||||
do not match local conventions. The ICU Collation Service provides string
|
||||
comparison capability with support for appropriate sort orderings for each of
|
||||
the locales you need. In the event that you have a very unusual requirement, you
|
||||
are also provided the facilities to customize orderings.
|
||||
|
||||
Starting in release 1.8, the ICU Collation Service is compliant to the Unicode
|
||||
Collation Algorithm (UCA) ([Unicode Technical Standard
|
||||
#10](http://www.unicode.org/reports/tr10/)) and based on the Default
|
||||
Unicode Collation Element Table (DUCET) which defines the same sort order as ISO
|
||||
14651.
|
||||
|
||||
The ICU Collation Service also contains several enhancements that are not
|
||||
available in UCA. These have been adopted into the [CLDR Collation
|
||||
Algorithm](http://www.unicode.org/reports/tr35/tr35-collation.html#CLDR_Collation_Algorithm).
|
||||
For example:
|
||||
|
||||
* Additional case handling (as specified by CLDR): ICU allows case differences
|
||||
to be ignored or flipped. Uppercase letters can be sorted before lowercase
|
||||
letters, or vice-versa.
|
||||
* Easy customization (as specified by CLDR): Services can be easily tailored
|
||||
to address a wide range of collation requirements.
|
||||
* The [default (root) sort
|
||||
order](http://www.unicode.org/reports/tr35/tr35-collation.html#Root_Collation)
|
||||
has been tailored slightly for improved functionality and performance.
|
||||
|
||||
In other words, ICU implements the CLDR Collation Algorithm which is an
|
||||
extension of the Unicode Collation Algorithm (UCA) which is an extension of ISO
|
||||
14651.
|
||||
|
||||
There are several benefits to using the collation algorithms defined in these
|
||||
standards, including:
|
||||
|
||||
* The algorithms have been designed and reviewed by experts in multilingual
|
||||
collation, and therefore are robust and comprehensive.
|
||||
|
||||
* Applications that share sorted data but do not agree on how the data should
|
||||
be ordered fail to perform correctly. By conforming to the CLDR/UCA/14651
|
||||
standards for collation and using CLDR language-specific collation data,
|
||||
independently developed applications sort data identically and perform
|
||||
properly.
|
||||
|
||||
In addition, Unicode contains a large set of characters. This can make it
|
||||
difficult for collation to be a fast operation or require collation to use
|
||||
significant memory or disk resources. The ICU collation implementation is
|
||||
designed to be fast, have a small memory footprint and be highly customizable.
|
||||
|
||||
There are many challenges when accommodating the world's languages and writing
|
||||
systems and the different orderings that are used. However, the ICU Collation
|
||||
Service provides an excellent means for comparing strings in a locale-sensitive
|
||||
fashion.
|
||||
|
||||
For example, here are some of the ways languages vary in ordering strings:
|
||||
|
||||
* The letters A-Z can be sorted in a different order than in English. For
|
||||
example, in Lithuanian, "y" is sorted between "i" and "k".
|
||||
|
||||
* Combinations of letters can be treated as if they were one letter. For
|
||||
example, in traditional Spanish "ch" is treated as a single letter, and
|
||||
sorted between "c" and "d".
|
||||
|
||||
* Accented letters can be treated as minor variants of the unaccented letter.
|
||||
For example, "é" can be treated equivalent to "e".
|
||||
|
||||
* Accented letters can be treated as distinct letters. For example, "Å" in
|
||||
Danish is treated as a separate letter that sorts just after "Z".
|
||||
|
||||
* Unaccented letters that are considered distinct in one language can be
|
||||
indistinct in another. For example, the letters "v" and "w" are two
|
||||
different letters according to English. However, "v" and "w" are
|
||||
traditionally considered variant forms of the same letter in Swedish.
|
||||
|
||||
* A letter can be treated as if it were two letters. For example, in German
|
||||
phonebook (or "lists of names") order "ä" is compared as if it were "ae".
|
||||
|
||||
* Thai requires that the order of certain letters be reversed.
|
||||
|
||||
* Some French dictionary ordering traditions sort accents in backwards order,
|
||||
from the end of the string. For example, the word "côte" sorts before "coté"
|
||||
because the acute accent on the final "e" is more significant than the
|
||||
circumflex on the "o".
|
||||
|
||||
* Sometimes lowercase letters sort before uppercase letters. The reverse is
|
||||
required in other situations. For example, lowercase letters are usually
|
||||
sorted before uppercase letters in English. Danish letters are the exact
|
||||
opposite.
|
||||
|
||||
* Even in the same language, different applications might require different
|
||||
sorting orders. For example, in German dictionaries, "öf" would come before
|
||||
"of". In phone books the situation is the exact opposite.
|
||||
|
||||
* Sorting orders can change over time due to government regulations or new
|
||||
characters/scripts in Unicode.
|
||||
|
||||
To accommodate the many languages and differing requirements, ICU collation
|
||||
supports customizing sort orderings - also known as **tailoring**. More details
|
||||
regarding tailoring are discussed in the [Customization
|
||||
chapter.](customization/index.md)
|
||||
|
||||
The basic ICU Collation Service is provided by two main categories of APIs:
|
||||
|
||||
* String comparison - most commonly used: APIs return result of comparing two
|
||||
strings (greater than, equal or less than). This is used as a comparator
|
||||
when sorting lists, building tree maps, etc.
|
||||
|
||||
* Sort key generation - used when a very large set of strings are
|
||||
compared/sorted repeatedly: APIs return a zero-terminated array of bytes per
|
||||
string known as a sort key. The keys can be compared directly using strcmp
|
||||
or memcmp standard library functions, saving repeated lookup and computation
|
||||
of each string's collation properties. For example, database applications
|
||||
use index tables of sort keys to index strings quickly. Note, however, that
|
||||
this only improves performance for large numbers of strings because sorting
|
||||
via the comparison functions is very fast. For more information, see
|
||||
[Sortkeys vs Comparison](concepts#sortkeys-vs-comparison).
|
||||
|
||||
ICU provides an AlphabeticIndex API for generating language-appropriate
|
||||
sorted-section labels like in dictionaries and phone books.
|
||||
|
||||
ICU also provides a higher-level [string search](string-search)
|
||||
API which can be used, for example, for case-insensitive or accent-insensitive
|
||||
search in an editor or in a web page. ICU string search is based on the
|
||||
low-level [collation element iteration](architecture).
|
||||
|
||||
## Programming Examples
|
||||
|
||||
Here are some [API usage conventions](api.md) for the ICU Collation Service
|
||||
APIs.
|
||||
@@ -1,337 +0,0 @@
|
||||
---
|
||||
layout: default
|
||||
title: String Search
|
||||
nav_order: 4
|
||||
parent: Collation
|
||||
---
|
||||
<!--
|
||||
© 2020 and later: Unicode, Inc. and others.
|
||||
License & terms of use: http://www.unicode.org/copyright.html
|
||||
-->
|
||||
|
||||
# String Search Service
|
||||
{: .no_toc }
|
||||
|
||||
## Contents
|
||||
{: .no_toc .text-delta }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
String searching, also known as string matching, is a very important subject in
|
||||
the wider domain of text processing and analysis. Many software applications use
|
||||
the basic string search algorithm in the implementations on most operating
|
||||
systems. With the popularity of Internet, the quantity of available data from
|
||||
different parts of the world has increased dramatically within a short time.
|
||||
Therefore, a string search algorithm that is language-aware has become more
|
||||
important. A bitwise match that uses the `u_strstr` (C), `UnicodeString::indexOf`
|
||||
(C++) or `String.indexOf` (Java) APIs will not yield the correct result specific
|
||||
to a particular language's requirements. The APIs will not yield the correct
|
||||
result because all the issues that are important to language-sensitive collation
|
||||
are also applicable to text searching. The following lists those issues which
|
||||
are applicable to text searching:
|
||||
|
||||
1. Accented letters\
|
||||
In English, accents are treated as minor variations of a letter. In French,
|
||||
accented letters have much more significance as they can actually change the
|
||||
meaning of a word. Very often, an accented letter is actually a distinct
|
||||
letter. For example, letter 'å' (\\u00e5) may be just a letter 'a' with an
|
||||
accent symbol to English speakers. However, it is actually a distinct letter
|
||||
in Danish; in Danish searching for 'a' should generally not match 'å' and
|
||||
vice versa. In some cases, such as in traditional German, an accented letter
|
||||
is short-hand for something longer. In sorting, an 'ä' (\\u00e4) is treated
|
||||
as 'ae'. Note that primary- and secondary-level distinctions for *searching*
|
||||
may not be the same as those for sorting; in ICU, many languages provide a
|
||||
special "search" collator with the appropriate level settings for search.
|
||||
|
||||
2. Conjoined letters\
|
||||
Special handling is required when a single letter is treated equivalent to
|
||||
two distinct letters and vice versa. For example, in German, the letter 'ß'
|
||||
(\\u00df) is treated as 'ss' in sorting. Also, in most languages, 'æ'
|
||||
(\\u00e6) is considered equivalent to the letter 'a' followed by the letter
|
||||
'e'. Also, the ligatures are often treated as distinct letters by
|
||||
themselves. For example, 'ch' is treated as a distinct letter between the
|
||||
letter 'c' and the letter 'd' in Spanish.
|
||||
|
||||
3. Ignorable punctuation\
|
||||
As in collation, it is important that the user is able to choose to ignore
|
||||
punctuation symbols while the user searches for a pattern in the string. For
|
||||
example, a user may search for "blackbird" and want to include entries such
|
||||
as "black-bird".
|
||||
|
||||
## ICU String Search Model
|
||||
|
||||
The ICU string search service provides similar APIs to the other text iterating
|
||||
services. Allowing users to specify the starting position and direction within
|
||||
the text string to be searched. For more information, please see the [Boundary
|
||||
Analysis](../boundaryanalysis/index.md) chapter. The user can locate one or all
|
||||
occurrences of a pattern in a string. For a given collator, a pattern match is
|
||||
located at the offsets <start, end> in a string if the collator finds that the
|
||||
sub-string between the start and end is equal.
|
||||
|
||||
The string search service supports two different types of canonical match
|
||||
behavior.
|
||||
|
||||
Let S' be the sub-string of a text string S between the offsets start and end
|
||||
<start, end>.
|
||||
A pattern string P matches a text string S at the offsets <start, end> if
|
||||
|
||||
1. option 1. P matches some canonical equivalent string of S'. Suppose the
|
||||
collator used for searching has a tertiary collation strength, all accents
|
||||
are non-ignorable. If the pattern "a\\u0300" is searched in the target text
|
||||
"a\\u0325\\u0300", a match will be found, since the target text is
|
||||
canonically equivalent to "a\\u0300\\u0325"
|
||||
|
||||
2. option 2. P matches S' and if P starts or ends with a combining mark, there
|
||||
exists no non-ignorable combining mark before or after S' in S respectively.
|
||||
Following the example above, the pattern "a\\u0300" will not find a match in
|
||||
"a\\u0325\\u0300", since there exists a non-ignorable accent '\\u0325' in
|
||||
the middle of 'a' and '\\u0300'. Even with a target text of
|
||||
"a\\u0300\\u0325" a match will not be found because of the non-ignorable
|
||||
trailing accent \\u0325.
|
||||
|
||||
One restriction is to be noted for option 1. Currently there are no composite
|
||||
characters that consists of a character with combining class greater than 0
|
||||
before a character with combining class equals to 0. However, if such a
|
||||
character exists in the future, the string search service may not work correctly
|
||||
with option 1 when such characters are encountered.
|
||||
|
||||
Furthermore, option 1 could generate more than one "encompassing" matches. For
|
||||
example, in Danish, 'å' (\\u00e5) and 'aa' are considered equivalent. So the
|
||||
pattern "baad" will match "a--båd--man" (a--b\\u00e5d--man) at the start offset
|
||||
at 3 and the end offset 5. However, the start offset can be 1 or 2 and the end
|
||||
offset can be 6 or 7, because "-" (hyphen) is ignorable for a certain collation.
|
||||
The ICU implementation always returns the offsets of the shortest match
|
||||
sub-string. To be more exact, the string search added a "tightest" match
|
||||
condition. In other words, if the pattern matches at offsets <start, end> as
|
||||
well as offsets <start + 1, end>, the offsets <start, end> are not considered a
|
||||
match. Likewise, if the pattern matches at offsets <start, end> as well as
|
||||
offsets <start, end + 1>, the offsets <start, end + 1> are not considered a
|
||||
match. Therefore, when the option 1 is chosen in Danish collator, 'baad' will
|
||||
match in the string "a--båd--man" (a--b\\u00e5d--man) ONLY at offsets <3,5>.
|
||||
|
||||
The default behavior is that described in option 2 above. To obtain the behavior
|
||||
described in option 1, you must set the normalization mode to ON in the collator
|
||||
used for search.
|
||||
|
||||
> :point_right: **Note**: The "tightest match" behavior described above
|
||||
> is defined as "Minimal Match" in
|
||||
> [Section 8 Searching and Matching in UTS #10 Unicode Collation Collation Algorithm](http://www.unicode.org/reports/tr10/#Searching).
|
||||
> "Medial Match" and "Maximal Match" are not yet implemented by the ICU String Search service.
|
||||
|
||||
The string search service also supports two varieties of “asymmetric search” as
|
||||
described in *[Section 8.2 Asymmetric Search in UTS #10 Unicode Collation
|
||||
Collation Algorithm](http://www.unicode.org/reports/tr10/#Asymmetric_Search)*.
|
||||
With asymmetric search, for example, unaccented characters are treated as
|
||||
“wildcards” that may match any character with the same primary weight, this
|
||||
behavior can be applied just to characters in the search pattern, or to
|
||||
characters in both the search pattern and the searched text. With the former
|
||||
behavior, searching with French behavior for 'e' might match 'e', 'è', 'é', 'ê',
|
||||
and so one, while search for 'é' would only match 'é'.
|
||||
|
||||
Both a locale or collator can be used to specify the language-sensitive rules
|
||||
for searches. When a locale is specified, a collator will be created internally
|
||||
and the StringSearch instance that is created is responsible for the ownership
|
||||
of the collator. All the collation attributes will be considered during the
|
||||
string search operation. However, the users only can set the collator attributes
|
||||
using the collator APIs. Normalization is usually done within collation and the
|
||||
process is outside the scope of the string search service.
|
||||
|
||||
As in other iterator interfaces, the string search service provides APIs to
|
||||
perform string matching for the first pattern occurrence, immediate next,
|
||||
previous match, and the last pattern occurrence. There are also options to allow
|
||||
for overlapping matching. For example, in English, if the string is "ababab" and
|
||||
the pattern is "abab", overlapping matching produces results of offsets <0, 3>
|
||||
and <2, 5>. Otherwise, the mutually exclusive matching produces the result
|
||||
offset <0, 3> only. To find a whole word match, the user can provide a
|
||||
locale-specific `BreakIterator` object to a `StringSearch` instance to correctly
|
||||
locate the word boundaries. For example, if "c" exists in the string "abc", a
|
||||
match is returned. However, the behavior can be overwritten by supplying a word
|
||||
`BreakIterator`.
|
||||
|
||||
The minimum unit of match is aligned to an extended grapheme cluster in the ICU
|
||||
string search service implementation defined by [UAX #29 Unicode Text
|
||||
Segmentation](http://www.unicode.org/reports/tr29/). Therefore, all matches will
|
||||
begin and end on extended grapheme cluster boundaries. If the given input search
|
||||
pattern starts with non-base character, no matches will be returned.
|
||||
When there are contractions in the collation sequence and the contraction
|
||||
happens to span across the boundary of a match, it is not considered a match.
|
||||
For example, in traditional Spanish where 'ch' is a contraction, the "har"
|
||||
pattern will not match in the string "uno charo". Boundaries that are
|
||||
discontiguous contractions will yield a match result similar to those described
|
||||
above, where the end of the match returned will be one character before the
|
||||
immediate following base letter. In addition, only the first match will be
|
||||
located if a pattern contains only combining marks and the search string
|
||||
contains more than one occurrences of the pattern consecutively. For example, if
|
||||
the user searches for the pattern "´" (\\u00b4) in the string "A´´B",
|
||||
(A\\u00b4\\u00b4B) the result will be offsets <1, 2>.
|
||||
|
||||
### Example
|
||||
|
||||
**In C:**
|
||||
|
||||
```c
|
||||
char *tgtstr = "The quick brown fox jumps over the lazy dog.";
|
||||
char *patstr = "fox";
|
||||
UChar target[64];
|
||||
|
||||
UChar pattern[16];
|
||||
int pos = 0;
|
||||
UErrorCode status = U_ZERO_ERROR;
|
||||
UStringSearch *search = NULL;
|
||||
|
||||
u_uastrcpy(target, tgtstr);
|
||||
u_uastrcpy(pattern, patstr);
|
||||
|
||||
|
||||
search = usearch_open(pattern, -1, target, -1, "en_US",
|
||||
NULL, &status);
|
||||
|
||||
|
||||
if (U_FAILURE(status)) {
|
||||
fprintf(stderr, "Could not create a UStringSearch.\n");
|
||||
return;
|
||||
}
|
||||
|
||||
for(pos = usearch_first(search, &status);
|
||||
U_SUCCESS(status) && pos != USEARCH_DONE;
|
||||
pos = usearch_next(search, &status))
|
||||
{
|
||||
fprintf(stdout, "Match found at position %d.\n", pos);
|
||||
}
|
||||
|
||||
if (U_FAILURE(status)) {
|
||||
fprintf(stderr, "Error searching for pattern.\n");
|
||||
}
|
||||
```
|
||||
|
||||
**In C++:**
|
||||
|
||||
```c++
|
||||
UErrorCode status = U_ZERO_ERROR;
|
||||
UnicodeString target("Jackdaws love my big sphinx of quartz.");
|
||||
UnicodeString pattern("sphinx");
|
||||
StringSearch search(pattern, target, Locale::getUS(), NULL, status);
|
||||
|
||||
|
||||
if (U_FAILURE(status)) {
|
||||
fprintf(stderr, "Could not create a StringSearch object.\n");
|
||||
return;
|
||||
}
|
||||
|
||||
for(int pos = search.first(status);
|
||||
U_SUCCESS(status) && pos != USEARCH_DONE;
|
||||
pos = search.next(status))
|
||||
{
|
||||
fprintf(stdout, "Match found at position %d.\n", pos);
|
||||
}
|
||||
|
||||
if (U_FAILURE(status)) {
|
||||
fprintf(stderr, "Error searching for pattern.\n");
|
||||
}
|
||||
```
|
||||
|
||||
**In Java:**
|
||||
|
||||
```java
|
||||
StringCharacterIterator target = new StringCharacterIterator(
|
||||
"Pack my box with five dozen liquor jugs.");
|
||||
String pattern = "box";
|
||||
|
||||
try {
|
||||
StringSearch search = new StringSearch(pattern, target, Locale.US);
|
||||
|
||||
|
||||
for(int pos = search.first();
|
||||
pos != StringSearch.DONE;
|
||||
pos = search.next())
|
||||
{
|
||||
System.out.println("Match found for pattern at position " + pos);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
System.err.println("StringSearch failure: " + e.toString());
|
||||
}
|
||||
```
|
||||
|
||||
## Performance and Other Implications
|
||||
|
||||
The ICU string search service is designed to be on top of the ICU collation
|
||||
service. Therefore, all the performance implications that apply to a collator
|
||||
are also applicable to the string search service. To obtain the best
|
||||
performance, use the default collator attributes described in the Performance
|
||||
and Storage Implications on Attributes section in the [Collation Service
|
||||
Architecture](architecture#performance-and-storage-implications-of-attributes)
|
||||
chapter. In addition, users need to be aware of
|
||||
the following `StringSearch` specific considerations:
|
||||
|
||||
### Search Algorithm
|
||||
|
||||
ICU4C (C/C++) releases up to 3.8 used the Boyer-Moore search algorithm in the string
|
||||
search service. There were some known issues in these previous releases.
|
||||
(See ICU tickets [ICU-5024](https://unicode-org.atlassian.net/browse/ICU-5024),
|
||||
[ICU-5382](https://unicode-org.atlassian.net/browse/ICU-5382),
|
||||
[ICU-5420](https://unicode-org.atlassian.net/browse/ICU-5420)).
|
||||
|
||||
In ICU4C 4.0, the string search service was updated to use a simple linear search
|
||||
algorithm, which locates a match by shifting a cursor in the target text one by one,
|
||||
and these issues were fixed.
|
||||
|
||||
In ICU4C 4.0.1, the Boyer-Moore search code was reintroduced as a separate API with
|
||||
technology preview status. However, in ICU4C 51.1, this was removed.
|
||||
(See ICU ticket [ICU-9573](https://unicode-org.atlassian.net/browse/ICU-9573)).
|
||||
|
||||
Similarly, in ICU4J 53 (Java) the Boyer-Moore search algorithm was replaced by the
|
||||
simple linear search algorithm, ported from ICU4C. (See ICU ticket [ICU-6288](https://unicode-org.atlassian.net/browse/ICU-6288)).
|
||||
|
||||
The Boyer-Moore search algorithm is based on automata or combinatorial properties of strings and
|
||||
pre-processes the pattern and known to be much faster than the linear search
|
||||
when search pattern length is longer. According to performance evaluation
|
||||
between these two implementations, the Boyer-Moore search is faster than the
|
||||
linear search when the pattern text is longer than 3 or 4 characters.
|
||||
However, it is very tricky to get correct results with a collation-based Boyer-Moore search.
|
||||
|
||||
### Change Iterating Direction
|
||||
|
||||
The ICU string search service provides a set of very dynamic APIs that allow
|
||||
users to change the iterating direction randomly. For example, users can search
|
||||
for a particular word going forward by calling the `usearch_next` (C),
|
||||
`StringSearch::next` (C++) or `StringSearch.next` (Java) APIs and then search
|
||||
backwards at any point of the search operation by calling the `usearch_previous`
|
||||
(C), `StringSearch::previous` (C++) or `StringSearch.previous` (Java) APIs. Another
|
||||
way to change the iterating direction is by calling the `usearch_reset` (C),
|
||||
`StringSearch::previous` (C++) or `StringSearch.previous` (Java) APIs. Though the
|
||||
direction change can occur without calling the reset APIs first, this operation
|
||||
comes with a reduction in speed.
|
||||
|
||||
> :point_right: **Note**: The backward search is not available with the
|
||||
> ICU4C Boyer-Moore search technology preview introduced in ICU4C 4.0.1
|
||||
> and only available with the linear search implementation.
|
||||
|
||||
### Thai and Lao Character Boundaries
|
||||
|
||||
In collation, certain Thai and Lao vowels are swapped with the next character.
|
||||
For example, the text string "A ขเ" (A \\u0e02\\u0e40) is processed internally
|
||||
in collation as
|
||||
"A เข" (A \\u0e40\\u0e02). Therefore, if the user searches for the pattern "Aเ"
|
||||
(A\\u0e40) in "A ขเ" (A \\u0e02\\u0e40) the string search service will match
|
||||
starting at offset 0. Since this normalization process is internal to collation,
|
||||
there is no notification that the swapping has happened. The return result
|
||||
offsets in this example will be <0, 2> even though the range would encompass one
|
||||
extra character.
|
||||
|
||||
### Case Level Search
|
||||
|
||||
Case level string search is currently done with the strength set to tertiary.
|
||||
When searching with the strength set to primary and the case level attribute
|
||||
turned on, results given may not be correct. The case level attribute is
|
||||
different from tertiary strength in that accents are ignored but case
|
||||
differences are not. Suppose you wanted to search for “A” in the text
|
||||
“ABC\\u00C5a”. The match found should be at 0 and 3 if using the case level
|
||||
attribute. However, searching with the case level attribute turned on finds
|
||||
matches at 0, 3, and 4, which includes the lower case 'a'. To ensure that case
|
||||
level differences are not ignored, string search must be done with at least
|
||||
tertiary strength.
|
||||
@@ -1,107 +0,0 @@
|
||||
---
|
||||
layout: default
|
||||
title: Compression
|
||||
nav_order: 4
|
||||
parent: Conversion
|
||||
---
|
||||
<!--
|
||||
© 2020 and later: Unicode, Inc. and others.
|
||||
License & terms of use: http://www.unicode.org/copyright.html
|
||||
-->
|
||||
|
||||
# Compression
|
||||
{: .no_toc }
|
||||
|
||||
## Contents
|
||||
{: .no_toc .text-delta }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
|
||||
---
|
||||
|
||||
## Overview of SCSU
|
||||
|
||||
Compressing Unicode text for transmission or storage results in minimal
|
||||
bandwidth usage and fewer storage devices. The compression scheme compresses
|
||||
Unicode text into a sequence of bytes by using characteristics of Unicode text.
|
||||
The compressed sequence can be used on its own or as further input to a general
|
||||
purpose file or disk-block based compression scheme. Note that the combination
|
||||
of the Unicode compression algorithm plus disk-block based compression produces
|
||||
better results than either method alone.
|
||||
|
||||
Strings in languages using small alphabets contain runs of characters that are
|
||||
coded close together in Unicode. These runs are typically interrupted only by
|
||||
punctuation characters, which are themselves coded in proximity to each other in
|
||||
Unicode (usually in the Basic Latin range).
|
||||
|
||||
For additional detail about the compression algorithm, which has been approved
|
||||
by the Unicode Consortium, please refer to [Unicode Technical Report #6 (A
|
||||
Standard Compression Scheme for
|
||||
Unicode)](https://www.unicode.org/reports/tr6/).
|
||||
|
||||
The Standard Compression Scheme for Unicode (SCSU) is used to:
|
||||
|
||||
* express all code points in Unicode
|
||||
|
||||
* approximate the storage size of traditional character sets
|
||||
|
||||
* facilitate the use of short strings
|
||||
|
||||
* provide transparency for characters between `U+0020`-`U+00FF`, as well as `CR`, `LF`
|
||||
and `TAB`
|
||||
|
||||
* support very simple decoders
|
||||
|
||||
* support simple as well as sophisticated encoders
|
||||
|
||||
It does not attempt to avoid the use of control bytes (including `NUL`) in the
|
||||
compressed stream.
|
||||
|
||||
The compression scheme is mainly intended for use with short to medium length
|
||||
Unicode strings. The resulting compressed format is intended for storage or
|
||||
transmission in bandwidth limited environments. It can be used stand-alone or as
|
||||
input to traditional general purpose data compression schemes. It is not
|
||||
intended as processing format or as general purpose interchange format.
|
||||
|
||||
## BOCU-1
|
||||
|
||||
A MIME compatible encoding called BOCU-1 is also available in ICU. Details about
|
||||
this encoding can be found in the [Unicode Technical Note
|
||||
#6](https://www.unicode.org/notes/tn6/). Both SCSU and BOCU-1 are IANA
|
||||
registered names.
|
||||
|
||||
## Usage
|
||||
|
||||
The compression service in ICU is a part of Conversion framework, and follows
|
||||
the semantics of converters. For more information on how to use ICU's conversion
|
||||
service, please refer to the Usage Model section in the [Using
|
||||
Converters](converters.md) chapter.
|
||||
|
||||
```c++
|
||||
uint16_t germanUTF16[]={
|
||||
0x00d6, 0x006c, 0x0020, 0x0066, 0x006c, 0x0069, 0x0065, 0x00df, 0x0074
|
||||
};
|
||||
|
||||
uint8_t germanSCSU[]={
|
||||
0xd6, 0x6c, 0x20, 0x66, 0x6c, 0x69, 0x65, 0xdf, 0x74
|
||||
};
|
||||
char target[100];
|
||||
UChar uTarget[100];
|
||||
UErrorCode status = U_ZERO_ERROR;
|
||||
UConverter *conv;
|
||||
int32_t len;
|
||||
|
||||
/* set up the SCSU converter */
|
||||
conv = ucnv_open("SCSU", &status);
|
||||
assert(U_SUCCESS(status));
|
||||
|
||||
/* compress the string using SCSU */
|
||||
len = ucnv_fromUChars(conv, target, 100, germanUTF16, -1, &status);
|
||||
assert(U_SUCCESS(status));
|
||||
|
||||
len = ucnv_toUChars(conv, uTarget, 100, germanSCSU, -1, &status);
|
||||
|
||||
/* close the converter */
|
||||
ucnv_close(conv);
|
||||
```
|
||||
@@ -1,835 +0,0 @@
|
||||
---
|
||||
layout: default
|
||||
title: Converter
|
||||
nav_order: 1
|
||||
parent: Conversion
|
||||
---
|
||||
<!--
|
||||
© 2020 and later: Unicode, Inc. and others.
|
||||
License & terms of use: http://www.unicode.org/copyright.html
|
||||
-->
|
||||
|
||||
# Using Converters
|
||||
{: .no_toc }
|
||||
|
||||
## Contents
|
||||
{: .no_toc .text-delta }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
When designing applications around Unicode characters, it is sometimes required
|
||||
to convert between Unicode encodings or between Unicode and legacy text data.
|
||||
The vast majority of modern Operating Systems support Unicode to some degree,
|
||||
but sometimes the legacy text data from older systems need to be converted to
|
||||
and from Unicode. This conversion process can be done with an ICU converter.
|
||||
|
||||
## ICU converters
|
||||
|
||||
ICU provides comprehensive character set conversion services, mapping tables,
|
||||
and implementations for many encodings. Since ICU uses Unicode (UTF-16)
|
||||
internally, all converters convert between UTF-16 (with the endianness according
|
||||
to the current platform) and another encoding. This includes Unicode encodings.
|
||||
In other words, internal text is 16-bit Unicode, while "external text" used as
|
||||
source or target for a conversion is always treated as a byte stream.
|
||||
|
||||
ICU converters are available for a wide range of encoding schemes. Most of them
|
||||
are based on mapping table data that is handled by few generic implementations.
|
||||
Some encodings are implemented algorithmically in addition to (or instead of)
|
||||
using mapping tables, especially Unicode encodings. The partly or entirely
|
||||
table-based encoding schemes include: All ICU converters map only single Unicode
|
||||
character code points to and from single codepage character code points. ICU
|
||||
converters **do not** deal directly with combining characters, bidirectional
|
||||
reordering, or Arabic shaping, for example. Such processes, if required, must be
|
||||
handled separately. For example, while in Unicode, the ICU BiDi APIs can be used
|
||||
for bidirectional reordering after a conversion to Unicode or before a
|
||||
conversion from Unicode.
|
||||
|
||||
ICU converters are not designed to perform any encoding autodetection. This
|
||||
means that the converters do not autodetect "endianness", the 6 Unicode encoding
|
||||
signatures, or the Shift-JIS vs. EUC-JP, etc. There are two exceptions: The
|
||||
UTF-16 and UTF-32 converters work according to Unicode's specification of their
|
||||
Character Encoding Schemes, that is, they read the BOM to figure out the actual
|
||||
"endianness".
|
||||
|
||||
The ICU mapping tables mostly come from an [IBM® codepage
|
||||
repository](http://www.ibm.com/software/globalization/cdra). For non-IBM
|
||||
codepages, there is typically an equivalent codepage registered with this
|
||||
repository. However, the textual data format (.ucm files) is generic, and data
|
||||
for other codepage mapping tables can also be added.
|
||||
|
||||
## Using the Default Codepage
|
||||
|
||||
ICU has code to determine the default codepage of the system or process. This
|
||||
default codepage can be used to convert `char *` strings to and from Unicode.
|
||||
|
||||
Depending on system design, setup and APIs, it may not always be possible to
|
||||
find a default codepage that fully works as expected. For example,
|
||||
|
||||
1. On Windows there are three encodings in use at the same time. Unicode
|
||||
(UTF-16) is always used inside of Windows, while for `char *` encodings there
|
||||
are two classes, called "ANSI" and "OEM" codepages. ICU will use the ANSI
|
||||
codepage. Note that the OEM codepage is used by default for console window
|
||||
output.
|
||||
|
||||
2. On some UNIX-type systems, non-standard names are used for encodings, or
|
||||
non-standard encodings are used altogether. Although ICU supports over 200
|
||||
encodings in its standard build and many more aliases for them, it will not
|
||||
be able to recognize such non-standard names.
|
||||
|
||||
3. Some systems do not have a notion of a system or process codepage, and may
|
||||
not have APIs for that.
|
||||
|
||||
If you have means of detecting a default codepage name that are more appropriate
|
||||
for your application, then you should set that name with `ucnv_setDefaultName()`
|
||||
as the first ICU function call. This makes sure that the internally cached
|
||||
default converter will be instantiated from your preferred name.
|
||||
|
||||
Starting in ICU 2.0, when a converter for the default codepage cannot be opened,
|
||||
a fallback default codepage name and converter will be used. On most platforms,
|
||||
this will be US-ASCII. For z/OS (OS/390), ibm-1047,swaplfnl is the default
|
||||
fallback codepage. For AS/400 (iSeries), ibm-37 is the default fallback
|
||||
codepage. This default fallback codepage is used when the operating system is
|
||||
using a non-standard name for a default codepage, or the converter was not
|
||||
packaged with ICU. The feature allows ICU to run in unusual computing
|
||||
environments without completely failing.
|
||||
|
||||
## Usage Model
|
||||
|
||||
A "Converter" refers to the C structure "UConverter". Converters are cheap to
|
||||
create. Any data that is shared between converters of the same kind (such as the
|
||||
mappings, the name and the properties) are automatically cached and shared in
|
||||
memory.
|
||||
|
||||
### Converter Names
|
||||
|
||||
Codepages with encoding schemes have been given many names by various vendors
|
||||
and platforms over the years. Vendors have different ways specify which codepage
|
||||
and encoding are being used. IBM uses a CCSID (Coded Character Set IDentifier).
|
||||
Windows uses a CPID (CodePage IDentifier). Macintosh has a TextEncoding. Many
|
||||
Unix vendors use [IANA](http://www.iana.org/assignments/character-sets)
|
||||
character set names. Many of these names are aliases to converters within ICU.
|
||||
|
||||
In order to help identify which names are recognized by certain platforms, ICU
|
||||
provides several converter alias functions. The complete description of these
|
||||
functions can be found in the [ICU API Reference](https://unicode-org.github.io/icu-docs/apidoc/released/icu4c/ucnv_8h.html) .
|
||||
|
||||
| Function Names | Short Description |
|
||||
| -------------- | ----------------- |
|
||||
| `ucnv_countAvailable`, `ucnv_getAvailableName` | Get a list of available converter names that can be opened. |
|
||||
| `ucnv_openAllNames` | Get a list of all known converter names. |
|
||||
| `ucnv_getName` | Get the name of an open converter. |
|
||||
| `ucnv_countAliases`, `ucnv_getAlias` | Get the list of aliases for the specified converter. |
|
||||
| `ucnv_countStandards`, `ucnv_getStandard` | Get the list of known standards. |
|
||||
| `ucnv_openStandardNames` | Get a filtered list of aliases for a converter that is known by the specified standard. |
|
||||
| `ucnv_getStandardName` | Get the preferred alias name specified by a given standard. |
|
||||
| `ucnv_getCanonicalName` | Get the converter name from the alias that is recognized by the specified standard. |
|
||||
| `ucnv_getDefaultName` | Get the default converter name that is currently used by ICU and the operating system. |
|
||||
| `ucnv_setDefaultName` | Use this function to override the default converter name. |
|
||||
|
||||
Even though IANA specifies a list of aliases, it usually does not specify the
|
||||
mappings or the actual character set for the aliases. Sometimes vendors will map
|
||||
similar glyph variants to different Unicode code points or sometimes they will
|
||||
assign completely different glyphs for the same codepage code point. Because of
|
||||
these ambiguities, you can sometimes get `U_AMBIGUOUS_ALIAS_WARNING` for the
|
||||
returned `UErrorCode` when more than one converter uses the requested alias. This
|
||||
is only a warning, and the results can still be used. This UErrorCode value is
|
||||
just a reminder that you may not get what you expected. The above functions can
|
||||
help you to determine which converter you actually wanted.
|
||||
|
||||
EBCDIC based converters do have the option to swap the newline and linefeed
|
||||
character mappings. This can be useful when transferring EBCDIC documents
|
||||
between z/OS (MVS, os/390 and the rest of the zSeries family) and another EBCDIC
|
||||
machine like OS/400 on iSeries. The ",swaplnlf" or `UCNV_SWAP_LFNL_OPTION_STRING`
|
||||
from ucnv.h can be appended to a converter alias in order to achieve this
|
||||
behavior. You can view other available options in ucnv.h.
|
||||
|
||||
You can always skip many of these aliasing and mapping problems by just using
|
||||
Unicode.
|
||||
|
||||
### Creating a Converter
|
||||
|
||||
There are four ways to create a converter:
|
||||
|
||||
1. **By name**: Converters can be created using different types of names. No
|
||||
distinction is made when the converter is created, as to which name is being
|
||||
employed. There are many types of aliases possible. Among these are
|
||||
[IANA](http://www.iana.org/assignments/character-sets) ("shift_jis",
|
||||
"koi8-r", or "iso-8859-3"), host specific names ("cp1252" which is the name
|
||||
for a Microsoft® Windows™ or a similar IBM® codepage). Finally, ICU's own
|
||||
internal canonical names for a converter can be used. These include "UTF-8"
|
||||
or "ISO-8859-1" for built-in conversion types, and names such as
|
||||
"ibm-949_P110-2000" (Shift-JIS with '\\' <-> '¥' mapping) or
|
||||
"ibm-949_P11A-2000" (Shift-JIS with '\\' <-> '\\' mapping) for data-file
|
||||
based conversions.
|
||||
|
||||
```c
|
||||
UConverter *conv = ucnv_open("shift_jis", &myError);
|
||||
```
|
||||
|
||||
As a convenience, converter names can be passed in as Unicode. (for example,
|
||||
if a user passed in the string from a Unicode-based user interface).
|
||||
However, the actual names are restricted to an invariant ASCII/EBCDIC
|
||||
subset.
|
||||
|
||||
```c
|
||||
UChar *name = ...; UConverter *conv = ucnv_openU(name, &myError);
|
||||
```
|
||||
|
||||
Converter names are case-insensitive. In addition, beginning with ICU 3.6,
|
||||
leading zeroes are ignored in sequences of digits (if further digits
|
||||
follow), and all non-alphanumeric characters are ignored. Thus the strings
|
||||
"UTF-8", "utf_8", "u\*T@f08" and "Utf 8" are equivalent. (Before ICU 3.6,
|
||||
leading zeroes were not ignored, and only spaces, dashes and underscores
|
||||
were ignored.) The `ucnv_compareNames()` function provides such string
|
||||
comparisons.
|
||||
|
||||
Unlike the names of resources or other types of ICU data, converter names
|
||||
can **not** be qualified with a path that indicates the directory or common
|
||||
data file containing the corresponding converter data. The requested
|
||||
converter's data must be present either in the main ICU data library or as a
|
||||
separate file located in the ICU data directory. However, you can always
|
||||
create a package of converters with pkgdata and open a converter from the
|
||||
package with `ucnv_openPackage()`
|
||||
|
||||
```c
|
||||
UConverter *conv = ucnv_openPackage("./myPackage.dat", "customConverter", &myError);
|
||||
```
|
||||
|
||||
2. **By number**: The design of the ICU is to accommodate codepages provided by
|
||||
different vendors. For example, the IBM CDRA (Character Data Representation
|
||||
Architecture which is an IBM architecture that defines a set of identifiers)
|
||||
has an ID type called the CCSID (Coded Character Set Identifier). The ICU
|
||||
API for opening a codepage by number must be given a vendor along with the
|
||||
number. Currently, only IBM (`UCNV_IBM`) is supported. For example, the US
|
||||
EBCDIC codepage (IBM #37) can be opened with the following code:
|
||||
|
||||
```c
|
||||
ucnv_openCCSID(37, UCNV_IBM, &myErr);
|
||||
```
|
||||
|
||||
3. **By iteration**: An application might not know ahead of time which codepage
|
||||
to use, and thus might need to query ICU to determine the entire list of
|
||||
installed converters. The ICU returns a list of its canonical (internal)
|
||||
names. From each names, the standard IANA name can be determined, and also a
|
||||
list of aliases which point to that name can be determined. For example, ICU
|
||||
might return among the canonical names "ibm-367". That name itself may or
|
||||
may not provide the application or its users with the information needed.
|
||||
(367 is actually the decimal form of a number that is calculated by
|
||||
appending certain hex digits together.) However, the IANA name can be
|
||||
requested from this canonical name, which should return something like
|
||||
"us-ascii". The alias list for ibm-367 can be iterated over as well, which
|
||||
returns additional names like "ascii", "646", "ansi_x3.4-1968" etc. If this
|
||||
is not sufficient information, once a converter is opened, it can be queried
|
||||
for its type, min and max char size, etc. This information is not available
|
||||
without actually opening the converter (a fairly lightweight process.)
|
||||
|
||||
```c
|
||||
/* Returns count of the number of available names */
|
||||
int count = ucnv_countAvailable();
|
||||
/* get the canonical name of the 36th available converter */
|
||||
const char *convName1 = ucnv_getAvailableName(36);
|
||||
/* get the 3rd alias for a given codepage. */
|
||||
const char *asciiAlias = ucnv_getAlias("ibm-367", 3, &myError);
|
||||
/* Get the IANA name of the converter */
|
||||
const char *ascii = ucnv_getStandardName("ibm-367", "IANA");
|
||||
/* Get the one of the non preferred IANA name of the converter. */
|
||||
UEnumeration *asciiEnum =
|
||||
ucnv_openStandardNames("ibm-367", "IANA", &myError);
|
||||
uenum_next(asciiEnum, &myError); /* skip preferred IANA alias */
|
||||
/* get one of the non-preferred IANA aliases */
|
||||
const char *ascii2 = uenum_next(asciiEnum, &myError);
|
||||
uenum_close(asciiEnum);
|
||||
```
|
||||
|
||||
4. **By using the default converter**: The default converter can be opened by
|
||||
passing a NULL as the name of the converter.
|
||||
|
||||
```c
|
||||
ucnv_open(NULL, &myErr);
|
||||
```
|
||||
|
||||
> :point_right: **Note**: ICU chooses this converter based on the best information available to it.
|
||||
> The purpose of this converter is to interface with the OS using a codepage (i.e. `char *`).
|
||||
> Do not use it as a way of determining the best overall converter to use.
|
||||
> Usually any Unicode encoding form is the best way to store and send text data,
|
||||
> so that important data does not get lost in the conversion.
|
||||
> Also, if the OS supports Unicode-based API's (such as Win32),
|
||||
> it is better to use only those Unicode API's.
|
||||
> As an example, the new Windows 2000 locales (such as Hindi) do not
|
||||
> define the default codepage to something that supports Hindi.
|
||||
> The default converter is used in expressions such as: `UnicodeString text("abc");`
|
||||
> to convert 'abc', and in the `u_uastrcpy()` C functions.
|
||||
> Code operating at the [OS level](../design.md) MAY choose to
|
||||
> change the default converter with `ucnv_setDefaultName()`.
|
||||
> However, be aware that this change has inconsistent results if it is done after
|
||||
> ICU components are initialized.
|
||||
|
||||
### Closing a Converter
|
||||
|
||||
Closing a converter frees memory occupied by that instance of the converter.
|
||||
However it does not release the larger shared data tables the converter might
|
||||
use. OS-level code may call `ucnv_flushCache()` to explicitly free memory occupied
|
||||
by [unused tables](../design.md).
|
||||
|
||||
```c
|
||||
ucnv_close(conv)
|
||||
```
|
||||
|
||||
### Converter Life Cycle
|
||||
|
||||
Note that a Converter is created with a certain type (for instance, ISO-8859-3)
|
||||
which does not change over the life of that [object](../design.md). Converters
|
||||
should be allocated one per thread. They are cheap to create, as the shared data
|
||||
doesn't need to be reallocated.
|
||||
|
||||
This is the typical life cycle of a converter, as shown step-by-step:
|
||||
|
||||
1. First, open up the converter with a specified name (or alias name).
|
||||
```c
|
||||
UConverter *conv = ucnv_open("shift_jis", &status);
|
||||
```
|
||||
|
||||
2. Target here is the `char s[]` to write into, and targetSize is how big the
|
||||
target buffer is. Source is the UChars that are being converted.
|
||||
```c
|
||||
int32_t len = ucnv_fromUChars(conv, target, targetSize, source, u_strlen(source), &status);
|
||||
```
|
||||
|
||||
3. Clean up the converter.
|
||||
```c
|
||||
ucnv_close(conv);
|
||||
```
|
||||
|
||||
### Sharing Converters Between Threads
|
||||
|
||||
A converter cannot be shared between threads at the same time. However, if it is
|
||||
reset it can be used for unrelated chunks of data. For example, use the same
|
||||
converter for converting data from Unicode to ISO-8859-3, and then reset it. Use
|
||||
the same converter for converting data from ISO-8859-3 back into Unicode.
|
||||
|
||||
### Converting Large Quantities of Data
|
||||
|
||||
If it is necessary to convert a large quantity of data in smaller buffers, use
|
||||
the same converter to convert each buffer. This will make sure any state is
|
||||
preserved from one chunk to the next. Doing this conversion is known as
|
||||
streaming or buffering, and is mentioned [Buffered or Streamed](#3-buffered-or-streamed)
|
||||
section (§) later in this chapter.
|
||||
|
||||
### Cloning a Converter
|
||||
|
||||
Cloning a converter returns a clone of the converter object along with any
|
||||
internal state that the converter might be storing. Cloning routines must be
|
||||
used with extreme care when using converters for stateful or multibyte
|
||||
encodings. If the converter object is carrying an internal state, and the
|
||||
newly-created clone is used to convert a new chunk of text, the converter
|
||||
produces incorrect results. Also note that the caller owns the cloned object and
|
||||
has to call `ucnv_close()` to dispose of the object. Calling `ucnv_reset()` before
|
||||
cloning will reset the converter to its original state.
|
||||
|
||||
```c
|
||||
UConverter* newCnv = ucnv_safeClone(oldCnv, 0, &bufferSize, &err)
|
||||
```
|
||||
|
||||
## Converter Behavior
|
||||
|
||||
### Conversion
|
||||
|
||||
1. The converters always consume the source buffer as far as possible, and
|
||||
advance the source pointer.
|
||||
|
||||
2. The converters write to the target all converted output as far as possible,
|
||||
and then write any remaining output to the internal services buffer. When
|
||||
the conversion routines are called again, the internal buffer is flushed out
|
||||
and written to the target buffer before proceeding with any further
|
||||
conversion.
|
||||
|
||||
3. In conversions to Unicode from Multi-byte encodings or conversions from
|
||||
Unicode involving surrogates, if (a) only a partial byte sequence is
|
||||
retrieved from the source buffer, (b) the "flush" parameter is set to "TRUE"
|
||||
and (c) the end of source is reached, then the callback is called with
|
||||
`U_TRUNCATED_CHAR_FOUND`.
|
||||
|
||||
### Reset
|
||||
|
||||
Converters can be reset explicitly or implicitly. Explicit reset is done by
|
||||
calling:
|
||||
|
||||
1. `ucnv_reset()`: Resets the converter to initial state in both directions.
|
||||
|
||||
2. `ucnv_resetToUnicode()`: Resets the converter to initial state to Unicode
|
||||
direction.
|
||||
|
||||
3. `ucnv_resetFromUnicode()`: Resets the converter to initial state from Unicode
|
||||
direction.
|
||||
|
||||
The converters are reset implicitly when the conversion functions are called
|
||||
with the "flush" parameter set to "TRUE" and the source is consumed.
|
||||
|
||||
### Error
|
||||
|
||||
#### Conversion from Unicode
|
||||
|
||||
Not all characters can be converted from Unicode to other codepages. In most
|
||||
cases, Unicode is a superset of the characters supported by any given codepage.
|
||||
|
||||
The default behavior of ICU in this case is to substitute the illegal or
|
||||
unmappable sequence, with the appropriate substitution sequence for that
|
||||
codepage. For example, ISO-8859-1, along with most ASCII-based codepages, has
|
||||
the character 0x1A (Control-Z) as the substitution sequence. When converting
|
||||
from Unicode to ISO-8859-1, any characters which cannot be converted would be
|
||||
replaced by 0x1A's.
|
||||
|
||||
SubChar1 is sometimes used as substitution character in MBCS conversions. For
|
||||
more information on SubChar1 please see the [Conversion Data](data.md) chapter.
|
||||
|
||||
In stateful converters like ISO-2022-JP, if a substitution character has to be
|
||||
written to the target, then an escape/shift sequence to change the state to
|
||||
single byte mode followed by a substitution character is written to the target.
|
||||
|
||||
The substitution character can be changed by calling the `ucnv_setSubstChars()`
|
||||
function with the desired codepage byte sequence. However, this has some
|
||||
limitations: It only allows setting a single character (although the character
|
||||
can consist of multiple bytes), and it may not work properly for some stateful
|
||||
converters (like HZ or ISO 2022 variants) when setting a multi-byte substitution
|
||||
character. (It will work for EBCDIC_STATEFUL ones.) Moreover, for setting a
|
||||
particular character, the caller needs to know the correct byte sequence for
|
||||
that character in the converter's codepage. (For example, a space (U+0020) is
|
||||
encoded as 0x20 in ASCII-based codepages, 0x40 in EBCDIC-based ones, 0x00 0x20
|
||||
or 0x20 0x00 in UTF-16 depending on the stream's endianness, etc.)
|
||||
|
||||
The `ucnv_setSubstString()` function (new in ICU 3.6) lifts these limitations. It
|
||||
takes a Unicode string and verifies that it can be converted to the codepage
|
||||
without error and that it is not too long (32 bytes as of ICU 3.6). The string
|
||||
can contain zero, one or more characters. An empty string has the effect of
|
||||
using the skip callback. See the Error Callbacks below. Stateful converters are
|
||||
fully supported. The same Unicode string will give equivalent results with all
|
||||
converters that support its conversion.
|
||||
|
||||
Internally, `ucnv_setSubstString()` stores the byte sequence from the test
|
||||
conversion if the converter is stateless, or the Unicode string itself if the
|
||||
converter is stateful. If the Unicode string is stored, then it is converted on
|
||||
the fly during substitution, handling all state transitions.
|
||||
|
||||
The function `ucnv_getSubstChars()` can be used to retrieve the substitution byte
|
||||
sequence if it is the default one, set by `ucnv_setSubstChars()`, or if
|
||||
`ucnv_setSubstString()` stored the byte sequence for a stateless converter. The
|
||||
Unicode string set for a stateful converter cannot be retrieved.
|
||||
|
||||
#### Conversion to Unicode
|
||||
|
||||
In conversion to Unicode, errors are normally due to ill-formed byte sequences:
|
||||
Unused byte values, or lead bytes not followed by trail bytes according to the
|
||||
encoding scheme. Well-formed but unmappable sequences are unusual but possible.
|
||||
|
||||
The ICU default behavior is to emit an `U+FFFD REPLACEMENT CHARACTER` per
|
||||
offending sequence.
|
||||
|
||||
If the conversion table .ucm file contains a `<subchar1>` entry (such as in the
|
||||
ibm-943 table), a U+001A C0 control ("SUB") is emitted for single-byte
|
||||
illegal/unmappable input rather than `U+FFFD REPLACEMENT CHARACTER`. For details
|
||||
on this behavior look for "001A" in the [Conversion Data](data.md) chapter.
|
||||
|
||||
* This behavior originates from mainframes with dedicated single-byte-to-single-byte
|
||||
and double-to-double conversions.
|
||||
* Emitting U+001A for single-byte errors can be avoided by (a) removing the
|
||||
`<subchar1>` mapping or (b) using a similar conversion table that does not
|
||||
have this mapping (e.g., windows-932 instead of ibm-943) or (c) writing a
|
||||
custom callback function.
|
||||
|
||||
### Error Codes
|
||||
|
||||
Here are some of the `UErrorCode`s which have significant meaning for conversion:
|
||||
|
||||
#### U_INDEX_OUTOFBOUNDS_ERROR
|
||||
|
||||
In `getNextUChar()` - all source data
|
||||
has been consumed without producing a Unicode character
|
||||
|
||||
#### U_INVALID_CHAR_FOUND
|
||||
No mapping was found from the source to the target encoding. For example, U+0398
|
||||
(Capital Theta) has no mapping into ISO-8859-1, and so U_INVALID_CHAR_FOUND
|
||||
will result.
|
||||
|
||||
#### U_TRUNCATED_CHAR_FOUND
|
||||
|
||||
All of the source data was read, and a
|
||||
character sequence was incomplete. For example, only half of a double-byte
|
||||
sequence may have been encountered. When converting FROM Unicode, this error
|
||||
would occur when a conversion ends with a low surrogate (U+D800) at the end of
|
||||
the source, with no corresponding high surrogate.
|
||||
|
||||
#### U_ILLEGAL_CHAR_FOUND
|
||||
|
||||
A character sequence was found in the source which is disallowed in the source
|
||||
encoding scheme. For example, many MBCS encodings have only certain byte
|
||||
sequences which are allowed as lead bytes. When converting from Unicode, if a
|
||||
low surrogate is NOT followed immediately by a high surrogate, or a high
|
||||
surrogate without its preceding low surrogate, an illegal sequence results.
|
||||
Note: Most, but not all, converters forbid surrogate code points or unpaired
|
||||
surrogate code units. (Lead surrogate without trail, or trail without lead.)
|
||||
Some converters permit surrogate code points/unpaired surrogates because their
|
||||
charset specification permits it. For example, LMBCS, SCSU and
|
||||
BOCU-1.
|
||||
|
||||
#### U_INVALID_TABLE_FORMAT
|
||||
|
||||
An error occurred trying to read the backing data
|
||||
for the converter. The data could be corrupt, or the wrong
|
||||
version.
|
||||
|
||||
#### U_BUFFER_OVERFLOW_ERROR
|
||||
|
||||
More output (target) characters were produced
|
||||
than fit in the target buffer. If in `to/fromUnicode()`, then process the target
|
||||
buffer and call the function again to retrieve the overflowed characters.
|
||||
|
||||
### Error Callbacks
|
||||
|
||||
What actually happens is that an "error callback function" is called at the
|
||||
point where the conversion failure occurred. The function can deal with the
|
||||
failed characters as it sees fit. Possible options at the callback's disposal
|
||||
include ignoring the bad sequence, converting it to a different sequence, and
|
||||
returning an error to the caller. The callback can also consume any data past
|
||||
where the error occurred, whether or not that data would have caused an error.
|
||||
Only one callback is installed at a time, per direction (to or from unicode).
|
||||
|
||||
A number of canned functions are provided by ICU, and an application can write
|
||||
new ones. The "callbacks" are either From Unicode (to codepage), or To Unicode
|
||||
(from codepage). Here is a list of the canned callbacks in ICU:
|
||||
|
||||
1. UCNV_**FROM_U**_CALLBACK_SUBSTITUTE: This callback is installed by default.
|
||||
It will write the codepage's substitute sequence or a user-set substitute
|
||||
sequence, or convert a user-set substitute UnicodeString to the codepage.
|
||||
See "Error / Conversion from Unicode" above.
|
||||
|
||||
2. UCNV_**TO_U**_CALLBACK_SUBSTITUTE: This callback is installed by default. It
|
||||
will write U+FFFD or sometimes U+001A. See "Error / Conversion to Unicode"
|
||||
above.
|
||||
|
||||
3. UCNV_FROM_U_CALLBACK_SKIP, UCNV_TO_U_CALLBACK_SKIP: Simply ignores any
|
||||
invalid characters in the input, no error is returned.
|
||||
|
||||
4. UCNV_FROM_U_CALLBACK_STOP, UCNV_TO_U_CALLBACK_STOP: Stop at the error.
|
||||
Return the error to the caller. (When using the 'BUFFER' mode of conversion,
|
||||
the source and target pointers returned can be examined to determine where
|
||||
the error occurred. `ucnv_getInvalidUChars()` and `ucnv_getInvalidChars()`
|
||||
return the actual text which failed).
|
||||
|
||||
5. UCNV_FROM_U_CALLBACK_ESCAPE, UCNV_TO_U_CALLBACK_ESCAPE: This callback is
|
||||
especially useful for debugging. Missing codepage characters are replaced by
|
||||
strings such as '%U094D' with the Unicode value, and missing Unicode chars
|
||||
are replaced with text of the form '%X0A' where the codepage had the
|
||||
unconvertible byte hex 0A.
|
||||
|
||||
When a callback is set, a "context" pointer is also provided. How this
|
||||
pointer is created depends on the specific callback. There is usually a
|
||||
`createContext()` function for that specific callback, where the caller can
|
||||
set certain options for the callback. Consult the documentation for the
|
||||
specific callback you are using. For ICU's canned callbacks, this pointer
|
||||
may be set to NULL. The functions for setting a different callback also
|
||||
return the old callback, and the old context pointer. These may be stored so
|
||||
that the old callback is re-installed when an operation is finished.
|
||||
|
||||
Additionally the following options can be passed as the context parameter to
|
||||
UCNV_FROM_U_CALLBACK_ESCAPE callback function to produce different outputs.
|
||||
|
||||
| UCNV_ESCAPE_ICU | %U12345 |
|
||||
| ------------------- | ------- |
|
||||
| UCNV_ESCAPE_JAVA | \\u1234 |
|
||||
| UCNV_ESCAPE_C | \\udbc9\\udd36 for Plane 1 and \\u1234 for Plane 0 codepoints |
|
||||
| UCNV_ESCAPE_XML_DEC | \ᅬ number expressed in Decimal |
|
||||
| UCNV_ESCAPE_XML_HEX | \ሴ number expressed in Hexadecimal |
|
||||
|
||||
Here are some examples of how to use callbacks.
|
||||
|
||||
```c
|
||||
UConverter *u;
|
||||
void *oldContext, *newContext;
|
||||
UConverterFromUCallback oldAction, newAction;
|
||||
u = ucnv_open("shift_jis", &myError);
|
||||
|
||||
... /* do some conversion with u from unicode.. */
|
||||
|
||||
ucnv_setFromUCallBack(
|
||||
u, MY_FROMU_CALLBACK, newContext, &oldAction, &oldContext, &myError);
|
||||
|
||||
... /* do some other conversion from unicode */
|
||||
|
||||
/* Now, set the callback back */
|
||||
ucnv_setFromUCallBack(
|
||||
u, oldAction, oldContext, &newAction, &newContext, &myError);
|
||||
|
||||
```
|
||||
|
||||
### Custom Callbacks
|
||||
|
||||
Writing a callback is somewhat involved, and will be covered more completely in
|
||||
a future version of this document. One might look at the source to the provided
|
||||
callbacks as a starting point, and address any further questions to the mailing
|
||||
list.
|
||||
|
||||
Basically, callback, unlike other ICU functions which expect to be called with
|
||||
`U_ZERO_ERROR` as the input, is called in an exceptional error condition. The
|
||||
callback is a kind of 'last ditch effort' to rectify the error which occurred,
|
||||
before it is returned back to the caller. This is why the implementation of STOP
|
||||
is very simple:
|
||||
|
||||
```c
|
||||
void UCNV_FROM_U_CALLBACK_STOP(...) { }
|
||||
```
|
||||
|
||||
The error code such as `U_INVALID_CHAR_FOUND` is returned to the user. If the
|
||||
callback determines that no error should be returned to the user, then the
|
||||
callback must set the error code to `U_ZERO_ERROR`. Note that this is a departure
|
||||
from most ICU functions, which are supposed to check the error code and return
|
||||
immediately if it is set.
|
||||
|
||||
> :point_right: **Note**: See the functions `ucnv_cb_write...()` for
|
||||
> functions which a callback may use to perform its task.
|
||||
|
||||
#### Ignore Default_Ignorable_Code_Point
|
||||
|
||||
Unicode has a number of characters that are not by themselves meaningful but
|
||||
assist with line breaking (e.g., U+00AD Soft Hyphen & U+200B Zero Width Space),
|
||||
bi-directional text layout (U+200E Left-To-Right Mark), collation and other
|
||||
algorithms (U+034F Combining Grapheme Joiner), or indicate a preference for a
|
||||
particular glyph variant (U+FE0F Variation Selector 16). These characters are
|
||||
"invisible" by default, that is, they should normally not be shown with a glyph
|
||||
of their own, except in special circumstances. Examples include showing a hyphen
|
||||
for when a Soft Hyphen was used for a line break, or modifying the glyph of a
|
||||
character preceding a Variation Selector.
|
||||
|
||||
Unicode has a character property to identify such characters, as well as
|
||||
currently-unassigned code points that are intended to be used for similar
|
||||
purposes: Default_Ignorable_Code_Point, or "DI" for short:
|
||||
http://www.unicode.org/cldr/utility/list-unicodeset.jsp?a=[:DI:]
|
||||
|
||||
Most charsets do not have most or any of these characters.
|
||||
|
||||
**ICU 54 and above by default skip default-ignorable code points if they are
|
||||
unmappable**. (Ticket #[10551](https://unicode-org.atlassian.net/browse/ICU-10551))
|
||||
|
||||
**Older versions of ICU** replaced unmappable default-ignorable code points like
|
||||
any other unmappable code points, by a question mark or whatever substitution
|
||||
character is defined for the charset.
|
||||
|
||||
For best results, a custom from-Unicode callback can be used to ignore
|
||||
Default_Ignorable_Code_Point characters that cannot be converted, so that they
|
||||
are removed from the charset output rather than replaced by a visible character.
|
||||
|
||||
This is a code snippet for use in a custom from-Unicode callback:
|
||||
|
||||
```c
|
||||
#include "unicode/uchar.h"
|
||||
// ...
|
||||
(from-Unicode callback)
|
||||
switch(reason) {
|
||||
case UCNV_UNASSIGNED:
|
||||
if(u_hasBinaryProperty(codePoint, UCHAR_DEFAULT_IGNORABLE_CODE_POINT)) {
|
||||
// Ignore/drop default ignorable code points that cannot be converted,
|
||||
// rather than treating them like errors/writing a substitution character etc.
|
||||
// For example, U+200B Zero Width Space,
|
||||
// U+200E Left-To-Right Mark, U+FE0F Variation Selector 16.
|
||||
*pErrorCode = U_ZERO_ERROR;
|
||||
return;
|
||||
} else {
|
||||
// ...
|
||||
```
|
||||
|
||||
## Modes of Conversion
|
||||
|
||||
When a converter is instantiated, it can be used to convert both in the Unicode
|
||||
to Codepage direction, and also in the Codepage to Unicode direction. There are
|
||||
three ways to use the converters, as well as a convenience function which does
|
||||
not require the instantiation of a converter.
|
||||
|
||||
1. **Single-String**: Simplest type of conversion to or from Unicode. The data
|
||||
is entirely contained within a single string.
|
||||
|
||||
2. **Character**: Converting from the codepage to a single Unicode codepoint,
|
||||
one at a time.
|
||||
|
||||
3. **Buffer**: Convert data which may not fit entirely within a single buffer.
|
||||
Usually the most efficient and flexible.
|
||||
|
||||
4. **Convenience**: Convert a single buffer from one codepage to another
|
||||
through Unicode, without requiring the instantiation of a converter.
|
||||
|
||||
### 1. Single-String
|
||||
|
||||
Data must be contained entirely within a single string or buffer.
|
||||
|
||||
```c
|
||||
conv = ucnv_open("shift_jis", &status);
|
||||
|
||||
/* Convert from Unicode to Shift JIS */
|
||||
len = ucnv_fromUChars(conv, target, targetLen, source, sourceLen, &status);
|
||||
ucnv_close(conv);
|
||||
|
||||
conv = ucnv_open("iso-8859-3", &status);
|
||||
/* Convert from ISO-8859-3 to Unicode */
|
||||
len = ucnv_toUChars(conv, target, targetSize, source, sourceLen, &status);
|
||||
ucnv_close(conv);
|
||||
```
|
||||
|
||||
### 2. Character
|
||||
|
||||
In this type, the input data is in the specified codepage. With each function
|
||||
call, only the next Unicode codepoint is converted at a time. This might be the
|
||||
most efficient way to scan for a certain character, or other processing of a
|
||||
single character at a time, because converters are stateful. This works even for
|
||||
multibyte charsets, and for stateful ones such as iso-2022-jp.
|
||||
|
||||
```c
|
||||
conv = ucnv_open("Big-5", &status);
|
||||
UChar32 target;
|
||||
while(source < sourceLimit) {
|
||||
target = ucnv_getNextUChar(conv, &source, sourceLimit, &status);
|
||||
ASSERT(status);
|
||||
processChar(target);
|
||||
}
|
||||
```
|
||||
|
||||
### 3. Buffered or Streamed
|
||||
|
||||
This is used in situations where a large document may be read in off of disk and
|
||||
processed. Also, many codepages take multiple bytes to encode a character, or
|
||||
have state. These factors make it impossible to convert arbitrary chunks of data
|
||||
without maintaining state across chunks. Even conversion from Unicode may
|
||||
encounter a leading surrogate at the end of one buffer, which needs to be paired
|
||||
with the trailing surrogate in the next buffer.
|
||||
|
||||
A basic API principle of the ICU to/from Unicode functions is that they will
|
||||
ALWAYS attempt to consume all of the input (source) data, unless the output
|
||||
buffer is full or some other error occurs. In other words, there is no need to
|
||||
ever test whether all of the source data has been consumed.
|
||||
|
||||
The basic loop that is used with the ICU buffer conversion routines is the same
|
||||
in the to and from Unicode directions. In the following pseudocode, either
|
||||
'source' (for fromUnicode) or 'target' (for toUnicode) are UTF-16 UChars.
|
||||
|
||||
```c
|
||||
UErrorCode err = U_ZERO_ERROR;
|
||||
|
||||
while (... /*input data available*/ ) {
|
||||
... /* read input data into buffer */
|
||||
|
||||
source = ... /* beginning of read data */;
|
||||
sourceLimit = source + readLength; // end + 1
|
||||
|
||||
UBool flush = (further input data still available) // (i.e. feof())
|
||||
|
||||
/* loop until all source has been processed */
|
||||
do {
|
||||
/* set up target pointers */
|
||||
target = ... /* beginning of output buffer */;
|
||||
targetLimit = target + sizeOfOutput;
|
||||
|
||||
err = U_ZERO_ERROR; /* so that the to/from does not fail */
|
||||
|
||||
ucnv_to/fromUnicode(converter, &target, targetLimit,
|
||||
&source, sourceLimit, NULL, flush, &err);
|
||||
|
||||
... /* write (target-beginningOfOutputBuffer) items
|
||||
starting at beginning of output buffer */
|
||||
} while (err == U_BUFFER_OVERFLOW_ERROR);
|
||||
if(U_FAILURE(error)) {
|
||||
... /* process error */
|
||||
break; /* out of the 'while' loop that reads source data */
|
||||
}
|
||||
}
|
||||
/* loop to read input data */
|
||||
if(U_FAILURE(error)) {
|
||||
... /* process error further */
|
||||
}
|
||||
```
|
||||
|
||||
The above code optimizes for processing entire chunks of input data. An
|
||||
efficient size for the output buffer can be calculated as follows. (in bytes):
|
||||
|
||||
```c
|
||||
ucnv_getMinCharSize() * inputBufferSize * sizeof(UChar)
|
||||
ucnv_getMaxCharSize() * inputBufferSize
|
||||
```
|
||||
|
||||
There are two loops used, an outer and an inner. The outer loop fetches input
|
||||
data to keep the source buffer full, and the inner loop 'writes' out data to
|
||||
keep the output buffer empty.
|
||||
|
||||
Note that while this efficiently handles data on the input side, there are some
|
||||
cases where the size of the output buffer is fixed. For instance, in network
|
||||
applications it is sometimes desirable to fill every output packet completely
|
||||
(not including the last packet in the sequence). The above loop does not ensure
|
||||
that every output buffer is completely full. For example, if a 4 UChar input
|
||||
buffer was used, and a 3 byte output buffer with `fromUnicode()`, the loop would
|
||||
typically write 3 bytes, then 1, then 3, and so on. If, instead of efficient use
|
||||
of the input data, the goal is filling output buffers, a slightly different loop
|
||||
can be used.
|
||||
|
||||
In such a scenario, the inner write does not occur unless a buffer overflow
|
||||
occurs OR 'flush' is true. So, the 'write' and resetting of the target and
|
||||
targetLimit pointers would only happen
|
||||
`if (err == U_BUFFER_OVERFLOW_ERROR || flush == TRUE)`
|
||||
|
||||
The flush parameter on each conversion call should be set to FALSE, until the
|
||||
conversion call is called for the last time for the buffer. This is because the
|
||||
conversion is stateful. On the last conversion call, the flush parameter should
|
||||
be set to TRUE. More details are mentioned in the API reference in
|
||||
[ucnv.h](https://unicode-org.github.io/icu-docs/apidoc/released/icu4c/ucnv_8h.html) .
|
||||
|
||||
### 4. Pre-flighting
|
||||
|
||||
Preflighting is the process of asking the conversion API for the size of target
|
||||
buffer required. (For a more general discussion, see the Preflighting section
|
||||
(§) in the [Strings](../strings/index.md) chapter.)
|
||||
|
||||
This is accomplished by calling the `ucnv_fromUChars` and `ucnv_toUChars` functions.
|
||||
|
||||
```c
|
||||
UChar uchar2;
|
||||
char input_char_buffer = "This is some text";
|
||||
|
||||
targetsize = ucnv_toUChars(myConverter, NULL, targetcapacity,
|
||||
input_char_buffer, sizeof(input_char_buffer), &err);
|
||||
|
||||
if(err==U_BUFFER_OVERFLOW_ERROR) {
|
||||
err=U_ZERO_ERROR;
|
||||
uchar2=(UChar*)malloc((targetsize) * sizeof(UChar));
|
||||
targetsize = ucnv_toUChars(myConverter, uchar2, targetsize,
|
||||
input_char_buffer, sizeof(input_char_buffer), &err);
|
||||
if(U_FAILURE(err)) {
|
||||
printf("ucnv_toUChars() FAILED %s\n", myErrorName(err));
|
||||
}
|
||||
else {
|
||||
printf("ucnv_toUChars() o.k.\n");
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
> :point_right: **Note**: *This is inefficient since the conversion is performed
|
||||
> **twice**, once for finding the size of target and once for writing to the target*.
|
||||
|
||||
### 5. Convenience
|
||||
|
||||
ICU provides some convenience functions for conversions:
|
||||
|
||||
```c
|
||||
ucnv_toUChars(myConverter, target_uchars, targetsize,
|
||||
input_char_buffer, sizeof(input_char_buffer), &err);
|
||||
ucnv_fromUChars(cnv, cTarget, (cTargetLimit-cTarget),
|
||||
uSource, (uSourceLimit-uSource), &errorCode);
|
||||
|
||||
char target[100];
|
||||
UnicodeString str("ABCDEF", "iso-8859-1");
|
||||
int32_t targetsize = str.extract(0, str.length(), target, sizeof(target), "SJIS");
|
||||
target[targetsize] = 0; /* NULL termination */
|
||||
```
|
||||
|
||||
## Conversion Examples
|
||||
|
||||
See the [ICU Conversion Examples](https://github.com/unicode-org/icu/blob/main/icu4c/source/samples/ucnv/convsamp.cpp) for more information.
|
||||
@@ -1,691 +0,0 @@
|
||||
---
|
||||
layout: default
|
||||
title: Conversion Data
|
||||
nav_order: 2
|
||||
parent: Conversion
|
||||
---
|
||||
<!--
|
||||
© 2020 and later: Unicode, Inc. and others.
|
||||
License & terms of use: http://www.unicode.org/copyright.html
|
||||
-->
|
||||
|
||||
# Conversion Data
|
||||
{: .no_toc }
|
||||
|
||||
## Contents
|
||||
{: .no_toc .text-delta }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
|
||||
---
|
||||
|
||||
## Introduction
|
||||
|
||||
### Algorithmic vs. Data-based
|
||||
|
||||
In a comprehensive conversion library, there are three kinds of codepage
|
||||
converter implementations: converters that use algorithms, mapping data, or
|
||||
those converters that use both.
|
||||
|
||||
1. Most codepages have a simple and straightforward structure but have an
|
||||
arbitrary relationship between input and output character codes. Mapping
|
||||
tables are necessary to define the conversion. If the codepage characters
|
||||
use more than one byte each, then the mapping table must also define the
|
||||
structure of the codepage.
|
||||
|
||||
2. Algorithmic converters work by transforming the input stream with built-in
|
||||
algorithms and possibly small, hard coded tables. The conversion can be
|
||||
complex, but the actual mapping of a character code is done numerically if
|
||||
the converter is purely algorithmic.
|
||||
|
||||
3. In some cases, a converter needs to be algorithmic for its basic operations
|
||||
but also relies on mapping data.
|
||||
|
||||
ICU provides converter implementations for all three groups of codepages. Since
|
||||
ICU always converts, to or from Unicode, the purely algorithmic converters are
|
||||
the ones for Unicode encodings (such as UTF-8, UTF-16BE, UTF-16LE, UTF-32BE,
|
||||
UTF-32LE, SCSU, BOCU-1 and UTF-7). Since Unicode is based on US-ASCII and
|
||||
ISO-8859-1 ("ISO Latin-1"), these encodings also use algorithmic converters for
|
||||
performance reasons.
|
||||
|
||||
Most other codepages use simple byte sequences but are not encodings of Unicode.
|
||||
They are converted with generic code using mapping data tables. ICU also
|
||||
supports a few encodings, like ISO-2022 and its variants, that employ an
|
||||
algorithmic structure to switch between a set of codepages. The converters for
|
||||
these encodings are algorithmic but use mapping tables for the embedded
|
||||
codepages.
|
||||
|
||||
### Stateful vs. Stateless
|
||||
|
||||
Character encodings are either stateful or stateless:
|
||||
|
||||
1. Stateless encodings define a byte sequence for each character. Complete
|
||||
character byte sequences can be used in any order, and the same complete
|
||||
character byte sequences always encodes the same characters. It is
|
||||
preferable to always encode one character using the same byte sequence.
|
||||
|
||||
2. Stateful encodings define byte sequences that change the state of the text
|
||||
stream. Depending on the current state, the same byte sequence may encode a
|
||||
different character and the same character may be encoded with different
|
||||
byte sequences.
|
||||
|
||||
This distinction between stateless and stateful encodings is important, because
|
||||
it determines if any available ICU converter implementation is used. The
|
||||
following are some more important considerations related to stateless versus
|
||||
stateful encodings:
|
||||
|
||||
1. A runtime converter object is always stateful, even for "stateless"
|
||||
encodings. They are always stateful because an input buffer may end with a
|
||||
partial byte sequence that is to be continued in the next input buffer in
|
||||
the following conversion call. The information about this is stored in the
|
||||
converter object. Similarly, if the input is Unicode text, then an input
|
||||
buffer may end with the first of a pair of surrogates. The converter object
|
||||
also stores overflow bytes or code units if the result of a character
|
||||
mapping did not fit entirely into the output buffer.
|
||||
|
||||
2. Stateless encodings are stateful in our converter implementation to
|
||||
interpret "complete byte sequences". They are "stateful" because many
|
||||
encodings can have the same byte value used in different positions of byte
|
||||
sequences for different characters; a specific byte value may be a lead byte
|
||||
or a trail byte. For instance, the lead and trail byte values overlap in
|
||||
codepages like Shift-JIS. If a program does not start reading at a character
|
||||
boundary, it may instead interpret the byte sequences from two or more
|
||||
separate characters as one character. Often, character boundaries can be
|
||||
detected reliably only by reading the non-Unicode text linearly from the
|
||||
beginning. This can be a problem for non-Unicode text processing, where text
|
||||
insertion, deletion, and searching are common. The UTF-8/16/32 encodings do
|
||||
not have this problem because the single, lead, or trail units have disjoint
|
||||
values and character boundary can be easily found.
|
||||
|
||||
3. Some stateful encodings only switch between two states: one with one byte
|
||||
per character and one with two bytes per character. This type of encoding is
|
||||
very common in mainframe systems based on Extended Binary Coded Decimal
|
||||
Interchange Code (EBCDIC) and is actually handled in ICU with almost the
|
||||
same code and type of mapping tables as stateless codepages.
|
||||
|
||||
4. The classifications of algorithmic vs. data-based converters and of
|
||||
stateless vs. stateful encodings are independent of each other: UTF-8,
|
||||
UTF-16, and UTF-32 encodings are algorithmic but stateless; UTF-7 and SCSU
|
||||
encodings are algorithmic and stateful; Windows-1252 and Shift-JIS encodings
|
||||
are data-based and stateless; ISO-2022-JP encoding is algorithmic,
|
||||
data-based, and stateful.
|
||||
|
||||
### Scope of this chapter
|
||||
|
||||
The following sections in this chapter discuss the mapping data tables that are
|
||||
used in ICU. For related material, please see:
|
||||
|
||||
1. [ICU character set collection](https://icu.unicode.org/charts/charset)
|
||||
|
||||
2. [Unicode Technical Report 22](http://www.unicode.org/reports/tr22/)
|
||||
|
||||
3. "Cross Mapping Tables" in [Unicode Online
|
||||
Data](http://www.unicode.org/onlinedat/online.html)
|
||||
|
||||
## ICU Mapping Table Data Files
|
||||
|
||||
### Overview
|
||||
|
||||
As stated above, most ICU converters rely on character mapping tables. ICU 1.8
|
||||
has one single data structure for all character mapping tables, which is used by
|
||||
a generic Multi-Byte Character Set (MBCS) converter implementation. The
|
||||
implementation is flexible enough to handle stateless encodings with the
|
||||
following parameters:
|
||||
|
||||
1. Support for variable-length, byte-based encodings with 1 to 4 bytes per
|
||||
character.
|
||||
|
||||
2. Support for all Unicode characters (code points 0..0x10ffff). Since ICU 1.8
|
||||
uses the UTF-16 encoding as its Unicode encoding form, surrogate pairs are
|
||||
completely supported.
|
||||
|
||||
3. Efficient distinction between unassigned (unmappable) and illegal byte
|
||||
sequences.
|
||||
|
||||
4. It is not possible to convert from Unicode to byte sequences with leading
|
||||
zero bytes.
|
||||
|
||||
5. Simple stateful encodings are also handled using only Shift-In and Shift-Out
|
||||
(SI/SO) codes and one single-byte and one double-byte state.
|
||||
|
||||
> :point_right: **Note**: *In the context of conversion tables, "unassigned" code
|
||||
> points or codepage byte sequences are valid but do not have a **mapping**. This
|
||||
> is different from "unassigned" code points in a character set like Unicode or
|
||||
> Shift-JIS which are codes that do not have assigned **characters**.*
|
||||
|
||||
Prior to version 1.8, ICU used more specific, more limited, converter
|
||||
implementations for Single Byte Character Set (SBCS), Double Byte Character Set
|
||||
(DBCS), and the stateful Extended Binary Coded Decimal Interchange Code (EBCDIC)
|
||||
codepages. Mapping table data is provided in text files. ICU comes with several
|
||||
dozen .ucm files (UniCode Mapping, in icu/source/data/mappings/) that are
|
||||
translated at build time by its makeconv tool (source code in
|
||||
icu/source/tools/makeconv). The makeconv tool writes one binary, memory-mappable
|
||||
.cnv file per .ucm file. The resulting .cnv files are included by default in the
|
||||
common data file for use at runtime.
|
||||
|
||||
The format of the .ucm files is similar to the format of the UPMAP files as
|
||||
provided by IBM® in the codepage repository and as used in the uconvdef tool on
|
||||
AIX. UPMAP is a text file that specifies the mapping of a codepage character to
|
||||
and from Unicode.
|
||||
|
||||
The format of the .cnv files is ICU-specific. The .cnv file format may change
|
||||
between ICU versions even for the same .ucm files. The .ucm file format may be
|
||||
extended to include more features.
|
||||
|
||||
The following sections concentrate on the .ucm file format. The .cnv file format
|
||||
is described in the source code in the `icu/source/common/ucnvmbcs.c` directory
|
||||
and is updated using the MBCS converter implementation.
|
||||
|
||||
These conversion tables can have more than one name. ICU allows multiple names
|
||||
("aliases") for the same encoding. It matches a requested encoding name against
|
||||
a list of names in `icu/source/data/mappings/convrtrs.txt` and when it finds a
|
||||
match, ICU opens a converter with the name in the leftmost position in the
|
||||
matching line. The name matching is not case-sensitive and ICU ignores spaces,
|
||||
dashes, and underscores. At build time, the gencnval tool located in the
|
||||
`icu/source/tools/gencnval` directory, generates a binary form of the convrtrs.txt
|
||||
file as a data file for runtime for the cnvalias.icu file ("Converter Aliases
|
||||
data file").
|
||||
|
||||
### .ucm File Format
|
||||
|
||||
.ucm files are line-oriented text files. Empty lines and comments starting with
|
||||
'`#`' are ignored.
|
||||
|
||||
A .ucm file contains two sections:
|
||||
|
||||
1. a header with general specifications of the codepage
|
||||
|
||||
2. a mapping table section between the "CHARMAP" and "END CHARMAP" lines.
|
||||
|
||||
For example:
|
||||
|
||||
```
|
||||
<code_set_name> "IBM-943"
|
||||
<char_name_mask> "AXXXX"
|
||||
<mb_cur_min> 1
|
||||
<mb_cur_max> 2
|
||||
<uconv_class> "MBCS"
|
||||
<subchar> \xFC\xFC
|
||||
<subchar1> \x7F
|
||||
<icu:state> 0-7f, 81-9f:1, a0-df, e0-fc:1
|
||||
<icu:state> 40-7e, 80-fc
|
||||
#
|
||||
CHARMAP
|
||||
#
|
||||
#
|
||||
#ISO 10646 IBM-943
|
||||
#_________ _________
|
||||
<U0000> \x00 |0
|
||||
<U0001> \x01 |0
|
||||
<U0002> \x02 |0
|
||||
<U0003> \x03 |0
|
||||
...
|
||||
<UFFE4> \xFA\x55 |1
|
||||
<UFFE5> \x81\x8F |0
|
||||
<UFFFD> \xFC\xFC |2
|
||||
END CHARMAP
|
||||
|
||||
```
|
||||
|
||||
The header fields are:
|
||||
|
||||
1. code_set_name - The name of the codepage. The makeconv tool generates the
|
||||
.cnv file name from the .ucm filename but uses this header field for the
|
||||
converter name that it writes into the .cnv file for ucnv_getName. The
|
||||
makeconv tool prints a warning message if this header field does not match
|
||||
the file name. The file name is not case-sensitive.
|
||||
|
||||
2. char_name_mask - This is ignored by makeconv tool. "AXXXX" specifies that
|
||||
the POSIX-style character "name" consists of one letter (Alpha) followed by
|
||||
4 hexadecimal digits. Since ICU only uses Unicode character "names" (for
|
||||
example, code points) the format is fixed (see below).
|
||||
|
||||
3. mb_cur_min - The minimum number of bytes per character.
|
||||
|
||||
4. mb_cur_max - The maximum number of bytes per character.
|
||||
|
||||
5. uconv_class - This can be either "SBCS", "DBCS", "MBCS", or
|
||||
"EBCDIC_STATEFUL"
|
||||
The most general converter class/type/category is MBCS, which requires that
|
||||
the codepage structure has the following <icu:state> lines. The other types
|
||||
of converters are subsets of MBCS. The makeconv tool uses predefined state
|
||||
tables for these other converters when their structure is not explicitly
|
||||
specified. The following describes how the converter types are interpreted:
|
||||
|
||||
a. MBCS: Generic ICU converter type, requires a state table
|
||||
|
||||
b. SBCS: Single-byte, 8-bit codepages
|
||||
|
||||
c. DBCS: Double-byte EBCDIC codepages
|
||||
|
||||
d. EBCDIC_STATEFUL: Mixed Single-Byte or Double-Byte EBCDIC codepages (stateful, using SI/SO)
|
||||
|
||||
The following shows the exact implied state tables for non-MBCS types. A state
|
||||
table may need to be overwritten in order to allow supplementary characters
|
||||
(U+10000 and up).
|
||||
|
||||
1. subchar - The substitution character byte sequence for this codepage. This sequence must be a valid byte sequence according to the codepage structure.
|
||||
|
||||
2. subchar1 - This is the single byte substitution character when subchar is defined. Some IBM converter libraries use different substitution characters for "narrow" and "wide" characters (single-byte and double-byte). ICU uses only one substitution character per codepage because it is common industry practice.
|
||||
|
||||
3. icu:state - See the "State Table Syntax in .ucm Files" section for a detailed description of how to specify a codepage structure.
|
||||
|
||||
4. icu:charsetFamily - This specifies if the codepage is ASCII or EBCDIC based.
|
||||
|
||||
The subchar and subchar1 fields have been known to cause some confusion. The
|
||||
following conditions outline when each are used:
|
||||
|
||||
1. Conversion from Unicode to a codepage occurs and an unassigned code point is
|
||||
found
|
||||
|
||||
a. If a subchar1 byte is defined and a subchar1 mapping is defined for the code point (with a |2 precision indicator),
|
||||
output the subchar1
|
||||
|
||||
b. Otherwise output the regular subchar
|
||||
|
||||
2. Conversion from a codepage to Unicode occurs and an unassigned codepoint is found
|
||||
|
||||
a. If the input sequence is of length 1 and a subchar1 byte is specified for the codepage, output U+001A
|
||||
|
||||
b. Otherwise output U+FFFD
|
||||
|
||||
In the CHARMAP section of a .ucm file, each line contains a Unicode code point
|
||||
(like <U(*1-6 hexadecimal digits for the code point*)> ), a codepage character
|
||||
byte sequence (each byte like `\xhh` (2 hexadecimal digits) ), and an optional
|
||||
"precision" or "fallback" indicator.
|
||||
|
||||
The precision indicator either must be present in all mappings or in none of
|
||||
them. The indicator is a pipe symbol `|` followed by a 0, 1, 2, 3, or 4 that has
|
||||
the following meaning:
|
||||
|
||||
* `|0` - A "normal", roundtrip mapping from a Unicode code point and back.
|
||||
* `|1` - A "fallback" mapping only from Unicode to the codepage, but not back.
|
||||
* `|2` - A subchar1 mapping. The code point is unmappable, and if a substitution
|
||||
is performed, then the subchar1 should be used rather than the subchar.
|
||||
Otherwise, such mappings are ignored.
|
||||
* `|3` - A "reverse fallback" mapping only from the codepage to Unicode, but not
|
||||
back to the codepage.
|
||||
* `|4` - A "good one-way" mapping only from Unicode to the codepage, but not
|
||||
back.
|
||||
|
||||
Fallback mappings from Unicode typically do not map codes for the same
|
||||
character, but for "similar" ones. This mapping is sometimes done if a character
|
||||
exists in Unicode but not in the codepage. To replace it, ICU maps a codepage
|
||||
code to a similar-looking code for human-readable output. This mapping feature
|
||||
is not useful for text data transmission especially in markup languages where a
|
||||
Unicode code point can be escaped with its code point value. The ICU application
|
||||
programming interface (API) `ucnv_setFallback()` controls this fallback behavior.
|
||||
|
||||
"Reverse fallbacks" are technically similar, but the same Unicode character can
|
||||
be encoded twice in the codepage. ICU always uses reverse fallbacks at runtime.
|
||||
|
||||
A subset of the fallback mappings from Unicode is always used at runtime: Those
|
||||
that map private-use Unicode code points. Fallbacks from private-use code points
|
||||
are often introduced as replacements for previous roundtrip mappings for the
|
||||
same pair of codes. These replacements are used when a Unicode version assigns a
|
||||
new character that was previously mapped to that private-use code point. The
|
||||
mapping table is then changed to map the same codepage byte sequence to the new
|
||||
Unicode code point (as a new roundtrip) and the mapping from the old private-use
|
||||
code point to the same codepage code is preserved as a fallback.
|
||||
|
||||
A "good one-way" mapping is like a fallback, but ICU always uses "good one-way"
|
||||
mappings at runtime, regardless of the fallback API flag.
|
||||
|
||||
The idea is that fallbacks normally lose information, such as mapping from a
|
||||
compatibility variant of a letter to the ASCII version; however, fallbacks from
|
||||
PUA and reverse fallbacks are assumed to be for "the same character", just an
|
||||
older code for it.
|
||||
|
||||
Something similar happens with from-Unicode Variation Selector sequences. It is
|
||||
possible to round-trip (`|0`) either the unadorned character or the sequence with
|
||||
a variation selector, and add a "good one-way" mapping (`|4`) from the other
|
||||
version. That "good one-way" mapping does not lose much information, and it is
|
||||
used even if the "use fallback" API flag is false. Alternatively, both mappings
|
||||
could be fallbacks (`|1`) that should be controlled by the "use fallback"
|
||||
attribute.
|
||||
|
||||
### State table syntax in .ucm files
|
||||
|
||||
The conversion to Unicode uses a state machine to achieve the above capabilities
|
||||
with reasonable data file sizes. The state machine information itself is loaded
|
||||
with the conversion data and defines the structure of the codepage, including
|
||||
which byte sequences are valid, unassigned, and illegal. This data cannot (or
|
||||
not easily) be computed from the pure mapping data. Instead, the .ucm files for
|
||||
MBCS encodings have additional entries that are specific to the ICU makeconv
|
||||
tool. The state tables for SBCS, DBCS, and EBCDIC_STATEFUL are implied, but they
|
||||
can be overridden (see the examples below). These state tables are specified in
|
||||
the header section of the .ucm file that contains the `<icu:state>` element. Each
|
||||
line defines one aspect of the state machine. The state machine uses a table of
|
||||
as many rows as there are states (= as many as there are `<icu:state>` lines).
|
||||
Each row has 256 entries; one for each possible byte value.
|
||||
|
||||
The state table lines in the .ucm header conform to the following Extended
|
||||
Backus-Naur Form (EBNF)-like grammar (whitespace is allowed between all tokens):
|
||||
|
||||
```
|
||||
row=[[firstentry ','] entry (',' entry)*]
|
||||
firstentry="initial" | "surrogates"
|
||||
(initial state (default for state 0), output is all surrogate pairs)
|
||||
```
|
||||
|
||||
Each state table row description (that follows the `<icu:state>`) begins with an
|
||||
optional initial or surrogates keyword and is followed by one or more column
|
||||
entries. For the purpose of codepage state tables, the states=rows in the table
|
||||
are numbered beginning at 0 for the first line in the .ucm file header. The
|
||||
numbers are assigned implicitly by the makeconv tool in order of the `<icu:state>`
|
||||
lines.
|
||||
|
||||
A row may be empty (nothing following the `<icu:state>`) - that is equivalent to
|
||||
"all illegal" or 0-ff.i and is useful for trail byte states for all-illegal byte
|
||||
sequences.
|
||||
|
||||
```
|
||||
entry=range [':' nextstate] ['.' [action]]
|
||||
range = number ['-' number]
|
||||
nextstate = number (0..7f)
|
||||
action = 'u' | 's' | 'p' | 'i'
|
||||
(unassigned, state change only, surrogate pair, illegal)
|
||||
number = (1- or 2-digit hexadecimal number)
|
||||
```
|
||||
|
||||
Each column entry contains at least one hexadecimal byte value or value range
|
||||
and is separated by a comma. The column entry specifies how to interpret an
|
||||
input byte in the row's state. If neither a next state nor an action is
|
||||
explicitly specified (only the byte range is given) then the byte value
|
||||
terminates the byte sequence, results in a valid mapping to a Unicode BMP
|
||||
character, and resets the state number to 0. The first line with `<icu:state>` is
|
||||
called state 0.
|
||||
|
||||
The next state can be explicitly specified with a separating colon ( : )
|
||||
followed by the number of the state (=number/index of the row, starting at 0).
|
||||
This specification is mostly used for intermediate byte values (such as bytes
|
||||
that are not the last ones in a sequence). The state machine needs to proceed to
|
||||
the next state and read another byte. In this case, no other action is
|
||||
specified.
|
||||
|
||||
If the byte value(s) terminate(s) a byte sequence, then the byte sequence
|
||||
results in the following depending on the action that is announced with a period
|
||||
( . ) followed by a letter:
|
||||
|
||||
| letter | meaning |
|
||||
|--|---------|
|
||||
| u | Unassigned. The byte sequence is valid but does not encode a character. |
|
||||
| none | (no letter) - Valid. If no action letter is specified, then the byte sequence is valid and encodes a Unicode character up to U+ffff |
|
||||
| p | Surrogate Pair. The byte sequence is valid and the result may map to a UTF-16 encoded surrogate pair |
|
||||
| i | Illegal. The byte sequence is illegal. This is the default for all byte values in a row that are not otherwise specified with column entries|
|
||||
| s | State change only. The byte sequence does not encode any character but may change the state number. This may be used with simple, stateful encodings (for example, SI/SO codes), but currently it is not used by ICU.|
|
||||
|
||||
If an action is specified without a next state, then the next state number
|
||||
defaults to 0. In other words, a byte value (range) terminates a sequence if
|
||||
there is an action specified for it, or when there is neither an action nor a
|
||||
next state. In this case, the byte value defaults to "valid, next state is 0"
|
||||
(equivalent to :0.).
|
||||
|
||||
If a byte value is not specified in any column entry row, then it is illegal in
|
||||
the current state. If a byte value is specified in more than one column entry of
|
||||
the same row, then ICU uses the last state. These specifications allow you to
|
||||
assign common properties for a wide byte value range followed by a few
|
||||
exceptions. This is easier than having to specify mutually exclusive ranges,
|
||||
especially if many of them have the same properties.
|
||||
|
||||
The optional keyword at the beginning of a state line has the following effect:
|
||||
|
||||
| keyword | effect |
|
||||
|---------|--------|
|
||||
| initial | The state machine can start reading byte sequences in this state. State 0 is always an initial state. Only initial states can be next states for final byte values. In an initial state, the Unicode mappings for all final bytes are also stored directly in the state table.
|
||||
| surrogates | All Unicode mappings for final bytes in non-initial states are stored in a separate table of 16-bit Unicode (UTF-16) code units. Since most legacy codepages map only to Unicode code points up to U+ffff (the Basic Multilingual Plane, BMP), the default allocation per mapping result is one 16-bit unit. Individual byte values can be specified to map to surrogate pairs (= two 16-bit units) with action letter p. The surrogates keyword specifies the values for the entire state (row). Surrogate pair mapping entries can still hold single units depending on the actual mapping data, but single-unit mapping entries cannot hold a pair of units. Mapping to single-unit entries is the default because the mapping is faster, uses half as much memory in the code units table, and is sufficient for most legacy codepages.|
|
||||
|
||||
When converting to Unicode, the state machine starts in state number 0. In each
|
||||
iteration, the state machine reads one input (codepage) byte and either proceeds
|
||||
to the next state as specified, or treats it as a final byte with the specified
|
||||
action and an optional non-0 next (initial) state. This means that a state table
|
||||
needs to have at least as many state rows as the maximum number of bytes per
|
||||
character, which is the maximum length of any byte sequence.
|
||||
|
||||
Exception: For EBCDIC_STATEFUL codepages, double-byte sequences start in state
|
||||
1, with the SI/SO bytes switching from state 0 to state 1 or from state 1 to
|
||||
state 0. See the default state table below.
|
||||
|
||||
### Extension and delta tables
|
||||
|
||||
ICU 2.8 adds an additional "extension" data structure to its conversion tables.
|
||||
The new data structure supports a number of new features. When any of the
|
||||
following features are used, then all mappings must use a precision indicator.
|
||||
|
||||
#### Converting multiple characters as a unit
|
||||
|
||||
Before ICU 2.8, only one Unicode code point could be converted to or from one
|
||||
complete codepage byte sequence. The new data structure supports the conversion
|
||||
between multiple Unicode code points and multiple complete codepage byte
|
||||
sequences. (A "complete codepage byte sequence" is a sequence of bytes which is
|
||||
valid according to the state table.)
|
||||
|
||||
Syntax: Simply write more than one Unicode code point on a mapping line, and/or
|
||||
more than one complete codepage byte sequence. Plus signs (+) are optional
|
||||
between code points and between bytes. For example,
|
||||
ibm-1390_P110-2003.ucm contains
|
||||
|
||||
<U304B><U309A> \xEC\xB5 |0
|
||||
|
||||
and test3.ucm contains
|
||||
|
||||
<U101234>+<U50005>+<U60006> \x07+\x00+\x01\x02\x0f+\x09 |0
|
||||
|
||||
For more examples see the ICU conversion data and the
|
||||
`icu/source/test/testdata/test*.ucm` test data files.
|
||||
|
||||
ICU 2.8 supports up to 19 UChars on the Unicode side of a mapping and up to 31
|
||||
bytes on the codepage side.
|
||||
|
||||
The longest match possible is converted in order to properly handle tables where
|
||||
the source sides of some mappings are prefixes of the source sides of other
|
||||
mappings.
|
||||
|
||||
As a side effect, if conversion offsets are written and a potential match
|
||||
crosses buffer boundaries, then some of the initial offsets for the following
|
||||
output may be unknown (-1) because their input was stored in the converter from
|
||||
a previous buffer while looking for a longer match.
|
||||
|
||||
Conversion tables for SI/SO-stateful (usually EBCDIC_STATEFUL) codepages cannot
|
||||
include mappings with SI or SO bytes or where there are SBCS characters in a
|
||||
multi-character byte sequence. In other words, for these tables there must be
|
||||
exactly one byte in a mapping or else a sequence of one or more DBCS characters.
|
||||
|
||||
#### Delta (extension-only) conversion table files
|
||||
|
||||
Physically, a binary conversion table (.cnv) file automatically contains both a
|
||||
traditional "base table" data structure for the 1:1 mappings and a new
|
||||
"extension table" for the m:n mappings if any are encountered in the .ucm file.
|
||||
An extension table can also be requested manually by splitting the CHARMAP into
|
||||
two. The first CHARMAP section will be used for the base table, and the second
|
||||
only for the extension table. M:n mappings in the first CHARMAP will be moved to
|
||||
the extension table.
|
||||
|
||||
In order to save space for very similar conversion tables, it is possible to
|
||||
create delta .cnv files that contain only an extension table and the name of
|
||||
another .cnv file with a base table. The base file must be split into two
|
||||
CHARMAPs such that the base file's base table does not contain any mappings that
|
||||
contradict any of the delta file's mappings.
|
||||
|
||||
The delta (extension-only) file uses only a single CHARMAP section. In addition,
|
||||
it nees a line in the header that both causes building just a delta file and
|
||||
specifies the name of the base file. For example, windows-936-2000.ucm contains
|
||||
|
||||
<icu:base> “ibm-1386_P100-2002”
|
||||
|
||||
makeconv ignores all mappings for the delta file that are also in the base
|
||||
file's base table. If the two conversion tables are sufficiently similar, then
|
||||
the delta file will contain only a relatively small set of mappings, which
|
||||
results in a small .cnv file. At runtime, both the delta file and its base file
|
||||
are loaded, and the base file's base table is used together with the extension
|
||||
file. The base file works as a standalone file, using its own extension table
|
||||
for its full set of mappings. The base file must be in the same ICU data package
|
||||
as the delta file.
|
||||
|
||||
The hard part is to split the base file's mappings into base and extension
|
||||
CHARMAPs such that the base table does not overlap with any delta file, while
|
||||
all shared mappings should be in the base table. (The base table data structure
|
||||
is more compact than the extension table data structure.)
|
||||
|
||||
ICU provides the ucmkbase tool in the
|
||||
[ucmtools](https://github.com/unicode-org/icu-data/tree/main/charset/source/ucmtools)
|
||||
collection to do this.
|
||||
|
||||
For example, the following illustrates how to use ucmkbase to make a base .ucm
|
||||
file for three Shift-JIS conversion table variants. (ibm-943_P15A-2003.ucm
|
||||
becomes the base.)
|
||||
|
||||
```
|
||||
C:\tmp\icu\ucm>ren ibm-943_P15A-2003.ucm ibm-943_P15A-2003.orig
|
||||
C:\tmp\icu\ucm>ucmkbase ibm-943_P15A-2003.orig ibm-943_P130-1999.ucm ibm-942_P12A-1999.ucm > ibm-943_P15A-2003.ucm
|
||||
```
|
||||
|
||||
After this, the two delta .ucm files only need to get the following line added
|
||||
before the start of their CHARMAPs:
|
||||
|
||||
```
|
||||
<icu:base> "ibm-943_P15A-2003"
|
||||
```
|
||||
|
||||
The ICU tools and runtime code handle DBCS-only conversion tables specially,
|
||||
allowing them to be built into delta files with MBCS or EBCDIC_STATEFUL base
|
||||
files without using their single-byte mappings, and without ucmkbase moving the
|
||||
single-byte mappings of the base file into the base file's extension table. See
|
||||
for example ibm-16684_P110-2003.ucm and ibm-1390_P110-2003.ucm.
|
||||
|
||||
#### Other enhancements
|
||||
|
||||
ICU 2.8 adds support for the specification of which unassigned Unicode code
|
||||
points should be mapped to subchar1 rather than the default subchar. See the
|
||||
discussion of subchar1 above for more details.
|
||||
|
||||
The extension table data structure also removes one minor limitation on ICU
|
||||
conversion tables: Fallback mappings to a single byte 00 are now allowed and
|
||||
handled properly. ICU versions before 2.8 could only handle roundtrips to/from
|
||||
00.
|
||||
|
||||
### Examples for codepage state tables
|
||||
|
||||
The following shows the exact implied state tables for non-MBCS types, A state
|
||||
table may need to be overwritten in order to allow supplementary characters
|
||||
(U+10000 and up).
|
||||
|
||||
US-ASCII
|
||||
```
|
||||
0-7f
|
||||
```
|
||||
|
||||
This single-row state table describes US-ASCII. Byte values from 0 to 0x7f are
|
||||
valid and map to Unicode characters up to U+ffff. Byte values from 0x80 to 0xff
|
||||
are illegal.
|
||||
|
||||
Shift-JIS
|
||||
```
|
||||
0-7f, 81-9f:1, a0-df, e0-fc:1
|
||||
40-7e, 80-fc
|
||||
```
|
||||
|
||||
This two-row state table describes the Shift-JIS structure which encodes some
|
||||
characters with one byte each and others with two bytes each. Bytes 0 to 0x7f
|
||||
and 0xa0 to 0xdf are valid single-byte encodings. Bytes 0x81 to 0x9f and 0xe0 to
|
||||
0xfc are lead bytes. (For example, they are followed by one of the bytes that is
|
||||
specified as valid in state 1). A byte sequence of 0x85 0x61 is valid while a
|
||||
single byte of 0x80 or 0xff is illegal. Similarly, a byte sequence of 0x85 0x31
|
||||
is illegal.
|
||||
|
||||
EUC-JP
|
||||
```
|
||||
0-8d, 8e:2, 8f:3, 90-9f, a1-fe:1
|
||||
a1-fe
|
||||
a1-e4
|
||||
a1-fe:1, a1:4, a3-af:4, b6:4, d6:4, da-db:4, ed-f2:4
|
||||
a1-fe.u
|
||||
```
|
||||
|
||||
This fairly complicated state table describes EUC-JP. Valid byte sequences are
|
||||
one, two, or three bytes long. Two-byte sequences have a lead byte of 0x8e and
|
||||
end in state 2, or have lead bytes 0xa1 to 0xfe and end in state 1. Three-byte
|
||||
sequences have a lead byte of 0x8f and continue in state 3. Some final byte
|
||||
value ranges are entirely unassigned, therefore they end in state 4 with an
|
||||
action letter of u for "unassigned" to save significant memory for the code
|
||||
units table. Assigned three-byte sequences end in state 1 like most two-byte
|
||||
sequences.
|
||||
|
||||
SBCS default state table:
|
||||
```
|
||||
0-ff
|
||||
```
|
||||
SBCS by default implies the structure for single-byte, 8-bit codepages.
|
||||
|
||||
DBCS default state table:
|
||||
```
|
||||
0-3f:3, 40:2, 41-fe:1, ff:3
|
||||
41-fe
|
||||
40
|
||||
|
||||
```
|
||||
|
||||
**Important**:
|
||||
These are four states — the fourth has an empty line (equivalent to 0-ff.i)!
|
||||
DBCS codepages, by default, are defined with the EBCDIC double-byte structure.
|
||||
Valid sequences are pairs of bytes from 0x41 to 0xfe and the one pair 0x40/0x40
|
||||
for the double-byte space. The structure is defined such that all illegal byte
|
||||
sequences are always two in length. Therefore, every byte in the initial state
|
||||
is a lead byte.
|
||||
|
||||
EBCDIC_STATEFUL default state table:
|
||||
```
|
||||
0-ff, e:1.s, f:0.s
|
||||
initial, 0-3f:4, e:1.s, f:0.s, 40:3, 41-fe:2, ff:4
|
||||
0-40:1.i, 41-fe:1., ff:1.i
|
||||
0-ff:1.i, 40:1.
|
||||
0-ff:1.i
|
||||
```
|
||||
|
||||
This is the structure of Mixed Single-byte and Double-byte EBCDIC codepages,
|
||||
which are stateful and use the Shift-In/Shift-Out (SI/SO) bytes 0x0f/0x0e. The
|
||||
initial state 0 is almost the same as for SBCS except for SI and SO. State 1 is
|
||||
also an initial state and is the basis for a state-shifted version of the DBCS
|
||||
structure above. All double-byte sequences return to state 1 and SI switches
|
||||
back to state 0. SI and SO are also allowed in their own states with no effect.
|
||||
|
||||
> :point_right: **Note**: *If a DBCS or EBCDIC_STATEFUL codepage maps supplementary (non-BMP) Unicode
|
||||
> characters, then a modified state table needs to be specified in the .ucm file.
|
||||
> The state table needs to use the surrogates designation for a table row or .p
|
||||
> for some entries.*
|
||||
>
|
||||
> *The reuse of a final or intermediate state (shown for EUC-JP) is valid for as
|
||||
> long as there is no circle in the state chain. The mappings will be unique
|
||||
> because of the different path to the shared state (sharing a state saves some
|
||||
> memory; each state table row occupies 1kB in the .cnv file). This table also
|
||||
> shows the redefinition of byte value ranges within one state row (State number
|
||||
> 3) as shorthand. State 3 defines bytes a1-fe to go to state 1, but the following
|
||||
> entries redefine and override certain bytes to go to state 4.*
|
||||
|
||||
An initial state never needs a surrogates designation or .p because Unicode
|
||||
mapping results in initial states that are stored directly in the state table,
|
||||
providing enough room in each cell. The size of a generated .cnv mapping table
|
||||
file depends primarily on the number and distribution of the mappings and on the
|
||||
number of valid, multi-byte sequences that the state table allows. Each state
|
||||
table row takes up one kilobyte.
|
||||
|
||||
For single-byte codepages, the state table cells contain all two-Unicode
|
||||
mappings. Code point results for multi-byte sequences are stored in an array
|
||||
with enough room for all valid byte sequences. For all byte sequences that end
|
||||
in a surrogates or .p state, Unicode allocates two code units.
|
||||
|
||||
If possible, valid state table entries may be changed to .u to reduce the number
|
||||
of valid, assignable sequences and to make the .cnv file smaller. If additional
|
||||
states are necessary, then each additional state itself adds 1kB to the file
|
||||
size, diminishing the file size savings. See the EUC-JP example above.
|
||||
|
||||
For codepages with up to two bytes per character, the makeconv tool
|
||||
automatically compacts the bytes, if possible, by introducing one more trail
|
||||
byte state. This state replaces valid entries in the original trail state with
|
||||
unassigned entries and changes each lead byte entry to work with the new state
|
||||
if there are no mappings with that lead byte.
|
||||
|
||||
For codepages with up to three or four bytes per character, compaction must be
|
||||
done manually. However, if the verbose option is set on the command line, the
|
||||
makeconv tool will print useful information about unassigned byte sequences.
|
||||
@@ -1,359 +0,0 @@
|
||||
---
|
||||
layout: default
|
||||
title: Charset Detection
|
||||
nav_order: 3
|
||||
parent: Conversion
|
||||
---
|
||||
<!--
|
||||
© 2020 and later: Unicode, Inc. and others.
|
||||
License & terms of use: http://www.unicode.org/copyright.html
|
||||
-->
|
||||
|
||||
# Character Set Detection
|
||||
{: .no_toc }
|
||||
|
||||
## Contents
|
||||
{: .no_toc .text-delta }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
Character set detection is the process of determining the character set, or
|
||||
encoding, of character data in an unknown format. This is, at best, an imprecise
|
||||
operation using statistics and heuristics. Because of this, detection works best
|
||||
if you supply at least a few hundred bytes of character data that's mostly in a
|
||||
single language. In some cases, the language can be determined along with the
|
||||
encoding.
|
||||
|
||||
Several different techniques are used for character set detection. For
|
||||
multi-byte encodings, the sequence of bytes is checked for legal patterns. The
|
||||
detected characters are also check against a list of frequently used characters
|
||||
in that encoding. For single byte encodings, the data is checked against a list
|
||||
of the most commonly occurring three letter groups for each language that can be
|
||||
written using that encoding. The detection process can be configured to
|
||||
optionally ignore html or xml style markup, which can interfere with the
|
||||
detection process by changing the statistics.
|
||||
|
||||
The input data can either be a Java input stream, or an array of bytes. The
|
||||
output of the detection process is a list of possible character sets, with the
|
||||
most likely one first. For simplicity, you can also ask for a Java Reader that
|
||||
will read the data in the detected encoding.
|
||||
|
||||
There is another character set detection C++ library, the [Compact Encoding
|
||||
Detector](https://github.com/google/compact_enc_det), that may have a lower
|
||||
error rate, particularly when working with short samples of text.
|
||||
|
||||
## CharsetMatch
|
||||
|
||||
The `CharsetMatch` class holds the result of comparing the input data to a
|
||||
particular encoding. You can use an instance of this class to get the name of
|
||||
the character set, the language, and how good the match is. You can also use
|
||||
this class to decode the input data.
|
||||
|
||||
To find out how good the match is, you use the `getConfidence()` method to get a
|
||||
*confidence value*. This is an integer from 0 to 100. The higher the value, the
|
||||
more confidence there is in the match For example:
|
||||
|
||||
```java
|
||||
CharsetMatch match = ...;
|
||||
int confidence;
|
||||
confidence = match.getConfidence();
|
||||
if (confidence < 50 ) {
|
||||
// handle a poor match...
|
||||
} else {
|
||||
// handle a good match...
|
||||
}
|
||||
```
|
||||
|
||||
In C, you can use the `ucsdet_getConfidence(const UCharsetMatch *ucsm, UErrorCode *status)`
|
||||
method to get a confidence value.
|
||||
|
||||
```c
|
||||
const UCharsetMatch *ucm;
|
||||
UErrorCode status = U_ZERO_ERROR;
|
||||
int32_t confidence = ucsdet_getConfidence(ucm, &status);
|
||||
if (confidence <50) {
|
||||
// handle a poor match...
|
||||
} else {
|
||||
// handle a good match...
|
||||
}
|
||||
```
|
||||
|
||||
To get the name of the character set, which can be used as an encoding name in
|
||||
Java, you use the `getName()` method:
|
||||
|
||||
```java
|
||||
CharsetMatch match = ...;
|
||||
byte characterData[] = ...;
|
||||
String charsetName;
|
||||
String unicodeData;
|
||||
charsetName = match.getName();
|
||||
unicodeData = new String(characterData, charsetName);
|
||||
```
|
||||
|
||||
To get the name of the character set in C:
|
||||
|
||||
```c
|
||||
const UCharsetMatch *ucm;
|
||||
UErrorCode status = U_ZERO_ERROR;
|
||||
const char *name = ucsdet_getName(ucm, &status);
|
||||
```
|
||||
|
||||
To get the three letter ISO code for the detected language, you use the
|
||||
`getLanguage()` method. If the language could not be determined, `getLanguage()`
|
||||
will return `null`. Note that language detection does not work with all charsets,
|
||||
and includes only a very small set of possible languages. It should not used if
|
||||
robust, reliable language detection is required.
|
||||
|
||||
```java
|
||||
CharsetMatch match = ...;
|
||||
String languageCode;
|
||||
languageCode = match.getLanguage();
|
||||
if (languageCode != null) {
|
||||
// handle the language code...
|
||||
}
|
||||
```
|
||||
|
||||
The `ucsdet_getLanguage(const UCharsetMatch *ucsm, UErrorCode *status)` method
|
||||
can be used in C to get the language code. If the language could not be
|
||||
determined, the method will return an empty string.
|
||||
|
||||
```c
|
||||
const UCharsetMatch *ucm;
|
||||
UErrorCode status = U_ZERO_ERROR;
|
||||
const char *language = ucsdet_getLanguage(ucm, &status);
|
||||
```
|
||||
|
||||
If you want to get a Java String containing the converted data you can use the
|
||||
`getString()` method:
|
||||
|
||||
```java
|
||||
CharsetMatch match = ...;
|
||||
String unicodeData;
|
||||
unicodeData = match.getString();
|
||||
```
|
||||
|
||||
If you want to limit the number of characters in the string, pass the maximum
|
||||
number of characters you want to the `getString()` method:
|
||||
|
||||
```java
|
||||
CharsetMatch match = ...;
|
||||
String unicodeData;
|
||||
unicodeData = match.getString(1024);
|
||||
```
|
||||
|
||||
To get a `java.io.Reader` to read the converted data, use the `getReader()` method:
|
||||
|
||||
```java
|
||||
CharsetMatch match = ...;
|
||||
Reader reader;
|
||||
StringBuffer sb = new StringBuffer();
|
||||
char[] buffer = new char[1024];
|
||||
int bytesRead = 0;
|
||||
reader = match.getReader();
|
||||
while ((bytesRead = reader.read(buffer, 0, 1024)) >= 0) {
|
||||
sb.append(buffer, 0, bytesRead);
|
||||
}
|
||||
reader.close();
|
||||
```
|
||||
|
||||
## CharsetDetector
|
||||
|
||||
The `CharsetDetector` class does the actual detection. It matches the input data
|
||||
against all character sets, and computes a list of `CharsetMatch` objects to hold
|
||||
the results. The input data can be supplied as an array of bytes, or as a
|
||||
`java.io.InputStream`.
|
||||
|
||||
To use a `CharsetDetector` object, first you construct it, and then you set the
|
||||
input data, using the `setText()` method. Because setting the input data is
|
||||
separate from the construction, it is easy to reuse a `CharsetDetector` object:
|
||||
|
||||
```java
|
||||
CharsetDetector detector;
|
||||
byte[] byteData = ...;
|
||||
InputStream streamData = ...;
|
||||
detector = new CharsetDetector();
|
||||
detector.setText(byteData);
|
||||
// use detector with byte data...
|
||||
detector.setText(streamData);
|
||||
// use detector with stream data...
|
||||
```
|
||||
|
||||
If you want to know which character set matches your input data with the highest
|
||||
confidence, you can use the `detect()` method, which will return a `CharsetMatch`
|
||||
object for the match with the highest confidence:
|
||||
|
||||
```java
|
||||
CharsetDetector detector;
|
||||
CharsetMatch match;
|
||||
byte[] byteData = ...;
|
||||
detector = new CharsetDetector();
|
||||
detector.setText(byteData);
|
||||
match = detector.detect();
|
||||
```
|
||||
|
||||
If you want to know which character set matches your input data in C, you can
|
||||
use the `ucsdet_detect(UCharsetDetector *csd , UErrorCode *status)` method.
|
||||
|
||||
```c
|
||||
UCharsetDetector *csd;
|
||||
const UCharsetMatch *ucm;
|
||||
static char buffer[BUFFER_SIZE] = {....};
|
||||
int32_t inputLength = ... // length of the input text
|
||||
UErrorCode status = U_ZERO_ERROR;
|
||||
ucsdet_setText(csd, buffer, inputLength, &status);
|
||||
ucm = ucsdet_detect(csd, &status);
|
||||
```
|
||||
|
||||
If you want to know all of the character sets that could match your input data
|
||||
with a non-zero confidence, you can use the `detectAll()` method, which will
|
||||
return an array of `CharsetMatch` objects sorted by confidence, from highest to
|
||||
lowest.:
|
||||
|
||||
```java
|
||||
CharsetDetector detector;
|
||||
CharsetMatch matches[];
|
||||
byte[] byteData = ...;
|
||||
detector = new CharsetDetector();
|
||||
detector.setText(byteData);
|
||||
matches = detector.detectAll();
|
||||
for (int m = 0; m < matches.length; m += 1) {
|
||||
// process this match...
|
||||
}
|
||||
```
|
||||
|
||||
> :point_right: **Note**: The `ucsdet_detectALL(UCharsetDetector *csd , int32_t *matchesFound, UErrorCode *status)`
|
||||
> method can be used in C in order to detect all of the character sets where `matchesFound` is a pointer
|
||||
> to a variable that will be set to the number of charsets identified that are consistent with the input data.
|
||||
|
||||
The `CharsetDetector` class also implements a crude *input filter* that can strip
|
||||
out html and xml style tags. If you want to enable the input filter, which is
|
||||
disabled when you construct a `CharsetDetector`, you use the `enableInputFilter()`
|
||||
method, which takes a boolean. Pass in true if you want to enable the input
|
||||
filter, and false if you want to disable it:
|
||||
|
||||
```java
|
||||
CharsetDetector detector;
|
||||
CharsetMatch match;
|
||||
byte[] byteDataWithTags = ...;
|
||||
detector = new CharsetDetector();
|
||||
detector.setText(byteDataWithTags);
|
||||
detector.enableInputFilter(true);
|
||||
match = detector.detect();
|
||||
```
|
||||
|
||||
To enable an input filter in C, you can use
|
||||
`ucsdet_enableInputFilter(UCharsetDetector *csd, UBool filter)` function.
|
||||
|
||||
```c
|
||||
UCharsetDetector *csd;
|
||||
const UCharsetMatch *ucm;
|
||||
static char buffer[BUFFER_SIZE] = {....};
|
||||
int32_t inputLength = ... // length of the input text
|
||||
UErrorCode status = U_ZERO_ERROR;
|
||||
ucsdet_setText(csd, buffer, inputLength, &status);
|
||||
ucsdet_enableInputFilter(csd, TRUE);
|
||||
ucm = ucsdet_detect(csd, &status);
|
||||
```
|
||||
|
||||
If you have more detailed knowledge about the structure of the input data, it is
|
||||
better to filter the data yourself before you pass it to CharsetDetector. For
|
||||
example, you might know that the data is from an html page that contains CSS
|
||||
styles, which will not be stripped by the input filter.
|
||||
|
||||
You can use the `inputFilterEnabled()` method to see if the input filter is
|
||||
enabled:
|
||||
|
||||
```java
|
||||
CharsetDetector detector;
|
||||
detector = new CharsetDetector();
|
||||
// do a bunch of stuff with detector
|
||||
// which may or may not enable the input filter...
|
||||
if (detector.inputFilterEnabled()) {
|
||||
// handle enabled input filter
|
||||
} else {
|
||||
// handle disabled input filter
|
||||
}
|
||||
```
|
||||
|
||||
> :point_right: **Note**: The ICU4C API provide `uscdet_isInputFilterEnabled(const UCharsetDetector* csd)` function
|
||||
> to check whether the input filter is enabled.
|
||||
|
||||
The `CharsetDetector` class also has two convenience methods that let you detect
|
||||
and convert the input data in one step: the `getReader()` and `getString()` methods:
|
||||
|
||||
```java
|
||||
CharsetDetector detector;
|
||||
byte[] byteData = ...;
|
||||
InputStream streamData = ...;
|
||||
String unicodeData;
|
||||
Reader unicodeReader;
|
||||
detector = new CharsetDetector();
|
||||
unicodeData = detector.getString(byteData, null);
|
||||
unicodeReader = detector.getReader(streamData, null);
|
||||
```
|
||||
|
||||
> :point_right: **Note**: The second argument to the `getReader()` and `getString()` methods
|
||||
> is a String called `declaredEncoding`, which is not currently used. There is also a
|
||||
> `setDeclaredEncoding()` method, which is also not currently used.
|
||||
|
||||
The following code is equivalent to using the convenience methods:
|
||||
|
||||
```java
|
||||
CharsetDetector detector;
|
||||
CharsetMatch match;
|
||||
byte[] byteData = ...;
|
||||
InputStream streamData = ...;
|
||||
String unicodeData;
|
||||
Reader unicodeReader;
|
||||
detector = new CharsetDetector();
|
||||
detector.setText(byteData);
|
||||
match = detector.detect();
|
||||
unicodeData = match.getString();
|
||||
detector.setText(streamData);
|
||||
match = detector.detect();
|
||||
unicodeReader = match.getReader();CharsetDetector
|
||||
```
|
||||
|
||||
## Detected Encodings
|
||||
|
||||
The following table shows all the encodings that can be detected. You can get
|
||||
this list (without the languages) by calling the `getAllDetectableCharsets()`
|
||||
method:
|
||||
|
||||
| **Character Set** | **Languages** |
|
||||
| ----------------- | ------------- |
|
||||
| UTF-8 | |
|
||||
| UTF-16BE | |
|
||||
| UTF-16LE | |
|
||||
| UTF-32BE | |
|
||||
| UTF-32LE | |
|
||||
| Shift_JIS | Japanese |
|
||||
| ISO-2022-JP | Japanese |
|
||||
| ISO-2022-CN | Simplified Chinese |
|
||||
| ISO-2022-KR | Korean |
|
||||
| GB18030 | Chinese |
|
||||
| Big5 | Traditional Chinese |
|
||||
| EUC-JP | Japanese |
|
||||
| EUC-KR | Korean |
|
||||
| ISO-8859-1 | Danish, Dutch, English, French, German, Italian, Norwegian, Portuguese, Swedish |
|
||||
| ISO-8859-2 | Czech, Hungarian, Polish, Romanian |
|
||||
| ISO-8859-5 | Russian |
|
||||
| ISO-8859-6 | Arabic |
|
||||
| ISO-8859-7 | Greek |
|
||||
| ISO-8859-8 | Hebrew |
|
||||
| ISO-8859-9 | Turkish |
|
||||
| windows-1250 | Czech, Hungarian, Polish, Romanian |
|
||||
| windows-1251 | Russian |
|
||||
| windows-1252 | Danish, Dutch, English, French, German, Italian, Norwegian, Portuguese, Swedish |
|
||||
| windows-1253 | Greek |
|
||||
| windows-1254 | Turkish |
|
||||
| windows-1255 | Hebrew |
|
||||
| windows-1256 | Arabic |
|
||||
| KOI8-R | Russian |
|
||||
| IBM420 | Arabic |
|
||||
| IBM424 | Hebrew |
|
||||
@@ -1,156 +0,0 @@
|
||||
---
|
||||
layout: default
|
||||
title: Conversion
|
||||
nav_order: 700
|
||||
has_children: true
|
||||
---
|
||||
<!--
|
||||
© 2020 and later: Unicode, Inc. and others.
|
||||
License & terms of use: http://www.unicode.org/copyright.html
|
||||
-->
|
||||
|
||||
# Conversion
|
||||
{: .no_toc }
|
||||
|
||||
## Contents
|
||||
{: .no_toc .text-delta }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
|
||||
---
|
||||
|
||||
## Conversion Overview
|
||||
|
||||
A converter is used to convert from one character encoding to another. In the
|
||||
case of ICU, the conversion is always between Unicode and another encoding, or
|
||||
vice-versa. A text encoding is a particular mapping from a given character set
|
||||
definition to the actual bits used to represent the data.
|
||||
|
||||
Unicode provides a single character set that covers the major languages of the
|
||||
world, and a small number of machine-friendly encoding forms and schemes to fit
|
||||
the needs of existing applications and protocols. It is designed for best
|
||||
interoperability with both ASCII and ISO-8859-1 (the most widely used character
|
||||
sets) to make it easier for Unicode to be used in almost all applications and
|
||||
protocols.
|
||||
|
||||
Hundreds of encodings have been developed over the years, each for small groups
|
||||
of languages and for special purposes. As a result, the interpretation of text,
|
||||
input, sorting, display, and storage depends on the knowledge of all the
|
||||
different types of character sets and their encodings. Programs have been
|
||||
written to handle either one single encoding at a time and switch between them,
|
||||
or to convert between external and internal encodings.
|
||||
|
||||
There is no single, authoritative source of precise definitions of many of the
|
||||
encodings and their names. However,
|
||||
[IANA](http://www.iana.org/assignments/character-sets) is the best source for
|
||||
names, and our Character Set repository is a good source of encoding definitions
|
||||
for each platform.
|
||||
|
||||
The transferring of text from one machine to another one often causes some loss
|
||||
of information. Some platforms have a different interpretation of the text than
|
||||
the other platforms. For example, Shift-JIS can be interpreted differently on
|
||||
Windows™ compared to UNIX®. Windows maps byte value 0x5C to the backslash
|
||||
symbol, while some UNIX machines map that byte value to the Yen symbol. Another
|
||||
problem arises when a character in the codepage looks like the Unicode Greek
|
||||
letter Mu or the Unicode micro symbol. Some platforms map this codepage byte
|
||||
sequence to one Unicode character, while another platform maps it to the other
|
||||
Unicode character. Fallbacks can partially fix this problem by mapping both
|
||||
Unicode characters to the same codepage byte sequence. Even though some
|
||||
character information is lost, the text is still readable.
|
||||
|
||||
ICU's converter API has the following main features:
|
||||
|
||||
1. Unicode surrogate support
|
||||
|
||||
2. Support for all major encodings
|
||||
|
||||
3. Consistent text conversion across all computer platforms
|
||||
|
||||
4. Text data can be streamed (buffered) through the API
|
||||
|
||||
5. Fast text conversion
|
||||
|
||||
6. Supports fallbacks to the codepage
|
||||
|
||||
7. Supports reverse fallbacks to Unicode
|
||||
|
||||
8. Allows callbacks for handling and substituting invalid or unmapped byte
|
||||
sequences
|
||||
|
||||
9. Allows a user to add support for unsupported encodings
|
||||
|
||||
This section deals with the processes of converting encodings to and from
|
||||
Unicode.
|
||||
|
||||
## Recommendations
|
||||
|
||||
1. **Use Unicode encodings whenever possible.** Together with Unicode for
|
||||
internal processing, it makes completely globalized systems possible and
|
||||
avoids the many problems with non-algorithmic conversions. (For a discussion
|
||||
of such problems, see for example ["Character Conversions and Mapping
|
||||
Tables"](http://icu-project.org/docs/papers/conversions_and_mappings_iuc19.ppt)
|
||||
on <http://icu-project.org/docs/> and the [XML Japanese
|
||||
Profile](http://www.w3.org/TR/japanese-xml/)).
|
||||
|
||||
1. Use UTF-8 and UTF-16.
|
||||
|
||||
2. Use UTF-16BE, SCSU and BOCU-1 as appropriate.
|
||||
|
||||
3. In special environments, other Unicode encodings may be used as well,
|
||||
such as UTF-16LE, UTF-32, UTF-32BE, UTF-32LE, UTF-7, UTF-EBCDIC, and
|
||||
CESU-8. (For turning Unicode filenames into ASCII-only filename strings,
|
||||
the IMAP-mailbox-name encoding can be used.)
|
||||
|
||||
4. Do not exchange text with single/unpaired surrogates.
|
||||
|
||||
2. **Use legacy charsets only when absolutely necessary**. For best data
|
||||
fidelity:
|
||||
|
||||
1. ISO-8859-1 is relatively unproblematic — if its limited character
|
||||
repertoire is sufficient — because it is converted trivially (1:1) to
|
||||
Unicode, avoiding conversion table problems for its small set of
|
||||
characters. (By contrast, proper conversion from US-ASCII requires a
|
||||
check for illegal byte values 0x80..0xff, which is an unnecessary
|
||||
complication for modern systems with 8-bit bytes. ISO-8859-1 is nearly
|
||||
as ubiquitous for modern systems as US-ASCII was for 7-bit systems.)
|
||||
|
||||
2. If you need to communicate with a certain platform, then use the same
|
||||
conversion tables as that platform itself, or at least ones that are
|
||||
very, very close.
|
||||
|
||||
3. ICU's conversion table repository contains hundreds of Unicode
|
||||
conversion tables from a number of common vendors and platforms as well
|
||||
as comparisons between these conversion tables:
|
||||
<https://icu.unicode.org/charts/charset> .
|
||||
|
||||
4. Do not trust codepage documentation that is not machine-readable, for
|
||||
example nice-looking charts: They are usually incomplete and out of
|
||||
date.
|
||||
|
||||
5. ICU's default build includes about 200 conversion tables. See the [ICU
|
||||
Data](../icudata.md) chapter for how to add or remove conversion tables
|
||||
and other data.
|
||||
|
||||
6. In ICU, you can (and should) also use APIs that map a charset name
|
||||
together with a standard/platform name. This allows you to get different
|
||||
converters for the same ambiguous charset name (like "Shift-JIS"),
|
||||
depending on the standard or platform specified. See the
|
||||
[convrtrs.txt](https://github.com/unicode-org/icu/blob/main/icu4c/source/data/mappings/convrtrs.txt)
|
||||
alias table, the [Using Converters](converters.md) chapter and [API
|
||||
references](https://unicode-org.github.io/icu-docs/apidoc/released/icu4c/ucnv_8h.html) .
|
||||
|
||||
7. For data exchange (rather than pure display), turn off fallback
|
||||
mappings: `ucnv_setFallback(cnv, FALSE)`;
|
||||
|
||||
8. For some text formats, especially XML and HTML, it is possible to set an
|
||||
"escape callback" function that turns unmappable Unicode code points
|
||||
into corresponding escape sequences, preventing data loss. See the API
|
||||
references and the [ucnv sample
|
||||
code](https://github.com/unicode-org/icu/tree/main/icu4c/source/samples/ucnv/)
|
||||
.
|
||||
|
||||
9. **Never modify a conversion table.** Instead, use existing ones that
|
||||
match precisely those in systems with which you communicate. "Modifying"
|
||||
a conversion table in reality just creates a new one, which makes the
|
||||
whole situation even less manageable.
|
||||
@@ -1,275 +0,0 @@
|
||||
---
|
||||
layout: default
|
||||
title: Calendar Examples
|
||||
nav_order: 2
|
||||
parent: Date/Time
|
||||
---
|
||||
<!--
|
||||
© 2020 and later: Unicode, Inc. and others.
|
||||
License & terms of use: http://www.unicode.org/copyright.html
|
||||
-->
|
||||
|
||||
# Calendar Examples
|
||||
{: .no_toc }
|
||||
|
||||
## Contents
|
||||
{: .no_toc .text-delta }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
|
||||
---
|
||||
|
||||
## Calendar for Default Time Zone
|
||||
|
||||
These C++, C , and Java examples get a Calendar based on the default time zone
|
||||
and add days to a date.
|
||||
|
||||
**C++**
|
||||
|
||||
```c++
|
||||
UErrorCode status = U_ZERO_ERROR;
|
||||
GregorianCalendar* gc = new GregorianCalendar(status);
|
||||
if (U_FAILURE(status)) {
|
||||
puts("Couldn't create GregorianCalendar");
|
||||
return;
|
||||
}
|
||||
// set up the date
|
||||
gc->set(2000, Calendar::FEBRUARY, 26);
|
||||
gc->set(Calendar::HOUR_OF_DAY, 23);
|
||||
gc->set(Calendar::MINUTE, 0);
|
||||
gc->set(Calendar::SECOND, 0);
|
||||
gc->set(Calendar::MILLISECOND, 0);
|
||||
// Iterate through the days and print it out.
|
||||
for (int32_t i = 0; i < 30; i++) {
|
||||
// print out the date.
|
||||
// You should use the DateFormat to properly format it
|
||||
printf("year: %d, month: %d (%d in the implementation), day: %d\n",
|
||||
gc->get(Calendar::YEAR, status),
|
||||
gc->get(Calendar::MONTH, status) + 1,
|
||||
gc->get(Calendar::MONTH, status),
|
||||
gc->get(Calendar::DATE, status));
|
||||
if (U_FAILURE(status)) {
|
||||
puts("Calendar::get failed");
|
||||
return;
|
||||
}
|
||||
// Add a day to the date
|
||||
gc->add(Calendar::DATE, 1, status);
|
||||
if (U_FAILURE(status)) {
|
||||
puts("Calendar::add failed");
|
||||
return;
|
||||
}
|
||||
}
|
||||
delete gc;
|
||||
```
|
||||
|
||||
**C**
|
||||
|
||||
```c
|
||||
UErrorCode status = U_ZERO_ERROR;
|
||||
int32_t i;
|
||||
UCalendar* cal = ucal_open(NULL, -1, NULL, UCAL_GREGORIAN, &status);
|
||||
if (U_FAILURE(status)) {
|
||||
puts("Couldn't create GregorianCalendar");
|
||||
return;
|
||||
}
|
||||
// set up the date
|
||||
ucal_set(cal, UCAL_YEAR, 2000);
|
||||
ucal_set(cal, UCAL_MONTH, UCAL_FEBRUARY); /* FEBRUARY */
|
||||
ucal_set(cal, UCAL_DATE, 26);
|
||||
ucal_set(cal, UCAL_HOUR_OF_DAY, 23);
|
||||
ucal_set(cal, UCAL_MINUTE, 0);
|
||||
ucal_set(cal, UCAL_SECOND, 0);
|
||||
ucal_set(cal, UCAL_MILLISECOND, 0);
|
||||
// Iterate through the days and print it out.
|
||||
for (i = 0; i < 30; i++) {
|
||||
// print out the date.
|
||||
// You should use the udat_* API to properly format it
|
||||
printf("year: %d, month: %d (%d in the implementation), day: %d\n",
|
||||
ucal_get(cal, UCAL_YEAR, &status),
|
||||
ucal_get(cal, UCAL_MONTH, &status) + 1,
|
||||
ucal_get(cal, UCAL_MONTH, &status),
|
||||
ucal_get(cal, UCAL_DATE, &status));
|
||||
if (U_FAILURE(status)) {
|
||||
puts("Calendar::get failed");
|
||||
return;
|
||||
}
|
||||
// Add a day to the date
|
||||
ucal_add(cal, UCAL_DATE, 1, &status);
|
||||
if (U_FAILURE(status)) {
|
||||
puts("Calendar::add failed");
|
||||
return;
|
||||
}
|
||||
}
|
||||
ucal_close(cal);
|
||||
```
|
||||
|
||||
**Java**
|
||||
|
||||
```java
|
||||
Calendar cal = new GregorianCalendar();
|
||||
if (cal == null) {
|
||||
System.out.println("Couldn't create GregorianCalendar");
|
||||
return;
|
||||
}
|
||||
// set up the date
|
||||
cal.set(Calendar.YEAR, 2000);
|
||||
cal.set(Calendar.MONTH, Calendar.FEBRUARY); /* FEBRUARY */
|
||||
cal.set(Calendar.DATE, 26);
|
||||
cal.set(Calendar.HOUR_OF_DAY, 23);
|
||||
cal.set(Calendar.MINUTE, 0);
|
||||
cal.set(Calendar.SECOND, 0);
|
||||
cal.set(Calendar.MILLISECOND, 0);
|
||||
// Iterate through the days and print it out.
|
||||
for (int i = 0; i < 30; i++) {
|
||||
// print out the date.
|
||||
System.out.println(" year: " + cal.get(Calendar.YEAR) +
|
||||
" month: " + (cal.get(Calendar.MONTH) + 1) +
|
||||
" day : " + cal.get(Calendar.DATE)
|
||||
);
|
||||
cal.add(Calendar.DATE, 1);
|
||||
}
|
||||
```
|
||||
|
||||
## Converting dates between calendars
|
||||
|
||||
These C++, C , and Java examples demonstrates converting dates from one calendar
|
||||
(Gregorian) to another calendar (Japanese).
|
||||
|
||||
**C++**
|
||||
|
||||
```c++
|
||||
UErrorCode status = U_ZERO_ERROR;
|
||||
UDate time;
|
||||
Calendar *cal1, *cal2;
|
||||
// Create a new Gregorian Calendar.
|
||||
cal1 = Calendar::createInstance("en_US@calendar=gregorian", status);
|
||||
if (U_FAILURE(status)) {
|
||||
printf("Error creating Gregorian calendar.\n");
|
||||
return;
|
||||
}
|
||||
// Set the Gregorian Calendar to a specific date for testing.
|
||||
cal1->set(1980, UCAL_SEPTEMBER, 3);
|
||||
// Display the date.
|
||||
printf("Gregorian Calendar:\t%d/%d/%d\n",
|
||||
cal1->get(UCAL_MONTH, status) + 1,
|
||||
cal1->get(UCAL_DATE, status),
|
||||
cal1->get(UCAL_YEAR, status));
|
||||
if (U_FAILURE(status)) {
|
||||
printf("Error getting Gregorian date.");
|
||||
return;
|
||||
}
|
||||
// Create a Japanese Calendar.
|
||||
cal2 = Calendar::createInstance("ja_JP@calendar=japanese", status);
|
||||
if (U_FAILURE(status)) {
|
||||
printf("Error creating Japnese calendar.\n");
|
||||
return;
|
||||
}
|
||||
// Set the date.
|
||||
time = cal1->getTime(status);
|
||||
if (U_FAILURE(status)) {
|
||||
printf("Error getting time.\n");
|
||||
return;
|
||||
}
|
||||
cal2->setTime(time, status);
|
||||
if (U_FAILURE(status)) {
|
||||
printf("Error setting the date for Japanese calendar.\n");
|
||||
return;
|
||||
}
|
||||
// Set the timezone
|
||||
cal2->setTimeZone(cal1->getTimeZone());
|
||||
// Display the date.
|
||||
printf("Japanese Calendar:\t%d/%d/%d\n",
|
||||
cal2->get(UCAL_MONTH, status) + 1,
|
||||
cal2->get(UCAL_DATE, status),
|
||||
cal2->get(UCAL_YEAR, status));
|
||||
if (U_FAILURE(status)) {
|
||||
printf("Error getting Japanese date.");
|
||||
return;
|
||||
}
|
||||
delete cal1;
|
||||
delete cal2;
|
||||
```
|
||||
|
||||
**C**
|
||||
|
||||
```c
|
||||
UErrorCode status = U_ZERO_ERROR;
|
||||
UDate time;
|
||||
UCalendar *cal1, *cal2;
|
||||
// Create a new Gregorian Calendar.
|
||||
cal1 = ucal_open(NULL, -1, "en_US@calendar=gregorian", UCAL_TRADITIONAL,
|
||||
&status);
|
||||
if (U_FAILURE(status)) {
|
||||
printf("Couldn't create Gregorian Calendar.");
|
||||
return;
|
||||
}
|
||||
// Set the Gregorian Calendar to a specific date for testing.
|
||||
ucal_setDate(cal1, 1980, UCAL_SEPTEMBER, 3, &status);
|
||||
if (U_FAILURE(status)) {
|
||||
printf("Error setting date.");
|
||||
return;
|
||||
}
|
||||
// Display the date.
|
||||
printf("Gregorian Calendar:\t%d/%d/%d\n",
|
||||
ucal_get(cal1, UCAL_MONTH, &status) + 1,
|
||||
ucal_get(cal1, UCAL_DATE, &status),
|
||||
ucal_get(cal1, UCAL_YEAR, &status));
|
||||
if (U_FAILURE(status)) {
|
||||
printf("Error getting Gregorian date.");
|
||||
return 1;
|
||||
}
|
||||
// Create a Japanese Calendar.
|
||||
cal2 = ucal_open(NULL, -1, "ja_J@calendar=japanese", UCAL_TRADITIONAL, &status);
|
||||
if (U_FAILURE(status)) {
|
||||
printf("Couldn't create Japanese Calendar.");
|
||||
return 1;
|
||||
}
|
||||
// Set the date.
|
||||
time = ucal_getMillis(cal1, &status);
|
||||
if (U_FAILURE(status)) {
|
||||
printf("Error getting time.\n");
|
||||
return;
|
||||
}
|
||||
ucal_setMillis(cal2, time, &status);
|
||||
if (U_FAILURE(status)) {
|
||||
printf("Error setting time.\n");
|
||||
return;
|
||||
}
|
||||
// Display the date.
|
||||
printf("Japanese Calendar:\t%d/%d/%d\n",
|
||||
ucal_get(cal2, UCAL_MONTH, &status) + 1,
|
||||
ucal_get(cal2, UCAL_DATE, &status),
|
||||
ucal_get(cal2, UCAL_YEAR, &status));
|
||||
if (U_FAILURE(status)) {
|
||||
printf("Error getting Japanese date.");
|
||||
return;
|
||||
}
|
||||
ucal_close(cal1);
|
||||
ucal_close(cal2);
|
||||
```
|
||||
|
||||
**Java**
|
||||
|
||||
```java
|
||||
Calendar cal1, cal2;
|
||||
// Create a new Gregorian Calendar.
|
||||
cal1 = new GregorianCalendar();
|
||||
// Set the Gregorian Calendar to a specific date for testing.
|
||||
cal1.set(1980, Calendar.SEPTEMBER, 3);
|
||||
// Display the date.
|
||||
System.out.println("Gregorian Calendar:\t" + (cal1.get(Calendar.MONTH) + 1) +
|
||||
"/" +
|
||||
cal1.get(Calendar.DATE) + "/" +
|
||||
cal1.get(Calendar.YEAR));
|
||||
// Create a Japanese Calendar.
|
||||
cal2 = new JapaneseCalendar();
|
||||
// Set the date and timezone
|
||||
cal2.setTime(cal1.getTime());
|
||||
cal2.setTimeZone(cal1.getTimeZone());
|
||||
// Display the date.
|
||||
System.out.println("Japanese Calendar:\t" + (cal2.get(Calendar.MONTH) + 1) +
|
||||
"/" +
|
||||
cal2.get(Calendar.DATE) + "/" +
|
||||
cal2.get(Calendar.YEAR));
|
||||
```
|
||||
@@ -1,323 +0,0 @@
|
||||
---
|
||||
layout: default
|
||||
title: Calendar Services
|
||||
nav_order: 1
|
||||
parent: Date/Time
|
||||
---
|
||||
<!--
|
||||
© 2020 and later: Unicode, Inc. and others.
|
||||
License & terms of use: http://www.unicode.org/copyright.html
|
||||
-->
|
||||
|
||||
# Calendar Classes
|
||||
{: .no_toc }
|
||||
|
||||
## Contents
|
||||
{: .no_toc .text-delta }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
ICU has two main calendar classes used for parsing and formatting Calendar
|
||||
information correctly:
|
||||
|
||||
1. `Calendar`
|
||||
|
||||
An abstract base class that defines the calendar API. This API supports
|
||||
UDate to fields conversion and field arithmetic.
|
||||
|
||||
2. `GregorianCalendar`
|
||||
|
||||
A concrete subclass of `Calendar` that implements the standard calendar used
|
||||
today internationally.
|
||||
|
||||
In addition to these, ICU has other `Calendar` subclasses to support
|
||||
non-gregorian calendars including:
|
||||
|
||||
* Buddhist
|
||||
|
||||
* Chinese
|
||||
|
||||
* Coptic
|
||||
|
||||
* Ethiopic
|
||||
|
||||
* Hebrew
|
||||
|
||||
* Indian
|
||||
|
||||
* Islamic
|
||||
|
||||
* Japanese
|
||||
|
||||
* Persian
|
||||
|
||||
The `Calendar` class is designed to support additional calendar systems in the future.
|
||||
|
||||
> :point_right: **Note**: *`Calendar` classes are related to `UDate`, the `TimeZone` classes, and the `DateFormat`
|
||||
classes.*
|
||||
|
||||
### Calendar locale and keyword handling
|
||||
|
||||
When a calendar object is created, via either `Calendar::create()`, or
|
||||
`ucal_open()`, or indirectly within a date formatter, ICU looks up the 'default'
|
||||
calendar type for that locale. At present, all locales default to a Gregorian
|
||||
calendar, except for the compatibility locales th_TH_TRADITIONAL and
|
||||
ja_JP_TRADITIONAL. If the "calendar" keyword is supplied, this value will
|
||||
override the default for that locale.
|
||||
|
||||
For instance, `Calendar::createInstance("fr_FR", status)` will create a Gregorian calendar,
|
||||
but `Calendar::createInstance("fr_FR@calendar=buddhist")` will create a Buddhist calendar.
|
||||
|
||||
It is an error to use an invalid calendar type. It will produce a missing resource error.
|
||||
|
||||
> :point_right: **Note**: *As of ICU 2.8, the above description applies to ICU4J only. ICU4J will have
|
||||
this behavior in 3.0*
|
||||
|
||||
## Usage
|
||||
|
||||
This section discusses how to use the `Calendar` class and the `GregorianCalendar` subclass.
|
||||
|
||||
### Calendar
|
||||
|
||||
`Calendar` is an abstract base class. It defines common protocols for a hierarchy
|
||||
of classes. Concrete subclasses of `Calendar`, for example the `GregorianCalendar`
|
||||
class, define specific operations that correspond to a real-world calendar
|
||||
system. `Calendar` objects (instantiations of concrete subclasses of `Calendar`),
|
||||
embody state that represents a specific context. They correspond to a real-world
|
||||
locale. They also contain state that specifies a moment in time.
|
||||
|
||||
The API defined by `Calendar` encompasses multiple functions:
|
||||
|
||||
1. Representation of a specific time as a `UDate`
|
||||
|
||||
2. Representation of a specific time as a set of integer fields, such as `YEAR`,
|
||||
`MONTH`, `HOUR`, etc.
|
||||
|
||||
3. Conversion from `UDate` to fields
|
||||
|
||||
4. Conversion from fields to `UDate`
|
||||
|
||||
5. Field arithmetic, including adding, rolling, and field difference
|
||||
|
||||
6. Context management
|
||||
|
||||
7. Factory methods
|
||||
|
||||
8. Miscellaneous: field meta-information, time comparison
|
||||
|
||||
#### Representation and Conversion
|
||||
|
||||
The basic function of the `Calendar` class is to convert between a `UDate` value and
|
||||
a set of integer fields. A `UDate` value is stored as UTC time in milliseconds,
|
||||
which means it is calendar and time zone independent. `UDate` is the most compact
|
||||
and portable way to store and transmit a date and time. `Integer` field values, on
|
||||
the other hand, depend on the calendar system (that is, the concrete subclass of
|
||||
`Calendar`) and the calendar object's context state.
|
||||
|
||||
> :point_right: **Note**: *`Integer` field values are needed when implementing a human interface that must
|
||||
display or input a date and/or time.*
|
||||
|
||||
At any given time, a calendar object uses (when `DateFormat` is not sufficient)
|
||||
either its internal `UDate` or its integer fields (depending on which has been set
|
||||
most recently via `setTime()` or `set()`), to represent a specific date and time.
|
||||
Whatever the current internal representation, when the caller requests a `UDate`
|
||||
or an integer field it is computed if necessary. The caller need never trigger
|
||||
the conversion explicitly. The caller must perform a conversion to set either
|
||||
the `UDate` or the integer fields, and then retrieve the desired data. This also
|
||||
applies in situations where the caller has some integer fields and wants to
|
||||
obtain others.
|
||||
|
||||
#### Field Arithmetic
|
||||
|
||||
Arithmetic with `UDate` values is straightforward. Since the values are
|
||||
millisecond scalar values, direct addition and subtraction is all that is
|
||||
required. Arithmetic with integer fields is more complicated. For example, what
|
||||
is the date June 4, 1999 plus 300 days? `Calendar` defines three basic methods (in
|
||||
several variants) that perform field arithmetic: `add()`, `roll()`, and
|
||||
`fieldDifference()`.
|
||||
|
||||
The `add()` method adds positive or negative values to a specified field. For
|
||||
example, calling `add(Calendar::MONTH, 2)` on a `GregorianCalendar` object set to
|
||||
March 15, 1999 sets the calendar to May 15, 1999. The `roll()` method is similar,
|
||||
but does not modify fields that are larger. For example, calling
|
||||
`roll(Calendar::HOUR, n)` changes the hour that a calendar is set to without
|
||||
changing the day. Calling `roll(Calendar::MONTH, n)` changes the month without
|
||||
changing the year.
|
||||
|
||||
The `fieldDifference()` method is the inverse of the `add()` method. It computes the
|
||||
difference between a calendar's currently set time and a specified `UDate` in
|
||||
terms of a specified field. Repeated calls to `fieldDifference()` compute the
|
||||
difference between two `UDate` objects in terms of whatever fields the caller specifies
|
||||
(for example, years, months, days, and hours). If the `add()` method is called
|
||||
with the results of `fieldDifference(when, n)`, then the calendar is moved toward
|
||||
field by field.
|
||||
|
||||
This is demonstrated in the following example:
|
||||
|
||||
```c++
|
||||
Calendar cal = Calendar.getInstance();
|
||||
cal.set(2000, Calendar.MARCH, 15);
|
||||
Date date = new Date(2000-1900, Calendar.JULY, 4);
|
||||
int yearDiff = cal.fieldDifference(date, Calendar.YEAR); // yearDiff <= 0
|
||||
int monthDiff = cal.fieldDifference(date, Calendar.MONTH); // monthDiff ;<= 3
|
||||
// At this point cal has been advanced 3 months to June 15, 2000.
|
||||
int dayDiff = cal.fieldDifference(date, Calendar.DAY_OF_MONTH); // dayDiff ;<=19
|
||||
// At this point cal has been advanced 19 days to July 4, 2000.
|
||||
```
|
||||
|
||||
#### Context Management
|
||||
|
||||
A `Calendar` object performs its computations within a specific context. The
|
||||
context affects the results of conversions and arithmetic computations. When a
|
||||
`Calendar` object is created, it establishes its context using either default
|
||||
values or values specified by the caller:
|
||||
|
||||
1. Locale-specific week data, including the first day of the week and the
|
||||
minimal days in the first week. Initially, this is retrieved from the locale
|
||||
resource data for the specified locale, or if none is specified, for the
|
||||
default locale.
|
||||
|
||||
2. A `TimeZone` object. Initially, this is set to the specified zone object, or
|
||||
if none is specified, the default `TimeZone`.
|
||||
|
||||
The context of a `Calendar` object can be queried after the calendar is created
|
||||
using calls such as `getMinimalDaysInFirstWeek()`, `getFirstDayOfWeek()`, and
|
||||
`getTimeZone()`. The context can be changed using calls such as
|
||||
`setMinimalDaysInFirstWeek()`, `setFirstDayOfWeek()`, and `setTimeZone()`.
|
||||
|
||||
#### Factory Methods
|
||||
|
||||
Like other format classes, the best way to create a calendar object is by using
|
||||
one of the factory methods. These are static methods on the `Calendar` class that
|
||||
create and return an instance of a concrete subclass. Factory methods should be
|
||||
used to enable the code to obtain the correct calendar for a locale without
|
||||
having to know specific details. The factory methods on `Calendar` are named
|
||||
`createInstance()`.
|
||||
|
||||
***`MONTH` field***
|
||||
> :point_right: **Note**: *Calendar numbers months starting from zero, so calling `cal.set(1998, 3, 5)`
|
||||
sets cal to April 15, 1998, not March 15, 1998. This follows the Java
|
||||
convention. To avoid mistakes, use the constants defined in the `Calendar` class
|
||||
for the months and days of the week. For example, `cal.set(1998, Calendar::APRIL, 15)`.*
|
||||
|
||||
#### Ambiguous Wall Clock Time Resolution
|
||||
|
||||
When the time offset from UTC has changed, it produces an ambiguous time slot
|
||||
around the transition. For example, many US locations observe daylight saving
|
||||
time. On the date of transition to daylight saving time in US, wall clock time
|
||||
jumps from 12:59 AM (standard) to 2:00 AM (daylight). Therefore, wall clock
|
||||
times from 1:00 AM to 1:59 AM do not exist on the date. When the input wall time
|
||||
falls into this missing time slot, the ICU Calendar resolves the time using the
|
||||
UTC offset before the transition by default. In this example, 1:30 AM is
|
||||
interpreted as 1:30 AM standard time (non-exist), so the final result will be
|
||||
2:30 AM daylight time.
|
||||
On the date of transition back to standard time, wall clock time is moved back
|
||||
one hour at 2:00 AM. So wall clock times from 1:00 AM to 1:59 AM occur twice. In
|
||||
this case, the ICU Calendar resolves the time using the UTC offset after the
|
||||
transition by default. For example, 1:30 AM on the date is resolved as 1:30 AM
|
||||
standard time.
|
||||
Ambiguous wall clock time resolution behaviors can be customized by Calendar
|
||||
APIs `setRepeatedWallTimeOption()` and `setSkippedWallTimeOption()`. These APIs are
|
||||
available in ICU 49 or later versions.
|
||||
|
||||
### `GregorianCalendar`
|
||||
|
||||
The `GregorianCalendar` class implements two calendar systems, the Gregorian
|
||||
calendar and the Julian calendar. These calendar systems are closely related,
|
||||
differing mainly in their definition of the leap year. The Julian calendar has
|
||||
leap years every four years; the Gregorian calendar refines this by excluding
|
||||
century years that are not divisible by 400. `GregorianCalendar` defines two eras,
|
||||
BC (B.C.E.) and AD (C.E.).
|
||||
|
||||
Historically, most western countries used the Julian calendar until the 16th to
|
||||
20th century, depending on the country. They then switched to the Gregorian
|
||||
calendar. The `GregorianCalendar` class mirrors this behavior by defining a
|
||||
cut-over date. Before this date, the Julian calendar algorithms are used. After
|
||||
it, the Gregorian calendar algorithms are used. By default, the cut-over date is
|
||||
set to October 4, 1582 C.E., which reflects the time when countries first began
|
||||
adopting the Gregorian calendar. The `GregorianCalendar` class does not attempt
|
||||
historical accuracy beyond this behavior, and does not vary its cut-over date by
|
||||
locale. However, users can modify the cut-over date by using the
|
||||
`setGregorianChange()` method.
|
||||
|
||||
Code that is written correctly instantiates calendar objects using the Calendar
|
||||
factory methods, and therefore holds a `Calendar*` pointer. Such code cannot
|
||||
directly access the GregorianCalendar-specific methods not present in `Calendar`.
|
||||
The correct way to handle this is to perform a dynamic cast, after testing the
|
||||
type of the object using `getDynamicClassID()`. For example:
|
||||
|
||||
```c++
|
||||
void setCutover(Calendar *cal, UDate myCutover) {
|
||||
if (cal->getDynamicClassID() == GregorianCalendar::getStaticClassID()) {
|
||||
GregorianCalendar *gc = (GregorianCalendar*)cal;
|
||||
gc->setGregorianChange(myCutover, status);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
> :point_right: **Note**: *This is a general technique that should be used throughout ICU in conjunction
|
||||
with the factory methods.*
|
||||
|
||||
### Disambiguation
|
||||
|
||||
When computing a `UDate` from fields, some special circumstances can arise. There
|
||||
might be insufficient information to compute the `UDate` (such as only year and
|
||||
month but no day in the month), there might be inconsistent information (such as
|
||||
"Tuesday, July 15, 1996" -— July 15, 1996, is actually a Monday), or the input
|
||||
time might be ambiguous because of time zone transition.
|
||||
|
||||
1. **Insufficient Information**
|
||||
ICU Calendar uses the default field values to specify missing fields. The
|
||||
default for a field is the same as that of the start of the epoch (that is,
|
||||
`YEAR = 1970`, `MONTH = JANUARY`, `DAY_OF_MONTH = 1`).
|
||||
|
||||
2. **Inconsistent Information**
|
||||
If fields conflict, the calendar gives preference to fields set more
|
||||
recently. For example, when determining the day, the calendar looks for one
|
||||
of the following combinations of fields:
|
||||
`MONTH + DAY_OF_MONTH`
|
||||
`MONTH + WEEK_OF_MONTH + DAY_OF_WEEK`
|
||||
`MONTH + DAY_OF_WEEK_IN_MONTH + DAY_OF_WEEK`
|
||||
`DAY_OF_YEAR`
|
||||
`DAY_OF_WEEK + WEEK_OF_YEAR`
|
||||
For the time of day, the calendar looks for one of the following
|
||||
combinations of fields:
|
||||
`HOUR_OF_DAY`
|
||||
`AM_PM + HOUR`
|
||||
|
||||
3. **Ambiguous Wall Clock Time**
|
||||
When time offset from UTC has changed, it produces ambiguous time slot
|
||||
around the transition. For example, many US locations observe daylight
|
||||
saving time. On the date switching to daylight saving time in US, wall clock
|
||||
time jumps from 1:00 AM (standard) to 2:00 AM (daylight). Therefore, wall
|
||||
clock time from 1:00 AM to 1:59 AM do not exist on the date. When the input
|
||||
wall time fall into this missing time slot, the ICU Calendar resolves the
|
||||
time using the UTC offset before the transition by default. In this example,
|
||||
1:30 AM is interpreted as 1:30 AM standard time (non-exist), so the final
|
||||
result will be 2:30 AM daylight time.
|
||||
On the date switching back to standard time, wall clock time is moved back
|
||||
one hour at 2:00 AM. So wall clock time from 1:00 AM to 1:59 AM occur twice.
|
||||
In this case, the ICU Calendar resolves the time using the UTC offset after
|
||||
the transition by default. For example, 1:30 AM on the date is resolved as
|
||||
1:30 AM standard time.
|
||||
|
||||
***Options for Ambiguous Time Resolution***
|
||||
> :point_right: **Note**: *Ambiguous wall clock time resolution behaviors can be customized by Calendar APIs `setRepeatedTimeOption()` and `setSkippedTimeOption()`. These methods are available in ICU 49 or later versions.*
|
||||
|
||||
***`WEEK_OF_YEAR` field***
|
||||
> :point_right: **Note**: *Values calculated for the `WEEK_OF_YEAR` field range from 1 to 53. Week 1 for a year is the first week that contains at least `getMinimalDaysInFirstWeek()` days from that year. It depends on the values of `getMinimalDaysInFirstWeek()`, `getFirstDayOfWeek()`, and the day of the week of January 1. Weeks between week 1 of one year and week 1 of the following year are numbered sequentially from 2 to 52 or 53 (if needed).
|
||||
For example, January 1, 1998 was a Thursday. If `getFirstDayOfWeek()` is `MONDAY`
|
||||
and `getMinimalDaysInFirstWeek()` is `4` (these are the values reflecting ISO 8601
|
||||
and many national standards), then week 1 of 1998 starts on December 29, 1997,
|
||||
and ends on January 4, 1998. However, if `getFirstDayOfWeek()` is `SUNDAY`, then
|
||||
week 1 of 1998 starts on January 4, 1998, and ends on January 10, 1998. The
|
||||
first three days of 1998 are then part of week 53 of 1997.*
|
||||
|
||||
## Programming Examples
|
||||
|
||||
Programming for calendar [examples in C++, C, and Java](examples.md) .
|
||||
@@ -1,156 +0,0 @@
|
||||
---
|
||||
layout: default
|
||||
title: Date/Time
|
||||
nav_order: 900
|
||||
has_children: true
|
||||
---
|
||||
<!--
|
||||
© 2020 and later: Unicode, Inc. and others.
|
||||
License & terms of use: http://www.unicode.org/copyright.html
|
||||
-->
|
||||
|
||||
# Date/Time Services
|
||||
{: .no_toc }
|
||||
|
||||
## Contents
|
||||
{: .no_toc .text-delta }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
|
||||
---
|
||||
|
||||
## Overview of ICU System Time Zones
|
||||
|
||||
A time zone represents an offset applied to Greenwich Mean Time (GMT) to obtain
|
||||
local time. The offset might vary throughout the year, if daylight savings time
|
||||
(DST) is used, or might be the same all year long. Typically, regions closer to
|
||||
the equator do not use DST. If DST is in use, then specific rules define the
|
||||
point at which the offset changes and the amount by which it changes. Thus, a
|
||||
time zone is described by the following information:
|
||||
|
||||
* An identifying string, or ID. This consists only of invariant characters
|
||||
(see the file `utypes.h`). It typically has the format continent / city. The
|
||||
city chosen is not the only city in which the zone applies, but rather a
|
||||
representative city for the region. Some IDs consist of three or four
|
||||
uppercase letters; these are legacy zone names that are aliases to standard
|
||||
zone names.
|
||||
|
||||
* An offset from GMT, either positive or negative. Offsets range from
|
||||
approximately minus half a day to plus half a day.
|
||||
|
||||
If DST is observed, then three additional pieces of information are needed:
|
||||
|
||||
1. The precise date and time during the year when DST begins. In the first half
|
||||
of the year it's in the northern hemisphere, and in the second half of the
|
||||
year it's in the southern hemisphere.
|
||||
|
||||
2. The precise date and time during the year when DST ends. In the first half
|
||||
of the year it's in the southern hemisphere, and in the second half of the
|
||||
year it's in the northern hemisphere.
|
||||
|
||||
3. The amount by which the GMT offset changes when DST is in effect. This is
|
||||
almost always one hour.
|
||||
|
||||
### System and User Time Zones
|
||||
|
||||
ICU supports local time zones through the classes `TimeZone` and `SimpleTimeZone` in
|
||||
the C++ API. In the C API, time zones are designated by their ID strings.
|
||||
|
||||
Users can construct their own time zone objects by specifying the above
|
||||
information to the C++ API. However, it is more typical for users to use a
|
||||
pre-existing system time zone since these represent all current international
|
||||
time zones in use. This document lists the system time zones, both in order of
|
||||
GMT offset and in alphabetical order of ID.
|
||||
|
||||
Since this list changes one or more times a year, *this document only represents
|
||||
a snapshot*. For the most current list of ICU system zones, use the method
|
||||
`TimeZone::getAvailableIDs()`.
|
||||
|
||||
*The zones are listed in binary sort order (that is, 'A' through 'Z' come before
|
||||
'a' through 'z'). This is the same order in which the zones are stored
|
||||
internally, and the same order in which they are returned by
|
||||
`TimeZone::getAvailableIDs()`. The reason for this is that ICU locates zones using
|
||||
a binary search, and the binary search relies on this sort order.*
|
||||
*You might notice that zones such as Etc/GMT+1 appear to have the wrong sign for
|
||||
their GMT offset. In fact, their sign is inverted since the the Etc zones follow
|
||||
the POSIX sign conventions. This is the way the original Olson data is set up,
|
||||
and ICU reproduces the Olson data faithfully. See the Olson files for more
|
||||
details.*
|
||||
|
||||
### References
|
||||
|
||||
The ICU system time zones are derived from the tz database (also known as the
|
||||
“Olson” database) at [ftp://elsie.nci.nih.gov/pub](ftp://elsie.nci.nih.gov/pub).
|
||||
This is the data used across much of the industry, including by UNIX systems,
|
||||
and is usually updated several times each year. ICU (since version 2.8) and base
|
||||
Java (since Java 1.4) contain code and tz data supporting both current and
|
||||
historic time zone usage.
|
||||
|
||||
## How ICU Represents Dates/Times
|
||||
|
||||
ICU represents dates and times using `UDate`s. A `UDate` is a scalar value that
|
||||
indicates a specific point in time, independent of calendar system and local
|
||||
time zone. It is stored as the number of milliseconds from a reference point
|
||||
known as the epoch. The epoch is midnight Universal Time Coordinated (UTC)
|
||||
January 1, 1970 A.D. Negative `UDate` values indicate times before the epoch.
|
||||
|
||||
*These classes have the same architecture as the Java classes.*
|
||||
|
||||
Most people only need to use the `DateFormat` classes for parsing and formatting
|
||||
dates and times. However, for those who need to convert dates and times or
|
||||
perform numeric calculations, the services described in this section can be very
|
||||
useful.
|
||||
|
||||
To translate a `UDate` to a useful form, a calendar system and local time zone
|
||||
must be specified. These are specified in the form of objects of the `Calendar`
|
||||
and `TimeZone` classes. Once these two objects are specified, they can be used to
|
||||
convert the `UDate` to and from its corresponding calendar fields. The different
|
||||
fields are defined in the `Calendar` class and include the year, month, day, hour,
|
||||
minute, second, and so on.
|
||||
|
||||
Specific `Calendar` objects correspond to calendar systems (such as Gregorian) and
|
||||
conventions (such as the first day of the week) in use in different parts of the
|
||||
world. To obtain a `Calendar` object for France, for example, call
|
||||
`Calendar::createInstance(Locale::getFrance(), status)`.
|
||||
|
||||
The `TimeZone` class defines the conversion between universal coordinated time
|
||||
(UTC), and local time, according to real-world rules. Different `TimeZone`
|
||||
objects correspond to different real-world time zones. For example, call
|
||||
`TimeZone::createTimeZone("America/Los_Angeles")` to obtain an object that
|
||||
implements the U.S. Pacific time zone, both Pacific Standard Time (PST) and
|
||||
Pacific Daylight Time (PDT).
|
||||
|
||||
As previously mentioned, the `Calendar` and `TimeZone` objects must be specified
|
||||
correctly together. One way of doing so is to create each independently, then
|
||||
use the `Calendar::setTimeZone()` method to associate the time zone with the
|
||||
calendar. Another is to use the `Calendar::createInstance()` method that takes a
|
||||
`TimeZone` object. For example, call `Calendar::createInstance(
|
||||
TimeZone::createInstance( "America/Los_Angeles"), Locale:getUS(), status)` to
|
||||
obtain a `Calendar` appropriate for use in the U.S. Pacific time zone.
|
||||
|
||||
ICU has four classes pertaining to calendars and timezones:
|
||||
|
||||
* [`Calendar`](calendar/index.md)
|
||||
|
||||
`Calendar` is an abstract base class that represents a calendar system.
|
||||
`Calendar` objects map `UDate` values to and from the individual fields used in
|
||||
a particular calendar system. `Calendar` also performs field computations such
|
||||
as advancing a date by two months.
|
||||
|
||||
* [`Gregorian Calendar`](calendar/index.md)
|
||||
|
||||
`GregorianCalendar` is a concrete subclass of `Calendar` that implements the
|
||||
rules of the Julian calendar and the Gregorian calendar, which is the common
|
||||
calendar in use internationally today.
|
||||
|
||||
* [`TimeZone`](timezone/index.md)
|
||||
|
||||
`TimeZone` is an abstract base class that represents a time zone. `TimeZone`
|
||||
objects map between universal coordinated time (UTC) and local time.
|
||||
|
||||
* [`SimpleTimeZone`](timezone/index.md)
|
||||
|
||||
`SimpleTimeZone` is a concrete subclass of `TimeZone` that implements standard
|
||||
time and daylight savings time according to real-world rules. Individual
|
||||
`SimpleTimeZone` objects correspond to real-world time zones.
|
||||
@@ -1,91 +0,0 @@
|
||||
---
|
||||
layout: default
|
||||
title: Date and Time Zone Examples
|
||||
nav_order: 4
|
||||
parent: Date/Time
|
||||
---
|
||||
<!--
|
||||
© 2020 and later: Unicode, Inc. and others.
|
||||
License & terms of use: http://www.unicode.org/copyright.html
|
||||
-->
|
||||
|
||||
# Date and Time Zone Examples
|
||||
{: .no_toc }
|
||||
|
||||
## Contents
|
||||
{: .no_toc .text-delta }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
|
||||
---
|
||||
|
||||
## C++ TimeZone example code
|
||||
|
||||
This example code illustrates some time zone operations.
|
||||
|
||||
```c++
|
||||
UErrorCode success = U_ZERO_ERROR;
|
||||
UnicodeString dateReturned, curTZNameEn, curTZNameFr;
|
||||
UDate curDate;
|
||||
int32_t stdOffset,dstOffset;
|
||||
|
||||
// Create a Time Zone with America/Los_Angeles
|
||||
TimeZone *tzWest = TimeZone::createTimeZone("America/Los_Angeles");
|
||||
|
||||
|
||||
// Print out the Time Zone Name, GMT offset etc.
|
||||
curTZNameEn = tzWest->getDisplayName(Locale::getEnglish(),curTZNameEn);
|
||||
u_printf("%s\n","Current Time Zone Name in English:");
|
||||
u_printf("%S\n", curTZNameEn.getTerminatedBuffer());
|
||||
|
||||
curTZNameFr = tzWest->getDisplayName(Locale::getCanadaFrench(),curTZNameFr);
|
||||
u_printf("%s\n","Current Time Zone Name in French:");
|
||||
u_printf("%S\n", curTZNameFr.getTerminatedBuffer());
|
||||
|
||||
|
||||
// Create a Calendar to get current date
|
||||
Calendar* calendar = Calendar::createInstance(success);
|
||||
curDate = calendar->getNow();
|
||||
|
||||
|
||||
// Print out the Current Date/Time in the given time zone
|
||||
DateFormat *dt = DateFormat::createDateInstance();
|
||||
dateReturned = dt->format(curDate,dateReturned,success);
|
||||
u_printf("%s\n", "Current Time:");
|
||||
u_printf("%S\n", dateReturned.getTerminatedBuffer());
|
||||
|
||||
|
||||
// Use getOffset to get the stdOffset and dstOffset for the given time
|
||||
tzWest->getOffset(curDate,true,stdOffset,dstOffset,success);
|
||||
u_printf("%s\n%d\n","Current Time Zone STD offset:",stdOffset/(1000*60*60));
|
||||
u_printf("%s\n%d\n","Current Time Zone DST offset:",dstOffset/(1000*60*60));
|
||||
u_printf("%s\n", "Current date/time is in daylight savings time?");
|
||||
u_printf("%s\n", (calendar->inDaylightTime(success))?"Yes":"No");
|
||||
|
||||
|
||||
// Use createTimeZoneIDEnumeration to get the specific Time Zone IDs
|
||||
// in United States with -5 hour standard offset from GMT
|
||||
stdOffset = (-5)*U_MILLIS_PER_HOUR; // U_MILLIS_PER_HOUR = 60*60*1000;
|
||||
StringEnumeration *ids = TimeZone::createTimeZoneIDEnumeration(UCAL_ZONE_TYPE_CANONICAL_LOCATION,"US",&stdOffset,success);
|
||||
for (int i=0; i<ids->count(success);i++) {
|
||||
u_printf("%s\n",ids->next(NULL,success));
|
||||
}
|
||||
|
||||
|
||||
// Use Calendar to get the hour of the day for different time zones
|
||||
int32_t hour1,hour2;
|
||||
TimeZone *tzEast = TimeZone::createTimeZone("America/New_York");
|
||||
Calendar * cal1 = Calendar::createInstance(tzWest,success);
|
||||
Calendar * cal2 = Calendar::createInstance(tzEast,success);
|
||||
hour1 = cal1->get(UCAL_HOUR_OF_DAY,success);
|
||||
hour2 = cal2->get(UCAL_HOUR_OF_DAY,success);
|
||||
u_printf("%s\n%d\n","Current hour of the day in North American West: ", hour1);
|
||||
u_printf("%s\n%d\n","Current hour of the day in North American East: ", hour2);
|
||||
|
||||
delete cal1;
|
||||
delete cal2;
|
||||
delete ids;
|
||||
delete calendar;
|
||||
delete dt;
|
||||
```
|
||||
@@ -1,257 +0,0 @@
|
||||
---
|
||||
layout: default
|
||||
title: TimeZone Classes
|
||||
nav_order: 3
|
||||
parent: Date/Time
|
||||
---
|
||||
<!--
|
||||
© 2020 and later: Unicode, Inc. and others.
|
||||
License & terms of use: http://www.unicode.org/copyright.html
|
||||
-->
|
||||
|
||||
# ICU TimeZone Classes
|
||||
{: .no_toc }
|
||||
|
||||
## Contents
|
||||
{: .no_toc .text-delta }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
A time zone is a system that is used for relating local times in different
|
||||
geographical areas to one another. For example, in the United States, Pacific
|
||||
Time is three hours earlier than Eastern Time; when it's 6 P.M. in San
|
||||
Francisco, it's 9 P.M. in Brooklyn. To make things simple, instead of relating
|
||||
time zones to one another, all time zones are related to a common reference
|
||||
point.
|
||||
|
||||
For historical reasons, the reference point is Greenwich, England. Local time in
|
||||
Greenwich is referred to as Greenwich Mean Time, or GMT. (This is similar, but
|
||||
not precisely identical, to Universal Coordinated Time, or UTC. We use the two
|
||||
terms interchangeably in ICU since ICU does not concern itself with either leap
|
||||
seconds or historical behavior.) Using this system, Pacific Time is expressed as
|
||||
GMT-8:00, or GMT-7:00 in the summer. The offset -8:00 indicates that Pacific
|
||||
Time is obtained from GMT by adding -8:00, that is, by subtracting 8 hours.
|
||||
|
||||
The offset differs in the summer because of daylight savings time, or DST. At
|
||||
this point it is useful to define three different flavors of local time:
|
||||
|
||||
* **Standard Time**:
|
||||
Standard Time is local time without a daylight savings time offset. For
|
||||
example, in California, standard time is GMT-8:00; that is, 8 hours before
|
||||
GMT.
|
||||
* **Daylight Savings Time**:
|
||||
Daylight savings time is local time with a daylight savings time offset.
|
||||
This offset is typically one hour, but is sometimes less. In California,
|
||||
daylight savings time is GMT-7:00. Daylight savings time is observed in most
|
||||
non-equatorial areas.
|
||||
* **Wall Time**:
|
||||
Wall time is what a local clock on the wall reads. In areas that observe
|
||||
daylight savings time for part of the year, wall time is either standard
|
||||
time or daylight savings time, depending on the date. In areas that do not
|
||||
observe daylight savings time, wall time is equivalent to standard time.
|
||||
|
||||
## Time Zones in ICU
|
||||
|
||||
ICU supports time zones through two classes:
|
||||
|
||||
* **TimeZone**:
|
||||
`TimeZone` is an abstract base class that defines the time zone API. This API
|
||||
supports conversion between GMT and local time.
|
||||
* **SimpleTimeZone**:
|
||||
`SimpleTimeZone` is a concrete subclass of TimeZone that implements the
|
||||
standard time zones used today internationally.
|
||||
|
||||
Timezone classes are related to `UDate`, the `Calendar` classes, and the
|
||||
`DateFormat` classes.
|
||||
|
||||
### Timezone Class in ICU
|
||||
|
||||
`TimeZone` is an abstract base class. It defines common protocol for a hierarchy
|
||||
of classes. This protocol includes:
|
||||
|
||||
* A programmatic ID, for example, "America/Los_Angeles". This ID is used to
|
||||
call up a specific real-world time zone. It corresponds to the IDs defined
|
||||
in the [IANA Time Zone datbase](https://www.iana.org/time-zones) used by UNIX
|
||||
and other systems, and has the format continent/city or ocean/city.
|
||||
* A raw offset. This is the difference, in milliseconds, between a time zone's
|
||||
standard time and GMT. Positive raw offsets are east of Greenwich.
|
||||
* Factory methods and methods for handling the default time zone.
|
||||
* Display name methods.
|
||||
* An API to compute the difference between local wall time and GMT.
|
||||
|
||||
#### Factory Methods and the Default Timezone
|
||||
|
||||
The TimeZone factory method `createTimeZone()` creates and returns a `TimeZone`
|
||||
object given a programmatic ID. The user does not know what the class of the
|
||||
returned object is, other than that it is a subclass of `TimeZone`.
|
||||
|
||||
The `createAvailableIDs()` methods return lists of the programmatic IDs of all
|
||||
zones known to the system. These IDs may then be passed to `createTimeZone()` to
|
||||
create the actual time zone objects. ICU maintains a comprehensive list of
|
||||
current international time zones, as derived from the Olson data.
|
||||
|
||||
`TimeZone` maintains a static time zone object known as the *default time zone*.
|
||||
This is the time zone that is used implicitly when the user does not specify
|
||||
one. ICU attempts to match this to the host OS time zone. The user may obtain a
|
||||
clone of the default time zone by calling `createDefault()` and may change the
|
||||
default time zone by calling `setDefault()` or `adoptDefault()`.
|
||||
|
||||
#### Display Name
|
||||
|
||||
When displaying the name of a time zone to the user, use the display name, not
|
||||
the programmatic ID. The display name is returned by the `getDisplayName()`
|
||||
method. A time zone may have three display names:
|
||||
|
||||
* Generic name, such as "Pacific Time".
|
||||
* Standard name, such as "Pacific Standard Time".
|
||||
* Daylight savings name, such as "Pacific Daylight Time".
|
||||
|
||||
Furthermore, each of these names may be LONG or SHORT. The SHORT form is
|
||||
typically an abbreviation, e.g., "PST", "PDT".
|
||||
|
||||
In addition to being available directly from the `TimeZone` API, the display name
|
||||
is used by the date format classes to format and parse time zones.
|
||||
|
||||
#### getOffset() API
|
||||
|
||||
`TimeZone` defines the API `getOffset()` by which the caller can determine the
|
||||
difference between local time and GMT. This is a pure virtual API, so it is
|
||||
implemented in the concrete subclasses of `TimeZone`.
|
||||
|
||||
## Updating the Time Zone Data
|
||||
|
||||
Time zone data changes often in response to governments around the world
|
||||
changing their local rules and the areas where they apply. ICU derives its tz
|
||||
data from the [IANA Time Zone Database](http://www.iana.org/time-zones).
|
||||
|
||||
The ICU project publishes updated timezone resource data in response to IANA
|
||||
updates, and these can be used to patch existing ICU installations. Several
|
||||
update strategies are possible, depending on the ICU version and configuration.
|
||||
|
||||
* ICU4J: Use the time zone update utility.
|
||||
* ICU4C 54 and newer: Drop in the binary update files.
|
||||
* ICU4C 36 and newer: the best update strategy will depend on how ICU data
|
||||
loading is configured for the specific ICU installation.
|
||||
* Data is loaded from a .dat package file: replace the time zone resources
|
||||
in the .dat file using the icupkg tool.
|
||||
* Data is loaded from a .dll or .so shared library: obtain the updated
|
||||
sources for the tz resources and rebuild the data library.
|
||||
* Data is loaded from individual files: drop in the updated binary .res
|
||||
files.
|
||||
|
||||
The [ICU Data](../../icudata.md) section of this user guide gives more
|
||||
information on how ICU loads resources.
|
||||
|
||||
The ICU resource files required for time zone data updates are posted at
|
||||
<https://github.com/unicode-org/icu-data/tree/main/tzdata/icunew>. The
|
||||
required resource files for ICU version 44 and newer are
|
||||
|
||||
* zoneinfo64.res
|
||||
* windowsZones.res
|
||||
* timezoneTypes.res
|
||||
* metaZones.res
|
||||
|
||||
### ICU4C TZ update of a .dat Package File
|
||||
|
||||
For ICU configurations that load data from a .dat package file, replace the time
|
||||
zone resources in that file.
|
||||
|
||||
1. Download the new .res files from
|
||||
`https://github.com/unicode-org/icu-data/tree/main/tzdata/icunew/<IANA tz version>/44/<platform directory>`.
|
||||
* `<IANA tz version>` is a combination of year and letter, such as "2019c".
|
||||
* *"44"* is the directory for updates to ICU version 4.4 and newer.
|
||||
* `<platform directory>` is "le" for little endian processors, including
|
||||
all Intel processors.
|
||||
* `<platform directory>` is "be" for big endian processors, including IBM
|
||||
Power and Sparc.
|
||||
* `<platform directory>` is "ee" for IBM mainframes using EBCDIC character
|
||||
sets.
|
||||
2. Check that the tool "icupkg" is available. If not already on your system,
|
||||
you can get it by [downloading](https://github.com/unicode-org/icu/releases)
|
||||
and building ICU, following the instructions in the ReadMe file included in
|
||||
the download. Alternatively, on many Linux systems, "apt-get install
|
||||
icu-devtools" will install the tool.
|
||||
3. Locate the .dat file to be updated, and do the update. The commands below
|
||||
are for a .dat file named icudt55l.dat.
|
||||
|
||||
```shell
|
||||
icupkg -a zoneinfo64.res icudt55l.dat
|
||||
icupkg -a windowsZones.res icudt55l.dat
|
||||
icupkg -a timezoneTypes.res icudt55l.dat
|
||||
icupkg -a metaZones.res icudt55l.dat
|
||||
```
|
||||
|
||||
In ICU versions older than 4.4 some of the time zone resources have slightly
|
||||
different names. The update procedure is the same, but substitute the names
|
||||
found in the desired download directory - 42, 40, 38 or 36.
|
||||
|
||||
### ICU4C TZ Update with Drop-in .res files (ICU 54 and newer)
|
||||
|
||||
With this approach, the four individual .res files are dropped in any convenient
|
||||
location in the file system, and ICU is given an absolute path to the directory
|
||||
containing them. For the time zone resources only, ICU will check this directory
|
||||
first when loading data. This approach will work even when all other ICU data
|
||||
loading is from a shared library or .dat file.
|
||||
|
||||
There are two ways to specify the directory:
|
||||
|
||||
* At ICU build time, by defining the C pre-processor variable
|
||||
`U_TIMEZONE_FILES_DIR` to the run time path to the directory containing the
|
||||
.res files.
|
||||
* At run time, by setting the environment variable `ICU_TIMEZONE_FILES_DIR` to
|
||||
the absolute path of the directory containing the .res files.
|
||||
|
||||
If both are defined, the environment variable `ICU_TIMEZONE_FILES_DIR` take
|
||||
precedence. If either is defined, the time zone directory will be checked first,
|
||||
meaning that time zone resource files placed there will override time zone
|
||||
resources that may exist in other ICU data locations.
|
||||
|
||||
To do the update, download the .res files appropriate for the platform, as
|
||||
described for the .dat file update above, and copy them into the time zone res
|
||||
file directory.
|
||||
|
||||
### ICU4C TZ update when ICU is configured for individual files
|
||||
|
||||
If the ICU-using application sets an ICU data path (or can be changed to set
|
||||
one), then the time zone .res file can be placed there. Download the files as
|
||||
described above and copy them to the specified directory. See the
|
||||
[ICU Data](../../icudata.md) page of the user guide for more information about
|
||||
the ICU data path.
|
||||
|
||||
### ICU4C TZ update when ICU data is built into a shared library
|
||||
|
||||
1. Set up the environment necessary to rebuild your specific configuration of
|
||||
ICU.
|
||||
2. Download the .txt file sources for the updated resources from
|
||||
`https://github.com/unicode-org/icu-data/tree/main/tzdata/icunew/<IANA tz version>/44`
|
||||
3. Copy the downloaded .txt files into the ICU sources for your installation,
|
||||
in the subdirectory source/data/misc/
|
||||
4. Rebuid ICU.
|
||||
5. Copy the freshly built ICU data shared library to the desired destination.
|
||||
|
||||
> :point_right: **Note**: The standard ICU download package contains pre-built
|
||||
> ICU data. To rebuild ICU data from .txt files, you will need to replace the
|
||||
> contents of `icu4c/source/data` with the contents of ICU4C data.zip. See
|
||||
> [ICU Data Build Tool](../../icu_data/buildtool.md) for more details.
|
||||
|
||||
There are too many possible platform variations to be more specific about how to
|
||||
rebuild ICU4C in these instructions. See the ReadMe file included with the ICU
|
||||
sources for general information on building ICU.
|
||||
|
||||
### Update the time zone data for ICU4J
|
||||
|
||||
The [ICU4J Time Zone Update Update
|
||||
Utility](https://icu.unicode.org/download/icutzu) automates the process of
|
||||
updating ICU4J jar files with the latest time zone data. Instructions for use
|
||||
are [here](https://htmlpreview.github.io/?https://github.com/unicode-org/icu-data/blob/main/tzdata/tzu/readme.html).
|
||||
|
||||
The updater will work with ICU version 3.4.2 and newer.
|
||||
|
||||
## Sample Code
|
||||
|
||||
See the [Date and Time Zone Examples](examples.md) subpage.
|
||||
@@ -1,271 +0,0 @@
|
||||
---
|
||||
layout: default
|
||||
title: Universal Time Scale
|
||||
nav_order: 5
|
||||
parent: Date/Time
|
||||
---
|
||||
<!--
|
||||
© 2020 and later: Unicode, Inc. and others.
|
||||
License & terms of use: http://www.unicode.org/copyright.html
|
||||
-->
|
||||
|
||||
# Universal Time Scale
|
||||
{: .no_toc }
|
||||
|
||||
## Contents
|
||||
{: .no_toc .text-delta }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
There are quite a few different conventions for binary datetime, depending on
|
||||
the platform or protocol. Some of these have severe drawbacks. For example,
|
||||
people using Unix time (seconds since Jan 1, 1970, usually in a 32-bit integer)
|
||||
think that they are safe until near the year 2038. But cases can and do arise
|
||||
where arithmetic manipulations causes serious problems. Consider the computation
|
||||
of the average of two datetimes, for example: if one calculates them with
|
||||
`averageTime = (time1 + time2)/2`, there will be overflow even with dates
|
||||
beginning in 2004. Moreover, even if these problems don't occur, there is the
|
||||
issue of conversion back and forth between different systems.
|
||||
|
||||
Binary datetimes differ in a number of ways: the data type, the unit, and the
|
||||
epoch (origin). We'll refer to these as time scales. For example: (Sorted by
|
||||
epoch and unit, descending. In Java, `int64_t`=`long` and `int32_t`=`int`.)
|
||||
|
||||
| Source | Data Type | Epoch | Unit |
|
||||
| ---------------------------------------------- | ---------------------------------------------------------------------------------- | ----------- | ------------------------------------------------------- |
|
||||
| MacOS X (`CFDate/NSDate`) | `double` (1.0=1s but fractional seconds are used as well; imprecise for 0.1s etc.) | 2001-Jan-01 | seconds (and fractions thereof) |
|
||||
| Unix `time_t` | `int32_t` or `int64_t` (`signed int32_t` limited to 1970..2038) | 1970-Jan-01 | seconds |
|
||||
| Java `Date` | `int64_t` | 1970-Jan-01 | milliseconds |
|
||||
| Joda `DateTime` | `int64_t` | 1970-Jan-01 | milliseconds |
|
||||
| ICU4C `UDate` | `double` (does not use fractional milliseconds) | 1970-Jan-01 | milliseconds |
|
||||
| JavaScript `Date` | `double` (does not use fractional milliseconds; JavaScript Number stores a double) | 1970-Jan-01 | milliseconds |
|
||||
| Unix `struct timeval (as in gettimeofday)` | `struct: time_t` (seconds); suseconds_t (microseconds) | 1970-Jan-01 | microseconds |
|
||||
| Gnome `g_get_real_time()` | `gint64` | 1970-Jan-01 | microseconds |
|
||||
| Unix `struct timespec` (as in `clock_gettime`) | `struct: time_t` (seconds); long (nanoseconds) | 1970-Jan-01 | nanoseconds |
|
||||
| MacOS (old) | `uint32_t` (1904..2040) | 1904-Jan-01 | seconds |
|
||||
| Excel | ? | 1899-Dec-31 | days |
|
||||
| DB2 | ? | 1899-Dec-31 | days |
|
||||
| Windows `FILETIME` | `int64_t` | 1601-Jan-01 | ticks (100 nanoseconds; finest granularity in industry) |
|
||||
| .NET `DateTime` | `uint62` (only 0001-9999; only 62 bits; also 2-bit field for UTC/local) | 0001-Jan-01 | ticks (100 nanoseconds; finest granularity in industry) |
|
||||
| ICU Universal Time Scale | `int64_t` | 0001-Jan-01 | same as .Net but allows 29000BC..29000AD |
|
||||
|
||||
All of the epochs start at 00:00 am (the earliest possible time on the day in
|
||||
question), and are usually assumed to be UTC.
|
||||
|
||||
The ranges, in years, for different data types are given in the following table.
|
||||
The range for integer types includes the entire range expressible with positive
|
||||
and negative values of the data type. The range for double is the range that
|
||||
would be allowed without losing precision to the corresponding unit.
|
||||
|
||||
| Units | 64-bit integer | Double | 32-bit integer |
|
||||
| ---------------------- | ----------------------- | -------------- | -------------- |
|
||||
| 1 second | 5.84542x10<sup>11</sup> | 285,420,920.94 | 136.10 |
|
||||
| 1 millisecond | 584,542,046.09 | 285,420.92 | 0.14 |
|
||||
| 1 microsecond | 584,542.05 | 285.42 | 0.00 |
|
||||
| 100 nanoseconds (tick) | 58,454.20 | 28.54 | 0.00 |
|
||||
| 1 nanosecond | 584.5420461 | 0.2854 | 0.00 |
|
||||
|
||||
ICU implements a universal time scale that is similar to the
|
||||
[.NET framework's System.DateTime](https://docs.microsoft.com/dotnet/api/system.datetime?view=netframework-4.8).
|
||||
The universal time scale is a 64-bit integer that holds ticks since midnight,
|
||||
January 1<sup>st</sup>, 0001. Negative values are supported. This has enough
|
||||
range to guarantee that calculations involving dates around the present are safe.
|
||||
|
||||
The universal time scale always measures time according to the proleptic
|
||||
Gregorian calendar. That is, the Gregorian calendar's leap year rules are used
|
||||
for all times, even before 1582 when it was introduced. (This is different from
|
||||
the default ICU calendar which switches from the Julian to the Gregorian
|
||||
calendar in 1582. See `GregorianCalendar::setGregorianChange()` and
|
||||
`ucal_setGregorianChange()`).
|
||||
|
||||
ICU provides conversion functions to and from all other major time scales,
|
||||
allowing datetimes in any time scale to be converted to the universal time
|
||||
scale, safely manipulated, and converted back to any other datetime time scale.
|
||||
|
||||
## Background
|
||||
|
||||
So how did we decide what to use for the universal time scale? Java time has
|
||||
plenty of range, but cannot represent a .NET `System.DateTime` value without
|
||||
severe loss of precision. ICU4C time addresses this by using a `double` that is
|
||||
otherwise equivalent to the Java time. However, there are disadvantages with
|
||||
doubles. They provide for much more graceful degradation in arithmetic
|
||||
operations. But they only have 53 bits of accuracy, which means that they will
|
||||
lose precision when converting back and forth to ticks. What would really be
|
||||
nice would be a `long double` (80 bits -- 64 bit mantissa), but that is not
|
||||
supported on most systems.
|
||||
|
||||
The Unix extended time uses a structure with two components: time in seconds and
|
||||
a fractional field (microseconds). However, this is clumsy, slow, and prone to
|
||||
error (you always have to keep track of overflow and underflow in the fractional
|
||||
field). `BigDecimal` would allow for arbitrary precision and arbitrary range, but
|
||||
we did not want to use this as the normal type, because it is slow and does not
|
||||
have a fixed size.
|
||||
|
||||
Because of these issues, we concluded that the .NET `System.DateTime` is the best
|
||||
timescale to use. However, we use the full range allowed by the data type,
|
||||
allowing for datetimes back to 29,000 BC and up to 29,000 AD. (`System.DateTime`
|
||||
uses only 62 bits and only supports dates from 0001 AD to 9999 AD). This time
|
||||
scale is very fine grained, does not lose precision, and covers a range that
|
||||
will meet almost all requirements. It will not handle the range that Java times
|
||||
do, but frankly, being able to handle dates before 29,000 BC or after 29,000 AD
|
||||
is of very limited interest.
|
||||
|
||||
## Constants
|
||||
|
||||
ICU provides routines to convert from other timescales to the universal time
|
||||
scale, to convert from the universal time scale to other timescales, and to get
|
||||
information about a particular timescale. In all of these routines, the
|
||||
timescales are referenced using an integer constant, according to the following
|
||||
table:
|
||||
|
||||
| Source | ICU4C | ICU4J |
|
||||
| ---------------------- | ----------------------------- | ------------------------ |
|
||||
| Java | `UDTS_JAVA_TIME` | `JAVA_TIME` |
|
||||
| Unix | `UDTS_UNIX_TIME` | `UNIX_TIME` |
|
||||
| ICU4C | `UDTS_ICU4C_TIME` | `ICU4C_TIME` |
|
||||
| Windows FILETIME | `UDTS_WINDOWS_FILE_TIME` | `WINDOWS_FILE_TIME` |
|
||||
| .NET DateTime | `UDTS_DOTNET_DATE_TIME` | `DOTNET_DATE_TIME` |
|
||||
| Macintosh (old) | `UDTS_MAC_OLD_TIME` | `MAC_OLD_TIME` |
|
||||
| Macintosh | `UDTS_MAC_TIME` | `MAC_TIME` |
|
||||
| Excel | `UDTS_EXCEL_TIME` | `EXCEL_TIME` |
|
||||
| DB2 | `UDTS_DB2_TIME` | `DB2_TIME` |
|
||||
| Unix with microseconds | `UDTS_UNIX_MICROSECONDS_TIME` | `UNIX_MICROSECONDS_TIME` |
|
||||
|
||||
The routine that gets a particular piece of information about a timescale takes
|
||||
an integer constant that identifies the particular piece of information,
|
||||
according to the following table:
|
||||
|
||||
| Value | ICU4C | ICU4J |
|
||||
| -------------------- | -------------------------- | -------------------- |
|
||||
| Precision | `UTSV_UNITS_VALUE` | `UNITS_VALUE` |
|
||||
| Epoch offset | `UTSV_EPOCH_OFFSET_VALUE` | `EPOCH_OFFSET_VALUE` |
|
||||
| Minimum "from" value | `UTSV_FROM_MIN_VALUE` | `FROM_MIN_VALUE` |
|
||||
| Maximum "from" value | `UTSV_FROM_MAX_VALUE` | `FROM_MAX_VALUE` |
|
||||
| Minimum "to" value | `UTSV_TO_MIN_VALUE` | `TO_MIN_VALUE` |
|
||||
| Maximum "to" value | `UTSV_TO_MAX_VALUE` | `TO_MAX_VALUE` |
|
||||
|
||||
Here is what the values mean:
|
||||
|
||||
* Precision -- the precision of the timescale, in ticks.
|
||||
* Epoch offset -- the distance from the universal timescale's epoch to the timescale's epoch, in the timescale's precision.
|
||||
* Minimum "from" value -- the minimum timescale value that can safely be converted to the universal timescale.
|
||||
* Maximum "from" value -- the maximum timescale value that can safely be converted to the universal timescale.
|
||||
* Minimum "to" value -- the minimum universal timescale value that can safely be converted to the timescale.
|
||||
* Maximum "to" value -- the maximum universal timescale value that can safely be converted to the timescale.
|
||||
|
||||
## Converting
|
||||
|
||||
You can convert from other timescale values to the universal timescale using the
|
||||
"from" methods. In ICU4C, you use `utmscale_fromInt64`:
|
||||
|
||||
```c
|
||||
UErrorCode err = U_ZERO_ERROR;
|
||||
int64_t unixTime = ...;
|
||||
int64_t universalTime;
|
||||
|
||||
universalTime = utmscale_fromInt64(unixTime, UDTS_UNIX_TIME, &err);
|
||||
```
|
||||
|
||||
In ICU4J, you use `UniversalTimeScale.from`:
|
||||
|
||||
```java
|
||||
long javaTime = ...;
|
||||
long universalTime;
|
||||
|
||||
universalTime = UniversalTimeScale.from(javaTime, UniversalTimeScale.JAVA_TIME);
|
||||
```
|
||||
|
||||
You can convert values in the universal timescale to other timescales using the
|
||||
"to" methods. In ICU4C, you use `utmscale_toInt64`:
|
||||
|
||||
```c
|
||||
UErrorCode err = U_ZERO_ERROR;
|
||||
int64_t universalTime = ...;
|
||||
int64_t unixTime;
|
||||
|
||||
unixTime = utmscale_toInt64(universalTime, UDTS_UNIX_TIME, &err);
|
||||
```
|
||||
|
||||
In ICU4J, you use `UniversalTimeScale.to`:
|
||||
|
||||
```java
|
||||
long universalTime = ...;
|
||||
long javaTime;
|
||||
|
||||
javaTime = UniversalTimeScale.to(universalTime, UniversalTimeScale.JAVA_TIME);
|
||||
```
|
||||
|
||||
That's all there is to it!
|
||||
|
||||
If the conversion is out of range, the ICU4C routines
|
||||
will set the error code to `U_ILLEGAL_ARGUMENT_ERROR`, and the ICU4J methods will
|
||||
throw `IllegalArgumentException`. In ICU4J, you can avoid out of range conversions
|
||||
by using the `BigDecimal` methods:
|
||||
|
||||
```java
|
||||
long fileTime = ...;
|
||||
double icu4cTime = ...;
|
||||
BigDecimal utICU4C, utFile, utUnix, unixTime, macTime;
|
||||
|
||||
utFile = UniversalTimeScale.bigDecimalFrom(fileTime, UniversalTime.WINDOWS_FILE_TIME);
|
||||
|
||||
utICU4C = UniversalTimeScale.bigDecimalFrom(icu4cTime, UniversalTimeScale.ICU4C_TIME);
|
||||
|
||||
unixTime = UniversalTimeScale.toBigDecimal(utFile, UniversalTime.UNIX_TIME);
|
||||
macTime = UniversalTimeScale.toBigDecimal(utICU4C, UniversalTime.MAC_TIME);
|
||||
|
||||
utUnix = UniversalTimeScale.bigDecimalFrom(unixTime, UniversalTime.UNIX_TIME);
|
||||
```
|
||||
|
||||
> :point_right: **Note**: Because the Universal Time Scale has a finer resolution
|
||||
> than some other time scales, time values that can be represented exactly in the
|
||||
> Universal Time Scale will be rounded when converting to these time scales, and
|
||||
> resolution will be lost. If you convert these values back to the Universal Time
|
||||
> Scale, you will not get the same time value that you started with. If the time
|
||||
> scale to which you are converting uses a double to represent the time value, you
|
||||
> may loose precision even though the double supports a range that is larger than
|
||||
> the range supported by the Universal Time Scale.
|
||||
|
||||
## Formatting and Parsing
|
||||
|
||||
Currently, ICU does not support direct formatting or parsing of Universal Time
|
||||
Scale values. If you want to format a Universal Time Scale value, you will need
|
||||
to convert it to an ICU time scale value first. Use `UTDS_ICU4C_TIME` with ICU4C,
|
||||
and `UniversalTimeScale.JAVA_TIME` with ICU4J.
|
||||
|
||||
When you parse a datetime string, the result will be an ICU time scale value.
|
||||
You can convert this value to a Universal Time Scale value using `UDTS_ICU4C_TIME`
|
||||
with ICU4C, and `UniversalTime.JAVA_TIME` for ICU4J.
|
||||
|
||||
See the previous section, *Converting*, for details of how to do the conversion.
|
||||
|
||||
## Getting Timescale Information
|
||||
|
||||
To get information about a particular timescale in ICU4C, use
|
||||
`utmscale_getTimeScaleValue`:
|
||||
|
||||
```c
|
||||
UErrorCode err = U_ZERO_ERROR;
|
||||
int64_t unixEpochOffset = utmscale_getTimeScaleValue(
|
||||
UDTS_UNIX_TIME,
|
||||
UTSV_EPOCH_OFFSET_VALUE,
|
||||
&err);
|
||||
```
|
||||
|
||||
In ICU4J, use `UniversalTimeScale.getTimeScaleValue`:
|
||||
|
||||
```java
|
||||
long javaEpochOffset = UniversalTimeScale.getTimeScaleValue(
|
||||
UniversalTimeScale.JAVA_TIME,
|
||||
UniversalTimeScale.EPOCH_OFFSET_VALUE);
|
||||
```
|
||||
|
||||
If the integer constants for selecting the timescale or the timescale value are
|
||||
out of range, the ICU4C routines will set the error code to
|
||||
`U_ILLEGAL_ARGUMENT_ERROR`, and the ICU4J methods will throw
|
||||
`IllegalArgumentException`.
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,137 +0,0 @@
|
||||
---
|
||||
layout: default
|
||||
title: Contributions
|
||||
nav_order: 4
|
||||
parent: Contributors
|
||||
---
|
||||
<!--
|
||||
© 2020 and later: Unicode, Inc. and others.
|
||||
License & terms of use: http://www.unicode.org/copyright.html
|
||||
-->
|
||||
|
||||
# Contributions to the ICU library
|
||||
{: .no_toc }
|
||||
|
||||
## Contents
|
||||
{: .no_toc .text-delta }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
|
||||
---
|
||||
|
||||
## Why Contribute?
|
||||
|
||||
ICU is an open source library that is a de-facto industry standard for
|
||||
internationalization libraries. Our goal is to provide top of the line i18n
|
||||
support on all widely used platforms. By contributing your code to the ICU
|
||||
library, you will get the benefit of continuing improvement by the ICU team and
|
||||
the community, as well as testing and multi-platform portability. In addition,
|
||||
it saves you from having to re-merge your own additions into ICU each time you
|
||||
upgrade to a new ICU release.
|
||||
|
||||
## Current Process
|
||||
|
||||
See <https://icu.unicode.org/processes/contribute>.
|
||||
|
||||
## Historical
|
||||
|
||||
### Legal Issues ICU 1.8.1-57
|
||||
|
||||
The following process was in place up to ICU 57, when the old ICU license was
|
||||
used.
|
||||
|
||||
### Old Process
|
||||
|
||||
In order for your code to be contributed, you need to assign to IBM joint
|
||||
copyright ownership in the contribution. You retain joint ownership in the
|
||||
contribution without restriction. (For the complete set of terms, please see the
|
||||
forms mentioned below.)
|
||||
|
||||
The sections below describe two processes, for one-time and ongoing
|
||||
contributors. In either case, please complete the form(s) electronically and
|
||||
send it/them to IBM for review. After review by IBM, please print and sign the
|
||||
form(s), send it/them by mail, and send the code. The code will then be
|
||||
evaluated.
|
||||
|
||||
Please consult a legal representative if you do not understand the implications
|
||||
of the copyright assignment.
|
||||
|
||||
### One-Time Contributors
|
||||
|
||||
If you would like to make a contribution only once or infrequently, please use
|
||||
the *Joint Copyright Assignment - One-time Contribution* form.
|
||||
(<https://github.com/unicode-org/icu-docs/blob/main/legal/contributions/Copyright_Assignment.rtf>).
|
||||
The contribution will be identified by a bug ID which is unique to the
|
||||
contribution and entered into the form. Therefore, please make sure that there
|
||||
is an appropriate bug (or Request For Enhancement) in the ICU bug database, or
|
||||
submit one.
|
||||
|
||||
The code contribution will be checked into a special part of the ICU source code
|
||||
repository and evaluated. The ICU team may request updates, for example for
|
||||
better conformance with the ICU [design](../design.md) principles,
|
||||
[coding](codingguidelines.md) and testing guidelines, or performance. (See also
|
||||
the Requirements (§) above.) Such updates can be contributed without exchanging
|
||||
another form: An ICU team member commits related materials into the ICU source
|
||||
code repository using the same bug ID that was entered into the copyright
|
||||
assignment form.
|
||||
|
||||
### Ongoing Contributors
|
||||
|
||||
If you are interested in making frequent contributions to ICU, then the ICU
|
||||
Project Management Committee may agree to invite you as an ongoing contributor.
|
||||
Ongoing contributors may be individuals but are more typically expected to be
|
||||
companies with one or more people ("authors") writing different parts of one or
|
||||
more contributions.
|
||||
|
||||
In this case, the relationship between the contributor and the ICU team is much
|
||||
closer: One or more authors belonging to the contributor will have commit access
|
||||
to the ICU source code repository. With this direct access come additional
|
||||
responsibilities including an understanding that the contributor will work to
|
||||
follow the technical Requirements (§) above for contributions, and agreement to
|
||||
adhere to the terms of the copyright assignment forms for all future
|
||||
contributions.
|
||||
|
||||
The process for ongoing contributors involves two types of forms: Initially, and
|
||||
only once, an ongoing contributor submits a *Joint Copyright Assignment by
|
||||
Ongoing Contributor* form, agreeing to essentially the same terms as in the
|
||||
one-time contributor form, for all future contributions. (See the form at
|
||||
<https://github.com/unicode-org/icu-docs/blob/main/legal/contributions/Copyright_Assignment_ongoing.rtf>).
|
||||
|
||||
The contributor must also send another form, *Addendum to Joint Copyright
|
||||
Assignment by Ongoing Contributor: Authors*, for the initial set and each
|
||||
addition of authors to ICU contributions, **before** any contributions from
|
||||
these authors are committed into the ICU source code repository. (Only new,
|
||||
additional authors need to be listed on each such form.) The contributor agrees
|
||||
to ensure that all of these authors agree to adhere to the terms of the
|
||||
associated *Joint Copyright Assignment by Ongoing Contributor Agreement*. (See
|
||||
the Authors Addendum form at
|
||||
<https://github.com/unicode-org/icu-docs/blob/main/legal/contributions/Copyright_Assignment_authors.rtf>).
|
||||
|
||||
Some of an ongoing contributor's authors will have commit access to the ICU
|
||||
source code repository. Their committer IDs need to be established before
|
||||
completing the Authors Addendum form, so that these committer IDs can be entered
|
||||
there. (The committer IDs should be activated only after the form is received.)
|
||||
|
||||
Committer authors commit materials directly into the appropriate parts of the
|
||||
ICU source code repository. Contributions from an ongoing contributor are
|
||||
identified by their association with the contributor's committer IDs.
|
||||
|
||||
### Previous Contributions
|
||||
|
||||
All previous "one-off" contributions from non-IBM sources to ICU are listed on
|
||||
the code contributions page in ICU's source code repository. The page contains
|
||||
links to the softcopies of the Joint Copyright Assignment forms. See
|
||||
<https://htmlpreview.github.io/?https://github.com/unicode-org/icu-docs/blob/main/legal/contributions/code_contributions.html>
|
||||
|
||||
In addition, the following non-IBM companies are registered as Ongoing
|
||||
Contributors:
|
||||
|
||||
* Apple
|
||||
* Google
|
||||
|
||||
See the repository folder that contains the contributions page for the full set
|
||||
of softcopies of contributor agreements including one-off contributions,
|
||||
ongoing-contributor agreements and author-addendum documents to
|
||||
ongoing-contributor agreements:
|
||||
<https://github.com/unicode-org/icu-docs/tree/main/legal/contributions>
|
||||
@@ -1,200 +0,0 @@
|
||||
---
|
||||
layout: default
|
||||
title: User Guide Editing
|
||||
nav_order: 5
|
||||
parent: Contributors
|
||||
---
|
||||
<!--
|
||||
© 2020 and later: Unicode, Inc. and others.
|
||||
License & terms of use: http://www.unicode.org/copyright.html
|
||||
-->
|
||||
|
||||
# Editing the ICU User Guide
|
||||
{: .no_toc }
|
||||
|
||||
## Contents
|
||||
{: .no_toc .text-delta }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
This version of the ICU User Guide is generated directly from the Markdown
|
||||
files in the `/docs` directory of the main repository. In particular, the
|
||||
Markdown files are customized to work with the Jekyll static site generator,
|
||||
which benefits from [GitHub's built-in support](https://docs.github.com/en/github/working-with-github-pages/setting-up-a-github-pages-site-with-jekyll)
|
||||
for Jekyll sites.
|
||||
|
||||
## How to Contribute
|
||||
|
||||
Because ICU User Guide source files are now contained within the main
|
||||
repository, changes can now be made through the same process for contributing
|
||||
code patches. See [Contributions to the ICU Library](constributions.md) for the
|
||||
current process of filing an issue and submitting a pull request.
|
||||
|
||||
In the case of a single file edit (ex: typo correction), a convenient way to
|
||||
create the pull request is to follow the "Edit this page on Github" link at
|
||||
the bottom of the page and [use GitHub to edit the file](https://docs.github.com/en/github/managing-files-in-a-repository/editing-files-in-your-repository)
|
||||
and create the new pull request.
|
||||
|
||||
After the source branch for a potential pull request is made, the changes can
|
||||
be previewed using the fork repository that contains the source branch. The
|
||||
fork repository must be [configured to serve GitHub Pages](https://docs.github.com/en/github/working-with-github-pages/configuring-a-publishing-source-for-your-github-pages-site#choosing-a-publishing-source)
|
||||
from the branch and the Markdown pages root directory (`/docs`).
|
||||
> :point_right: **Note**: there is a
|
||||
latency of several minutes before GitHub Pages renders new changes, and
|
||||
[errors](https://docs.github.com/en/github/working-with-github-pages/about-jekyll-build-errors-for-github-pages-sites)
|
||||
prevent the changes from being rendered.
|
||||
|
||||
## Jekyll Theme Configuration
|
||||
|
||||
Jekyll site-wide configurations are stored
|
||||
in the `_config.yml` file. The current theme being used is
|
||||
[Just the Docs](https://pmarsceill.github.io/just-the-docs/).
|
||||
|
||||
Due to incompatibilities between this theme and GitHub’s built-in implementation of the
|
||||
CommonMark Markdown parser, the default Jekyll Markdown parser, Kramdown, is
|
||||
used.
|
||||
|
||||
## Document Structure
|
||||
|
||||
Major chapters have Introduction pages, and further sections in a chapter are
|
||||
subpages of that main chapter page. The navigation bar is generated from the
|
||||
information provided in the header section of the Markdown file according to
|
||||
the theme's [navigation annotations](https://pmarsceill.github.io/just-the-docs/docs/navigation-structure/).
|
||||
|
||||
## Syntax
|
||||
|
||||
[Markdown](https://kramdown.gettalong.org/syntax.html) should be preferred over custom
|
||||
HTML to create content wherever possible.
|
||||
|
||||
## Common Styles
|
||||
|
||||
Styles are provided by the theme's settings, which can be modified in `_config.yml`.
|
||||
|
||||
## Code
|
||||
|
||||
Use the markdown notation of [single backtick](https://kramdown.gettalong.org/quickref.html#inline-code)
|
||||
code spans (inline) and [tilde line-demarcated](https://kramdown.gettalong.org/quickref.html#code-blocks)
|
||||
fenced code blocks (multi-line).
|
||||
|
||||
Unlike Github-flavored Markdown, in multi-line fenced code
|
||||
blocks, the programming language in the [info string](https://spec.commonmark.org/0.29/#info-string)
|
||||
should be lowercased.
|
||||
|
||||
## Notes
|
||||
|
||||
> :point_right: **Note**: Notes should be made by starting the line with the
|
||||
following Markdown:
|
||||
|
||||
~~~
|
||||
> :point_right: **Note**:
|
||||
~~~
|
||||
|
||||
## TODOs
|
||||
|
||||
"TODO" items should be marked by starting the line with the
|
||||
following Markdown:
|
||||
|
||||
~~~
|
||||
> :construction: **TODO**:
|
||||
~~~
|
||||
|
||||
This will result in a "TODO" item like this:
|
||||
|
||||
> :construction: **TODO**: Adjust this page for use of GitHub Markdown (since 2020)
|
||||
|
||||
## Emoji
|
||||
|
||||
GitHub-flavored emoji are enabled using the
|
||||
[Jemoji](https://github.com/jekyll/jemoji) Ruby gem.
|
||||
|
||||
## Bookmarks & Links
|
||||
|
||||
For internal links, please used relative paths to the corresponding Markdown
|
||||
file as the Markdown link path rather than specifying the full URL.
|
||||
|
||||
Anchors are generated by Jekyll for sections using the section header text in
|
||||
lowercase with hyphens instead of spaces.
|
||||
|
||||
> :construction: **TODO**: search our pages for "(§)" and fix the links.
|
||||
|
||||
## Previewing Changes Locally
|
||||
|
||||
Because of the latency in GitHub Pages rendering changes, it can be useful to
|
||||
run a Jekyll instance locally to preview changes.
|
||||
|
||||
### Setup
|
||||
|
||||
GitHub Pages provides a list of
|
||||
[dependency versions](https://pages.github.com/versions/) used in order
|
||||
to allow local environments to match exactly. Since Jekyll is a Ruby project,
|
||||
setup details depend on gems (code packages), Bundler (dependency management for
|
||||
execution), and the version of Ruby.
|
||||
|
||||
[rbenv](https://www.ruby-lang.org/en/documentation/installation/#rbenv) is one
|
||||
of the few different ways to install and control the version of Ruby. To
|
||||
install it and the required `ruby-build` plugin:
|
||||
```bash
|
||||
git clone https://github.com/rbenv/rbenv.git ~/.rbenv
|
||||
cd ~/.rbenv && src/configure && make -C src # Optional but greatly speeds up certain operations
|
||||
echo 'export PATH="$HOME/.rbenv/bin:$PATH"' >> ~/.bashrc
|
||||
```
|
||||
```bash
|
||||
mkdir -p "$(rbenv root)"/plugins
|
||||
git clone https://github.com/rbenv/ruby-build.git "$(rbenv root)"/plugins/ruby-build
|
||||
```
|
||||
|
||||
To install specific Ruby version that GitHub Pages uses:
|
||||
```bash
|
||||
rbenv versions # list all Ruby versions available
|
||||
rbenv install <version-num> # install GH Pages Ruby version
|
||||
rbenv versions # list all Ruby versions after installing
|
||||
```
|
||||
|
||||
To activate the version of Ruby:
|
||||
```bash
|
||||
rbenv init # OR: eval "$(rbenv init -)"
|
||||
rbenv shell <version-num>
|
||||
rbenv versions # verify the specified version is in use
|
||||
```
|
||||
|
||||
To install [Bundler](https://bundler.io/):
|
||||
```bash
|
||||
gem install bundler
|
||||
```
|
||||
|
||||
The simplest way to ensure that
|
||||
transitive dependency versions are included correctly is to just specify the
|
||||
`github-pages` gem. This is already configured in `Gemfile` in the root
|
||||
directory. Thus, to ensure that the Ruby shell has the correct versions of
|
||||
dependencies downloaded and cached, in the root directory (`/docs`), run:
|
||||
```bash
|
||||
cd <ICU>/docs # change to User Guide docs root directory
|
||||
bundle update
|
||||
```
|
||||
|
||||
### Running
|
||||
|
||||
Ensure that the specific version of Ruby has been activated. Then use Bundler
|
||||
to ensure that the dependencies are downloaded in the current Ruby "shell"
|
||||
instance. Then use Bundler to execute the Jekyll server.
|
||||
```bash
|
||||
rbenv init # OR: eval "$(rbenv init -)"
|
||||
rbenv shell <version-num>
|
||||
cd <ICU>/docs # change to User Guide docs root directory
|
||||
bundle update
|
||||
bundle exec jekyll server
|
||||
```
|
||||
|
||||
Jekyll will take several seconds to generate HTML from the Markdown, so the
|
||||
Jekyll server will wait for that before being ready. Then, the rendered pages
|
||||
will be available at <http://localhost:4000/icu/>.
|
||||
|
||||
## Previous version
|
||||
|
||||
The previous version of the ICU User Guide has been maintained via Google Sites. The Site
|
||||
address is <http://sites.google.com/site/icuprojectuserguide/>
|
||||
@@ -1,21 +0,0 @@
|
||||
---
|
||||
layout: default
|
||||
title: Contributors
|
||||
nav_order: 1800
|
||||
has_children: true
|
||||
---
|
||||
<!--
|
||||
© 2020 and later: Unicode, Inc. and others.
|
||||
License & terms of use: http://www.unicode.org/copyright.html
|
||||
-->
|
||||
|
||||
# Contributors
|
||||
|
||||
Top-level page for topics for contributors to ICU. See the subpages listed below for
|
||||
details:
|
||||
|
||||
[Coding Guidelines](codingguidelines.md)
|
||||
|
||||
[Contributions to the ICU library](contributions.md)
|
||||
|
||||
[Synchronization Issues](sync/index.md)
|
||||
@@ -1,245 +0,0 @@
|
||||
---
|
||||
layout: default
|
||||
title: Custom ICU4C Synchronization
|
||||
nav_order: 3
|
||||
parent: Contributors
|
||||
---
|
||||
<!--
|
||||
© 2020 and later: Unicode, Inc. and others.
|
||||
License & terms of use: http://www.unicode.org/copyright.html
|
||||
-->
|
||||
|
||||
# Custom ICU4C Synchronization
|
||||
{: .no_toc }
|
||||
|
||||
## Contents
|
||||
{: .no_toc .text-delta }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
|
||||
---
|
||||
|
||||
> :warning: ***Support for including an alternate implementation of atomic and mutex
|
||||
> operations has been withdrawn and removed from ICU4C.***
|
||||
> See issue [ICU-20185](https://unicode-org.atlassian.net/browse/ICU-20185).
|
||||
|
||||
### Build Time User Provided Synchronization
|
||||
|
||||
Build time user synchronization provides a mechanism for platforms with special
|
||||
requirements to provide their own mutex and one-time initialization
|
||||
implementations to ICU. This facility was introduced in ICU 53. It may change
|
||||
over time.
|
||||
|
||||
The alternative implementations are compiled directly into the ICU libraries.
|
||||
Alternative implementations cannot be plugged in at run time.
|
||||
|
||||
The tables below show the items that must be defined by a custom ICU
|
||||
synchronization implementation. The list includes both functions that are used
|
||||
throughout ICU code and additional functions are for internal by other ICU
|
||||
synchronization primitives.
|
||||
|
||||
**Low Level Atomics**, a set of platform or compiler dependent typedefs and
|
||||
inlines. Provided in the internal header file
|
||||
[`umutex.h`](https://github.com/unicode-org/icu/blob/main/icu4c/source/common/umutex.h).
|
||||
|
||||
| Type/Function | Description |
|
||||
|---------------------------------------------------------|-------------------------------------------------------------------------------|
|
||||
| `typedef u_atomic_int32_t` | A 32 bit integer that will work with low level atomic operations. (`typedef`) |
|
||||
| `umtx_loadAcquire(u_atomic_int32_t &var)` | |
|
||||
| `umtx_storeRelease(u_atomic_int32_t &var, int32_t val)` | |
|
||||
| `umtx_atomic_inc(u_atomic_int32_t &var)` | |
|
||||
| `umtx_atomic_dec(u_atomic_int32_t &var)` | |
|
||||
|
||||
**Mutexes**. Type declarations for ICU mutex wrappers. Provided in a header file.
|
||||
|
||||
| Type | Description |
|
||||
|-----------------------|---------------------------------------------------------------------------------------------------|
|
||||
| `struct UMutex` | An ICU mutex. All instances will be static. Typically just contains an underlying platform mutex. |
|
||||
| `U_MUTEX_INITIALIZER` | A C style initializer for a static instance of a `UMutex`. |
|
||||
|
||||
**Mutex and InitOnce implementations**. Out-of-line platform-specific code.
|
||||
Provided in a .cpp file.
|
||||
|
||||
| Function | Description |
|
||||
|-----------------------------------------|--------------------------------------------|
|
||||
| `umtx_lock(UMutex *mutex)` | Lock a mutex. |
|
||||
| `umtx_unlock(UMutex* mutex)` | Unlock a mutex. |
|
||||
| `umtx_initImplPreInit(UInitOnce &uio)` | `umtx_initOnce()` implementation function. |
|
||||
| `umtx_initImplPostInit(UInitOnce &uio)` | `umtx_initOnce()` implementation function. |
|
||||
|
||||
`UInitOnce` and `umtx_initOnce()` are used internally by ICU for thread-safe
|
||||
one-time initialization. Their implementation is split into a
|
||||
platform-independent part (contained in
|
||||
[`umutex.h`](https://github.com/unicode-org/icu/blob/main/icu4c/source/common/umutex.h)),
|
||||
and the pair of platform-dependent implementation functions listed above.
|
||||
|
||||
**Build Setup**
|
||||
|
||||
Compiler preprocessor variables are used to name the custom files to be included
|
||||
in the ICU build. If defined, the files are included at the top of the normal
|
||||
platform `#ifdef` chains in the ICU sources, and effectively define a new
|
||||
platform.
|
||||
|
||||
| Macro | Description |
|
||||
|--------------------|---------------------------------------------------------|
|
||||
| `U_USER_ATOMICS_H` | Set to the name of the low level atomics header file. |
|
||||
| `U_USER_MUTEX_H` | Mutexes header file. |
|
||||
| `U_USER_MUTEX_CPP` | Mutexes and `InitOnce` implementation file. |
|
||||
|
||||
It is possible (and reasonable) to supply only the two mutex files, while
|
||||
retaining the ICU default implementation for the low level atomics.
|
||||
|
||||
Example ICU configure with user mutexes specified:
|
||||
|
||||
CPPFLAGS='-DU_USER_ATOMICS_H=atomic_c11.h -DU_USER_MUTEX_H=mutex_c11.h -DU_USER_MUTEX_CPP=mutex_c11.cpp' ./runConfigureICU --enable-debug Linux
|
||||
|
||||
**Stability**
|
||||
|
||||
This interface may change between ICU releases. The required set of functions
|
||||
may be extended, or details of the behavior required may be altered.
|
||||
|
||||
The types and functions defined by this interface reach deeply into the ICU
|
||||
implementation, and we need to retain the ability to make changes should the
|
||||
need arise.
|
||||
|
||||
**Examples**
|
||||
|
||||
The code below shows a complete set of ICU user synchronization files.
|
||||
|
||||
This implementation uses C++11 language mutexes and atomics. These make for a
|
||||
convenient reference implementation because the C++11 constructs are well
|
||||
defined and straight forward to use.
|
||||
|
||||
Similar implementations for POSIX and Windows can be found in files
|
||||
`common/umutex.h` and `common/umutex.cpp`, in the platform `#ifdef` chains; these are
|
||||
part of the standard ICU distribution.
|
||||
|
||||
**Mutex Header**
|
||||
```c++
|
||||
// Example of an ICU build time customized mutex header.
|
||||
//
|
||||
// Must define struct UMutex and an initializer that will work with static instances.
|
||||
// All UMutex instances in ICU code will be static.
|
||||
|
||||
#ifndef ICU_MUTEX_C11_H
|
||||
#define ICU_MUTEX_C11_H
|
||||
#include <mutex>
|
||||
#include <condition_variable>
|
||||
struct UMutex {
|
||||
std::mutex fMutex;
|
||||
};
|
||||
#define U_MUTEX_INITIALIZER {}
|
||||
#endif
|
||||
```
|
||||
|
||||
**Atomics Header**
|
||||
```c++
|
||||
#include <atomic>
|
||||
|
||||
typedef std::atomic<int32_t> u_atomic_int32_t;
|
||||
#define ATOMIC_INT32_T_INITIALIZER(val) ATOMIC_VAR_INIT(val)
|
||||
|
||||
inline int32_t umtx_loadAcquire(u_atomic_int32_t &var) {
|
||||
return var.load(std::memory_order_acquire);
|
||||
}
|
||||
|
||||
inline void umtx_storeRelease(u_atomic_int32_t &var, int32_t val) {
|
||||
var.store(val, std::memory_order_release);
|
||||
}
|
||||
|
||||
inline int32_t umtx_atomic_inc(u_atomic_int32_t &var) {
|
||||
return var.fetch_add(1) + 1;
|
||||
}
|
||||
|
||||
inline int32_t umtx_atomic_dec(u_atomic_int32_t &var) {
|
||||
return var.fetch_sub(1) - 1;
|
||||
}
|
||||
```
|
||||
|
||||
**Mutex and InitOnce implementations**
|
||||
```c++
|
||||
//
|
||||
// Example ICU build time custom mutex cpp file.
|
||||
//
|
||||
// Must implement these functions:
|
||||
// umtx_lock(UMutex *mutex);
|
||||
// umtx_unlock(UMutex *mutex);
|
||||
// umtx_initImplPreInit(UInitOnce &uio);
|
||||
// umtx_initImplPostInit(UInitOnce &uio);
|
||||
|
||||
U_CAPI void U_EXPORT2
|
||||
umtx_lock(UMutex *mutex) {
|
||||
if (mutex == NULL) {
|
||||
// Note: globalMutex is pre-defined in the platform-independent ICU code.
|
||||
mutex = &globalMutex;
|
||||
}
|
||||
mutex->fMutex.lock();
|
||||
}
|
||||
|
||||
U_CAPI void U_EXPORT2
|
||||
umtx_unlock(UMutex* mutex) `{
|
||||
if (mutex == NULL) {
|
||||
mutex = &globalMutex;
|
||||
}
|
||||
mutex->fMutex.unlock();
|
||||
}
|
||||
|
||||
// A mutex and a condition variable are used by the implementation of umtx_initOnce()
|
||||
// The mutex is held only while the state of the InitOnce object is being changed or
|
||||
// tested. It is not held while initialization functions are running.
|
||||
// Threads needing to block, waiting for an initialization to complete, will wait
|
||||
// on the condition variable.
|
||||
// All InitOnce objects share a common mutex and condition variable. This means that
|
||||
// all blocked threads will wake if any (possibly unrelated) initialization completes.
|
||||
// Which does no harm, it should be statistically rare, and any spuriously woken
|
||||
// threads will check their state and promptly wait again.
|
||||
|
||||
static std::mutex initMutex;
|
||||
static std::condition_variable initCondition;
|
||||
|
||||
// This function is called from umtx_initOnce() when an initial test of a UInitOnce::fState flag
|
||||
// reveals that initialization has not completed, that we either need to call the
|
||||
// function on this thread, or wait for some other thread to complete the initialization.
|
||||
//
|
||||
// The actual call to the init function is made inline by template code
|
||||
// that knows the C++ types involved. This function returns TRUE if
|
||||
// the inline code needs to invoke the Init function, or FALSE if the initialization
|
||||
// has completed on another thread.
|
||||
//
|
||||
// UInitOnce::fState values:
|
||||
// 0: Initialization has not yet begun.
|
||||
// 1: Initialization is in progress, not yet complete.
|
||||
// 2: Initialization is complete.
|
||||
//
|
||||
UBool umtx_initImplPreInit(UInitOnce &uio) {
|
||||
std::unique_lock<std::mutex> initLock(initMutex);
|
||||
int32_t state = uio.fState;
|
||||
if (state == 0) {
|
||||
umtx_storeRelease(uio.fState, 1);
|
||||
return TRUE; // Caller will next call the init function.
|
||||
} else {
|
||||
while (uio.fState == 1) {
|
||||
// Another thread is currently running the initialization.
|
||||
// Wait until it completes.
|
||||
initCondition.wait(initLock);
|
||||
}
|
||||
U_ASSERT(uio.fState == 2);
|
||||
return FALSE;
|
||||
}
|
||||
}
|
||||
|
||||
// This function is called from umtx_initOnce() just after an initializationfunction completes.
|
||||
// Its purpose is to set the state of the UInitOnce object to initialized, and to
|
||||
// unblock any threads that may be waiting on the initialization.
|
||||
//
|
||||
// Some threads may be waiting on the condition variable, requiring the notify_all().
|
||||
// Some threads may be racing to test the fState flag outside of the mutex,
|
||||
// requiring the use of store-release when changing its value.
|
||||
|
||||
void umtx_initImplPostInit(UInitOnce &uio) {
|
||||
std::unique_lock<std::mutex> initLock(initMutex);
|
||||
umtx_storeRelease(uio.fState, 2);
|
||||
initCondition.notify_all();
|
||||
}
|
||||
```
|
||||
@@ -1,86 +0,0 @@
|
||||
---
|
||||
layout: default
|
||||
title: Synchronization
|
||||
nav_order: 2
|
||||
parent: Contributors
|
||||
---
|
||||
<!--
|
||||
© 2020 and later: Unicode, Inc. and others.
|
||||
License & terms of use: http://www.unicode.org/copyright.html
|
||||
-->
|
||||
|
||||
# Synchronization Issues
|
||||
{: .no_toc }
|
||||
|
||||
## Contents
|
||||
{: .no_toc .text-delta }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
ICU is designed for use in multi-threaded environments. Guidelines for
|
||||
developers using ICU are in the [ICU Design](../../design.md) section of the
|
||||
user guide.
|
||||
|
||||
Within the ICU implementation, access to shared or global data sometimes must be
|
||||
protected in order to provide the threading model promised by the ICU design.
|
||||
The information on this page is intended for developers of ICU library code
|
||||
itself.
|
||||
|
||||
ICU4J uses normal JDK synchronization services.
|
||||
|
||||
ICU4C faces a more difficult problem, as there is no standard, fully portable
|
||||
set of C or C++ synchronization primitives. Internally, ICU4C provides a small
|
||||
set of synchronization operations, and requires that all synchronization needed
|
||||
within the ICU library code be implemented using them.
|
||||
|
||||
The ICU4C synchronization primitives are for internal use only; they are not
|
||||
exported as API to normal users of ICU.
|
||||
|
||||
ICU provides implementations of its synchronization functions for Windows, POSIX
|
||||
and C++11 platforms, and provides a build-time interface to allow [custom
|
||||
implementations](custom.md) for other platforms.
|
||||
|
||||
## ICU4C Synchronization Primitives
|
||||
|
||||
The functions and types listed below are intended for use throughout the ICU
|
||||
library code, where ever synchronization is required. They are defined in the
|
||||
internal header
|
||||
[umutex.h](https://github.com/unicode-org/icu/blob/main/icu4c/source/common/umutex.h).
|
||||
|
||||
All synchronization within ICU4C implementation code must use these, and avoid
|
||||
direct use of functions provided by a particular operating system or compiler.
|
||||
|
||||
For examples of use, search the ICU library code.
|
||||
|
||||
**Low Level Atomics**
|
||||
|
||||
| Type/Function | Description |
|
||||
|------------------------------------------|-----------------------------------------------------------------|
|
||||
| `typedef u_atomic_int32_t` | A 32 bit integer type for use with low level atomic operations. |
|
||||
| `umtx_atomic_inc(u_atomic_int32_t &var)` | |
|
||||
| `umtx_atomic_dec(u_atomic_int32_t &var)` | |
|
||||
|
||||
**Mutexes**
|
||||
|
||||
| Type/Function | Description |
|
||||
|------------------------------|-----------------------------------------------------------------------|
|
||||
| `struct UMutex` | An ICU mutex. All instances must be `static`. |
|
||||
| `U_MUTEX_INITIALIZER` | A C style initializer for a `UMutex`. |
|
||||
| `umtx_lock(UMutex *mutex)` | Lock a mutex. |
|
||||
| `umtx_unlock(UMutex* mutex)` | Unlock a mutex. |
|
||||
| `class Mutex` | C++ Mutex wrapper with automatic lock & unlock. See header `mutex.h.` |
|
||||
|
||||
**One Time Initialization**
|
||||
|
||||
| Type/Function | Description |
|
||||
|---------------------------------|-----------------------------------------------------------------------------------------|
|
||||
| `struct UInitOnce` | Provides an efficient facility for one-time initialization of static or global objects. |
|
||||
| `umtx_initOnce(UInitOnce, ...)` | A family of initialization functions. |
|
||||
|
||||
All of these functions are for internal ICU implementation use only. They are
|
||||
not exported, and not intended for external use.
|
||||
@@ -1,293 +0,0 @@
|
||||
---
|
||||
layout: default
|
||||
title: Date and Time Formatting Examples
|
||||
nav_order: 1
|
||||
grand_parent: Formatting
|
||||
parent: Formatting Dates and Times
|
||||
---
|
||||
<!--
|
||||
© 2020 and later: Unicode, Inc. and others.
|
||||
License & terms of use: http://www.unicode.org/copyright.html
|
||||
-->
|
||||
|
||||
# Date and Time Formatting Examples
|
||||
{: .no_toc }
|
||||
|
||||
## Contents
|
||||
{: .no_toc .text-delta }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
|
||||
---
|
||||
|
||||
## Format
|
||||
|
||||
The ICU DateFormat interface enables you to format a date in milliseconds into a
|
||||
string representation of the date. Also, the interface enables you to parse the
|
||||
string back to the internal date representation in milliseconds.
|
||||
|
||||
### C++
|
||||
|
||||
```cpp
|
||||
DateFormat* df = DateFormat::createDateInstance();
|
||||
UnicodeString myString;
|
||||
UDate myDateArr[] = { 0.0, 100000000.0, 2000000000.0 };
|
||||
for (int32_t i = 0; i < 3; ++i) {
|
||||
myString.remove();
|
||||
cout << df->format( myDateArr[i], myString ) << endl;
|
||||
}
|
||||
```
|
||||
|
||||
### C
|
||||
|
||||
```c
|
||||
/* 1st example: format the dates in millis 100000000 and 2000000000 */
|
||||
UErrorCode status=U_ZERO_ERROR;
|
||||
int32_t i, myStrlen=0;
|
||||
UChar* myString;
|
||||
UDate myDateArr[] = { 0.0, 100000000.0, 2000000000.0 }; // test values
|
||||
UDateFormat* df = udat_open(UCAL_DEFAULT, UCAL_DEFAULT, NULL, "GMT", &status);
|
||||
for (i = 0; i < 3; ++i) {
|
||||
myStrlen = udat_format(df, myDateArr[i], NULL, myStrlen, NULL, &status);
|
||||
if(status==U_BUFFER_OVERFLOW_ERROR){
|
||||
status=U_ZERO_ERROR;
|
||||
myString=(UChar*)malloc(sizeof(UChar) * (myStrlen+1) );
|
||||
udat_format(df, myDateArr[i], myString, myStrlen+1, NULL, &status);
|
||||
printf("%s\n", austrdup(myString) );
|
||||
/* austrdup( a function used to convert UChar* to char*) */
|
||||
free(myString);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Parse
|
||||
|
||||
To parse a date for a different locale, specify it in the locale call. This call
|
||||
creates a formatting object.
|
||||
|
||||
### C++
|
||||
|
||||
```cpp
|
||||
DateFormat* df = DateFormat::createDateInstance
|
||||
( DateFormat::SHORT, Locale::getFrance());
|
||||
```
|
||||
|
||||
### C
|
||||
|
||||
```c
|
||||
/* 2nd example: parse a date with short French date/time formatter */
|
||||
UDateFormat* df = udat_open(UDAT_SHORT, UDAT_SHORT, "fr_FR", "GMT", &status);
|
||||
UErrorCode status = U_ZERO_ERROR;
|
||||
int32_t parsepos=0;
|
||||
UDate myDate = udat_parse(df, myString, u_strlen(myString), &parsepos,
|
||||
&status);
|
||||
```
|
||||
|
||||
### Java
|
||||
|
||||
```java
|
||||
import java.text.FieldPosition;
|
||||
import java.text.ParseException;
|
||||
import java.util.Calendar;
|
||||
import java.util.Date;
|
||||
import java.util.Locale;
|
||||
|
||||
import com.ibm.icu.text.DateFormat;
|
||||
|
||||
public class TestDateTimeFormat {
|
||||
public void run() {
|
||||
|
||||
// Formatting Dates
|
||||
|
||||
DateFormat dfUS = DateFormat.getDateInstance(DateFormat.FULL, Locale.US);
|
||||
DateFormat dfFrance = DateFormat.getDateInstance(DateFormat.FULL, Locale.FRANCE);
|
||||
StringBuffer sb = new StringBuffer();
|
||||
Calendar c = Calendar.getInstance();
|
||||
Date d = c.getTime();
|
||||
sb = dfUS.format(d, sb, new FieldPosition(0));
|
||||
System.out.println(sb.toString());
|
||||
|
||||
StringBuffer sbf = new StringBuffer();
|
||||
sbf = dfFrance.format(d, sbf, new FieldPosition(0));
|
||||
System.out.println(sbf.toString());
|
||||
|
||||
StringBuffer sbg = new StringBuffer();
|
||||
DateFormat dfg = DateFormat.getDateTimeInstance(DateFormat.FULL, DateFormat.SHORT);
|
||||
FieldPosition pos = new FieldPosition(DateFormat.MINUTE_FIELD);
|
||||
sbg = dfg.format(d, sbg, pos);
|
||||
System.out.println(sbg.toString());
|
||||
System.out.println(sbg.toString().substring(pos.getBeginIndex(), pos.getEndIndex()));
|
||||
|
||||
// Parsing Dates
|
||||
|
||||
String dateString_US = "Thursday, February 7, 2008";
|
||||
String dateString_FRANCE = "jeudi 7 février 2008";
|
||||
try {
|
||||
Date parsedDate_US = dfUS.parse(dateString_US);
|
||||
Date parsedDate_FRANCE = dfFrance.parse(dateString_FRANCE);
|
||||
System.out.println(parsedDate_US.toString());
|
||||
System.out.println(parsedDate_FRANCE.toString());
|
||||
} catch (ParseException pe) {
|
||||
System.out.println("Exception while parsing :" + pe);
|
||||
}
|
||||
}
|
||||
|
||||
public static void main(String args[]) {
|
||||
new TestDateTimeFormat().run();
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Getting Specific Date Fields
|
||||
|
||||
To get specific fields of a date, you can use the FieldPosition function for C++
|
||||
or UFieldPosition function for C.
|
||||
|
||||
### C++
|
||||
|
||||
```cpp
|
||||
UErrorCode status = U_ZERO_ERROR;
|
||||
FieldPosition pos(DateFormat::YEAR_FIELD)
|
||||
UDate myDate = Calendar::getNow();
|
||||
UnicodeString str;
|
||||
DateFormat* df = DateFormat::createDateInstance
|
||||
( DateFormat::LONG, Locale::getFrance());
|
||||
|
||||
df->format(myDate, str, pos, status);
|
||||
cout << pos.getBeginIndex() << "," << pos. getEndIndex() << endl;
|
||||
```
|
||||
|
||||
### C
|
||||
|
||||
```c
|
||||
UErrorCode status = U_ZERO_ERROR;
|
||||
UFieldPosition pos;
|
||||
UChar *myString;
|
||||
int32_t myStrlen = 0;
|
||||
char buffer[1024];
|
||||
|
||||
|
||||
pos.field = 1; /* Same as the DateFormat::EField enum */
|
||||
UDateFormat* dfmt = udat_open(UCAL_DEFAULT, UCAL_DEFAULT, NULL, "PST",
|
||||
&status);
|
||||
myStrlen = udat_format(dfmt, myDate, NULL, myStrlen, &pos, &status);
|
||||
if (status==U_BUFFER_OVERFLOW_ERROR){
|
||||
status=U_ZERO_ERROR;
|
||||
myString=(UChar*)malloc(sizeof(UChar) * (myStrlen+1) );
|
||||
udat_format(dfmt, myDate, myString, myStrlen+1, &pos, &status);
|
||||
}
|
||||
printf("date format: %s\n", u_austrcpy(buffer, myString));
|
||||
buffer[pos.endIndex] = 0; // NULL terminate the string.
|
||||
printf("UFieldPosition position equals %s\n", &buffer[pos.beginIndex]);
|
||||
```
|
||||
|
||||
## DateTimePatternGenerator
|
||||
|
||||
This class lets you get a different variety of patterns, such as month+day. The
|
||||
following illustrates this in Java, C++ and C.
|
||||
|
||||
### Java
|
||||
|
||||
```java
|
||||
// set up the generator
|
||||
DateTimePatternGenerator generator
|
||||
= DateTimePatternGenerator.getInstance(locale);
|
||||
|
||||
// get a pattern for an abbreviated month and day
|
||||
final String pattern = generator.getBestPattern("MMMd");
|
||||
SimpleDateFormat formatter = new SimpleDateFormat(pattern, locale);
|
||||
|
||||
// use it to format (or parse)
|
||||
String formatted = formatter.format(new Date());
|
||||
// for French, the result is "13 sept."
|
||||
```
|
||||
|
||||
### C++
|
||||
|
||||
```cpp
|
||||
// set up the generator
|
||||
status = U_ZERO_ERROR;
|
||||
DateTimePatternGenerator *generator = DateTimePatternGenerator::createInstance( locale, status);
|
||||
if (U_FAILURE(status)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// get a pattern for an abbreviated month and day
|
||||
UnicodeString pattern = generator->getBestPattern(UnicodeString("MMMd"), status);
|
||||
SimpleDateFormat *formatter = new SimpleDateFormat(pattern, locale, status);
|
||||
|
||||
// use it to format (or parse)
|
||||
UnicodeString formatted;
|
||||
formatted = formatter->format(Calendar::getNow(), formatted, status);
|
||||
// for French, the result is "13 sept."
|
||||
```
|
||||
|
||||
### C
|
||||
|
||||
```c
|
||||
const UChar skeleton[]= {'M', 'M', 'M', 'd', 0};
|
||||
|
||||
status=U_ZERO_ERROR;
|
||||
generator=udatpg_open(locale, &status);
|
||||
if(U_FAILURE(status)) {
|
||||
return;
|
||||
|
||||
}
|
||||
|
||||
/* get a pattern for an abbreviated month and day */
|
||||
length = udatpg_getBestPattern(generator, skeleton, 4,
|
||||
pattern, patternCapacity, &status);
|
||||
formatter = udat_open(UDAT_IGNORE, UDAT_DEFAULT, locale, NULL, -1,
|
||||
pattern, length, &status);
|
||||
|
||||
/* use it to format (or parse) */
|
||||
formattedCapacity = (int32_t)(sizeof(formatted)/sizeof((formatted)[0]));
|
||||
resultLen=udat_format(formatter, ucal_getNow(), formatted, formattedCapacity,
|
||||
NULL, &status);
|
||||
/* for French, the result is "13 sept." */
|
||||
```
|
||||
|
||||
## Changing the TimeZone Formatting Style
|
||||
|
||||
It also contains some helper functions for parsing patterns. Here's an example
|
||||
of replacing the kind of timezone used in a pattern.
|
||||
|
||||
### Java
|
||||
|
||||
```cpp
|
||||
/**
|
||||
* Replace the zone string with a different type, eg v's for z's, etc.
|
||||
* <p>Called with a pattern, such as one gotten from
|
||||
* <pre>
|
||||
* String pattern = ((SimpleDateFormat)
|
||||
* DateFormat.getTimeInstance(style, locale)).toPattern();
|
||||
* </pre>
|
||||
* @param pattern original pattern to change, such as "HH:mm zzzz"
|
||||
* @param newZone Must be: z, zzzz, Z, ZZZZ, v, vvvv, V, or VVVV
|
||||
* @return
|
||||
*/
|
||||
public String replaceZoneString(String pattern, String newZone) {
|
||||
DateTimePatternGenerator.FormatParser formatParser =
|
||||
new DateTimePatternGenerator.FormatParser();
|
||||
final List itemList = formatParser.set(pattern).getItems();
|
||||
boolean found = false;
|
||||
for (int i = 0; i < itemList.size(); ++i) {
|
||||
Object item = itemList.get(i);
|
||||
if (item instanceof VariableField) {
|
||||
// the first character of the variable field determines the type,
|
||||
// according to CLDR.
|
||||
String variableField = item.toString();
|
||||
switch (variableField.charAt(0)) {
|
||||
case 'z': case 'Z': case 'v': case 'V':
|
||||
if (!variableField.equals(newZone)) {
|
||||
found = true;
|
||||
itemList.set(i, new VariableField(newZone));
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return found ? formatParser.toString() : pattern;
|
||||
}
|
||||
```
|
||||
@@ -1,386 +0,0 @@
|
||||
---
|
||||
layout: default
|
||||
title: Formatting Dates and Times
|
||||
nav_order: 2
|
||||
parent: Formatting
|
||||
has_children: true
|
||||
---
|
||||
<!--
|
||||
© 2020 and later: Unicode, Inc. and others.
|
||||
License & terms of use: http://www.unicode.org/copyright.html
|
||||
-->
|
||||
|
||||
# Formatting Dates and Times
|
||||
{: .no_toc }
|
||||
|
||||
## Contents
|
||||
{: .no_toc .text-delta }
|
||||
|
||||
1. TOC
|
||||
{:toc}
|
||||
|
||||
---
|
||||
|
||||
## Formatting Dates and Times Overview
|
||||
|
||||
Date and time formatters are used to convert dates and times from their internal
|
||||
representations to textual form and back again in a language-independent manner.
|
||||
The date and time formatters use `UDate`, which is the internal representation.
|
||||
Converting from the internal representation (milliseconds since midnight,
|
||||
January 1, 1970) to text is known as "formatting," and converting from text to
|
||||
milliseconds is known as "parsing." These processes involve two mappings:
|
||||
|
||||
* A mapping between a point in time (`UDate`) and a set of calendar fields,
|
||||
which in turn depends on:
|
||||
* The rules of a particular calendar system (e.g. Gregorian, Buddhist,
|
||||
Chinese Lunar)
|
||||
* The time zone
|
||||
* A mapping between a set of calendar fields and a formatted textual
|
||||
representation, which depends on the fields selected for display, their
|
||||
display style, and the conventions of a particular locale.
|
||||
|
||||
## `DateFormat`
|
||||
|
||||
`DateFormat` helps format and parse dates for any locale. Your code can be
|
||||
completely independent of the locale conventions for months, days of the week,
|
||||
or calendar format.
|
||||
|
||||
### Formatting Dates
|
||||
|
||||
The `DateFormat` interface in ICU enables you to format a `Date` in milliseconds
|
||||
into a string representation of the date. It also parses the string back to the
|
||||
internal `Date` representation in milliseconds.
|
||||
|
||||
```cpp
|
||||
DateFormat* df = DateFormat::createDateInstance();
|
||||
UnicodeString myString;
|
||||
UDate myDateArr[] = { 0.0, 100000000.0, 2000000000.0 };
|
||||
for (int32_t i = 0; i < 3; ++i) {
|
||||
myString.remove();
|
||||
cout << df->format( myDateArr[i], myString ) << endl;
|
||||
}
|
||||
```
|
||||
|
||||
To format a date for a different `Locale`, specify it in the call to:
|
||||
|
||||
```cpp
|
||||
DateFormat* df = DateFormat::createDateInstance
|
||||
( DateFormat::SHORT, Locale::getFrance());
|
||||
```
|
||||
|
||||
### Parsing Dates
|
||||
|
||||
Use a `DateFormat` to parse also:
|
||||
|
||||
```cpp
|
||||
UErrorCode status = ZERO_ERROR;
|
||||
UDate myDate = df->parse(myString, status);
|
||||
```
|
||||
|
||||
When numeric fields abut one another directly, with no intervening delimiter
|
||||
characters, they constitute a run of abutting numeric fields. Such runs are
|
||||
parsed specially. For example, the format `"HHmmss"` parses the input text
|
||||
"123456" to 12:34:56, parses the input text "12345" to 1:23:45, and fails to
|
||||
parse "1234". In other words, the leftmost field of the run is flexible, while
|
||||
the others keep a fixed width. If the parse fails anywhere in the run, then the
|
||||
leftmost field is shortened by one character, and the entire run is parsed
|
||||
again. This is repeated until either the parse succeeds or the leftmost field is
|
||||
one character in length. If the parse still fails at that point, the parse of
|
||||
the run fails.
|
||||
|
||||
### Producing Normal Date Formats for a Locale
|
||||
|
||||
Use `createDateInstance` to produce the normal date format for that country. There
|
||||
are other static factory methods available. Use `createTimeInstance` to produce
|
||||
the normal time format for that country. Use `createDateTimeInstance` to produce a
|
||||
`DateFormat` that formats both date and time. You can pass different options to
|
||||
these factory methods to control the length of the result; from `SHORT` to `MEDIUM`
|
||||
to `LONG` to `FULL`. The exact result depends on the locale, but generally:
|
||||
|
||||
1. `SHORT` is numeric, such as 12/13/52 or 3:30pm
|
||||
|
||||
2. `MEDIUM` is longer, such as Jan. 12, 1952
|
||||
|
||||
3. `LONG` is longer, such as January 12, 1952 or 3:30:32pm
|
||||
|
||||
4. `FULL` is completely specified, such as Tuesday, April 12, 1952 AD or
|
||||
3:30:42pm PST
|
||||
|
||||
For more general flexibility, the [`DateTimePatternGenerator`](index#datetimepatterngenerator)
|
||||
can map a custom selection of time and date fields, along with various display styles for
|
||||
those fields, to a locale-appropriate format that can then be set as the format to use by the `DateFormat`.
|
||||
|
||||
### Producing Relative Date Formats for a Locale
|
||||
|
||||
ICU currently provides limited support for formatting dates using a “relative”
|
||||
style, specified using `RELATIVE_SHORT`, `RELATIVE_MEDIUM`, `RELATIVE_LONG` or
|
||||
`RELATIVE_FULL`. As currently implemented, relative date formatting only affects
|
||||
the formatting of dates within a limited range of calendar days before or after
|
||||
the current date, based on the CLDR `<field type="day">`/`<relative>` data: For
|
||||
example, in English, "Yesterday", "Today", and "Tomorrow". Within this range,
|
||||
the specific relative style currently makes no difference. Outside of this
|
||||
range, relative dates are formatted using the corresponding non-relative style
|
||||
(`SHORT`, `MEDIUM`, etc.). Relative time styles are not currently supported, and
|
||||
behave just like the corresponding non-relative style.
|
||||
|
||||
### Setting Time Zones
|
||||
|
||||
You can set the time zone on the format. If you want more control over the
|
||||
format or parsing, cast the `DateFormat` you get from the factory methods to a
|
||||
`SimpleDateFormat`. This works for the majority of countries.
|
||||
|
||||
> :point_right: **Note**: *Remember to check `getDynamicClassID()` before carrying out the cast.*
|
||||
|
||||
### Working with Positions
|
||||
|
||||
You can also use forms of the parse and format methods with `ParsePosition` and
|
||||
`FieldPosition` to enable you to:
|
||||
|
||||
1. Progressively parse through pieces of a string.
|
||||
|
||||
2. Align any particular field, or find out where it is for selection on the
|
||||
screen.
|
||||
|
||||
## `SimpleDateFormat`
|
||||
|
||||
`SimpleDateFormat` is a concrete class used for formatting and parsing dates in a
|
||||
language-independent manner. It allows for formatting, parsing, and
|
||||
normalization. It formats or parses a date or time, which is the standard
|
||||
milliseconds since 24:00 GMT, Jan. 1, 1970.
|
||||
|
||||
`SimpleDateFormat` is the only built-in implementation of `DateFormat`. It provides
|
||||
a programmable interface that can be used to produce formatted dates and times
|
||||
in a wide variety of formats. The formats include almost all of the most common
|
||||
ones.
|
||||
|
||||
Create a date-time formatter using the following methods rather than
|
||||
constructing an instance of `SimpleDateFormat`. In this way, the program is
|
||||
guaranteed to get an appropriate formatting pattern of the locale.
|
||||
|
||||
1. `DateFormat::getInstance()`
|
||||
|
||||
2. `getDateInstance()`
|
||||
|
||||
3. `getDateTimeInstance()`
|
||||
|
||||
If you need a more unusual pattern, construct a `SimpleDateFormat` directly and
|
||||
give it an appropriate pattern.
|
||||
|
||||
### Date/Time Format Syntax
|
||||
|
||||
A date pattern is a string of characters, where specific strings of characters
|
||||
are replaced with date and time data from a calendar when formatting or used to
|
||||
generate data for a calendar when parsing.
|
||||
|
||||
The Date Field Symbol Table below contains the characters used in patterns to
|
||||
show the appropriate formats for a given locale, such as `yyyy` for the year.
|
||||
Characters may be used multiple times. For example, if `y` is used for the year,
|
||||
`"yy"` might produce "99", whereas `"yyyy"` produces "1999". For most numerical
|
||||
fields, the number of characters specifies the field width. For example, if `h` is
|
||||
the hour, `"h"` might produce "5", but `"hh"` produces "05". For some characters,
|
||||
the count specifies whether an abbreviated or full form should be used, but may
|
||||
have other choices, as given below.
|
||||
|
||||
Two single quotes represents a literal single quote, either inside or outside
|
||||
single quotes. Text within single quotes is not interpreted in any way (except
|
||||
for two adjacent single quotes). Otherwise all ASCII letter from **a** to **z** and **A** to
|
||||
**Z** are reserved as syntax characters, and require quoting if they are to
|
||||
represent literal characters. In addition, certain ASCII punctuation characters
|
||||
may become variable in the future (eg `':'` being interpreted as the time
|
||||
separator and `'/'` as a date separator, and replaced by respective
|
||||
locale-sensitive characters in display).
|
||||
|
||||
"Stand-alone" values refer to those designed to stand on their own independently, as opposed
|
||||
to being with other formatted values. "2nd quarter" would use the wide stand-alone
|
||||
format `"qqqq"`, whereas "2nd quarter 2007" would use the regular format `"QQQQ
|
||||
yyyy"`. For more information about format and stand-alone forms, see [CLDR Calendar Elements](https://www.unicode.org/reports/tr35/tr35-dates.html#months_days_quarters_eras).
|
||||
|
||||
The pattern characters used in the Date Field Symbol Table are defined by CLDR; for more information see [CLDR Date Field Symbol Table](https://www.unicode.org/reports/tr35/tr35-dates.html#Date_Field_Symbol_Table).
|
||||
|
||||
Note that the examples may not reflect current CLDR data.
|
||||
|
||||
#### Date Field Symbol Table
|
||||
|
||||
| Symbol | Meaning | Pattern | Example Output |
|
||||
| --- | --- | --- | --- |
|
||||
| `G` | era designator | `G`, `GG`, or `GGG`<br/>`GGGG`<br/>`GGGGG` | AD<br/>Anno Domini<br/>A |
|
||||
| `y` | year | `yy`<br/>`y` or `yyyy` | 96<br/>1996 |
|
||||
| `Y` | year of "Week of Year" | `Y` | 1997 |
|
||||
| `u` | extended year | `u` | 4601 |
|
||||
| `U` | cyclic year name, as in Chinese lunar calendar | `U` | 甲子 |
|
||||
| `r` | related Gregorian year | `r` | 1996 |
|
||||
| `Q` | quarter | `Q`<br/>`QQ`<br/>`QQQ`<br/>`QQQQ`<br/>`QQQQQ` | 2<br/>02<br/>Q2<br/>2nd quarter<br/>2 |
|
||||
| `q` | stand-alone quarter | `q`<br/>`qq`<br/>`qqq`<br/>`qqqq`<br/>`qqqqq` | 2<br/>02<br/>Q2<br/>2nd quarter<br/>2 |
|
||||
| `M` | month in year | `M`<br/>`MM`<br/>`MMM`<br/>`MMMM`<br/>`MMMMM` | 9<br/>09<br/>Sep<br/>September<br/>S |
|
||||
| `L` | stand-alone month in year | `L`<br/>`LL`<br/>`LLL`<br/>`LLLL`<br/>`LLLLL` | 9<br/>09<br/>Sep<br/>September<br/>S |
|
||||
| `w` | week of year | `w`<br/>`ww` | 27<br/>27 |
|
||||
| `W` | week of month | `W` | 2 |
|
||||
| `d` | day in month | `d`<br/>`dd` | 2<br/>02 |
|
||||
| `D` | day of year | `D` | 189 |
|
||||
| `F` | day of week in month | `F` | 2 (2nd Wed in July) |
|
||||
| `g` | modified julian day | `g` | 2451334 |
|
||||
| `E` | day of week | `E,` `EE`, or `EEE`<br/>`EEEE`<br/>`EEEEE`<br/>`EEEEEE` | Tue<br/>Tuesday<br/>T<br/>Tu |
|
||||
| `e` | local day of week<br/>example: if Monday is 1st day, Tuesday is 2nd ) | `e` or `ee`<br/>`eee`<br/>`eeee`<br/>`eeeee`<br/>`eeeeee` | 2<br/>Tue<br/>Tuesday<br/>T<br/>Tu |
|
||||
| `c` | stand-alone local day of week | `c` or `cc`<br/>`ccc`<br/>`cccc`<br/>`ccccc`<br/>`cccccc` | 2<br/>Tue<br/>Tuesday<br/>T<br/>Tu |
|
||||
| `a` | am/pm marker | `a` | pm |
|
||||
| `h` | hour in am/pm (1~12) | `h`<br/>`hh` | 7<br/>07 |
|
||||
| `H` | hour in day (0~23) | `H`<br/>`HH` | 0<br/>00 |
|
||||
| `k` | hour in day (1~24) | `k`<br/>`kk` | 24<br/>24 |
|
||||
| `K` | hour in am/pm (0~11) | `K`<br/>`KK` | 0<br/>00 |
|
||||
| `m` | minute in hour | `m`<br/>`mm` | 4<br/>04 |
|
||||
| `s` | second in minute | `s`<br/>`ss` | 5<br/>05 |
|
||||
| `S` | fractional second - truncates (like other time fields)<br/>to the count of letters when formatting. Appends<br/>zeros if more than 3 letters specified. Truncates at<br/>three significant digits when parsing. | `S`<br/>`SS`<br/>`SSS`<br/>`SSSS` | 2<br/>23<br/>235<br/>2350 |
|
||||
| `A` | milliseconds in day | `A` | 61201235 |
|
||||
| `z` | Time Zone: specific non-location | `z`, `zz`, or `zzz`<br/>`zzzz` | PDT<br/>Pacific Daylight Time |
|
||||
| `Z` | Time Zone: ISO8601 basic hms? / RFC 822<br/>Time Zone: long localized GMT (=OOOO)<br/>TIme Zone: ISO8601 extended hms? (=XXXXX) | `Z`, `ZZ`, or `ZZZ`<br/>`ZZZZ`<br/>`ZZZZZ` | -0800<br/>GMT-08:00<br/>-08:00, -07:52:58, Z |
|
||||
| `O` | Time Zone: short localized GMT<br/>Time Zone: long localized GMT (=ZZZZ) | `O`<br/>`OOOO` | GMT-8<br/>GMT-08:00 |
|
||||
| `v` | Time Zone: generic non-location<br/>(falls back first to VVVV) | `v`<br/>`vvvv` | PT<br/>Pacific Time or Los Angeles Time |
|
||||
| `V` | Time Zone: short time zone ID<br/>Time Zone: long time zone ID<br/>Time Zone: time zone exemplar city<br/>Time Zone: generic location (falls back to OOOO) | `V`<br/>`VV`<br/>`VVV`<br/>`VVVV` | uslax<br/>America/Los_Angeles<br/>Los Angeles<br/>Los Angeles Time |
|
||||
| `X` | Time Zone: ISO8601 basic hm?, with Z for 0<br/>Time Zone: ISO8601 basic hm, with Z<br/>Time Zone: ISO8601 extended hm, with Z<br/>Time Zone: ISO8601 basic hms?, with Z<br/>Time Zone: ISO8601 extended hms?, with Z | `X`<br/>`XX`<br/>`XXX`<br/>`XXXX`<br/>`XXXXX` | -08, +0530, Z<br/>-0800, Z<br/>-08:00, Z<br/>-0800, -075258, Z<br/>-08:00, -07:52:58, Z |
|
||||
| `x` | Time Zone: ISO8601 basic hm?, without Z for 0<br/>Time Zone: ISO8601 basic hm, without Z<br/>Time Zone: ISO8601 extended hm, without Z<br/>Time Zone: ISO8601 basic hms?, without Z<br/>Time Zone: ISO8601 extended hms?, without Z | `x`<br/>`xx`<br/>`xxx`<br/>`xxxx`<br/>`xxxxx` | -08, +0530<br/>-0800<br/>-08:00<br/>-0800, -075258<br/>-08:00, -07:52:58 |
|
||||
| `'` | escape for text | `'` | (nothing) |
|
||||
| `' '` | two single quotes produce one | `' '` | ' |
|
||||
|
||||
> :point_right: **Note**: *Any characters in the pattern that are not in the ranges of
|
||||
['a'..'z'] and ['A'..'Z'] will be treated as quoted text. For instance,
|
||||
characters like `':'`, `'.'`, `' '`, `'#'` and `'@'` will appear in the resulting time
|
||||
text even they are not enclosed within single quotes. The single quote is used to
|
||||
'escape' letters. Two single quotes in a row, whether inside or outside a quoted
|
||||
sequence, represent a 'real' single quote.*
|
||||
|
||||
> :point_right: **Note**: *A pattern containing any invalid pattern letter results in a failing `UErrorCode`
|
||||
result during formatting or parsing.*
|
||||
|
||||
| Format Pattern | Result |
|
||||
| --- | --- |
|
||||
| `"yyyy.MM.dd G 'at' HH:mm:ss zzz"` | "1996.07.10 AD at 15:08:56 PDT" |
|
||||
| `"EEE, MMM d, ''yy"` | "Wed, July 10, '96" |
|
||||
| `"h:mm a"` | "12:08 PM" |
|
||||
| `"hh 'o''clock' a, zzzz"` | "12 o'clock PM, Pacific Daylight Time" |
|
||||
| `"K:mm a, z"` | "0:00 PM, PST" |
|
||||
| `"yyyyy.MMMM.dd GGG hh:mm aaa"` | "01996.July.10 AD 12:08 PM" |
|
||||
|
||||
### Time Zone Display Names
|
||||
|
||||
ICU supports time zone display names defined by the LDML ([Unicode Locale Data
|
||||
Markup Language](http://www.unicode.org/reports/tr35/) ) specification. Since
|
||||
ICU 3.8, the vast majority of localized time zone names are no longer associated
|
||||
with individual time zones. Instead, a set of localized time zone names are
|
||||
associated with a *metazone* and one or more individual time zones are mapped to
|
||||
the same *metazone*. For example, *metazone* “America_Pacific” has its own
|
||||
display name data such as “PST” “PDT” “PT” “Pacific Standard Time” “Pacific
|
||||
Daylight Time” “Pacific Time” and these names are shared by multiple individual
|
||||
time zones “America/Los_Angeles”, “America/Vancouver”, “America/Tijuana” and so
|
||||
on. The mapping from individual time zone to *metazone* is not a simple 1-to-1
|
||||
mapping, but it changes time to time. For example, time zone
|
||||
“America/Indiana/Tell_City” uses name data from *metazone* “America_Eastern”
|
||||
until April 2, 2006, but it changes to *metazone* “America_Central” after the
|
||||
date. So the display name used for “America/Indiana/Tell_City” before the date
|
||||
(e.g. “Eastern Time”) differs from the one after the date (e.g. “Central Time).
|
||||
|
||||
> :point_right: **Note**: *Prior to ICU 3.8, a localized time zone name (except GMT format) and a time
|
||||
zone ID was in 1-to-1 relationship always. Therefore, a time zone name produced
|
||||
by `DateFormat` can be parsed back to the original time zone. This assumption no
|
||||
longer applies to ICU 3.8 and later releases for all time zone format types. If
|
||||
you program requires to roundtrip specific time zone ID, you must use the
|
||||
generic location format `VVVV` explained below.*
|
||||
|
||||
There are several different display name types available in the LDML
|
||||
specification.
|
||||
|
||||
#### Time Zone Display Name Types
|
||||
|
||||
| Type | Description | Examples |
|
||||
| --- | --- | --- |
|
||||
| Generic non-location | Reflects wall time, suited for displaying recurring events, meetings or anywhere people do not want to be overly specific. Available in two length options – long and short. | Pacific Time<br/>PT |
|
||||
| Generic partial location | Reflects wall time, used as a fallback format when the generic non-location format is not specific enough. A generic partial location name is constructed from a generic non-location name with a location name. For example, “PT” is shared by multiple time zones via metazone “America_Pacific”. When GMT offset in the time zone at the given time differs from the preferred time zone of the metazone for the locale, location name is appended to generic non-location name to distinguish the time zone from the preferred zone. Available in two length options – long and short. | Pacific Time (Canada)<br/>PT (Yellowknife) |
|
||||
| Generic location | Reflects wall time, suited for populating choice list for time zones. If the time zone is the single time zone available in the region (country), the generic location name is constructed with the region name. Otherwise, the name is constructed from the region name and the city name. Unlike other format types, this name is unique per time zone. | United States (Los Angeles) Time<br/>Italy Time |
|
||||
| Specific non-location | Reflects a specific standard or daylight time. Available in two length options – long and short. | Pacific Standard Time<br/>PDT |
|
||||
| Localized GMT | A constant, specific offset from GMT in a localized form. | GMT-08:00 |
|
||||
| RFC822 GMT | A constant, specific offset from GMT in a locale insensitive format. | -0800 |
|
||||
|
||||
Each format type in the above table is used as a primary type or a fallback in
|
||||
`SimpleDateFormat`. The table below explains how ICU time zone format pattern work
|
||||
and its characteristics.
|
||||
|
||||
#### Time Zone Pattern Usage
|
||||
|
||||
| Pattern | Behavior | Round-trip time at daylight transitions(\*) | Round-trip Time Zone | Suggested Usage |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| `z`, `zz`, `zzz` | Short specific non-location format (e.g. “PST”). If the localized data is not available or the short abbreviation is not commonly used for the locale, localized GMT format is used (e.g. "GMT-08:00"). | yes | no | For displaying a time with a user friendly time zone name. |
|
||||
| `zzzz` | Long specific non-location format (e.g. “Pacific Standard Time”). If the localized data is not available, localized GMT format is used (e.g. "GMT-08:00"). | yes | no | Same as `z`, but longer format. |
|
||||
| `v` | Short generic non-location format (e.g. “PT”). If the localized data is not available or the short abbreviation is not commonly used for the locale, generic location format (e.g. “United States(Los Angeles) Time”) is used. If the localized data comes from metazone and the GMT offset at the given time in the specified time zone differs from the preferred time zone of the metazone for the locale, generic partial location format (e.g. “PT (Canada)”) is used. | no | no | For displaying a recurring wall time (e.g. events, meetings) or anywhere people do not want to be overly specific. |
|
||||
| `vvvv` | Long generic non-location format (e.g. “Pacific Time”). If the localized data is not available, generic location format (e.g. “United States(Los Angeles) Time”) is used. | no | no | Same as `v`, but longer format. |
|
||||
| `V` | Same as `z`, except using the short abbreviation even it is not commonly used for the locale. | yes | no | Same as `z`. |
|
||||
| `VVVV` | Generic location format (e.g. “United States (Los Angeles) Time”). | no | yes | For populating a choice list for time zones, because it supports 1-to-1 name/zone ID mapping and is more uniform than other text formats. Also, this is only the pattern supporting time zone round-trip. If your program requires to preserve the original time zone information, use this pattern. |
|
||||
| `Z`, `ZZ`, `ZZZ` | Localized GMT format (e.g. “GMT-08:00”). | yes | no | For displaying a time in UI in a uniformed manner. |
|
||||
| `ZZZZ` | RFC822 GMT format (e.g. “-0800”). | yes | no | For formatting a time for non-user-facing data. |
|
||||
|
||||
\* At a transition from daylight saving time to standard time, there is a wall
|
||||
time interval occurs twice.
|
||||
|
||||
## `DateTimePatternGenerator`
|
||||
|
||||
The `DateTimePatternGenerator` class provides a way to map a request for a set of
|
||||
date/time fields, along with their width, to a locale-appropriate format
|
||||
pattern. The request is in the form of a “skeleton” which just contains pattern
|
||||
letters for the desired fields using the representation for the desired width.
|
||||
In a skeleton, anything other than a pattern letter is ignored, field order is
|
||||
insignificant, and there are two special additional pattern letters that may be
|
||||
used: `j` requests the preferred hour-cycle type for the locale (it gets mapped
|
||||
to one of `H`, `h`, `k`, or `K`); `J` is similar but requests no AM/PM marker
|
||||
even if the locale’s preferred hour-cycle type is `h` or `K`.
|
||||
|
||||
For example, a skeleton of `“MMMMdjmm”` might result in the following format
|
||||
patterns for different locales:
|
||||
|
||||
| locale | format pattern for skeleton `“MMMMdjmm”` | example |
|
||||
| ------ | ---------------------------------------- | -------------------- |
|
||||
| en_US | `"MMMM d 'at' h:mm a"` | "April 2 at 5:00 PM" |
|
||||
| es_ES | `"d 'de' MMMM, H:mm"` | "2 de abril, 17:00" |
|
||||
| ja_JP | `"M月d日 H:mm"` | "4月2日 17:00" |
|
||||
|
||||
The most important DateTimePatternGenerator methods are the varieties of
|
||||
getBestPattern.
|
||||
|
||||
Note that the fields in the format pattern may be adjusted as appropriate for
|
||||
the locale and may not exactly match those in the skeleton. For example:
|
||||
|
||||
* In Russian (locale "ru"), the skeleton `"yMMMM"` will produce the format
|
||||
pattern `"LLLL y"` (or `"LLLL y 'г'."`) since a month name without a day number
|
||||
must be in nominative form, as indicated by `LLLL`.
|
||||
* When using the Japanese calendar in the Japanese locale (locale
|
||||
"ja@calendar=japanese"), the skeleton `"yMMMd"` will produce the format
|
||||
pattern `"Gy年M月d日"` since the era must always be shown with the year in the
|
||||
Japanese calendar.
|
||||
|
||||
## `DateFormatSymbols`
|
||||
|
||||
`DateFormatSymbols` is a public class for encapsulating localizable date-time
|
||||
formatting data, including time zone data. `DateFormatSymbols` is used by
|
||||
`DateFormat` and `SimpleDateFormat`.
|
||||
|
||||
`DateFormatSymbols` specifies the exact character strings to use for various parts
|
||||
of a date or time For example, the names of the months and days of the week, the
|
||||
strings for AM and PM and the day of the week considered to be the first day of
|
||||
the week (used in drawing calendar grids) are controlled by `DateFormatSymbols`.
|
||||
|
||||
Create a date-time formatter using the `createTimeInstance`, `createDateInstance`,
|
||||
or `createDateTimeInstance` methods in `DateFormat`. Each of these methods can
|
||||
return a date/time formatter initialized with a default format pattern, along
|
||||
with the date-time formatting data for a given or default locale. After a
|
||||
formatter is created, modify the format pattern using `applyPattern`.
|
||||
|
||||
If you want to create a date-time formatter with a particular format pattern and
|
||||
locale, use one of the `SimpleDateFormat` constructors:
|
||||
|
||||
```cpp
|
||||
UnicodeString aPattern("GyyyyMMddHHmmssSSZ", "");
|
||||
new SimpleDateFormat(aPattern, new DateFormatSymbols(Locale::getUS())
|
||||
```
|
||||
|
||||
This loads the appropriate date-time formatting data from the locale.
|
||||
|
||||
## Programming Examples
|
||||
|
||||
See [date and time formatting examples](examples.md).
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user