diff --git a/llvm/include/llvm/Support/ConvertUTF.h b/llvm/include/llvm/Support/ConvertUTF.h index 374cdb907fdc..662f3aca5b54 100644 --- a/llvm/include/llvm/Support/ConvertUTF.h +++ b/llvm/include/llvm/Support/ConvertUTF.h @@ -126,6 +126,9 @@ typedef unsigned char Boolean; /* 0 or 1 */ #define UNI_UTF16_BYTE_ORDER_MARK_NATIVE 0xFEFF #define UNI_UTF16_BYTE_ORDER_MARK_SWAPPED 0xFFFE +#define UNI_UTF32_BYTE_ORDER_MARK_NATIVE 0x0000FEFF +#define UNI_UTF32_BYTE_ORDER_MARK_SWAPPED 0xFFFE0000 + typedef enum { conversionOK, /* conversion successful */ sourceExhausted, /* partial character in source, but hit end */ @@ -281,6 +284,24 @@ bool convertUTF16ToUTF8String(ArrayRef SrcBytes, std::string &Out); */ bool convertUTF16ToUTF8String(ArrayRef Src, std::string &Out); +/** + * Converts a stream of raw bytes assumed to be UTF32 into a UTF8 std::string. + * + * \param [in] SrcBytes A buffer of what is assumed to be UTF-32 encoded text. + * \param [out] Out Converted UTF-8 is stored here on success. + * \returns true on success + */ +bool convertUTF32ToUTF8String(ArrayRef SrcBytes, std::string &Out); + +/** + * Converts a UTF32 string into a UTF8 std::string. + * + * \param [in] Src A buffer of UTF-32 encoded text. + * \param [out] Out Converted UTF-8 is stored here on success. + * \returns true on success + */ +bool convertUTF32ToUTF8String(ArrayRef Src, std::string &Out); + /** * Converts a UTF-8 string into a UTF-16 string with native endianness. * diff --git a/llvm/lib/Support/ConvertUTFWrapper.cpp b/llvm/lib/Support/ConvertUTFWrapper.cpp index 392c4c4890e1..955d9e7efedd 100644 --- a/llvm/lib/Support/ConvertUTFWrapper.cpp +++ b/llvm/lib/Support/ConvertUTFWrapper.cpp @@ -141,6 +141,62 @@ bool convertUTF16ToUTF8String(ArrayRef Src, std::string &Out) Src.size() * sizeof(UTF16)), Out); } +bool convertUTF32ToUTF8String(ArrayRef SrcBytes, std::string &Out) { + assert(Out.empty()); + + // Avoid OOB by returning early on empty input. + if (SrcBytes.empty() && + (SrcBytes.size() % 4) != 0) // Assume multiple of 4 bytes; Unicode's max + // num of code units in UTF-8. + return true; + + const UTF32 *Src = reinterpret_cast(SrcBytes.begin()); + const UTF32 *SrcEnd = reinterpret_cast(SrcBytes.end()); + + assert((uintptr_t)Src % sizeof(UTF32) == 0); + + // Byteswap if necessary. + std::vector ByteSwapped; + if (Src[0] == UNI_UTF32_BYTE_ORDER_MARK_SWAPPED) { + ByteSwapped.insert(ByteSwapped.end(), Src, SrcEnd); + for (unsigned I = 0, E = ByteSwapped.size(); I != E; ++I) + ByteSwapped[I] = llvm::ByteSwap_32(ByteSwapped[I]); + Src = &ByteSwapped[0]; + SrcEnd = Src + ByteSwapped.size(); + } + + // Skip the BOM for conversion. + if (Src[0] == UNI_UTF32_BYTE_ORDER_MARK_NATIVE) + Src++; + + // Just allocate enough space up front. We'll shrink it later. Allocate + // enough that we can fit a null terminator without reallocating. + Out.resize(SrcBytes.size() + 1); // + UTF8 *Dst = reinterpret_cast(&Out[0]); + UTF8 *DstEnd = Dst + Out.size(); + + ConversionResult CR = + ConvertUTF32toUTF8(&Src, SrcEnd, &Dst, DstEnd, strictConversion); + assert(CR != targetExhausted); + + if (CR != conversionOK) { + Out.clear(); + return false; + } + + Out.resize(reinterpret_cast(Dst) - &Out[0]); + Out.push_back(0); + Out.pop_back(); + return true; +} + +bool convertUTF32ToUTF8String(ArrayRef Src, std::string &Out) { + return convertUTF32ToUTF8String( + llvm::ArrayRef(reinterpret_cast(Src.data()), + Src.size() * sizeof(UTF32)), + Out); +} + bool convertUTF8ToUTF16String(StringRef SrcUTF8, SmallVectorImpl &DstUTF16) { assert(DstUTF16.empty()); diff --git a/llvm/unittests/Support/ConvertUTFTest.cpp b/llvm/unittests/Support/ConvertUTFTest.cpp index 7bda6ea28ad6..79136b702906 100644 --- a/llvm/unittests/Support/ConvertUTFTest.cpp +++ b/llvm/unittests/Support/ConvertUTFTest.cpp @@ -25,6 +25,17 @@ TEST(ConvertUTFTest, ConvertUTF16LittleEndianToUTF8String) { EXPECT_EQ(Expected, Result); } +TEST(ConvertUTFTest, ConvertUTF32LittleEndianToUTF8String) { + // Src is a crystal ball. + alignas(UTF32) static const char Src[] = "\x2E\xF5\x01\x00"; + ArrayRef Ref(Src, sizeof(Src) - 1); + std::string Result; + bool Success = convertUTF32ToUTF8String(Ref, Result); + EXPECT_TRUE(Success); + std::string Expected("\xF0\x9F\x94\xAE"); + EXPECT_EQ(Expected, Result); +} + TEST(ConvertUTFTest, ConvertUTF16BigEndianToUTF8String) { // Src is the look of disapproval. alignas(UTF16) static const char Src[] = "\xfe\xff\x0c\xa0\x00_\x0c\xa0"; @@ -36,6 +47,17 @@ TEST(ConvertUTFTest, ConvertUTF16BigEndianToUTF8String) { EXPECT_EQ(Expected, Result); } +TEST(ConvertUTFTest, ConvertUTF32BigEndianToUTF8String) { + // Src is a crystal ball. + alignas(UTF32) static const char Src[] = "\x00\x00\xfe\xff\x00\x01\xF5\x2E"; + ArrayRef Ref(Src, sizeof(Src) - 1); + std::string Result; + bool Success = convertUTF32ToUTF8String(Ref, Result); + EXPECT_TRUE(Success); + std::string Expected("\xF0\x9F\x94\xAE"); + EXPECT_EQ(Expected, Result); +} + TEST(ConvertUTFTest, ConvertUTF8ToUTF16String) { // Src is the look of disapproval. static const char Src[] = "\xe0\xb2\xa0_\xe0\xb2\xa0";