Google Git Google Git
Sign in
gnu/gcc/0b6038c17cabdbc8b9bfd0d13e2dd6f74db78734/./libphobos/src/std/utf.d
blob: 7db120befd5ec74fdc0310ca511831f27b304e3a [file]
// Written in the D programming language.
/++
Encode and decode UTF-8, UTF-16 and UTF-32 strings.
UTF character support is restricted to
$(D '\u0000' <= character <= '\U0010FFFF').
$(SCRIPT inhibitQuickIndex = 1;)
$(DIVC quickindex,
$(BOOKTABLE,
$(TR $(TH Category) $(TH Functions))
$(TR $(TD Decode) $(TD
$(LREF decode)
$(LREF decodeFront)
))
$(TR $(TD Lazy decode) $(TD
$(LREF byCodeUnit)
$(LREF byChar)
$(LREF byWchar)
$(LREF byDchar)
$(LREF byUTF)
))
$(TR $(TD Encode) $(TD
$(LREF encode)
$(LREF toUTF8)
$(LREF toUTF16)
$(LREF toUTF32)
$(LREF toUTFz)
$(LREF toUTF16z)
))
$(TR $(TD Length) $(TD
$(LREF codeLength)
$(LREF count)
$(LREF stride)
$(LREF strideBack)
))
$(TR $(TD Index) $(TD
$(LREF toUCSindex)
$(LREF toUTFindex)
))
$(TR $(TD Validation) $(TD
$(LREF isValidDchar)
$(LREF isValidCodepoint)
$(LREF validate)
))
$(TR $(TD Miscellaneous) $(TD
$(LREF replacementDchar)
$(LREF UseReplacementDchar)
$(LREF UTFException)
))
))
See_Also:
$(LINK2 http://en.wikipedia.org/wiki/Unicode, Wikipedia)<br>
$(LINK http://www.cl.cam.ac.uk/~mgk25/unicode.html#utf-8)<br>
$(LINK https://web.archive.org/web/20100113043530/https://anubis.dkuug.dk/JTC1/SC2/WG2/docs/n1335)
Copyright: Copyright The D Language Foundation 2000 - 2012.
License: $(HTTP www.boost.org/LICENSE_1_0.txt, Boost License 1.0).
Authors: $(HTTP digitalmars.com, Walter Bright) and
$(HTTP jmdavisprog.com, Jonathan M Davis)
Source: $(PHOBOSSRC std/utf.d)
+/
module std.utf;
import std.exception : basicExceptionCtors;
import core.exception : UnicodeException;
import std.meta : AliasSeq;
import std.range;
import std.traits : isAutodecodableString, isConvertibleToString,
isSomeChar, isSomeString, isStaticArray, Unqual;
import std.typecons : Flag, Yes, No;
/++
Exception thrown on errors in std.utf functions.
+/
class UTFException : UnicodeException
{
import core.internal.string : unsignedToTempString, UnsignedStringBuf;
uint[4] sequence;
size_t len;
@safe pure nothrow @nogc
UTFException setSequence(scope uint[] data...) return
{
assert(data.length <= 4);
len = data.length < 4 ? data.length : 4;
sequence[0 .. len] = data[0 .. len];
return this;
}
// FIXME: Use std.exception.basicExceptionCtors here once
// https://issues.dlang.org/show_bug.cgi?id=11500 is fixed
/**
Standard exception constructors.
*/
this(string msg, string file = __FILE__, size_t line = __LINE__,
Throwable next = null) @nogc @safe pure nothrow
{
super(msg, 0, file, line, next);
}
/// ditto
this(string msg, size_t index, string file = __FILE__,
size_t line = __LINE__, Throwable next = null) @safe pure nothrow
{
UnsignedStringBuf buf = void;
msg ~= " (at index " ~ unsignedToTempString(index, buf) ~ ")";
super(msg, index, file, line, next);
}
/**
Returns:
A `string` detailing the invalid UTF sequence.
*/
override string toString() const
{
if (len == 0)
{
/* Exception.toString() is not marked as const, although
* it is const-compatible.
*/
//return super.toString();
auto e = () @trusted { return cast(Exception) super; } ();
return e.toString();
}
string result = "Invalid UTF sequence:";
foreach (i; sequence[0 .. len])
{
UnsignedStringBuf buf = void;
result ~= ' ';
auto h = unsignedToTempString!16(i, buf);
if (h.length == 1)
result ~= '0';
result ~= h;
result ~= 'x';
}
if (super.msg.length > 0)
{
result ~= " - ";
result ~= super.msg;
}
return result;
}
}
///
@safe unittest
{
import std.exception : assertThrown;
char[4] buf;
assertThrown!UTFException(encode(buf, cast(dchar) 0xD800));
assertThrown!UTFException(encode(buf, cast(dchar) 0xDBFF));
assertThrown!UTFException(encode(buf, cast(dchar) 0xDC00));
assertThrown!UTFException(encode(buf, cast(dchar) 0xDFFF));
assertThrown!UTFException(encode(buf, cast(dchar) 0x110000));
}
/*
Provide array of invalidly encoded UTF strings. Useful for testing.
Params:
Char = char, wchar, or dchar
Returns:
an array of invalidly encoded UTF strings
*/
package auto invalidUTFstrings(Char)() @safe pure @nogc nothrow
if (isSomeChar!Char)
{
static if (is(Char == char))
{
enum x = 0xDC00; // invalid surrogate value
enum y = 0x110000; // out of range
static immutable string[8] result =
[
"\x80", // not a start byte
"\xC0", // truncated
"\xC0\xC0", // invalid continuation
"\xF0\x82\x82\xAC", // overlong
[
0xE0 | (x >> 12),
0x80 | ((x >> 6) & 0x3F),
0x80 | (x & 0x3F)
],
[
cast(char)(0xF0 | (y >> 18)),
cast(char)(0x80 | ((y >> 12) & 0x3F)),
cast(char)(0x80 | ((y >> 6) & 0x3F)),
cast(char)(0x80 | (y & 0x3F))
],
[
cast(char)(0xF8 | 3), // 5 byte encoding
cast(char)(0x80 | 3),
cast(char)(0x80 | 3),
cast(char)(0x80 | 3),
cast(char)(0x80 | 3),
],
[
cast(char)(0xFC | 3), // 6 byte encoding
cast(char)(0x80 | 3),
cast(char)(0x80 | 3),
cast(char)(0x80 | 3),
cast(char)(0x80 | 3),
cast(char)(0x80 | 3),
],
];
return result[];
}
else static if (is(Char == wchar))
{
static immutable wstring[5] result =
[
[
cast(wchar) 0xDC00,
],
[
cast(wchar) 0xDFFF,
],
[
cast(wchar) 0xDBFF,
cast(wchar) 0xDBFF,
],
[
cast(wchar) 0xDBFF,
cast(wchar) 0xE000,
],
[
cast(wchar) 0xD800,
],
];
return result[];
}
else static if (is(Char == dchar))
{
static immutable dstring[3] result =
[
[ cast(dchar) 0x110000 ],
[ cast(dchar) 0x00D800 ],
[ cast(dchar) 0x00DFFF ],
];
return result;
}
else
static assert(0);
}
/++
Check whether the given Unicode code point is valid.
Params:
c = code point to check
Returns:
`true` if and only if `c` is a valid Unicode code point
Note:
`'\uFFFE'` and `'\uFFFF'` are considered valid by `isValidDchar`,
as they are permitted for internal use by an application, but they are
not allowed for interchange by the Unicode standard.
+/
bool isValidDchar(dchar c) pure nothrow @safe @nogc
{
return c < 0xD800 || (c > 0xDFFF && c <= 0x10FFFF);
}
///
@safe @nogc pure nothrow unittest
{
assert( isValidDchar(cast(dchar) 0x41));
assert( isValidDchar(cast(dchar) 0x00));
assert(!isValidDchar(cast(dchar) 0xD800));
assert(!isValidDchar(cast(dchar) 0x11FFFF));
}
pure nothrow @safe @nogc unittest
{
import std.exception;
assertCTFEable!(
{
assert( isValidDchar(cast(dchar)'a') == true);
assert( isValidDchar(cast(dchar) 0x1FFFFF) == false);
assert(!isValidDchar(cast(dchar) 0x00D800));
assert(!isValidDchar(cast(dchar) 0x00DBFF));
assert(!isValidDchar(cast(dchar) 0x00DC00));
assert(!isValidDchar(cast(dchar) 0x00DFFF));
assert( isValidDchar(cast(dchar) 0x00FFFE));
assert( isValidDchar(cast(dchar) 0x00FFFF));
assert( isValidDchar(cast(dchar) 0x01FFFF));
assert( isValidDchar(cast(dchar) 0x10FFFF));
assert(!isValidDchar(cast(dchar) 0x110000));
});
}
/**
Checks if a single character forms a valid code point.
When standing alone, some characters are invalid code points. For
example the `wchar` `0xD800` is a so called high surrogate, which can
only be interpreted together with a low surrogate following it. As a
standalone character it is considered invalid.
See $(LINK2 http://www.unicode.org/versions/Unicode13.0.0/,
Unicode Standard, D90, D91 and D92) for more details.
Params:
c = character to test
Char = character type of `c`
Returns:
`true`, if `c` forms a valid code point.
*/
bool isValidCodepoint(Char)(Char c)
if (isSomeChar!Char)
{
alias UChar = typeof(cast() c);
static if (is(UChar == char))
{
return c <= 0x7F;
}
else static if (is(UChar == wchar))
{
return c <= 0xD7FF || c >= 0xE000;
}
else static if (is(UChar == dchar))
{
return isValidDchar(c);
}
else
static assert(false, "unknown character type: `" ~ Char.stringof ~ "`");
}
///
@safe pure nothrow unittest
{
assert( isValidCodepoint(cast(char) 0x40));
assert(!isValidCodepoint(cast(char) 0x80));
assert( isValidCodepoint(cast(wchar) 0x1234));
assert(!isValidCodepoint(cast(wchar) 0xD800));
assert( isValidCodepoint(cast(dchar) 0x0010FFFF));
assert(!isValidCodepoint(cast(dchar) 0x12345678));
}
/++
Calculate the length of the UTF sequence starting at `index`
in `str`.
Params:
str = $(REF_ALTTEXT input range, isInputRange, std,range,primitives)
of UTF code units. Must be random access if `index` is passed
index = starting index of UTF sequence (default: `0`)
Returns:
The number of code units in the UTF sequence. For UTF-8, this is a
value between 1 and 4 (as per $(HTTP tools.ietf.org/html/rfc3629#section-3, RFC 3629$(COMMA) section 3)).
For UTF-16, it is either 1 or 2. For UTF-32, it is always 1.
Throws:
May throw a `UTFException` if `str[index]` is not the start of a
valid UTF sequence.
Note:
`stride` will only analyze the first `str[index]` element. It
will not fully verify the validity of the UTF sequence, nor even verify
the presence of the sequence: it will not actually guarantee that
$(D index + stride(str, index) <= str.length).
+/
uint stride(S)(auto ref S str, size_t index)
if (is(S : const char[]) ||
(isRandomAccessRange!S && is(immutable ElementType!S == immutable char)))
{
static if (is(typeof(str.length) : ulong))
assert(index < str.length, "Past the end of the UTF-8 sequence");
immutable c = str[index];
if (c < 0x80)
return 1;
else
return strideImpl(c, index);
}
/// Ditto
uint stride(S)(auto ref S str)
if (is(S : const char[]) ||
(isInputRange!S && is(immutable ElementType!S == immutable char)))
{
static if (is(S : const char[]))
immutable c = str[0];
else
immutable c = str.front;
if (c < 0x80)
return 1;
else
return strideImpl(c, 0);
}
@system unittest
{
import core.exception : AssertError;
import std.conv : to;
import std.exception;
import std.string : format;
import std.traits : FunctionAttribute, functionAttributes, isSafe;
static void test(string s, dchar c, size_t i = 0, size_t line = __LINE__)
{
enforce(stride(s, i) == codeLength!char(c),
new AssertError(format("Unit test failure string: %s", s), __FILE__, line));
enforce(stride(RandomCU!char(s), i) == codeLength!char(c),
new AssertError(format("Unit test failure range: %s", s), __FILE__, line));
auto refRandom = new RefRandomCU!char(s);
immutable randLen = refRandom.length;
enforce(stride(refRandom, i) == codeLength!char(c),
new AssertError(format("Unit test failure rand ref range: %s", s), __FILE__, line));
enforce(refRandom.length == randLen,
new AssertError(format("Unit test failure rand ref range length: %s", s), __FILE__, line));
if (i == 0)
{
enforce(stride(s) == codeLength!char(c),
new AssertError(format("Unit test failure string 0: %s", s), __FILE__, line));
enforce(stride(InputCU!char(s)) == codeLength!char(c),
new AssertError(format("Unit test failure range 0: %s", s), __FILE__, line));
auto refBidir = new RefBidirCU!char(s);
immutable bidirLen = refBidir.length;
enforce(stride(refBidir) == codeLength!char(c),
new AssertError(format("Unit test failure bidir ref range code length: %s", s), __FILE__, line));
enforce(refBidir.length == bidirLen,
new AssertError(format("Unit test failure bidir ref range length: %s", s), __FILE__, line));
}
}
assertCTFEable!(
{
test("a", 'a');
test(" ", ' ');
test("\u2029", '\u2029'); //paraSep
test("\u0100", '\u0100');
test("\u0430", '\u0430');
test("\U00010143", '\U00010143');
test("abcdefcdef", 'a');
test("hello\U00010143\u0100\U00010143", 'h', 0);
test("hello\U00010143\u0100\U00010143", 'e', 1);
test("hello\U00010143\u0100\U00010143", 'l', 2);
test("hello\U00010143\u0100\U00010143", 'l', 3);
test("hello\U00010143\u0100\U00010143", 'o', 4);
test("hello\U00010143\u0100\U00010143", '\U00010143', 5);
test("hello\U00010143\u0100\U00010143", '\u0100', 9);
test("hello\U00010143\u0100\U00010143", '\U00010143', 11);
foreach (S; AliasSeq!(char[], const char[], string))
{
enum str = to!S("hello world");
static assert(isSafe!({ stride(str, 0); }));
static assert(isSafe!({ stride(str); }));
static assert((functionAttributes!({ stride(str, 0); }) & FunctionAttribute.pure_) != 0);
static assert((functionAttributes!({ stride(str); }) & FunctionAttribute.pure_) != 0);
}
});
}
@safe unittest // invalid start bytes
{
import std.exception : assertThrown;
immutable char[] invalidStartBytes = [
0b1111_1000, // indicating a sequence length of 5
0b1111_1100, // 6
0b1111_1110, // 7
0b1111_1111, // 8
0b1000_0000, // continuation byte
];
foreach (c; invalidStartBytes)
assertThrown!UTFException(stride([c]));
}
/// Ditto
uint stride(S)(auto ref S str, size_t index)
if (is(S : const wchar[]) ||
(isRandomAccessRange!S && is(immutable ElementType!S == immutable wchar)))
{
static if (is(typeof(str.length) : ulong))
assert(index < str.length, "Past the end of the UTF-16 sequence");
immutable uint u = str[index];
return 1 + (u >= 0xD800 && u <= 0xDBFF);
}
/// Ditto
uint stride(S)(auto ref S str) @safe pure
if (is(S : const wchar[]))
{
return stride(str, 0);
}
/// Ditto
uint stride(S)(auto ref S str)
if (isInputRange!S && is(immutable ElementType!S == immutable wchar) &&
!is(S : const wchar[]))
{
assert(!str.empty, "UTF-16 sequence is empty");
immutable uint u = str.front;
return 1 + (u >= 0xD800 && u <= 0xDBFF);
}
@system unittest
{
import core.exception : AssertError;
import std.conv : to;
import std.exception;
import std.string : format;
import std.traits : FunctionAttribute, functionAttributes, isSafe;
static void test(wstring s, dchar c, size_t i = 0, size_t line = __LINE__)
{
enforce(stride(s, i) == codeLength!wchar(c),
new AssertError(format("Unit test failure string: %s", s), __FILE__, line));
enforce(stride(RandomCU!wchar(s), i) == codeLength!wchar(c),
new AssertError(format("Unit test failure range: %s", s), __FILE__, line));
auto refRandom = new RefRandomCU!wchar(s);
immutable randLen = refRandom.length;
enforce(stride(refRandom, i) == codeLength!wchar(c),
new AssertError(format("Unit test failure rand ref range: %s", s), __FILE__, line));
enforce(refRandom.length == randLen,
new AssertError(format("Unit test failure rand ref range length: %s", s), __FILE__, line));
if (i == 0)
{
enforce(stride(s) == codeLength!wchar(c),
new AssertError(format("Unit test failure string 0: %s", s), __FILE__, line));
enforce(stride(InputCU!wchar(s)) == codeLength!wchar(c),
new AssertError(format("Unit test failure range 0: %s", s), __FILE__, line));
auto refBidir = new RefBidirCU!wchar(s);
immutable bidirLen = refBidir.length;
enforce(stride(refBidir) == codeLength!wchar(c),
new AssertError(format("Unit test failure bidir ref range code length: %s", s), __FILE__, line));
enforce(refBidir.length == bidirLen,
new AssertError(format("Unit test failure bidir ref range length: %s", s), __FILE__, line));
}
}
assertCTFEable!(
{
test("a", 'a');
test(" ", ' ');
test("\u2029", '\u2029'); //paraSep
test("\u0100", '\u0100');
test("\u0430", '\u0430');
test("\U00010143", '\U00010143');
test("abcdefcdef", 'a');
test("hello\U00010143\u0100\U00010143", 'h', 0);
test("hello\U00010143\u0100\U00010143", 'e', 1);
test("hello\U00010143\u0100\U00010143", 'l', 2);
test("hello\U00010143\u0100\U00010143", 'l', 3);
test("hello\U00010143\u0100\U00010143", 'o', 4);
test("hello\U00010143\u0100\U00010143", '\U00010143', 5);
test("hello\U00010143\u0100\U00010143", '\u0100', 7);
test("hello\U00010143\u0100\U00010143", '\U00010143', 8);
foreach (S; AliasSeq!(wchar[], const wchar[], wstring))
{
enum str = to!S("hello world");
static assert(isSafe!(() => stride(str, 0)));
static assert(isSafe!(() => stride(str) ));
static assert((functionAttributes!(() => stride(str, 0)) & FunctionAttribute.pure_) != 0);
static assert((functionAttributes!(() => stride(str) ) & FunctionAttribute.pure_) != 0);
}
});
}
/// Ditto
uint stride(S)(auto ref S str, size_t index = 0)
if (is(S : const dchar[]) ||
(isInputRange!S && is(immutable ElementEncodingType!S == immutable dchar)))
{
static if (is(typeof(str.length) : ulong))
assert(index < str.length, "Past the end of the UTF-32 sequence");
else
assert(!str.empty, "UTF-32 sequence is empty.");
return 1;
}
///
@safe unittest
{
assert("a".stride == 1);
assert("λ".stride == 2);
assert("aλ".stride == 1);
assert("aλ".stride(1) == 2);
assert("𐐷".stride == 4);
}
@system unittest
{
import core.exception : AssertError;
import std.conv : to;
import std.exception;
import std.string : format;
import std.traits : FunctionAttribute, functionAttributes, isSafe;
static void test(dstring s, dchar c, size_t i = 0, size_t line = __LINE__)
{
enforce(stride(s, i) == codeLength!dchar(c),
new AssertError(format("Unit test failure string: %s", s), __FILE__, line));
enforce(stride(RandomCU!dchar(s), i) == codeLength!dchar(c),
new AssertError(format("Unit test failure range: %s", s), __FILE__, line));
auto refRandom = new RefRandomCU!dchar(s);
immutable randLen = refRandom.length;
enforce(stride(refRandom, i) == codeLength!dchar(c),
new AssertError(format("Unit test failure rand ref range: %s", s), __FILE__, line));
enforce(refRandom.length == randLen,
new AssertError(format("Unit test failure rand ref range length: %s", s), __FILE__, line));
if (i == 0)
{
enforce(stride(s) == codeLength!dchar(c),
new AssertError(format("Unit test failure string 0: %s", s), __FILE__, line));
enforce(stride(InputCU!dchar(s)) == codeLength!dchar(c),
new AssertError(format("Unit test failure range 0: %s", s), __FILE__, line));
auto refBidir = new RefBidirCU!dchar(s);
immutable bidirLen = refBidir.length;
enforce(stride(refBidir) == codeLength!dchar(c),
new AssertError(format("Unit test failure bidir ref range code length: %s", s), __FILE__, line));
enforce(refBidir.length == bidirLen,
new AssertError(format("Unit test failure bidir ref range length: %s", s), __FILE__, line));
}
}
assertCTFEable!(
{
test("a", 'a');
test(" ", ' ');
test("\u2029", '\u2029'); //paraSep
test("\u0100", '\u0100');
test("\u0430", '\u0430');
test("\U00010143", '\U00010143');
test("abcdefcdef", 'a');
test("hello\U00010143\u0100\U00010143", 'h', 0);
test("hello\U00010143\u0100\U00010143", 'e', 1);
test("hello\U00010143\u0100\U00010143", 'l', 2);
test("hello\U00010143\u0100\U00010143", 'l', 3);
test("hello\U00010143\u0100\U00010143", 'o', 4);
test("hello\U00010143\u0100\U00010143", '\U00010143', 5);
test("hello\U00010143\u0100\U00010143", '\u0100', 6);
test("hello\U00010143\u0100\U00010143", '\U00010143', 7);
foreach (S; AliasSeq!(dchar[], const dchar[], dstring))
{
enum str = to!S("hello world");
static assert(isSafe!(() => stride(str, 0)));
static assert(isSafe!(() => stride(str) ));
static assert((functionAttributes!(() => stride(str, 0)) & FunctionAttribute.pure_) != 0);
static assert((functionAttributes!(() => stride(str) ) & FunctionAttribute.pure_) != 0);
}
});
}
private uint strideImpl(char c, size_t index) @trusted pure
in { assert(c & 0x80); }
do
{
import core.bitop : bsr;
immutable msbs = 7 - bsr((~uint(c)) & 0xFF);
if (c == 0xFF || msbs < 2 || msbs > 4)
throw new UTFException("Invalid UTF-8 sequence", index);
return msbs;
}
/++
Calculate the length of the UTF sequence ending one code unit before
`index` in `str`.
Params:
str = bidirectional range of UTF code units. Must be random access if
`index` is passed
index = index one past end of UTF sequence (default: `str.length`)
Returns:
The number of code units in the UTF sequence. For UTF-8, this is a
value between 1 and 4 (as per $(HTTP tools.ietf.org/html/rfc3629#section-3, RFC 3629$(COMMA) section 3)).
For UTF-16, it is either 1 or 2. For UTF-32, it is always 1.
Throws:
May throw a `UTFException` if `str[index]` is not one past the
end of a valid UTF sequence.
Note:
`strideBack` will only analyze the element at $(D str[index - 1])
element. It will not fully verify the validity of the UTF sequence, nor
even verify the presence of the sequence: it will not actually
guarantee that $(D strideBack(str, index) <= index).
+/
uint strideBack(S)(auto ref S str, size_t index)
if (is(S : const char[]) ||
(isRandomAccessRange!S && is(immutable ElementType!S == immutable char)))
{
static if (is(typeof(str.length) : ulong))
assert(index <= str.length, "Past the end of the UTF-8 sequence");
assert(index > 0, "Not the end of the UTF-8 sequence");
if ((str[index-1] & 0b1100_0000) != 0b1000_0000)
return 1;
if (index >= 4) //single verification for most common case
{
static foreach (i; 2 .. 5)
{
if ((str[index-i] & 0b1100_0000) != 0b1000_0000)
return i;
}
}
else
{
static foreach (i; 2 .. 4)
{
if (index >= i && (str[index-i] & 0b1100_0000) != 0b1000_0000)
return i;
}
}
throw new UTFException("Not the end of the UTF sequence", index);
}
/// Ditto
uint strideBack(S)(auto ref S str)
if (is(S : const char[]) ||
(isRandomAccessRange!S && hasLength!S && is(immutable ElementType!S == immutable char)))
{
return strideBack(str, str.length);
}
/// Ditto
uint strideBack(S)(auto ref S str)
if (isBidirectionalRange!S && is(immutable ElementType!S == immutable char) && !isRandomAccessRange!S)
{
assert(!str.empty, "Past the end of the UTF-8 sequence");
auto temp = str.save;
foreach (i; AliasSeq!(1, 2, 3, 4))
{
if ((temp.back & 0b1100_0000) != 0b1000_0000)
return i;
temp.popBack();
if (temp.empty)
break;
}
throw new UTFException("The last code unit is not the end of the UTF-8 sequence");
}
@system unittest
{
import core.exception : AssertError;
import std.conv : to;
import std.exception;
import std.string : format;
import std.traits : FunctionAttribute, functionAttributes, isSafe;
static void test(string s, dchar c, size_t i = size_t.max, size_t line = __LINE__)
{
enforce(strideBack(s, i == size_t.max ? s.length : i) == codeLength!char(c),
new AssertError(format("Unit test failure string: %s", s), __FILE__, line));
enforce(strideBack(RandomCU!char(s), i == size_t.max ? s.length : i) == codeLength!char(c),
new AssertError(format("Unit test failure range: %s", s), __FILE__, line));
auto refRandom = new RefRandomCU!char(s);
immutable randLen = refRandom.length;
enforce(strideBack(refRandom, i == size_t.max ? s.length : i) == codeLength!char(c),
new AssertError(format("Unit test failure rand ref range: %s", s), __FILE__, line));
enforce(refRandom.length == randLen,
new AssertError(format("Unit test failure rand ref range length: %s", s), __FILE__, line));
if (i == size_t.max)
{
enforce(strideBack(s) == codeLength!char(c),
new AssertError(format("Unit test failure string code length: %s", s), __FILE__, line));
enforce(strideBack(BidirCU!char(s)) == codeLength!char(c),
new AssertError(format("Unit test failure range code length: %s", s), __FILE__, line));
auto refBidir = new RefBidirCU!char(s);
immutable bidirLen = refBidir.length;
enforce(strideBack(refBidir) == codeLength!char(c),
new AssertError(format("Unit test failure bidir ref range code length: %s", s), __FILE__, line));
enforce(refBidir.length == bidirLen,
new AssertError(format("Unit test failure bidir ref range length: %s", s), __FILE__, line));
}
}
assertCTFEable!(
{
test("a", 'a');
test(" ", ' ');
test("\u2029", '\u2029'); //paraSep
test("\u0100", '\u0100');
test("\u0430", '\u0430');
test("\U00010143", '\U00010143');
test("abcdefcdef", 'f');
test("\U00010143\u0100\U00010143hello", 'o', 15);
test("\U00010143\u0100\U00010143hello", 'l', 14);
test("\U00010143\u0100\U00010143hello", 'l', 13);
test("\U00010143\u0100\U00010143hello", 'e', 12);
test("\U00010143\u0100\U00010143hello", 'h', 11);
test("\U00010143\u0100\U00010143hello", '\U00010143', 10);
test("\U00010143\u0100\U00010143hello", '\u0100', 6);
test("\U00010143\u0100\U00010143hello", '\U00010143', 4);
foreach (S; AliasSeq!(char[], const char[], string))
{
enum str = to!S("hello world");
static assert(isSafe!({ strideBack(str, 0); }));
static assert(isSafe!({ strideBack(str); }));
static assert((functionAttributes!({ strideBack(str, 0); }) & FunctionAttribute.pure_) != 0);
static assert((functionAttributes!({ strideBack(str); }) & FunctionAttribute.pure_) != 0);
}
});
}
//UTF-16 is self synchronizing: The length of strideBack can be found from
//the value of a single wchar
/// Ditto
uint strideBack(S)(auto ref S str, size_t index)
if (is(S : const wchar[]) ||
(isRandomAccessRange!S && is(immutable ElementType!S == immutable wchar)))
{
static if (is(typeof(str.length) : ulong))
assert(index <= str.length, "Past the end of the UTF-16 sequence");
assert(index > 0, "Not the end of a UTF-16 sequence");
immutable c2 = str[index-1];
return 1 + (0xDC00 <= c2 && c2 < 0xE000);
}
/// Ditto
uint strideBack(S)(auto ref S str)
if (is(S : const wchar[]) ||
(isBidirectionalRange!S && is(immutable ElementType!S == immutable wchar)))
{
assert(!str.empty, "UTF-16 sequence is empty");
static if (is(S : const(wchar)[]))
immutable c2 = str[$ - 1];
else
immutable c2 = str.back;
return 1 + (0xDC00 <= c2 && c2 <= 0xE000);
}
@system unittest
{
import core.exception : AssertError;
import std.conv : to;
import std.exception;
import std.string : format;
import std.traits : FunctionAttribute, functionAttributes, isSafe;
static void test(wstring s, dchar c, size_t i = size_t.max, size_t line = __LINE__)
{
enforce(strideBack(s, i == size_t.max ? s.length : i) == codeLength!wchar(c),
new AssertError(format("Unit test failure string: %s", s), __FILE__, line));
enforce(strideBack(RandomCU!wchar(s), i == size_t.max ? s.length : i) == codeLength!wchar(c),
new AssertError(format("Unit test failure range: %s", s), __FILE__, line));
auto refRandom = new RefRandomCU!wchar(s);
immutable randLen = refRandom.length;
enforce(strideBack(refRandom, i == size_t.max ? s.length : i) == codeLength!wchar(c),
new AssertError(format("Unit test failure rand ref range: %s", s), __FILE__, line));
enforce(refRandom.length == randLen,
new AssertError(format("Unit test failure rand ref range length: %s", s), __FILE__, line));
if (i == size_t.max)
{
enforce(strideBack(s) == codeLength!wchar(c),
new AssertError(format("Unit test failure string code length: %s", s), __FILE__, line));
enforce(strideBack(BidirCU!wchar(s)) == codeLength!wchar(c),
new AssertError(format("Unit test failure range code length: %s", s), __FILE__, line));
auto refBidir = new RefBidirCU!wchar(s);
immutable bidirLen = refBidir.length;
enforce(strideBack(refBidir) == codeLength!wchar(c),
new AssertError(format("Unit test failure bidir ref range code length: %s", s), __FILE__, line));
enforce(refBidir.length == bidirLen,
new AssertError(format("Unit test failure bidir ref range length: %s", s), __FILE__, line));
}
}
assertCTFEable!(
{
test("a", 'a');
test(" ", ' ');
test("\u2029", '\u2029'); //paraSep
test("\u0100", '\u0100');
test("\u0430", '\u0430');
test("\U00010143", '\U00010143');
test("abcdefcdef", 'f');
test("\U00010143\u0100\U00010143hello", 'o', 10);
test("\U00010143\u0100\U00010143hello", 'l', 9);
test("\U00010143\u0100\U00010143hello", 'l', 8);
test("\U00010143\u0100\U00010143hello", 'e', 7);
test("\U00010143\u0100\U00010143hello", 'h', 6);
test("\U00010143\u0100\U00010143hello", '\U00010143', 5);
test("\U00010143\u0100\U00010143hello", '\u0100', 3);
test("\U00010143\u0100\U00010143hello", '\U00010143', 2);
foreach (S; AliasSeq!(wchar[], const wchar[], wstring))
{
enum str = to!S("hello world");
static assert(isSafe!(() => strideBack(str, 0)));
static assert(isSafe!(() => strideBack(str) ));
static assert((functionAttributes!(() => strideBack(str, 0)) & FunctionAttribute.pure_) != 0);
static assert((functionAttributes!(() => strideBack(str) ) & FunctionAttribute.pure_) != 0);
}
});
}
/// Ditto
uint strideBack(S)(auto ref S str, size_t index)
if (isRandomAccessRange!S && is(immutable ElementEncodingType!S == immutable dchar))
{
static if (is(typeof(str.length) : ulong))
assert(index <= str.length, "Past the end of the UTF-32 sequence");
assert(index > 0, "Not the end of the UTF-32 sequence");
return 1;
}
/// Ditto
uint strideBack(S)(auto ref S str)
if (isBidirectionalRange!S && is(immutable ElementEncodingType!S == immutable dchar))
{
assert(!str.empty, "Empty UTF-32 sequence");
return 1;
}
///
@safe unittest
{
assert("a".strideBack == 1);
assert("λ".strideBack == 2);
assert("aλ".strideBack == 2);
assert("aλ".strideBack(1) == 1);
assert("𐐷".strideBack == 4);
}
@system unittest
{
import core.exception : AssertError;
import std.conv : to;
import std.exception;
import std.string : format;
import std.traits : FunctionAttribute, functionAttributes, isSafe;
static void test(dstring s, dchar c, size_t i = size_t.max, size_t line = __LINE__)
{
enforce(strideBack(s, i == size_t.max ? s.length : i) == codeLength!dchar(c),
new AssertError(format("Unit test failure string: %s", s), __FILE__, line));
enforce(strideBack(RandomCU!dchar(s), i == size_t.max ? s.length : i) == codeLength!dchar(c),
new AssertError(format("Unit test failure range: %s", s), __FILE__, line));
auto refRandom = new RefRandomCU!dchar(s);
immutable randLen = refRandom.length;
enforce(strideBack(refRandom, i == size_t.max ? s.length : i) == codeLength!dchar(c),
new AssertError(format("Unit test failure rand ref range: %s", s), __FILE__, line));
enforce(refRandom.length == randLen,
new AssertError(format("Unit test failure rand ref range length: %s", s), __FILE__, line));
if (i == size_t.max)
{
enforce(strideBack(s) == codeLength!dchar(c),
new AssertError(format("Unit test failure string code length: %s", s), __FILE__, line));
enforce(strideBack(BidirCU!dchar(s)) == codeLength!dchar(c),
new AssertError(format("Unit test failure range code length: %s", s), __FILE__, line));
auto refBidir = new RefBidirCU!dchar(s);
immutable bidirLen = refBidir.length;
enforce(strideBack(refBidir) == codeLength!dchar(c),
new AssertError(format("Unit test failure bidir ref range code length: %s", s), __FILE__, line));
enforce(refBidir.length == bidirLen,
new AssertError(format("Unit test failure bidir ref range length: %s", s), __FILE__, line));
}
}
assertCTFEable!(
{
test("a", 'a');
test(" ", ' ');
test("\u2029", '\u2029'); //paraSep
test("\u0100", '\u0100');
test("\u0430", '\u0430');
test("\U00010143", '\U00010143');
test("abcdefcdef", 'f');
test("\U00010143\u0100\U00010143hello", 'o', 8);
test("\U00010143\u0100\U00010143hello", 'l', 7);
test("\U00010143\u0100\U00010143hello", 'l', 6);
test("\U00010143\u0100\U00010143hello", 'e', 5);
test("\U00010143\u0100\U00010143hello", 'h', 4);
test("\U00010143\u0100\U00010143hello", '\U00010143', 3);
test("\U00010143\u0100\U00010143hello", '\u0100', 2);
test("\U00010143\u0100\U00010143hello", '\U00010143', 1);
foreach (S; AliasSeq!(dchar[], const dchar[], dstring))
{
enum str = to!S("hello world");
static assert(isSafe!(() => strideBack(str, 0)));
static assert(isSafe!(() => strideBack(str) ));
static assert((functionAttributes!(() => strideBack(str, 0)) & FunctionAttribute.pure_) != 0);
static assert((functionAttributes!(() => strideBack(str) ) & FunctionAttribute.pure_) != 0);
}
});
}
/++
Given `index` into `str` and assuming that `index` is at the start
of a UTF sequence, `toUCSindex` determines the number of UCS characters
up to `index`. So, `index` is the index of a code unit at the
beginning of a code point, and the return value is how many code points into
the string that that code point is.
+/
size_t toUCSindex(C)(const(C)[] str, size_t index) @safe pure
if (isSomeChar!C)
{
static if (is(immutable C == immutable dchar))
return index;
else
{
size_t n = 0;
size_t j = 0;
for (; j < index; ++n)
j += stride(str, j);
if (j > index)
{
static if (is(immutable C == immutable char))
throw new UTFException("Invalid UTF-8 sequence", index);
else
throw new UTFException("Invalid UTF-16 sequence", index);
}
return n;
}
}
///
@safe unittest
{
assert(toUCSindex(`hello world`, 7) == 7);
assert(toUCSindex(`hello world`w, 7) == 7);
assert(toUCSindex(`hello world`d, 7) == 7);
assert(toUCSindex(`Ma Chérie`, 7) == 6);
assert(toUCSindex(`Ma Chérie`w, 7) == 7);
assert(toUCSindex(`Ma Chérie`d, 7) == 7);
assert(toUCSindex(`さいごの果実 / ミツバチと科学者`, 9) == 3);
assert(toUCSindex(`さいごの果実 / ミツバチと科学者`w, 9) == 9);
assert(toUCSindex(`さいごの果実 / ミツバチと科学者`d, 9) == 9);
}
/++
Given a UCS index `n` into `str`, returns the UTF index.
So, `n` is how many code points into the string the code point is, and
the array index of the code unit is returned.
+/
size_t toUTFindex(C)(const(C)[] str, size_t n) @safe pure
if (isSomeChar!C)
{
static if (is(immutable C == immutable dchar))
{
return n;
}
else
{
size_t i;
while (n--)
{
i += stride(str, i);
}
return i;
}
}
///
@safe unittest
{
assert(toUTFindex(`hello world`, 7) == 7);
assert(toUTFindex(`hello world`w, 7) == 7);
assert(toUTFindex(`hello world`d, 7) == 7);
assert(toUTFindex(`Ma Chérie`, 6) == 7);
assert(toUTFindex(`Ma Chérie`w, 7) == 7);
assert(toUTFindex(`Ma Chérie`d, 7) == 7);
assert(toUTFindex(`さいごの果実 / ミツバチと科学者`, 3) == 9);
assert(toUTFindex(`さいごの果実 / ミツバチと科学者`w, 9) == 9);
assert(toUTFindex(`さいごの果実 / ミツバチと科学者`d, 9) == 9);
}
/* =================== Decode ======================= */
/// Whether or not to replace invalid UTF with $(LREF replacementDchar)
alias UseReplacementDchar = Flag!"useReplacementDchar";
/++
Decodes and returns the code point starting at `str[index]`. `index`
is advanced to one past the decoded code point. If the code point is not
well-formed, then a `UTFException` is thrown and `index` remains
unchanged.
decode will only work with strings and random access ranges of code units
with length and slicing, whereas $(LREF decodeFront) will work with any
input range of code units.
Params:
useReplacementDchar = if invalid UTF, return replacementDchar rather than throwing
str = input string or indexable Range
index = starting index into s[]; incremented by number of code units processed
Returns:
decoded character
Throws:
$(LREF UTFException) if `str[index]` is not the start of a valid UTF
sequence and useReplacementDchar is `No.useReplacementDchar`
+/
dchar decode(UseReplacementDchar useReplacementDchar = No.useReplacementDchar, S)(auto ref S str, ref size_t index)
if (!isSomeString!S &&
isRandomAccessRange!S && hasSlicing!S && hasLength!S && isSomeChar!(ElementType!S))
in
{
assert(index < str.length, "Attempted to decode past the end of a string");
}
out (result)
{
assert(isValidDchar(result));
}
do
{
if (str[index] < codeUnitLimit!S)
return str[index++];
else
return decodeImpl!(true, useReplacementDchar)(str, index);
}
/// ditto
dchar decode(UseReplacementDchar useReplacementDchar = No.useReplacementDchar, S)(
auto ref scope S str, ref size_t index) @trusted pure
if (isSomeString!S)
in
{
assert(index < str.length, "Attempted to decode past the end of a string");
}
out (result)
{
assert(isValidDchar(result));
}
do
{
if (str[index] < codeUnitLimit!S)
return str[index++];
else static if (is(immutable S == immutable C[], C))
return decodeImpl!(true, useReplacementDchar)(cast(const(C)[]) str, index);
}
///
@safe pure unittest
{
size_t i;
assert("a".decode(i) == 'a' && i == 1);
i = 0;
assert("å".decode(i) == 'å' && i == 2);
i = 1;
assert("aå".decode(i) == 'å' && i == 3);
i = 0;
assert("å"w.decode(i) == 'å' && i == 1);
// ë as a multi-code point grapheme
i = 0;
assert("e\u0308".decode(i) == 'e' && i == 1);
// ë as a single code point grapheme
i = 0;
assert("ë".decode(i) == 'ë' && i == 2);
i = 0;
assert("ë"w.decode(i) == 'ë' && i == 1);
}
@safe pure unittest // https://issues.dlang.org/show_bug.cgi?id=22867
{
import std.conv : hexString;
string data = hexString!"f787a598";
size_t offset = 0;
try data.decode(offset);
catch (UTFException ex) assert(offset == 0);
}
/++
`decodeFront` is a variant of $(LREF decode) which specifically decodes
the first code point. Unlike $(LREF decode), `decodeFront` accepts any
$(REF_ALTTEXT input range, isInputRange, std,range,primitives)
of code units (rather than just a string or random access
range). It also takes the range by `ref` and pops off the elements as it
decodes them. If `numCodeUnits` is passed in, it gets set to the number
of code units which were in the code point which was decoded.
Params:
useReplacementDchar = if invalid UTF, return replacementDchar rather than throwing
str = input string or indexable Range
numCodeUnits = set to number of code units processed
Returns:
decoded character
Throws:
$(LREF UTFException) if `str.front` is not the start of a valid UTF
sequence. If an exception is thrown, then there is no guarantee as to
the number of code units which were popped off, as it depends on the
type of range being used and how many code units had to be popped off
before the code point was determined to be invalid.
+/
dchar decodeFront(UseReplacementDchar useReplacementDchar = No.useReplacementDchar, S)(
ref S str, out size_t numCodeUnits)
if (!isSomeString!S && isInputRange!S && isSomeChar!(ElementType!S))
in
{
assert(!str.empty);
}
out (result)
{
assert(isValidDchar(result));
}
do
{
immutable fst = str.front;
if (fst < codeUnitLimit!S)
{
str.popFront();
numCodeUnits = 1;
return fst;
}
else
{
// https://issues.dlang.org/show_bug.cgi?id=14447 forces canIndex to be
// done outside of decodeImpl, which is undesirable, since not all
// overloads of decodeImpl need it. So, it should be moved back into
// decodeImpl once https://issues.dlang.org/show_bug.cgi?id=8521
// has been fixed.
enum canIndex = is(S : const char[]) || isRandomAccessRange!S && hasSlicing!S && hasLength!S;
immutable retval = decodeImpl!(canIndex, useReplacementDchar)(str, numCodeUnits);
// The other range types were already popped by decodeImpl.
static if (isRandomAccessRange!S && hasSlicing!S && hasLength!S)
str = str[numCodeUnits .. str.length];
return retval;
}
}
/// ditto
dchar decodeFront(UseReplacementDchar useReplacementDchar = No.useReplacementDchar, S)(
ref scope S str, out size_t numCodeUnits) @trusted pure
if (isSomeString!S)
in
{
assert(!str.empty);
}
out (result)
{
assert(isValidDchar(result));
}
do
{
if (str[0] < codeUnitLimit!S)
{
numCodeUnits = 1;
immutable retval = str[0];
str = str[1 .. $];
return retval;
}
else static if (is(immutable S == immutable C[], C))
{
immutable retval = decodeImpl!(true, useReplacementDchar)(cast(const(C)[]) str, numCodeUnits);
str = str[numCodeUnits .. $];
return retval;
}
}
/++ Ditto +/
dchar decodeFront(UseReplacementDchar useReplacementDchar = No.useReplacementDchar, S)(ref S str)
if (isInputRange!S && isSomeChar!(ElementType!S))
{
size_t numCodeUnits;
return decodeFront!useReplacementDchar(str, numCodeUnits);
}
///
@safe pure unittest
{
import std.range.primitives;
string str = "Hello, World!";
assert(str.decodeFront == 'H' && str == "ello, World!");
str = "å";
assert(str.decodeFront == 'å' && str.empty);
str = "å";
size_t i;
assert(str.decodeFront(i) == 'å' && i == 2 && str.empty);
}
/++
`decodeBack` is a variant of $(LREF decode) which specifically decodes
the last code point. Unlike $(LREF decode), `decodeBack` accepts any
bidirectional range of code units (rather than just a string or random access
range). It also takes the range by `ref` and pops off the elements as it
decodes them. If `numCodeUnits` is passed in, it gets set to the number
of code units which were in the code point which was decoded.
Params:
useReplacementDchar = if invalid UTF, return `replacementDchar` rather than throwing
str = input string or bidirectional Range
numCodeUnits = gives the number of code units processed
Returns:
A decoded UTF character.
Throws:
$(LREF UTFException) if `str.back` is not the end of a valid UTF
sequence. If an exception is thrown, the `str` itself remains unchanged,
but there is no guarantee as to the value of `numCodeUnits` (when passed).
+/
dchar decodeBack(UseReplacementDchar useReplacementDchar = No.useReplacementDchar, S)(
ref S str, out size_t numCodeUnits)
if (isSomeString!S)
in
{
assert(!str.empty);
}
out (result)
{
assert(isValidDchar(result));
}
do
{
if (str[$ - 1] < codeUnitLimit!S)
{
numCodeUnits = 1;
immutable retval = str[$ - 1];
str = str[0 .. $ - 1];
return retval;
}
else static if (is(immutable S == immutable C[], C))
{
numCodeUnits = strideBack(str);
immutable newLength = str.length - numCodeUnits;
size_t index = newLength;
immutable retval = decodeImpl!(true, useReplacementDchar)(cast(const(C)[]) str, index);
str = str[0 .. newLength];
return retval;
}
}
/++ Ditto +/
dchar decodeBack(UseReplacementDchar useReplacementDchar = No.useReplacementDchar, S)(
ref S str, out size_t numCodeUnits)
if (!isSomeString!S && isSomeChar!(ElementType!S) && isBidirectionalRange!S
&& ((isRandomAccessRange!S && hasLength!S) || !isRandomAccessRange!S))
in
{
assert(!str.empty);
}
out (result)
{
assert(isValidDchar(result));
}
do
{
if (str.back < codeUnitLimit!S)
{
numCodeUnits = 1;
immutable retval = str.back;
str.popBack();
return retval;
}
else
{
numCodeUnits = strideBack(str);
static if (isRandomAccessRange!S)
{
size_t index = str.length - numCodeUnits;
immutable retval = decodeImpl!(true, useReplacementDchar)(str, index);
str.popBackExactly(numCodeUnits);
return retval;
}
else
{
alias Char = typeof(cast() ElementType!S.init);
Char[4] codeUnits = void;
S tmp = str.save;
for (size_t i = numCodeUnits; i > 0; )
{
codeUnits[--i] = tmp.back;
tmp.popBack();
}
const Char[] codePoint = codeUnits[0 .. numCodeUnits];
size_t index = 0;
immutable retval = decodeImpl!(true, useReplacementDchar)(codePoint, index);
str = tmp;
return retval;
}
}
}
/++ Ditto +/
dchar decodeBack(UseReplacementDchar useReplacementDchar = No.useReplacementDchar, S)(ref S str)
if (isSomeString!S
|| (isRandomAccessRange!S && hasLength!S && isSomeChar!(ElementType!S))
|| (!isRandomAccessRange!S && isBidirectionalRange!S && isSomeChar!(ElementType!S)))
in
{
assert(!str.empty);
}
out (result)
{
assert(isValidDchar(result));
}
do
{
size_t numCodeUnits;
return decodeBack!useReplacementDchar(str, numCodeUnits);
}
///
@system pure unittest
{
import std.range.primitives;
string str = "Hello, World!";
assert(str.decodeBack == '!' && str == "Hello, World");
str = "å";
assert(str.decodeBack == 'å' && str.empty);
str = "å";
size_t i;
assert(str.decodeBack(i) == 'å' && i == 2 && str.empty);
}
// For the given range, code unit values less than this
// are guaranteed to be valid single-codepoint encodings.
package template codeUnitLimit(S)
if (isSomeChar!(ElementEncodingType!S))
{
static if (is(immutable ElementEncodingType!S == immutable char))
enum char codeUnitLimit = 0x80;
else static if (is(immutable ElementEncodingType!S == immutable wchar))
enum wchar codeUnitLimit = 0xD800;
else
enum dchar codeUnitLimit = 0xD800;
}
/*
* For strings, this function does its own bounds checking to give a
* more useful error message when attempting to decode past the end of a string.
* Subsequently it uses a pointer instead of an array to avoid
* redundant bounds checking.
*
* The three overloads of this operate on chars, wchars, and dchars.
*
* Params:
* canIndex = if S is indexable
* useReplacementDchar = if invalid UTF, return replacementDchar rather than throwing
* str = input string or Range
* index = starting index into s[]; incremented by number of code units processed
*
* Returns:
* decoded character
*/
private dchar decodeImpl(bool canIndex, UseReplacementDchar useReplacementDchar = No.useReplacementDchar, S)(
auto ref S str, ref size_t index)
if (
is(S : const char[]) || (isInputRange!S && is(immutable ElementEncodingType!S == immutable char)))
{
/* The following encodings are valid, except for the 5 and 6 byte
* combinations:
* 0xxxxxxx
* 110xxxxx 10xxxxxx
* 1110xxxx 10xxxxxx 10xxxxxx
* 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
* 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
* 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
*/
/* Dchar bitmask for different numbers of UTF-8 code units.
*/
alias bitMask = AliasSeq!((1 << 7) - 1, (1 << 11) - 1, (1 << 16) - 1, (1 << 21) - 1);
static if (is(S : const char[]))
auto pstr = str.ptr + index; // this is what makes decodeImpl() @system code
else static if (isRandomAccessRange!S && hasSlicing!S && hasLength!S)
auto pstr = str[index .. str.length];
else
alias pstr = str;
// https://issues.dlang.org/show_bug.cgi?id=14447 forces this to be done
// outside of decodeImpl
//enum canIndex = is(S : const char[]) || (isRandomAccessRange!S && hasSlicing!S && hasLength!S);
static if (canIndex)
{
immutable length = str.length - index;
ubyte fst = pstr[0];
}
else
{
ubyte fst = pstr.front;
pstr.popFront();
}
static if (!useReplacementDchar)
{
static if (canIndex)
{
static UTFException exception(S)(S str, string msg)
{
uint[4] sequence = void;
size_t i;
do
{
sequence[i] = str[i];
} while (++i < str.length && i < 4 && (str[i] & 0xC0) == 0x80);
return new UTFException(msg, i).setSequence(sequence[0 .. i]);
}
}
UTFException invalidUTF()
{
static if (canIndex)
return exception(pstr[0 .. length], "Invalid UTF-8 sequence");
else
{
//We can't include the invalid sequence with input strings without
//saving each of the code units along the way, and we can't do it with
//forward ranges without saving the entire range. Both would incur a
//cost for the decoding of every character just to provide a better
//error message for the (hopefully) rare case when an invalid UTF-8
//sequence is encountered, so we don't bother trying to include the
//invalid sequence here, unlike with strings and sliceable ranges.
return new UTFException("Invalid UTF-8 sequence");
}
}
UTFException outOfBounds()
{
static if (canIndex)
return exception(pstr[0 .. length], "Attempted to decode past the end of a string");
else
return new UTFException("Attempted to decode past the end of a string");
}
}
if ((fst & 0b1100_0000) != 0b1100_0000)
{
static if (useReplacementDchar)
{
++index; // always consume bad input to avoid infinite loops
return replacementDchar;
}
else
throw invalidUTF(); // starter must have at least 2 first bits set
}
ubyte tmp = void;
dchar d = fst; // upper control bits are masked out later
fst <<= 1;
foreach (i; AliasSeq!(1, 2, 3))
{
static if (canIndex)
{
if (i == length)
{
static if (useReplacementDchar)
{
index += i;
return replacementDchar;
}
else
throw outOfBounds();
}
}
else
{
if (pstr.empty)
{
static if (useReplacementDchar)
{
index += i;
return replacementDchar;
}
else
throw outOfBounds();
}
}
static if (canIndex)
tmp = pstr[i];
else
{
tmp = pstr.front;
pstr.popFront();
}
if ((tmp & 0xC0) != 0x80)
{
static if (useReplacementDchar)
{
index += i + 1;
return replacementDchar;
}
else
throw invalidUTF();
}
d = (d << 6) | (tmp & 0x3F);
fst <<= 1;
if (!(fst & 0x80)) // no more bytes
{
d &= bitMask[i]; // mask out control bits
// overlong, could have been encoded with i bytes
if ((d & ~bitMask[i - 1]) == 0)
{
static if (useReplacementDchar)
{
index += i + 1;
return replacementDchar;
}
else
throw invalidUTF();
}
// check for surrogates only needed for 3 bytes
static if (i == 2)
{
if (!isValidDchar(d))
{
static if (useReplacementDchar)
{
index += i + 1;
return replacementDchar;
}
else
throw invalidUTF();
}
}
static if (i == 3)
{
if (d > dchar.max)
{
static if (useReplacementDchar)
d = replacementDchar;
else
throw invalidUTF();
}
}
index += i + 1;
return d;
}
}
static if (useReplacementDchar)
{
index += 4; // read 4 chars by now
return replacementDchar;
}
else
throw invalidUTF();
}
@safe pure @nogc nothrow
unittest
{
// Add tests for useReplacemendDchar == yes path
static struct R
{
@safe pure @nogc nothrow:
this(string s) { this.s = s; }
@property bool empty() { return idx == s.length; }
@property char front() { return s[idx]; }
void popFront() { ++idx; }
size_t idx;
string s;
}
foreach (s; invalidUTFstrings!char())
{
auto r = R(s);
size_t index;
dchar dc = decodeImpl!(false, Yes.useReplacementDchar)(r, index);
assert(dc == replacementDchar);
assert(1 <= index && index <= s.length);
}
}
private dchar decodeImpl(bool canIndex, UseReplacementDchar useReplacementDchar = No.useReplacementDchar, S)
(auto ref S str, ref size_t index)
if (is(S : const wchar[]) || (isInputRange!S && is(immutable ElementEncodingType!S == immutable wchar)))
{
static if (is(S : const wchar[]))
auto pstr = str.ptr + index;
else static if (isRandomAccessRange!S && hasSlicing!S && hasLength!S)
auto pstr = str[index .. str.length];
else
alias pstr = str;
// https://issues.dlang.org/show_bug.cgi?id=14447 forces this to be done
// outside of decodeImpl
//enum canIndex = is(S : const wchar[]) || (isRandomAccessRange!S && hasSlicing!S && hasLength!S);
static if (canIndex)
{
immutable length = str.length - index;
uint u = pstr[0];
}
else
{
uint u = pstr.front;
pstr.popFront();
}
static if (!useReplacementDchar)
{
UTFException exception(string msg)
{
static if (canIndex)
return new UTFException(msg).setSequence(pstr[0]);
else
return new UTFException(msg);
}
}
// The < case must be taken care of before decodeImpl is called.
assert(u >= 0xD800);
if (u <= 0xDBFF)
{
static if (canIndex)
immutable onlyOneCodeUnit = length == 1;
else
immutable onlyOneCodeUnit = pstr.empty;
if (onlyOneCodeUnit)
{
static if (useReplacementDchar)
{
++index;
return replacementDchar;
}
else
throw exception("surrogate UTF-16 high value past end of string");
}
static if (canIndex)
immutable uint u2 = pstr[1];
else
{
immutable uint u2 = pstr.front;
pstr.popFront();
}
if (u2 < 0xDC00 || u2 > 0xDFFF)
{
static if (useReplacementDchar)
u = replacementDchar;
else
throw exception("surrogate UTF-16 low value out of range");
}
else
u = ((u - 0xD7C0) << 10) + (u2 - 0xDC00);
++index;
}
else if (u >= 0xDC00 && u <= 0xDFFF)
{
static if (useReplacementDchar)
u = replacementDchar;
else
throw exception("unpaired surrogate UTF-16 value");
}
++index;
// Note: u+FFFE and u+FFFF are specifically permitted by the
// Unicode standard for application internal use (see isValidDchar)
return cast(dchar) u;
}
@safe pure @nogc nothrow
unittest
{
// Add tests for useReplacemendDchar == true path
static struct R
{
@safe pure @nogc nothrow:
this(wstring s) { this.s = s; }
@property bool empty() { return idx == s.length; }
@property wchar front() { return s[idx]; }
void popFront() { ++idx; }
size_t idx;
wstring s;
}
foreach (s; invalidUTFstrings!wchar())
{
auto r = R(s);
size_t index;
dchar dc = decodeImpl!(false, Yes.useReplacementDchar)(r, index);
assert(dc == replacementDchar);
assert(1 <= index && index <= s.length);
}
}
private dchar decodeImpl(bool canIndex, UseReplacementDchar useReplacementDchar = No.useReplacementDchar, S)(
auto ref S str, ref size_t index)
if (is(S : const dchar[]) || (isInputRange!S && is(immutable ElementEncodingType!S == immutable dchar)))
{
static if (is(S : const dchar[]))
auto pstr = str.ptr;
else
alias pstr = str;
static if (is(S : const dchar[]) || isRandomAccessRange!S)
{
dchar dc = pstr[index];
if (!isValidDchar(dc))
{
static if (useReplacementDchar)
dc = replacementDchar;
else
throw new UTFException("Invalid UTF-32 value").setSequence(dc);
}
++index;
return dc;
}
else
{
dchar dc = pstr.front;
if (!isValidDchar(dc))
{
static if (useReplacementDchar)
dc = replacementDchar;
else
throw new UTFException("Invalid UTF-32 value").setSequence(dc);
}
++index;
pstr.popFront();
return dc;
}
}
@safe pure @nogc nothrow
unittest
{
// Add tests for useReplacemendDchar == true path
static struct R
{
@safe pure @nogc nothrow:
this(dstring s) { this.s = s; }
@property bool empty() { return idx == s.length; }
@property dchar front() { return s[idx]; }
void popFront() { ++idx; }
size_t idx;
dstring s;
}
foreach (s; invalidUTFstrings!dchar())
{
auto r = R(s);
size_t index;
dchar dc = decodeImpl!(false, Yes.useReplacementDchar)(r, index);
assert(dc == replacementDchar);
assert(1 <= index && index <= s.length);
}
}
version (StdUnittest) private void testDecode(R)(R range,
size_t index,
dchar expectedChar,
size_t expectedIndex,
size_t line = __LINE__)
{
import core.exception : AssertError;
import std.exception : enforce;
import std.string : format;
import std.traits : isNarrowString;
static if (hasLength!R)
immutable lenBefore = range.length;
static if (isRandomAccessRange!R && !isNarrowString!R)
{
{
immutable result = decode(range, index);
enforce(result == expectedChar,
new AssertError(format("decode: Wrong character: %s", result), __FILE__, line));
enforce(index == expectedIndex,
new AssertError(format("decode: Wrong index: %s", index), __FILE__, line));
static if (hasLength!R)
{
enforce(range.length == lenBefore,
new AssertError(format("decode: length changed: %s", range.length), __FILE__, line));
}
}
}
}
version (StdUnittest) private void testDecodeFront(R)(ref R range,
dchar expectedChar,
size_t expectedNumCodeUnits,
size_t line = __LINE__)
{
import core.exception : AssertError;
import std.exception : enforce;
import std.string : format;
static if (hasLength!R)
immutable lenBefore = range.length;
size_t numCodeUnits;
immutable result = decodeFront(range, numCodeUnits);
enforce(result == expectedChar,
new AssertError(format("decodeFront: Wrong character: %s", result), __FILE__, line));
enforce(numCodeUnits == expectedNumCodeUnits,
new AssertError(format("decodeFront: Wrong numCodeUnits: %s", numCodeUnits), __FILE__, line));
static if (hasLength!R)
{
enforce(range.length == lenBefore - numCodeUnits,
new AssertError(format("decodeFront: wrong length: %s", range.length), __FILE__, line));
}
}
version (StdUnittest) private void testDecodeBack(R)(ref R range,
dchar expectedChar,
size_t expectedNumCodeUnits,
size_t line = __LINE__)
{
// This condition is to allow unit testing all `decode` functions together
static if (!isBidirectionalRange!R)
return;
else
{
import core.exception : AssertError;
import std.exception : enforce;
import std.string : format;
static if (hasLength!R)
immutable lenBefore = range.length;
size_t numCodeUnits;
immutable result = decodeBack(range, numCodeUnits);
enforce(result == expectedChar,
new AssertError(format("decodeBack: Wrong character: %s", result), __FILE__, line));
enforce(numCodeUnits == expectedNumCodeUnits,
new AssertError(format("decodeBack: Wrong numCodeUnits: %s", numCodeUnits), __FILE__, line));
static if (hasLength!R)
{
enforce(range.length == lenBefore - numCodeUnits,
new AssertError(format("decodeBack: wrong length: %s", range.length), __FILE__, line));
}
}
}
version (StdUnittest) private void testAllDecode(R)(R range,
dchar expectedChar,
size_t expectedIndex,
size_t line = __LINE__)
{
testDecode(range, 0, expectedChar, expectedIndex, line);
static if (isBidirectionalRange!R)
{
auto rangeCopy = range.save;
testDecodeBack(rangeCopy, expectedChar, expectedIndex, line);
}
testDecodeFront(range, expectedChar, expectedIndex, line);
}
version (StdUnittest) private void testBadDecode(R)(R range, size_t index, size_t line = __LINE__)
{
import core.exception : AssertError;
import std.exception : assertThrown, enforce;
import std.string : format;
immutable initialIndex = index;
static if (hasLength!R)
immutable lenBefore = range.length;
static if (isRandomAccessRange!R)
{
assertThrown!UTFException(decode(range, index), null, __FILE__, line);
enforce(index == initialIndex,
new AssertError(format("decode: Wrong index: %s", index), __FILE__, line));
static if (hasLength!R)
{
enforce(range.length == lenBefore,
new AssertError(format("decode: length changed:", range.length), __FILE__, line));
}
}
if (initialIndex == 0)
assertThrown!UTFException(decodeFront(range, index), null, __FILE__, line);
}
version (StdUnittest) private void testBadDecodeBack(R)(R range, size_t line = __LINE__)
{
// This condition is to allow unit testing all `decode` functions together
static if (!isBidirectionalRange!R)
return;
else
{
import core.exception : AssertError;
import std.exception : assertThrown, enforce;
import std.string : format;
static if (hasLength!R)
immutable lenBefore = range.length;
static if (isRandomAccessRange!R)
{
assertThrown!UTFException(decodeBack(range), null, __FILE__, line);
static if (hasLength!R)
{
enforce(range.length == lenBefore,
new AssertError(format("decodeBack: length changed:", range.length), __FILE__, line));
}
}
}
}
@system unittest
{
import std.conv : to;
import std.exception;
assertCTFEable!(
{
foreach (S; AliasSeq!(to!string, InputCU!char, RandomCU!char,
(string s) => new RefBidirCU!char(s),
(string s) => new RefRandomCU!char(s)))
{
enum sHasLength = hasLength!(typeof(S("abcd")));
{
auto range = S("abcd");
testDecode(range, 0, 'a', 1);
testDecode(range, 1, 'b', 2);
testDecodeFront(range, 'a', 1);
testDecodeFront(range, 'b', 1);
assert(decodeFront(range) == 'c');
assert(decodeFront(range) == 'd');
}
{
auto range = S("ウェブサイト");
testDecode(range, 0, 'ウ', 3);
testDecode(range, 3, 'ェ', 6);
testDecodeFront(range, 'ウ', 3);
testDecodeFront(range, 'ェ', 3);
assert(decodeFront(range) == 'ブ');
assert(decodeFront(range) == 'サ');
}
{
auto range = S("abcd");
testDecodeBack(range, 'd', 1);
testDecodeBack(range, 'c', 1);
testDecodeBack(range, 'b', 1);
testDecodeBack(range, 'a', 1);
}
{
auto range = S("ウェブサイト");
testDecodeBack(range, 'ト', 3);
testDecodeBack(range, 'イ', 3);
testDecodeBack(range, 'サ', 3);
testDecodeBack(range, 'ブ', 3);
}
testAllDecode(S("\xC2\xA9"), '\u00A9', 2);
testAllDecode(S("\xE2\x89\xA0"), '\u2260', 3);
foreach (str; ["\xE2\x89", // too short
"\xC0\x8A",
"\xE0\x80\x8A",
"\xF0\x80\x80\x8A",
"\xF8\x80\x80\x80\x8A",
"\xFC\x80\x80\x80\x80\x8A"])
{
testBadDecode(S(str), 0);
testBadDecode(S(str), 1);
testBadDecodeBack(S(str));
}
//Invalid UTF-8 sequence where the first code unit is valid.
testAllDecode(S("\xEF\xBF\xBE"), cast(dchar) 0xFFFE, 3);
testAllDecode(S("\xEF\xBF\xBF"), cast(dchar) 0xFFFF, 3);
//Invalid UTF-8 sequence where the first code unit isn't valid.
foreach (str; ["\xED\xA0\x80",
"\xED\xAD\xBF",
"\xED\xAE\x80",
"\xED\xAF\xBF",
"\xED\xB0\x80",
"\xED\xBE\x80",
"\xED\xBF\xBF"])
{
testBadDecode(S(str), 0);
testBadDecodeBack(S(str));
}
}
});
}
@system unittest
{
import std.exception;
assertCTFEable!(
{
foreach (S; AliasSeq!((wstring s) => s, InputCU!wchar, RandomCU!wchar,
(wstring s) => new RefBidirCU!wchar(s),
(wstring s) => new RefRandomCU!wchar(s)))
{
testAllDecode(S([cast(wchar) 0x1111]), cast(dchar) 0x1111, 1);
testAllDecode(S([cast(wchar) 0xD800, cast(wchar) 0xDC00]), cast(dchar) 0x10000, 2);
testAllDecode(S([cast(wchar) 0xDBFF, cast(wchar) 0xDFFF]), cast(dchar) 0x10FFFF, 2);
testAllDecode(S([cast(wchar) 0xFFFE]), cast(dchar) 0xFFFE, 1);
testAllDecode(S([cast(wchar) 0xFFFF]), cast(dchar) 0xFFFF, 1);
testBadDecode(S([ cast(wchar) 0xD801 ]), 0);
testBadDecode(S([ cast(wchar) 0xD800, cast(wchar) 0x1200 ]), 0);
testBadDecodeBack(S([ cast(wchar) 0xD801 ]));
testBadDecodeBack(S([ cast(wchar) 0x0010, cast(wchar) 0xD800 ]));
{
auto range = S("ウェブサイト");
testDecode(range, 0, 'ウ', 1);
testDecode(range, 1, 'ェ', 2);
testDecodeFront(range, 'ウ', 1);
testDecodeFront(range, 'ェ', 1);
assert(decodeFront(range) == 'ブ');
assert(decodeFront(range) == 'サ');
}
{
auto range = S("ウェブサイト");
testDecodeBack(range, 'ト', 1);
testDecodeBack(range, 'イ', 1);
testDecodeBack(range, 'サ', 1);
testDecodeBack(range, 'ブ', 1);
}
}
foreach (S; AliasSeq!((wchar[] s) => s.idup, RandomCU!wchar, (wstring s) => new RefRandomCU!wchar(s)))
{
auto str = S([cast(wchar) 0xD800, cast(wchar) 0xDC00,
cast(wchar) 0x1400,
cast(wchar) 0xDAA7, cast(wchar) 0xDDDE]);
testDecode(str, 0, cast(dchar) 0x10000, 2);
testDecode(str, 2, cast(dchar) 0x1400, 3);
testDecode(str, 3, cast(dchar) 0xB9DDE, 5);
testDecodeBack(str, cast(dchar) 0xB9DDE, 2);
testDecodeBack(str, cast(dchar) 0x1400, 1);
testDecodeBack(str, cast(dchar) 0x10000, 2);
}
});
}
@system unittest
{
import std.exception;
assertCTFEable!(
{
foreach (S; AliasSeq!((dstring s) => s, RandomCU!dchar, InputCU!dchar,
(dstring s) => new RefBidirCU!dchar(s),
(dstring s) => new RefRandomCU!dchar(s)))
{
testAllDecode(S([cast(dchar) 0x1111]), cast(dchar) 0x1111, 1);
testAllDecode(S([cast(dchar) 0x10000]), cast(dchar) 0x10000, 1);
testAllDecode(S([cast(dchar) 0x10FFFF]), cast(dchar) 0x10FFFF, 1);
testAllDecode(S([cast(dchar) 0xFFFE]), cast(dchar) 0xFFFE, 1);
testAllDecode(S([cast(dchar) 0xFFFF]), cast(dchar) 0xFFFF, 1);
testBadDecode(S([cast(dchar) 0xD800]), 0);
testBadDecode(S([cast(dchar) 0xDFFE]), 0);
testBadDecode(S([cast(dchar) 0x110000]), 0);
testBadDecodeBack(S([cast(dchar) 0xD800]));
testBadDecodeBack(S([cast(dchar) 0xDFFE]));
testBadDecodeBack(S([cast(dchar) 0x110000]));
{
auto range = S("ウェブサイト");
testDecode(range, 0, 'ウ', 1);
testDecode(range, 1, 'ェ', 2);
testDecodeFront(range, 'ウ', 1);
testDecodeFront(range, 'ェ', 1);
assert(decodeFront(range) == 'ブ');
assert(decodeFront(range) == 'サ');
}
{
auto range = S("ウェブサイト");
testDecodeBack(range, 'ト', 1);
testDecodeBack(range, 'イ', 1);
testDecodeBack(range, 'サ', 1);
testDecodeBack(range, 'ブ', 1);
}
}
foreach (S; AliasSeq!((dchar[] s) => s.idup, RandomCU!dchar, (dstring s) => new RefRandomCU!dchar(s)))
{
auto str = S([cast(dchar) 0x10000, cast(dchar) 0x1400, cast(dchar) 0xB9DDE]);
testDecode(str, 0, 0x10000, 1);
testDecode(str, 1, 0x1400, 2);
testDecode(str, 2, 0xB9DDE, 3);
testDecodeBack(str, cast(dchar) 0xB9DDE, 1);
testDecodeBack(str, cast(dchar) 0x1400, 1);
testDecodeBack(str, cast(dchar) 0x10000, 1);
}
});
}
@safe unittest
{
import std.exception;
import std.traits : FunctionAttribute, functionAttributes, isSafe;
assertCTFEable!(
{
foreach (S; AliasSeq!( char[], const( char)[], string,
wchar[], const(wchar)[], wstring,
dchar[], const(dchar)[], dstring))
{
static assert(isSafe!({ S str; size_t i = 0; decode(str, i); }));
static assert(isSafe!({ S str; size_t i = 0; decodeFront(str, i); }));
static assert(isSafe!({ S str; decodeFront(str); }));
static assert((functionAttributes!({ S str; size_t i = 0; decode(str, i); }) & FunctionAttribute.pure_) != 0);
static assert((functionAttributes!({
S str; size_t i = 0; decodeFront(str, i);
}) & FunctionAttribute.pure_) != 0);
static assert((functionAttributes!({ S str; decodeFront(str); }) & FunctionAttribute.pure_) != 0);
static assert((functionAttributes!({
S str; size_t i = 0; decodeBack(str, i);
}) & FunctionAttribute.pure_) != 0);
static assert((functionAttributes!({ S str; decodeBack(str); }) & FunctionAttribute.pure_) != 0);
}
});
}
@safe unittest
{
import std.exception;
char[4] val;
val[0] = 0b1111_0111;
val[1] = 0b1011_1111;
val[2] = 0b1011_1111;
val[3] = 0b1011_1111;
size_t i = 0;
assertThrown!UTFException((){ dchar ch = decode(val[], i); }());
}
/* =================== Encode ======================= */
private dchar _utfException(UseReplacementDchar useReplacementDchar)(string msg, dchar c)
{
static if (useReplacementDchar)
return replacementDchar;
else
throw new UTFException(msg).setSequence(c);
}
/++
Encodes `c` into the static array, `buf`, and returns the actual
length of the encoded character (a number between `1` and `4` for
`char[4]` buffers and a number between `1` and `2` for
`wchar[2]` buffers).
Throws:
`UTFException` if `c` is not a valid UTF code point.
+/
size_t encode(UseReplacementDchar useReplacementDchar = No.useReplacementDchar)(
out char[4] buf, dchar c) @safe pure
{
if (c <= 0x7F)
{
assert(isValidDchar(c));
buf[0] = cast(char) c;
return 1;
}
if (c <= 0x7FF)
{
assert(isValidDchar(c));
buf[0] = cast(char)(0xC0 | (c >> 6));
buf[1] = cast(char)(0x80 | (c & 0x3F));
return 2;
}
if (c <= 0xFFFF)
{
if (0xD800 <= c && c <= 0xDFFF)
c = _utfException!useReplacementDchar("Encoding a surrogate code point in UTF-8", c);
assert(isValidDchar(c));
L3:
buf[0] = cast(char)(0xE0 | (c >> 12));
buf[1] = cast(char)(0x80 | ((c >> 6) & 0x3F));
buf[2] = cast(char)(0x80 | (c & 0x3F));
return 3;
}
if (c <= 0x10FFFF)
{
assert(isValidDchar(c));
buf[0] = cast(char)(0xF0 | (c >> 18));
buf[1] = cast(char)(0x80 | ((c >> 12) & 0x3F));
buf[2] = cast(char)(0x80 | ((c >> 6) & 0x3F));
buf[3] = cast(char)(0x80 | (c & 0x3F));
return 4;
}
assert(!isValidDchar(c));
c = _utfException!useReplacementDchar("Encoding an invalid code point in UTF-8", c);
goto L3;
}
///
@safe unittest
{
import std.exception : assertThrown;
import std.typecons : Yes;
char[4] buf;
assert(encode(buf, '\u0000') == 1 && buf[0 .. 1] == "\u0000");
assert(encode(buf, '\u007F') == 1 && buf[0 .. 1] == "\u007F");
assert(encode(buf, '\u0080') == 2 && buf[0 .. 2] == "\u0080");
assert(encode(buf, '\uE000') == 3 && buf[0 .. 3] == "\uE000");
assert(encode(buf, 0xFFFE) == 3 && buf[0 .. 3] == "\xEF\xBF\xBE");
assertThrown!UTFException(encode(buf, cast(dchar) 0x110000));
encode!(Yes.useReplacementDchar)(buf, cast(dchar) 0x110000);
auto slice = buf[];
assert(slice.decodeFront == replacementDchar);
}
///
@safe unittest
{
import std.exception : assertThrown;
import std.typecons : Yes;
wchar[2] buf;
assert(encode(buf, '\u0000') == 1 && buf[0 .. 1] == "\u0000");
assert(encode(buf, '\uD7FF') == 1 && buf[0 .. 1] == "\uD7FF");
assert(encode(buf, '\uE000') == 1 && buf[0 .. 1] == "\uE000");
assert(encode(buf, '\U00010000') == 2 && buf[0 .. 2] == "\U00010000");
assert(encode(buf, '\U0010FFFF') == 2 && buf[0 .. 2] == "\U0010FFFF");
assertThrown!UTFException(encode(buf, cast(dchar) 0xD800));
encode!(Yes.useReplacementDchar)(buf, cast(dchar) 0x110000);
auto slice = buf[];
assert(slice.decodeFront == replacementDchar);
}
///
@safe unittest
{
import std.exception : assertThrown;
import std.typecons : Yes;
dchar[1] buf;
assert(encode(buf, '\u0000') == 1 && buf[0] == '\u0000');
assert(encode(buf, '\uD7FF') == 1 && buf[0] == '\uD7FF');
assert(encode(buf, '\uE000') == 1 && buf[0] == '\uE000');
assert(encode(buf, '\U0010FFFF') == 1 && buf[0] == '\U0010FFFF');
assertThrown!UTFException(encode(buf, cast(dchar) 0xD800));
encode!(Yes.useReplacementDchar)(buf, cast(dchar) 0x110000);
assert(buf[0] == replacementDchar);
}
@safe unittest
{
import std.exception;
assertCTFEable!(
{
char[4] buf;
assert(encode(buf, '\u0000') == 1 && buf[0 .. 1] == "\u0000");
assert(encode(buf, '\u007F') == 1 && buf[0 .. 1] == "\u007F");
assert(encode(buf, '\u0080') == 2 && buf[0 .. 2] == "\u0080");
assert(encode(buf, '\u07FF') == 2 && buf[0 .. 2] == "\u07FF");
assert(encode(buf, '\u0800') == 3 && buf[0 .. 3] == "\u0800");
assert(encode(buf, '\uD7FF') == 3 && buf[0 .. 3] == "\uD7FF");
assert(encode(buf, '\uE000') == 3 && buf[0 .. 3] == "\uE000");
assert(encode(buf, 0xFFFE) == 3 && buf[0 .. 3] == "\xEF\xBF\xBE");
assert(encode(buf, 0xFFFF) == 3 && buf[0 .. 3] == "\xEF\xBF\xBF");
assert(encode(buf, '\U00010000') == 4 && buf[0 .. 4] == "\U00010000");
assert(encode(buf, '\U0010FFFF') == 4 && buf[0 .. 4] == "\U0010FFFF");
assertThrown!UTFException(encode(buf, cast(dchar) 0xD800));
assertThrown!UTFException(encode(buf, cast(dchar) 0xDBFF));
assertThrown!UTFException(encode(buf, cast(dchar) 0xDC00));
assertThrown!UTFException(encode(buf, cast(dchar) 0xDFFF));
assertThrown!UTFException(encode(buf, cast(dchar) 0x110000));
assert(encode!(Yes.useReplacementDchar)(buf, cast(dchar) 0x110000) == buf.stride);
enum replacementDcharString = "\uFFFD";
assert(buf[0 .. replacementDcharString.length] == replacementDcharString);
});
}
/// Ditto
size_t encode(UseReplacementDchar useReplacementDchar = No.useReplacementDchar)(
out wchar[2] buf, dchar c) @safe pure
{
if (c <= 0xFFFF)
{
if (0xD800 <= c && c <= 0xDFFF)
c = _utfException!useReplacementDchar("Encoding an isolated surrogate code point in UTF-16", c);
assert(isValidDchar(c));
L1:
buf[0] = cast(wchar) c;
return 1;
}
if (c <= 0x10FFFF)
{
assert(isValidDchar(c));
buf[0] = cast(wchar)((((c - 0x10000) >> 10) & 0x3FF) + 0xD800);
buf[1] = cast(wchar)(((c - 0x10000) & 0x3FF) + 0xDC00);
return 2;
}
c = _utfException!useReplacementDchar("Encoding an invalid code point in UTF-16", c);
goto L1;
}
@safe unittest
{
import std.exception;
assertCTFEable!(
{
wchar[2] buf;
assert(encode(buf, '\u0000') == 1 && buf[0 .. 1] == "\u0000");
assert(encode(buf, '\uD7FF') == 1 && buf[0 .. 1] == "\uD7FF");
assert(encode(buf, '\uE000') == 1 && buf[0 .. 1] == "\uE000");
assert(encode(buf, 0xFFFE) == 1 && buf[0] == 0xFFFE);
assert(encode(buf, 0xFFFF) == 1 && buf[0] == 0xFFFF);
assert(encode(buf, '\U00010000') == 2 && buf[0 .. 2] == "\U00010000");
assert(encode(buf, '\U0010FFFF') == 2 && buf[0 .. 2] == "\U0010FFFF");
assertThrown!UTFException(encode(buf, cast(dchar) 0xD800));
assertThrown!UTFException(encode(buf, cast(dchar) 0xDBFF));
assertThrown!UTFException(encode(buf, cast(dchar) 0xDC00));
assertThrown!UTFException(encode(buf, cast(dchar) 0xDFFF));
assertThrown!UTFException(encode(buf, cast(dchar) 0x110000));
assert(encode!(Yes.useReplacementDchar)(buf, cast(dchar) 0x110000) == buf.stride);
assert(buf.front == replacementDchar);
});
}
/// Ditto
size_t encode(UseReplacementDchar useReplacementDchar = No.useReplacementDchar)(
out dchar[1] buf, dchar c) @safe pure
{
if ((0xD800 <= c && c <= 0xDFFF) || 0x10FFFF < c)
c = _utfException!useReplacementDchar("Encoding an invalid code point in UTF-32", c);
else
assert(isValidDchar(c));
buf[0] = c;
return 1;
}
@safe unittest
{
import std.exception;
assertCTFEable!(
{
dchar[1] buf;
encode(buf, '\u0000'); assert(buf[0] == '\u0000');
encode(buf, '\uD7FF'); assert(buf[0] == '\uD7FF');
encode(buf, '\uE000'); assert(buf[0] == '\uE000');
encode(buf, 0xFFFE); assert(buf[0] == 0xFFFE);
encode(buf, 0xFFFF); assert(buf[0] == 0xFFFF);
encode(buf, '\U0010FFFF'); assert(buf[0] == '\U0010FFFF');
assertThrown!UTFException(encode(buf, cast(dchar) 0xD800));
assertThrown!UTFException(encode(buf, cast(dchar) 0xDBFF));
assertThrown!UTFException(encode(buf, cast(dchar) 0xDC00));
assertThrown!UTFException(encode(buf, cast(dchar) 0xDFFF));
assertThrown!UTFException(encode(buf, cast(dchar) 0x110000));
assert(encode!(Yes.useReplacementDchar)(buf, cast(dchar) 0x110000) == buf.stride);
assert(buf.front == replacementDchar);
});
}
/++
Encodes `c` in `str`'s encoding and appends it to `str`.
Throws:
`UTFException` if `c` is not a valid UTF code point.
+/
void encode(UseReplacementDchar useReplacementDchar = No.useReplacementDchar)(
ref scope char[] str, dchar c) @safe pure
{
if (c <= 0x7F)
{
assert(isValidDchar(c));
str ~= cast(char) c;
}
else
{
char[4] buf;
uint L;
if (c <= 0x7FF)
{
assert(isValidDchar(c));
buf[0] = cast(char)(0xC0 | (c >> 6));
buf[1] = cast(char)(0x80 | (c & 0x3F));
L = 2;
}
else if (c <= 0xFFFF)
{
if (0xD800 <= c && c <= 0xDFFF)
c = _utfException!useReplacementDchar("Encoding a surrogate code point in UTF-8", c);
assert(isValidDchar(c));
L3:
buf[0] = cast(char)(0xE0 | (c >> 12));
buf[1] = cast(char)(0x80 | ((c >> 6) & 0x3F));
buf[2] = cast(char)(0x80 | (c & 0x3F));
L = 3;
}
else if (c <= 0x10FFFF)
{
assert(isValidDchar(c));
buf[0] = cast(char)(0xF0 | (c >> 18));
buf[1] = cast(char)(0x80 | ((c >> 12) & 0x3F));
buf[2] = cast(char)(0x80 | ((c >> 6) & 0x3F));
buf[3] = cast(char)(0x80 | (c & 0x3F));
L = 4;
}
else
{
assert(!isValidDchar(c));
c = _utfException!useReplacementDchar("Encoding an invalid code point in UTF-8", c);
goto L3;
}
str ~= buf[0 .. L];
}
}
///
@safe unittest
{
char[] s = "abcd".dup;
dchar d1 = 'a';
dchar d2 = 'ø';
encode(s, d1);
assert(s.length == 5);
assert(s == "abcda");
encode(s, d2);
assert(s.length == 7);
assert(s == "abcdaø");
}
@safe unittest
{
import std.exception;
assertCTFEable!(
{
char[] s = "abcd".dup;
encode(s, cast(dchar)'a');
assert(s.length == 5);
assert(s == "abcda");
encode(s, cast(dchar)'\u00A9');
assert(s.length == 7);
assert(s == "abcda\xC2\xA9");
//assert(s == "abcda\u00A9"); // BUG: fix compiler
encode(s, cast(dchar)'\u2260');
assert(s.length == 10);
assert(s == "abcda\xC2\xA9\xE2\x89\xA0");
});
}
@safe unittest
{
import std.exception;
assertCTFEable!(
{
char[] buf;
encode(buf, '\u0000'); assert(buf[0 .. $] == "\u0000");
encode(buf, '\u007F'); assert(buf[1 .. $] == "\u007F");
encode(buf, '\u0080'); assert(buf[2 .. $] == "\u0080");
encode(buf, '\u07FF'); assert(buf[4 .. $] == "\u07FF");
encode(buf, '\u0800'); assert(buf[6 .. $] == "\u0800");
encode(buf, '\uD7FF'); assert(buf[9 .. $] == "\uD7FF");
encode(buf, '\uE000'); assert(buf[12 .. $] == "\uE000");
encode(buf, 0xFFFE); assert(buf[15 .. $] == "\xEF\xBF\xBE");
encode(buf, 0xFFFF); assert(buf[18 .. $] == "\xEF\xBF\xBF");
encode(buf, '\U00010000'); assert(buf[21 .. $] == "\U00010000");
encode(buf, '\U0010FFFF'); assert(buf[25 .. $] == "\U0010FFFF");
assertThrown!UTFException(encode(buf, cast(dchar) 0xD800));
assertThrown!UTFException(encode(buf, cast(dchar) 0xDBFF));
assertThrown!UTFException(encode(buf, cast(dchar) 0xDC00));
assertThrown!UTFException(encode(buf, cast(dchar) 0xDFFF));
assertThrown!UTFException(encode(buf, cast(dchar) 0x110000));
enum replacementDcharString = "\uFFFD";
enum rdcslen = replacementDcharString.length;
assert(buf[$ - rdcslen .. $] != replacementDcharString);
encode!(Yes.useReplacementDchar)(buf, cast(dchar) 0x110000);
assert(buf[$ - rdcslen .. $] == replacementDcharString);
});
}
/// ditto
void encode(UseReplacementDchar useReplacementDchar = No.useReplacementDchar)(
ref scope wchar[] str, dchar c) @safe pure
{
if (c <= 0xFFFF)
{
if (0xD800 <= c && c <= 0xDFFF)
c = _utfException!useReplacementDchar("Encoding an isolated surrogate code point in UTF-16", c);
assert(isValidDchar(c));
L1:
str ~= cast(wchar) c;
}
else if (c <= 0x10FFFF)
{
wchar[2] buf;
assert(isValidDchar(c));
buf[0] = cast(wchar)((((c - 0x10000) >> 10) & 0x3FF) + 0xD800);
buf[1] = cast(wchar)(((c - 0x10000) & 0x3FF) + 0xDC00);
str ~= buf;
}
else
{
assert(!isValidDchar(c));
c = _utfException!useReplacementDchar("Encoding an invalid code point in UTF-16", c);
goto L1;
}
}
@safe unittest
{
import std.exception;
assertCTFEable!(
{
wchar[] buf;
encode(buf, '\u0000'); assert(buf[0] == '\u0000');
encode(buf, '\uD7FF'); assert(buf[1] == '\uD7FF');
encode(buf, '\uE000'); assert(buf[2] == '\uE000');
encode(buf, 0xFFFE); assert(buf[3] == 0xFFFE);
encode(buf, 0xFFFF); assert(buf[4] == 0xFFFF);
encode(buf, '\U00010000'); assert(buf[5 .. $] == "\U00010000");
encode(buf, '\U0010FFFF'); assert(buf[7 .. $] == "\U0010FFFF");
assertThrown!UTFException(encode(buf, cast(dchar) 0xD800));
assertThrown!UTFException(encode(buf, cast(dchar) 0xDBFF));
assertThrown!UTFException(encode(buf, cast(dchar) 0xDC00));
assertThrown!UTFException(encode(buf, cast(dchar) 0xDFFF));
assertThrown!UTFException(encode(buf, cast(dchar) 0x110000));
assert(buf.back != replacementDchar);
encode!(Yes.useReplacementDchar)(buf, cast(dchar) 0x110000);
assert(buf.back == replacementDchar);
});
}
/// ditto
void encode(UseReplacementDchar useReplacementDchar = No.useReplacementDchar)(
ref scope dchar[] str, dchar c) @safe pure
{
if ((0xD800 <= c && c <= 0xDFFF) || 0x10FFFF < c)
c = _utfException!useReplacementDchar("Encoding an invalid code point in UTF-32", c);
else
assert(isValidDchar(c));
str ~= c;
}
@safe unittest
{
import std.exception;
assertCTFEable!(
{
dchar[] buf;
encode(buf, '\u0000'); assert(buf[0] == '\u0000');
encode(buf, '\uD7FF'); assert(buf[1] == '\uD7FF');
encode(buf, '\uE000'); assert(buf[2] == '\uE000');
encode(buf, 0xFFFE); assert(buf[3] == 0xFFFE);
encode(buf, 0xFFFF); assert(buf[4] == 0xFFFF);
encode(buf, '\U0010FFFF'); assert(buf[5] == '\U0010FFFF');
assertThrown!UTFException(encode(buf, cast(dchar) 0xD800));
assertThrown!UTFException(encode(buf, cast(dchar) 0xDBFF));
assertThrown!UTFException(encode(buf, cast(dchar) 0xDC00));
assertThrown!UTFException(encode(buf, cast(dchar) 0xDFFF));
assertThrown!UTFException(encode(buf, cast(dchar) 0x110000));
assert(buf.back != replacementDchar);
encode!(Yes.useReplacementDchar)(buf, cast(dchar) 0x110000);
assert(buf.back == replacementDchar);
});
}
/++
Returns the number of code units that are required to encode the code point
`c` when `C` is the character type used to encode it.
+/
ubyte codeLength(C)(dchar c) @safe pure nothrow @nogc
if (isSomeChar!C)
{
static if (C.sizeof == 1)
{
if (c <= 0x7F) return 1;
if (c <= 0x7FF) return 2;
if (c <= 0xFFFF) return 3;
if (c <= 0x10FFFF) return 4;
assert(false);
}
else static if (C.sizeof == 2)
{
return c <= 0xFFFF ? 1 : 2;
}
else
{
static assert(C.sizeof == 4);
return 1;
}
}
///
@safe pure nothrow @nogc unittest
{
assert(codeLength!char('a') == 1);
assert(codeLength!wchar('a') == 1);
assert(codeLength!dchar('a') == 1);
assert(codeLength!char('\U0010FFFF') == 4);
assert(codeLength!wchar('\U0010FFFF') == 2);
assert(codeLength!dchar('\U0010FFFF') == 1);
}
/++
Returns the number of code units that are required to encode `str`
in a string whose character type is `C`. This is particularly useful
when slicing one string with the length of another and the two string
types use different character types.
Params:
C = the character type to get the encoding length for
input = the $(REF_ALTTEXT input range, isInputRange, std,range,primitives)
to calculate the encoding length from
Returns:
The number of code units in `input` when encoded to `C`
+/
size_t codeLength(C, InputRange)(InputRange input)
if (isSomeFiniteCharInputRange!InputRange)
{
alias EncType = typeof(cast() ElementEncodingType!InputRange.init);
static if (isSomeString!InputRange && is(EncType == C) && is(typeof(input.length)))
return input.length;
else
{
size_t total = 0;
foreach (c; input.byDchar)
total += codeLength!C(c);
return total;
}
}
///
@safe unittest
{
assert(codeLength!char("hello world") ==
"hello world".length);
assert(codeLength!wchar("hello world") ==
"hello world"w.length);
assert(codeLength!dchar("hello world") ==
"hello world"d.length);
assert(codeLength!char(`プログラミング`) ==
`プログラミング`.length);
assert(codeLength!wchar(`プログラミング`) ==
`プログラミング`w.length);
assert(codeLength!dchar(`プログラミング`) ==
`プログラミング`d.length);
string haystack = `Être sans la verité, ça, ce ne serait pas bien.`;
wstring needle = `Être sans la verité`;
assert(haystack[codeLength!char(needle) .. $] ==
`, ça, ce ne serait pas bien.`);
}
@safe unittest
{
import std.algorithm.iteration : filter;
import std.conv : to;
import std.exception;
assertCTFEable!(
{
foreach (S; AliasSeq!( char[], const char[], string,
wchar[], const wchar[], wstring,
dchar[], const dchar[], dstring))
{
foreach (C; AliasSeq!(char, wchar, dchar))
{
assert(codeLength!C(to!S("Walter Bright")) == to!(C[])("Walter Bright").length);
assert(codeLength!C(to!S(`言語`)) == to!(C[])(`言語`).length);
assert(codeLength!C(to!S(`ウェブサイト@La_Verité.com`)) ==
to!(C[])(`ウェブサイト@La_Verité.com`).length);
assert(codeLength!C(to!S(`ウェブサイト@La_Verité.com`).filter!(x => true)()) ==
to!(C[])(`ウェブサイト@La_Verité.com`).length);
}
}
});
}
/+
Internal helper function:
Returns true if it is safe to search for the Codepoint `c` inside
code units, without decoding.
This is a runtime check that is used an optimization in various functions,
particularly, in `std.string`.
+/
package bool canSearchInCodeUnits(C)(dchar c)
if (isSomeChar!C)
{
static if (C.sizeof == 1)
return c <= 0x7F;
else static if (C.sizeof == 2)
return c <= 0xD7FF || (0xE000 <= c && c <= 0xFFFF);
else static if (C.sizeof == 4)
return true;
else
static assert(0);
}
@safe unittest
{
assert( canSearchInCodeUnits! char('a'));
assert( canSearchInCodeUnits!wchar('a'));
assert( canSearchInCodeUnits!dchar('a'));
assert(!canSearchInCodeUnits! char('ö')); //Important test: ö <= 0xFF
assert(!canSearchInCodeUnits! char(cast(char)'ö')); //Important test: ö <= 0xFF
assert( canSearchInCodeUnits!wchar('ö'));
assert( canSearchInCodeUnits!dchar('ö'));
assert(!canSearchInCodeUnits! char('日'));
assert( canSearchInCodeUnits!wchar('日'));
assert( canSearchInCodeUnits!dchar('日'));
assert(!canSearchInCodeUnits!wchar(cast(wchar) 0xDA00));
assert( canSearchInCodeUnits!dchar(cast(dchar) 0xDA00));
assert(!canSearchInCodeUnits! char('\U00010001'));
assert(!canSearchInCodeUnits!wchar('\U00010001'));
assert( canSearchInCodeUnits!dchar('\U00010001'));
}
/* =================== Validation ======================= */
/++
Checks to see if `str` is well-formed unicode or not.
Throws:
`UTFException` if `str` is not well-formed.
+/
void validate(S)(in S str) @safe pure
if (isSomeString!S)
{
immutable len = str.length;
for (size_t i = 0; i < len; )
{
decode(str, i);
}
}
///
@safe unittest
{
import std.exception : assertThrown;
char[] a = [167, 133, 175];
assertThrown!UTFException(validate(a));
}
// https://issues.dlang.org/show_bug.cgi?id=12923
@safe unittest
{
import std.exception;
assertThrown((){
char[3]a=[167, 133, 175];
validate(a[]);
}());
}
/**
* Encodes the elements of `s` to UTF-8 and returns a newly allocated
* string of the elements.
*
* Params:
* s = the string to encode
* Returns:
* A UTF-8 string
* See_Also:
* For a lazy, non-allocating version of these functions, see $(LREF byUTF).
*/
string toUTF8(S)(S s)
if (isSomeFiniteCharInputRange!S)
{
return toUTFImpl!string(s);
}
///
@safe pure unittest
{
import std.algorithm.comparison : equal;
// The ö is represented by two UTF-8 code units
assert("Hellø"w.toUTF8.equal(['H', 'e', 'l', 'l', 0xC3, 0xB8]));
// 𐐷 is four code units in UTF-8
assert("𐐷"d.toUTF8.equal([0xF0, 0x90, 0x90, 0xB7]));
}
@system pure unittest
{
import std.algorithm.comparison : equal;
import std.internal.test.dummyrange : ReferenceInputRange;
alias RT = ReferenceInputRange!(ElementType!(string));
auto r1 = new RT("Hellø");
auto r2 = new RT("𐐷");
assert(r1.toUTF8.equal(['H', 'e', 'l', 'l', 0xC3, 0xB8]));
assert(r2.toUTF8.equal([0xF0, 0x90, 0x90, 0xB7]));
}
/**
* Encodes the elements of `s` to UTF-16 and returns a newly GC allocated
* `wstring` of the elements.
*
* Params:
* s = the range to encode
* Returns:
* A UTF-16 string
* See_Also:
* For a lazy, non-allocating version of these functions, see $(LREF byUTF).
*/
wstring toUTF16(S)(S s)
if (isSomeFiniteCharInputRange!S)
{
return toUTFImpl!wstring(s);
}
///
@safe pure unittest
{
import std.algorithm.comparison : equal;
// these graphemes are two code units in UTF-16 and one in UTF-32
assert("𤭢"d.length == 1);
assert("𐐷"d.length == 1);
assert("𤭢"d.toUTF16.equal([0xD852, 0xDF62]));
assert("𐐷"d.toUTF16.equal([0xD801, 0xDC37]));
}
@system pure unittest
{
import std.algorithm.comparison : equal;
import std.internal.test.dummyrange : ReferenceInputRange;
alias RT = ReferenceInputRange!(ElementType!(string));
auto r1 = new RT("𤭢");
auto r2 = new RT("𐐷");
assert(r1.toUTF16.equal([0xD852, 0xDF62]));
assert(r2.toUTF16.equal([0xD801, 0xDC37]));
}
/**
* Encodes the elements of `s` to UTF-32 and returns a newly GC allocated
* `dstring` of the elements.
*
* Params:
* s = the range to encode
* Returns:
* A UTF-32 string
* See_Also:
* For a lazy, non-allocating version of these functions, see $(LREF byUTF).
*/
dstring toUTF32(S)(scope S s)
if (isSomeFiniteCharInputRange!S)
{
return toUTFImpl!dstring(s);
}
///
@safe pure unittest
{
import std.algorithm.comparison : equal;
// these graphemes are two code units in UTF-16 and one in UTF-32
assert("𤭢"w.length == 2);
assert("𐐷"w.length == 2);
assert("𤭢"w.toUTF32.equal([0x00024B62]));
assert("𐐷"w.toUTF32.equal([0x00010437]));
}
private T toUTFImpl(T, S)(scope S s)
{
static if (is(S : T))
{
return s.idup;
}
else
{
import std.array : appender;
auto app = appender!T();
static if (is(S == C[], C) || hasLength!S)
app.reserve(s.length);
ElementEncodingType!T e = void;
foreach (c; s.byUTF!(typeof(cast() ElementEncodingType!T.init)))
app.put(c);
return app.data;
}
}
/* =================== toUTFz ======================= */
/++
Returns a C-style zero-terminated string equivalent to `str`. `str`
must not contain embedded `'\0'`'s as any C function will treat the first
`'\0'` that it sees as the end of the string. If `str.empty` is
`true`, then a string containing only `'\0'` is returned.
`toUTFz` accepts any type of string and is templated on the type of
character pointer that you wish to convert to. It will avoid allocating a
new string if it can, but there's a decent chance that it will end up having
to allocate a new string - particularly when dealing with character types
other than `char`.
$(RED Warning 1:) If the result of `toUTFz` equals `str.ptr`, then if
anything alters the character one past the end of `str` (which is the
`'\0'` character terminating the string), then the string won't be
zero-terminated anymore. The most likely scenarios for that are if you
append to `str` and no reallocation takes place or when `str` is a
slice of a larger array, and you alter the character in the larger array
which is one character past the end of `str`. Another case where it could
occur would be if you had a mutable character array immediately after
`str` in memory (for example, if they're member variables in a
user-defined type with one declared right after the other) and that
character array happened to start with `'\0'`. Such scenarios will never
occur if you immediately use the zero-terminated string after calling
`toUTFz` and the C function using it doesn't keep a reference to it.
Also, they are unlikely to occur even if you save the zero-terminated string
(the cases above would be among the few examples of where it could happen).
However, if you save the zero-terminate string and want to be absolutely
certain that the string stays zero-terminated, then simply append a
`'\0'` to the string and use its `ptr` property rather than calling
`toUTFz`.
$(RED Warning 2:) When passing a character pointer to a C function, and the
C function keeps it around for any reason, make sure that you keep a
reference to it in your D code. Otherwise, it may go away during a garbage
collection cycle and cause a nasty bug when the C code tries to use it.
+/
template toUTFz(P)
if (is(P == C*, C) && isSomeChar!C)
{
P toUTFz(S)(S str) @safe pure
if (isSomeString!S)
{
return toUTFzImpl!(P, S)(str);
}
}
///
@safe pure unittest
{
auto p1 = toUTFz!(char*)("hello world");
auto p2 = toUTFz!(const(char)*)("hello world");
auto p3 = toUTFz!(immutable(char)*)("hello world");
auto p4 = toUTFz!(char*)("hello world"d);
auto p5 = toUTFz!(const(wchar)*)("hello world");
auto p6 = toUTFz!(immutable(dchar)*)("hello world"w);
}
private P toUTFzImpl(P, S)(return scope S str) @safe pure
if (is(immutable typeof(*P.init) == typeof(str[0])))
//immutable(C)[] -> C*, const(C)*, or immutable(C)*
{
if (str.empty)
{
typeof(*P.init)[] retval = ['\0'];
auto trustedPtr() @trusted { return retval.ptr; }
return trustedPtr();
}
alias C = typeof(cast() ElementEncodingType!S.init);
//If the P is mutable, then we have to make a copy.
static if (is(typeof(cast() *P.init) == typeof(*P.init)))
{
return toUTFzImpl!(P, const(C)[])(cast(const(C)[])str);
}
else
{
if (!__ctfe)
{
auto trustedPtrAdd(S s) @trusted { return s.ptr + s.length; }
immutable p = trustedPtrAdd(str);
// Peek past end of str, if it's 0, no conversion necessary.
// Note that the compiler will put a 0 past the end of static
// strings, and the storage allocator will put a 0 past the end
// of newly allocated char[]'s.
// Is p dereferenceable? A simple test: if the p points to an
// address multiple of 4, then conservatively assume the pointer
// might be pointing to a new block of memory, which might be
// unreadable. Otherwise, it's definitely pointing to valid
// memory.
if ((cast(size_t) p & 3) && *p == '\0')
return &str[0];
}
return toUTFzImpl!(P, const(C)[])(cast(const(C)[])str);
}
}
private P toUTFzImpl(P, S)(return scope S str) @safe pure
if (is(typeof(str[0]) C) && is(immutable typeof(*P.init) == immutable C) && !is(C == immutable))
//C[] or const(C)[] -> C*, const(C)*, or immutable(C)*
{
alias InChar = typeof(str[0]);
alias UInChar = typeof(cast() str[0]); // unqualified version of InChar
alias OutChar = typeof(*P.init);
alias UOutChar = typeof(cast() *P.init); // unqualified version
//const(C)[] -> const(C)* or
//C[] -> C* or const(C)*
static if (( is(const(UInChar) == InChar) && is( const(UOutChar) == OutChar)) ||
(!is(const(UInChar) == InChar) && !is(immutable(UOutChar) == OutChar)))
{
if (!__ctfe)
{
auto trustedPtrAdd(S s) @trusted { return s.ptr + s.length; }
auto p = trustedPtrAdd(str);
if ((cast(size_t) p & 3) && *p == '\0')
return &str[0];
}
str ~= '\0';
return &str[0];
}
//const(C)[] -> C* or immutable(C)* or
//C[] -> immutable(C)*
else
{
import std.array : uninitializedArray;
auto copy = uninitializedArray!(UOutChar[])(str.length + 1);
copy[0 .. $ - 1] = str[];
copy[$ - 1] = '\0';
auto trustedCast(typeof(copy) c) @trusted { return cast(P) c.ptr; }
return trustedCast(copy);
}
}
private P toUTFzImpl(P, S)(S str) @safe pure
if (!is(immutable typeof(*P.init) == immutable typeof(str[0])))
//C1[], const(C1)[], or immutable(C1)[] -> C2*, const(C2)*, or immutable(C2)*
{
import std.array : appender;
auto retval = appender!(typeof(*P.init)[])();
foreach (dchar c; str)
retval.put(c);
retval.put('\0');
return () @trusted { return cast(P) retval.data.ptr; } ();
}
@safe pure unittest
{
import core.exception : AssertError;
import std.algorithm;
import std.conv : to;
import std.exception;
import std.string : format;
assertCTFEable!(
{
foreach (S; AliasSeq!(string, wstring, dstring))
{
alias C = Unqual!(ElementEncodingType!S);
auto s1 = to!S("hello\U00010143\u0100\U00010143");
auto temp = new C[](s1.length + 1);
temp[0 .. $ - 1] = s1[0 .. $];
temp[$ - 1] = '\n';
--temp.length;
auto trustedAssumeUnique(T)(T t) @trusted { return assumeUnique(t); }
auto s2 = trustedAssumeUnique(temp);
assert(s1 == s2);
void trustedCStringAssert(P, S)(S s) @trusted
{
auto p = toUTFz!P(s);
assert(p[0 .. s.length] == s);
assert(p[s.length] == '\0');
}
foreach (P; AliasSeq!(C*, const(C)*, immutable(C)*))
{
trustedCStringAssert!P(s1);
trustedCStringAssert!P(s2);
}
}
});
static void test(P, S)(S s, size_t line = __LINE__) @trusted
{
static size_t zeroLen(C)(const(C)* ptr) @trusted
{
size_t len = 0;
while (*ptr != '\0') { ++ptr; ++len; }
return len;
}
auto p = toUTFz!P(s);
immutable len = zeroLen(p);
enforce(cmp(s, p[0 .. len]) == 0,
new AssertError(format("Unit test failed: %s %s", P.stringof, S.stringof),
__FILE__, line));
}
assertCTFEable!(
{
foreach (P; AliasSeq!(wchar*, const(wchar)*, immutable(wchar)*,
dchar*, const(dchar)*, immutable(dchar)*))
{
test!P("hello\U00010143\u0100\U00010143");
}
foreach (P; AliasSeq!( char*, const( char)*, immutable( char)*,
dchar*, const(dchar)*, immutable(dchar)*))
{
test!P("hello\U00010143\u0100\U00010143"w);
}
foreach (P; AliasSeq!( char*, const( char)*, immutable( char)*,
wchar*, const(wchar)*, immutable(wchar)*))
{
test!P("hello\U00010143\u0100\U00010143"d);
}
foreach (S; AliasSeq!( char[], const( char)[],
wchar[], const(wchar)[],
dchar[], const(dchar)[]))
{
auto s = to!S("hello\U00010143\u0100\U00010143");
foreach (P; AliasSeq!( char*, const( char)*, immutable( char)*,
wchar*, const(wchar)*, immutable(wchar)*,
dchar*, const(dchar)*, immutable(dchar)*))
{
test!P(s);
}
}
});
}
/++
`toUTF16z` is a convenience function for `toUTFz!(const(wchar)*)`.
Encodes string `s` into UTF-16 and returns the encoded string.
`toUTF16z` is suitable for calling the 'W' functions in the Win32 API
that take an `LPCWSTR` argument.
+/
const(wchar)* toUTF16z(C)(const(C)[] str) @safe pure
if (isSomeChar!C)
{
return toUTFz!(const(wchar)*)(str);
}
///
@system unittest
{
string str = "Hello, World!";
const(wchar)* p = str.toUTF16z;
assert(p[str.length] == '\0');
}
@safe pure unittest
{
import std.conv : to;
//toUTFz is already thoroughly tested, so this will just verify that
//toUTF16z compiles properly for the various string types.
foreach (S; AliasSeq!(string, wstring, dstring))
assert(toUTF16z(to!S("hello world")) !is null);
}
/* ================================ tests ================================== */
@safe pure unittest
{
import std.exception;
assertCTFEable!(
{
assert(toUTF16("hello"c) == "hello");
assert(toUTF32("hello"c) == "hello");
assert(toUTF8 ("hello"w) == "hello");
assert(toUTF32("hello"w) == "hello");
assert(toUTF8 ("hello"d) == "hello");
assert(toUTF16("hello"d) == "hello");
assert(toUTF16("hel\u1234o"c) == "hel\u1234o");
assert(toUTF32("hel\u1234o"c) == "hel\u1234o");
assert(toUTF8 ("hel\u1234o"w) == "hel\u1234o");
assert(toUTF32("hel\u1234o"w) == "hel\u1234o");
assert(toUTF8 ("hel\u1234o"d) == "hel\u1234o");
assert(toUTF16("hel\u1234o"d) == "hel\u1234o");
assert(toUTF16("he\U0010AAAAllo"c) == "he\U0010AAAAllo");
assert(toUTF32("he\U0010AAAAllo"c) == "he\U0010AAAAllo");
assert(toUTF8 ("he\U0010AAAAllo"w) == "he\U0010AAAAllo");
assert(toUTF32("he\U0010AAAAllo"w) == "he\U0010AAAAllo");
assert(toUTF8 ("he\U0010AAAAllo"d) == "he\U0010AAAAllo");
assert(toUTF16("he\U0010AAAAllo"d) == "he\U0010AAAAllo");
});
}
/++
Returns the total number of code points encoded in `str`.
Supercedes: This function supercedes $(LREF toUCSindex).
Standards: Unicode 5.0, ASCII, ISO-8859-1, WINDOWS-1252
Throws:
`UTFException` if `str` is not well-formed.
+/
size_t count(C)(const(C)[] str) @safe pure nothrow @nogc
if (isSomeChar!C)
{
return walkLength(str.byDchar);
}
///
@safe pure nothrow @nogc unittest
{
assert(count("") == 0);
assert(count("a") == 1);
assert(count("abc") == 3);
assert(count("\u20AC100") == 4);
}
@safe pure nothrow @nogc unittest
{
import std.exception;
assertCTFEable!(
{
assert(count("") == 0);
assert(count("a") == 1);
assert(count("abc") == 3);
assert(count("\u20AC100") == 4);
});
}
// Ranges of code units for testing.
version (StdUnittest)
{
private:
struct InputCU(C)
{
import std.conv : to;
@property bool empty() { return _str.empty; }
@property C front() { return _str[0]; }
void popFront() { _str = _str[1 .. $]; }
this(inout(C)[] str)
{
_str = to!(C[])(str);
}
C[] _str;
}
struct BidirCU(C)
{
import std.conv : to;
@property bool empty() { return _str.empty; }
@property C front() { return _str[0]; }
void popFront() { _str = _str[1 .. $]; }
@property C back() { return _str[$ - 1]; }
void popBack() { _str = _str[0 .. $ - 1]; }
@property auto save() { return BidirCU(_str); }
@property size_t length() { return _str.length; }
this(inout(C)[] str)
{
_str = to!(C[])(str);
}
C[] _str;
}
struct RandomCU(C)
{
import std.conv : to;
@property bool empty() { return _str.empty; }
@property C front() { return _str[0]; }
void popFront() { _str = _str[1 .. $]; }
@property C back() { return _str[$ - 1]; }
void popBack() { _str = _str[0 .. $ - 1]; }
@property auto save() { return RandomCU(_str); }
@property size_t length() { return _str.length; }
C opIndex(size_t i) { return _str[i]; }
auto opSlice(size_t i, size_t j) { return RandomCU(_str[i .. j]); }
this(inout(C)[] str)
{
_str = to!(C[])(str);
}
C[] _str;
}
class RefBidirCU(C)
{
import std.conv : to;
@property bool empty() { return _str.empty; }
@property C front() { return _str[0]; }
void popFront() { _str = _str[1 .. $]; }
@property C back() { return _str[$ - 1]; }
void popBack() { _str = _str[0 .. $ - 1]; }
@property auto save() { return new RefBidirCU(_str); }
@property size_t length() { return _str.length; }
this(inout(C)[] str)
{
_str = to!(C[])(str);
}
C[] _str;
}
class RefRandomCU(C)
{
import std.conv : to;
@property bool empty() { return _str.empty; }
@property C front() { return _str[0]; }
void popFront() { _str = _str[1 .. $]; }
@property C back() { return _str[$ - 1]; }
void popBack() { _str = _str[0 .. $ - 1]; }
@property auto save() { return new RefRandomCU(_str); }
@property size_t length() { return _str.length; }
C opIndex(size_t i) { return _str[i]; }
auto opSlice(size_t i, size_t j) { return new RefRandomCU(_str[i .. j]); }
this(inout(C)[] str)
{
_str = to!(C[])(str);
}
C[] _str;
}
}
/**
* Inserted in place of invalid UTF sequences.
*
* References:
* $(LINK http://en.wikipedia.org/wiki/Replacement_character#Replacement_character)
*/
enum dchar replacementDchar = '\uFFFD';
/********************************************
* Iterate a range of char, wchar, or dchars by code unit.
*
* The purpose is to bypass the special case decoding that
* $(REF front, std,range,primitives) does to character arrays. As a result,
* using ranges with `byCodeUnit` can be `nothrow` while
* $(REF front, std,range,primitives) throws when it encounters invalid Unicode
* sequences.
*
* A code unit is a building block of the UTF encodings. Generally, an
* individual code unit does not represent what's perceived as a full
* character (a.k.a. a grapheme cluster in Unicode terminology). Many characters
* are encoded with multiple code units. For example, the UTF-8 code units for
* `ø` are `0xC3 0xB8`. That means, an individual element of `byCodeUnit`
* often does not form a character on its own. Attempting to treat it as
* one while iterating over the resulting range will give nonsensical results.
*
* Params:
* r = an $(REF_ALTTEXT input range, isInputRange, std,range,primitives)
* of characters (including strings) or a type that implicitly converts to a string type.
* Returns:
* If `r` is not an auto-decodable string (i.e. a narrow string or a
* user-defined type that implicitly converts to a string type), then `r`
* is returned.
*
* Otherwise, `r` is converted to its corresponding string type (if it's
* not already a string) and wrapped in a random-access range where the
* element encoding type of the string (its code unit) is the element type
* of the range, and that range returned. The range has slicing.
*
* If `r` is quirky enough to be a struct or class which is an input range
* of characters on its own (i.e. it has the input range API as member
* functions), $(I and) it's implicitly convertible to a string type, then
* `r` is returned, and no implicit conversion takes place.
*
* If `r` is wrapped in a new range, then that range has a `source`
* property for returning the string that's currently contained within that
* range.
*
* See_Also:
* Refer to the $(MREF std, uni) docs for a reference on Unicode
* terminology.
*
* For a range that iterates by grapheme cluster (written character) see
* $(REF byGrapheme, std,uni).
*/
auto byCodeUnit(R)(R r)
if ((isConvertibleToString!R && !isStaticArray!R) ||
(isInputRange!R && isSomeChar!(ElementEncodingType!R)))
{
import std.traits : StringTypeOf;
static if (// This would be cleaner if we had a way to check whether a type
// was a range without any implicit conversions.
(isAutodecodableString!R && !__traits(hasMember, R, "empty") &&
!__traits(hasMember, R, "front") && !__traits(hasMember, R, "popFront")))
{
static struct ByCodeUnitImpl
{
@safe pure nothrow @nogc:
@property bool empty() const { return source.length == 0; }
@property auto ref front() inout { return source[0]; }
void popFront() { source = source[1 .. $]; }
@property auto save() { return ByCodeUnitImpl(source.save); }
@property auto ref back() inout { return source[$ - 1]; }
void popBack() { source = source[0 .. $-1]; }
auto ref opIndex(size_t index) inout { return source[index]; }
auto opSlice(size_t lower, size_t upper) { return ByCodeUnitImpl(source[lower .. upper]); }
@property size_t length() const { return source.length; }
alias opDollar = length;
StringTypeOf!R source;
}
static assert(isRandomAccessRange!ByCodeUnitImpl);
return ByCodeUnitImpl(r);
}
else static if (!isInputRange!R ||
(is(R : const dchar[]) && !__traits(hasMember, R, "empty") &&
!__traits(hasMember, R, "front") && !__traits(hasMember, R, "popFront")))
{
return cast(StringTypeOf!R) r;
}
else
{
// byCodeUnit for ranges and dchar[] is a no-op
return r;
}
}
///
@safe unittest
{
import std.range.primitives;
import std.traits : isAutodecodableString;
auto r = "Hello, World!".byCodeUnit();
static assert(hasLength!(typeof(r)));
static assert(hasSlicing!(typeof(r)));
static assert(isRandomAccessRange!(typeof(r)));
static assert(is(ElementType!(typeof(r)) == immutable char));
// contrast with the range capabilities of standard strings (with or
// without autodecoding enabled).
auto s = "Hello, World!";
static assert(isBidirectionalRange!(typeof(r)));
static if (isAutodecodableString!(typeof(s)))
{
// with autodecoding enabled, strings are non-random-access ranges of
// dchar.
static assert(is(ElementType!(typeof(s)) == dchar));
static assert(!isRandomAccessRange!(typeof(s)));
static assert(!hasSlicing!(typeof(s)));
static assert(!hasLength!(typeof(s)));
}
else
{
// without autodecoding, strings are normal arrays.
static assert(is(ElementType!(typeof(s)) == immutable char));
static assert(isRandomAccessRange!(typeof(s)));
static assert(hasSlicing!(typeof(s)));
static assert(hasLength!(typeof(s)));
}
}
/// `byCodeUnit` does no Unicode decoding
@safe unittest
{
string noel1 = "noe\u0308l"; // noël using e + combining diaeresis
assert(noel1.byCodeUnit[2] != 'ë');
assert(noel1.byCodeUnit[2] == 'e');
string noel2 = "no\u00EBl"; // noël using a precomposed ë character
// Because string is UTF-8, the code unit at index 2 is just
// the first of a sequence that encodes 'ë'
assert(noel2.byCodeUnit[2] != 'ë');
}
/// `byCodeUnit` exposes a `source` property when wrapping narrow strings.
@safe unittest
{
import std.algorithm.comparison : equal;
import std.range : popFrontN;
import std.traits : isAutodecodableString;
{
auto range = byCodeUnit("hello world");
range.popFrontN(3);
assert(equal(range.save, "lo world"));
static if (isAutodecodableString!string) // only enabled with autodecoding
{
string str = range.source;
assert(str == "lo world");
}
}
// source only exists if the range was wrapped
{
auto range = byCodeUnit("hello world"d);
static assert(!__traits(compiles, range.source));
}
}
@safe pure nothrow @nogc unittest
{
import std.range;
{
enum testStr = "𐁄𐂌𐃯 hello ディラン";
char[testStr.length] s;
int i;
foreach (c; testStr.byCodeUnit().byCodeUnit())
{
s[i++] = c;
}
assert(s == testStr);
}
{
enum testStr = "𐁄𐂌𐃯 hello ディラン"w;
wchar[testStr.length] s;
int i;
foreach (c; testStr.byCodeUnit().byCodeUnit())
{
s[i++] = c;
}
assert(s == testStr);
}
{
enum testStr = "𐁄𐂌𐃯 hello ディラン"d;
dchar[testStr.length] s;
int i;
foreach (c; testStr.byCodeUnit().byCodeUnit())
{
s[i++] = c;
}
assert(s == testStr);
}
{
auto bcu = "hello".byCodeUnit();
assert(bcu.length == 5);
assert(bcu[3] == 'l');
assert(bcu[2 .. 4][1] == 'l');
}
{
char[5] orig = "hello";
auto bcu = orig[].byCodeUnit();
bcu.front = 'H';
assert(bcu.front == 'H');
bcu[1] = 'E';
assert(bcu[1] == 'E');
}
{
auto bcu = "hello".byCodeUnit().byCodeUnit();
static assert(isForwardRange!(typeof(bcu)));
static assert(is(typeof(bcu) == struct) == isAutodecodableString!string);
auto s = bcu.save;
bcu.popFront();
assert(s.front == 'h');
}
{
auto bcu = "hello".byCodeUnit();
static assert(hasSlicing!(typeof(bcu)));
static assert(isBidirectionalRange!(typeof(bcu)));
static assert(is(typeof(bcu) == struct) == isAutodecodableString!string);
static assert(is(typeof(bcu) == typeof(bcu.byCodeUnit())));
auto ret = bcu.retro;
assert(ret.front == 'o');
ret.popFront();
assert(ret.front == 'l');
}
{
auto bcu = "κόσμε"w.byCodeUnit();
static assert(hasSlicing!(typeof(bcu)));
static assert(isBidirectionalRange!(typeof(bcu)));
static assert(is(typeof(bcu) == struct) == isAutodecodableString!wstring);
static assert(is(typeof(bcu) == typeof(bcu.byCodeUnit())));
auto ret = bcu.retro;
assert(ret.front == 'ε');
ret.popFront();
assert(ret.front == 'μ');
}
{
static struct Stringish
{
string s;
alias s this;
}
auto orig = Stringish("\U0010fff8 𐁊 foo 𐂓");
auto bcu = orig.byCodeUnit();
static assert(is(typeof(bcu) == struct));
static assert(!is(typeof(bcu) == Stringish) == isAutodecodableString!Stringish);
static assert(is(typeof(bcu) == typeof(bcu.byCodeUnit())));
static assert(is(ElementType!(typeof(bcu)) == immutable char));
assert(bcu.front == cast(char) 244);
}
{
static struct WStringish
{
wstring s;
alias s this;
}
auto orig = WStringish("\U0010fff8 𐁊 foo 𐂓"w);
auto bcu = orig.byCodeUnit();
static assert(is(typeof(bcu) == struct));
static assert(!is(typeof(bcu) == WStringish) == isAutodecodableString!WStringish);
static assert(is(typeof(bcu) == typeof(bcu.byCodeUnit())));
static assert(is(ElementType!(typeof(bcu)) == immutable wchar));
assert(bcu.front == cast(wchar) 56319);
}
{
static struct DStringish
{
dstring s;
alias s this;
}
auto orig = DStringish("\U0010fff8 𐁊 foo 𐂓"d);
auto bcu = orig.byCodeUnit();
static assert(is(typeof(bcu) == dstring));
static assert(is(typeof(bcu) == typeof(bcu.byCodeUnit())));
static assert(is(ElementType!(typeof(bcu)) == immutable dchar));
assert(bcu.front == cast(dchar) 1114104);
}
{
static struct FuncStringish
{
string str;
string s() pure nothrow @nogc { return str; }
alias s this;
}
auto orig = FuncStringish("\U0010fff8 𐁊 foo 𐂓");
auto bcu = orig.byCodeUnit();
static if (isAutodecodableString!FuncStringish)
static assert(is(typeof(bcu) == struct));
else
static assert(is(typeof(bcu) == string));
static assert(!is(typeof(bcu) == FuncStringish));
static assert(is(typeof(bcu) == typeof(bcu.byCodeUnit())));
static assert(is(ElementType!(typeof(bcu)) == immutable char));
assert(bcu.front == cast(char) 244);
}
{
static struct Range
{
string data;
bool empty() pure nothrow @nogc { return data.empty; }
char front() pure nothrow @nogc { return data[0]; }
void popFront() pure nothrow @nogc { data = data[1 .. $]; }
}
auto orig = Range("\U0010fff8 𐁊 foo 𐂓");
auto bcu = orig.byCodeUnit();
static assert(is(typeof(bcu) == Range));
static assert(is(typeof(bcu) == typeof(bcu.byCodeUnit())));
static assert(is(ElementType!(typeof(bcu)) == char));
assert(bcu.front == cast(char) 244);
}
{
static struct WRange
{
wstring data;
bool empty() pure nothrow @nogc { return data.empty; }
wchar front() pure nothrow @nogc { return data[0]; }
void popFront() pure nothrow @nogc { data = data[1 .. $]; }
}
auto orig = WRange("\U0010fff8 𐁊 foo 𐂓"w);
auto bcu = orig.byCodeUnit();
static assert(is(typeof(bcu) == WRange));
static assert(is(typeof(bcu) == typeof(bcu.byCodeUnit())));
static assert(is(ElementType!(typeof(bcu)) == wchar));
assert(bcu.front == 56319);
}
{
static struct DRange
{
dstring data;
bool empty() pure nothrow @nogc { return data.empty; }
dchar front() pure nothrow @nogc { return data[0]; }
void popFront() pure nothrow @nogc { data = data[1 .. $]; }
}
auto orig = DRange("\U0010fff8 𐁊 foo 𐂓"d);
auto bcu = orig.byCodeUnit();
static assert(is(typeof(bcu) == DRange));
static assert(is(typeof(bcu) == typeof(bcu.byCodeUnit())));
static assert(is(ElementType!(typeof(bcu)) == dchar));
assert(bcu.front == 1114104);
}
{
static struct RangeAndStringish
{
bool empty() pure nothrow @nogc { return data.empty; }
char front() pure nothrow @nogc { return data[0]; }
void popFront() pure nothrow @nogc { data = data[1 .. $]; }
string data;
string s;
alias s this;
}
auto orig = RangeAndStringish("test.d", "other");
auto bcu = orig.byCodeUnit();
static assert(is(typeof(bcu) == RangeAndStringish));
static assert(is(typeof(bcu) == typeof(bcu.byCodeUnit())));
static assert(is(ElementType!(typeof(bcu)) == char));
assert(bcu.front == 't');
}
{
static struct WRangeAndStringish
{
bool empty() pure nothrow @nogc { return data.empty; }
wchar front() pure nothrow @nogc { return data[0]; }
void popFront() pure nothrow @nogc { data = data[1 .. $]; }
wstring data;
wstring s;
alias s this;
}
auto orig = WRangeAndStringish("test.d"w, "other"w);
auto bcu = orig.byCodeUnit();
static assert(is(typeof(bcu) == WRangeAndStringish));
static assert(is(typeof(bcu) == typeof(bcu.byCodeUnit())));
static assert(is(ElementType!(typeof(bcu)) == wchar));
assert(bcu.front == 't');
}
{
static struct DRangeAndStringish
{
bool empty() pure nothrow @nogc { return data.empty; }
dchar front() pure nothrow @nogc { return data[0]; }
void popFront() pure nothrow @nogc { data = data[1 .. $]; }
dstring data;
dstring s;
alias s this;
}
auto orig = DRangeAndStringish("test.d"d, "other"d);
auto bcu = orig.byCodeUnit();
static assert(is(typeof(bcu) == DRangeAndStringish));
static assert(is(typeof(bcu) == typeof(bcu.byCodeUnit())));
static assert(is(ElementType!(typeof(bcu)) == dchar));
assert(bcu.front == 't');
}
{
enum Enum : string { a = "test.d" }
auto orig = Enum.a;
auto bcu = orig.byCodeUnit();
static assert(!is(typeof(bcu) == Enum));
static if (isAutodecodableString!Enum)
static assert(is(typeof(bcu) == struct));
else
static assert(is(typeof(bcu) == string));
static assert(is(ElementType!(typeof(bcu)) == immutable char));
assert(bcu.front == 't');
}
{
enum WEnum : wstring { a = "test.d"w }
auto orig = WEnum.a;
auto bcu = orig.byCodeUnit();
static assert(!is(typeof(bcu) == WEnum));
static if (isAutodecodableString!WEnum)
static assert(is(typeof(bcu) == struct));
else
static assert(is(typeof(bcu) == wstring));
static assert(is(ElementType!(typeof(bcu)) == immutable wchar));
assert(bcu.front == 't');
}
{
enum DEnum : dstring { a = "test.d"d }
auto orig = DEnum.a;
auto bcu = orig.byCodeUnit();
static assert(is(typeof(bcu) == dstring));
static assert(is(ElementType!(typeof(bcu)) == immutable dchar));
assert(bcu.front == 't');
}
static if (autodecodeStrings)
{
static assert(!is(typeof(byCodeUnit("hello")) == string));
static assert(!is(typeof(byCodeUnit("hello"w)) == wstring));
}
else
{
static assert(is(typeof(byCodeUnit("hello")) == string));
static assert(is(typeof(byCodeUnit("hello"w)) == wstring));
}
static assert(is(typeof(byCodeUnit("hello"d)) == dstring));
static assert(!__traits(compiles, byCodeUnit((char[5]).init)));
static assert(!__traits(compiles, byCodeUnit((wchar[5]).init)));
static assert(!__traits(compiles, byCodeUnit((dchar[5]).init)));
enum SEnum : char[5] { a = "hello" }
enum WSEnum : wchar[5] { a = "hello"w }
enum DSEnum : dchar[5] { a = "hello"d }
static assert(!__traits(compiles, byCodeUnit(SEnum.a)));
static assert(!__traits(compiles, byCodeUnit(WSEnum.a)));
static assert(!__traits(compiles, byCodeUnit(DSEnum.a)));
}
/****************************
* Iterate an $(REF_ALTTEXT input range, isInputRange, std,range,primitives)
* of characters by char, wchar, or dchar.
* These aliases simply forward to $(LREF byUTF) with the
* corresponding C argument.
*
* Params:
* r = input range of characters, or array of characters
*/
alias byChar = byUTF!char;
/// Ditto
alias byWchar = byUTF!wchar;
/// Ditto
alias byDchar = byUTF!dchar;
@safe pure nothrow @nogc unittest
{
{
char[5] s;
int i;
foreach (c; "hello".byChar.byChar())
{
//writefln("[%d] '%c'", i, c);
s[i++] = c;
}
assert(s == "hello");
}
{
char[5+2+3+4+3+3] s;
int i;
dchar[10] a;
a[0 .. 8] = "hello\u07FF\uD7FF\U0010FFFF"d;
a[8] = 0xD800; // invalid
a[9] = cast(dchar) 0x110000; // invalid
foreach (c; a[].byChar())
{
//writefln("[%d] '%c'", i, c);
s[i++] = c;
}
assert(s == "hello\u07FF\uD7FF\U0010FFFF\uFFFD\uFFFD");
}
{
auto r = "hello"w.byChar();
r.popFront();
r.popFront();
assert(r.front == 'l');
}
{
auto r = "hello"d.byChar();
r.popFront();
r.popFront();
assert(r.front == 'l');
}
{
auto r = "hello"d.byChar();
assert(isForwardRange!(typeof(r)));
auto s = r.save;
r.popFront();
assert(s.front == 'h');
}
}
@safe pure nothrow @nogc unittest
{
{
wchar[11] s;
int i;
dchar[10] a;
a[0 .. 8] = "hello\u07FF\uD7FF\U0010FFFF"d;
a[8] = 0xD800; // invalid
a[9] = cast(dchar) 0x110000; // invalid
foreach (c; a[].byWchar())
{
//writefln("[%d] '%c' x%x", i, c, c);
s[i++] = c;
}
foreach (j, wchar c; "hello\u07FF\uD7FF\U0010FFFF\uFFFD\uFFFD"w)
{
//writefln("[%d] '%c' x%x", j, c, c);
}
assert(s == "hello\u07FF\uD7FF\U0010FFFF\uFFFD\uFFFD"w);
}
{
auto r = "hello".byWchar();
r.popFront();
r.popFront();
assert(r.front == 'l');
}
{
auto r = "hello"d.byWchar();
r.popFront();
r.popFront();
assert(r.front == 'l');
}
{
auto r = "hello"d.byWchar();
assert(isForwardRange!(typeof(r)));
auto s = r.save;
r.popFront();
assert(s.front == 'h');
}
}
@safe pure nothrow @nogc unittest
{
{
dchar[9] s;
int i;
string a = "hello\u07FF\uD7FF\U00010000\U0010FFFF"; // 1,2,3,4 byte sequences
foreach (c; a.byDchar())
{
s[i++] = c;
}
assert(s == "hello\u07FF\uD7FF\U00010000\U0010FFFF"d);
}
{
foreach (s; invalidUTFstrings!char())
{
auto r = s.byDchar();
assert(!r.empty);
assert(r.front == r.front);
dchar c = r.front;
assert(c == replacementDchar);
}
}
{
auto r = "hello".byDchar();
r.popFront();
r.popFront();
assert(r.front == 'l');
}
{
dchar[8] s;
int i;
wstring a = "hello\u07FF\uD7FF\U0010FFFF"w;
foreach (c; a.byDchar())
{
//writefln("[%d] '%c' x%x", i, c, c);
s[i++] = c;
}
assert(s == "hello\u07FF\uD7FF\U0010FFFF"d);
}
{
foreach (s; invalidUTFstrings!wchar())
{
auto r = s.byDchar();
assert(!r.empty);
assert(r.front == r.front);
dchar c = r.front;
assert(c == replacementDchar);
}
}
{
wchar[2] ws;
ws[0] = 0xD800;
ws[1] = 0xDD00; // correct surrogate pair
auto r = ws[].byDchar();
assert(!r.empty);
assert(r.front == r.front);
dchar c = r.front;
assert(c == '\U00010100');
}
{
auto r = "hello"w.byDchar();
r.popFront();
r.popFront();
assert(r.front == 'l');
}
{
dchar[5] s;
int i;
dstring a = "hello"d;
foreach (c; a.byDchar.byDchar())
{
//writefln("[%d] '%c' x%x", i, c, c);
s[i++] = c;
}
assert(s == "hello"d);
}
{
auto r = "hello".byDchar();
assert(isForwardRange!(typeof(r)));
auto s = r.save;
r.popFront();
assert(s.front == 'h');
}
{
auto r = "hello"w.byDchar();
assert(isForwardRange!(typeof(r)));
auto s = r.save;
r.popFront();
assert(s.front == 'h');
}
}
// test pure, @safe, nothrow, @nogc correctness of byChar/byWchar/byDchar,
// which needs to support ranges with and without those attributes
pure @safe nothrow @nogc unittest
{
dchar[5] s = "hello"d;
foreach (c; s[].byChar()) { }
foreach (c; s[].byWchar()) { }
foreach (c; s[].byDchar()) { }
}
version (StdUnittest)
private int impureVariable;
@system unittest
{
static struct ImpureThrowingSystemRange(Char)
{
@property bool empty() const { return true; }
@property Char front() const { return Char.init; }
void popFront()
{
impureVariable++;
throw new Exception("only for testing nothrow");
}
}
foreach (Char; AliasSeq!(char, wchar, dchar))
{
ImpureThrowingSystemRange!Char range;
foreach (c; range.byChar()) { }
foreach (c; range.byWchar()) { }
foreach (c; range.byDchar()) { }
}
}
/****************************
* Iterate an $(REF_ALTTEXT input range, isInputRange, std,range,primitives)
* of characters by char type `C` by encoding the elements of the range.
*
* UTF sequences that cannot be converted to the specified encoding are either
* replaced by U+FFFD per "5.22 Best Practice for U+FFFD Substitution"
* of the Unicode Standard 6.2 or result in a thrown UTFException.
* Hence byUTF is not symmetric.
* This algorithm is lazy, and does not allocate memory.
* `@nogc`, `pure`-ity, `nothrow`, and `@safe`-ty are inferred from the
* `r` parameter.
*
* Params:
* C = `char`, `wchar`, or `dchar`
* useReplacementDchar = UseReplacementDchar.yes means replace invalid UTF with `replacementDchar`,
* UseReplacementDchar.no means throw `UTFException` for invalid UTF
*
* Throws:
* `UTFException` if invalid UTF sequence and `useReplacementDchar` is set to `UseReplacementDchar.no`
*
* GC:
* Does not use GC if `useReplacementDchar` is set to `UseReplacementDchar.yes`
*
* Returns:
* A bidirectional range if `R` is a bidirectional range and not auto-decodable,
* as defined by $(REF isAutodecodableString, std, traits).
*
* A forward range if `R` is a forward range and not auto-decodable.
*
* Or, if `R` is a range and it is auto-decodable and
* `is(ElementEncodingType!typeof(r) == C)`, then the range is passed
* to $(LREF byCodeUnit).
*
* Otherwise, an input range of characters.
*/
template byUTF(C, UseReplacementDchar useReplacementDchar = Yes.useReplacementDchar)
if (isSomeChar!C)
{
static if (is(immutable C == immutable UC, UC) && !is(C == UC))
alias byUTF = byUTF!UC;
else:
auto ref byUTF(R)(R r)
if (isAutodecodableString!R && isInputRange!R && isSomeChar!(ElementEncodingType!R))
{
return byUTF(r.byCodeUnit());
}
auto ref byUTF(R)(R r)
if (!isAutodecodableString!R && isInputRange!R && isSomeChar!(ElementEncodingType!R))
{
static if (is(immutable ElementEncodingType!R == immutable RC, RC) && is(RC == C))
{
return r.byCodeUnit();
}
else static if (is(C == dchar))
{
static struct Result
{
enum Empty = uint.max; // range is empty or just constructed
this(return scope R r)
{
this.r = r;
}
this(return scope R r, uint buff)
{
this.r = r;
this.buff = buff;
}
static if (isBidirectionalRange!R)
{
this(return scope R r, uint frontBuff, uint backBuff)
{
this.r = r;
this.buff = frontBuff;
this.backBuff = backBuff;
}
}
@property bool empty()
{
static if (isBidirectionalRange!R)
return buff == Empty && backBuff == Empty && r.empty;
else
return buff == Empty && r.empty;
}
@property dchar front() scope // 'scope' required by call to decodeFront() below
{
if (buff == Empty)
{
auto c = r.front;
static if (is(RC == wchar))
enum firstMulti = 0xD800; // First high surrogate.
else
enum firstMulti = 0x80; // First non-ASCII.
if (c < firstMulti)
{
r.popFront;
buff = cast(dchar) c;
}
else
{
buff = () @trusted { return decodeFront!(useReplacementDchar)(r); }();
}
}
return cast(dchar) buff;
}
void popFront()
{
if (buff == Empty)
front();
buff = Empty;
}
static if (isForwardRange!R)
{
@property auto save()
{
static if (isBidirectionalRange!R)
{
return Result(r.save, buff, backBuff);
}
else
{
return Result(r.save, buff);
}
}
}
static if (isBidirectionalRange!R)
{
@property dchar back() scope // 'scope' required by call to decodeBack() below
{
if (backBuff != Empty)
return cast(dchar) backBuff;
auto c = r.back;
static if (is(RC == wchar))
enum firstMulti = 0xD800; // First high surrogate.
else
enum firstMulti = 0x80; // First non-ASCII.
if (c < firstMulti)
{
r.popBack;
backBuff = cast(dchar) c;
}
else
{
backBuff = () @trusted { return decodeBack!useReplacementDchar(r); }();
}
return cast(dchar) backBuff;
}
void popBack()
{
if (backBuff == Empty)
back();
backBuff = Empty;
}
}
private:
R r;
uint buff = Empty; // one character lookahead buffer
static if (isBidirectionalRange!R)
uint backBuff = Empty;
}
return Result(r);
}
else
{
static struct Result
{
this(return scope R r)
{
this.r = r;
}
this(return scope R r, ushort pos, ushort fill, C[4 / C.sizeof] buf)
{
this.r = r;
this.pos = pos;
this.fill = fill;
this.buf = buf;
}
static if (isBidirectionalRange!R)
{
this(return scope R r, ushort frontPos, ushort frontFill,
ushort backPos, ushort backFill, C[4 / C.sizeof] buf)
{
this.r = r;
this.pos = frontPos;
this.fill = frontFill;
this.backPos = backPos;
this.backFill = backFill;
this.buf = buf;
}
}
@property bool empty()
{
static if (isBidirectionalRange!R)
return pos == fill && backPos == backFill && r.empty;
else
return pos == fill && r.empty;
}
@property auto front() scope // 'scope' required by call to decodeFront() below
{
if (pos == fill)
{
pos = 0;
auto c = r.front;
static if (C.sizeof >= 2 && RC.sizeof >= 2)
enum firstMulti = 0xD800; // First high surrogate.
else
enum firstMulti = 0x80; // First non-ASCII.
if (c < firstMulti)
{
fill = 1;
r.popFront;
buf[pos] = cast(C) c;
}
else
{
static if (is(RC == dchar))
{
r.popFront;
dchar dc = c;
}
else
dchar dc = () @trusted { return decodeFront!(useReplacementDchar)(r); }();
fill = cast(ushort) encode!(useReplacementDchar)(buf, dc);
}
}
return buf[pos];
}
void popFront()
{
if (pos == fill)
front;
++pos;
}
static if (isForwardRange!R)
{
@property auto save()
{
static if (isBidirectionalRange!R)
{
return Result(r.save, pos, fill, backPos, backFill, buf);
}
else
{
return Result(r.save, pos, fill, buf);
}
}
}
static if (isBidirectionalRange!R)
{
@property auto back() scope // 'scope' required by call to decodeBack() below
{
if (backPos != backFill)
return buf[cast(ushort) (backFill - backPos - 1)];
backPos = 0;
auto c = r.back;
static if (C.sizeof >= 2 && RC.sizeof >= 2)
enum firstMulti = 0xD800; // First high surrogate.
else
enum firstMulti = 0x80; // First non-ASCII.
if (c < firstMulti)
{
backFill = 1;
r.popBack;
buf[cast(ushort) (backFill - backPos - 1)] = cast(C) c;
}
else
{
static if (is(RC == dchar))
{
r.popBack;
dchar dc = c;
}
else
dchar dc = () @trusted { return decodeBack!(useReplacementDchar)(r); }();
backFill = cast(ushort) encode!(useReplacementDchar)(buf, dc);
}
return buf[cast(ushort) (backFill - backPos - 1)];
}
void popBack()
{
if (backPos == backFill)
back;
++backPos;
}
}
private:
R r;
ushort pos, fill;
static if (isBidirectionalRange!R)
ushort backPos, backFill;
C[4 / C.sizeof] buf = void;
}
return Result(r);
}
}
}
///
@safe pure nothrow unittest
{
import std.algorithm.comparison : equal;
// hellö as a range of `char`s, which are UTF-8
assert("hell\u00F6".byUTF!char().equal(['h', 'e', 'l', 'l', 0xC3, 0xB6]));
// `wchar`s are able to hold the ö in a single element (UTF-16 code unit)
assert("hell\u00F6".byUTF!wchar().equal(['h', 'e', 'l', 'l', 'ö']));
// 𐐷 is four code units in UTF-8, two in UTF-16, and one in UTF-32
assert("𐐷".byUTF!char().equal([0xF0, 0x90, 0x90, 0xB7]));
assert("𐐷".byUTF!wchar().equal([0xD801, 0xDC37]));
assert("𐐷".byUTF!dchar().equal([0x00010437]));
}
///
@safe unittest
{
import std.algorithm.comparison : equal;
import std.exception : assertThrown;
assert("hello\xF0betty".byChar.byUTF!(dchar, UseReplacementDchar.yes).equal("hello\uFFFDetty"));
assertThrown!UTFException("hello\xF0betty".byChar.byUTF!(dchar, UseReplacementDchar.no).equal("hello betty"));
}
@safe unittest
{
{
wchar[] s = ['a', 'b', 0x219];
auto r = s.byUTF!char;
assert(isBidirectionalRange!(typeof(r)));
assert(r.back == 0x99);
r.popBack;
assert(r.back == 0xc8);
r.popBack;
assert(r.back == 'b');
}
{
wchar[] s = ['a', 'b', 0x219];
auto r = s.byUTF!wchar;
uint i;
assert(isBidirectionalRange!(typeof(r)));
assert(r.back == 0x219);
r.popBack;
assert(r.back == 'b');
}
{
wchar[] s = ['a', 'b', 0x219];
auto r = s.byUTF!dchar;
assert(isBidirectionalRange!(typeof(r)));
assert(r.back == 0x219);
r.popBack;
assert(r.back == 'b');
}
{
dchar[] s = ['𐐷', '😁'];
auto r = s.byUTF!wchar;
assert(r.back == 0xde01);
r.popBack;
assert(r.back == 0xd83d);
r.popBack;
assert(r.back == 0xdc37);
r.popBack;
assert(r.back == 0xd801);
}
{
dchar[] s = ['𐐷', '😁'];
auto r = s.byUTF!char;
char[] res;
while (!r.empty)
{
res ~= r.back;
r.popBack;
}
import std.algorithm.comparison : equal;
assert(res.equal([0x81, 0x98, 0x9f, 0xf0, 0xb7, 0x90, 0x90, 0xf0]));
}
{
dchar[] res;
auto r = ['a', 'b', 'c', 'd', 'e'].byUTF!dchar;
while (!r.empty)
{
res ~= r.back;
r.popBack;
}
import std.algorithm.comparison : equal;
assert(res.equal(['e', 'd', 'c', 'b', 'a']));
}
{
//testing the save() function
wchar[] s = ['Ă','ț'];
auto rc = s.byUTF!char;
rc.popBack;
auto rcCopy = rc.save;
assert(rc.back == rcCopy.back);
assert(rcCopy.back == 0xc8);
auto rd = s.byUTF!dchar;
rd.popBack;
auto rdCopy = rd.save;
assert(rd.back == rdCopy.back);
assert(rdCopy.back == 'Ă');
}
}
///
@safe pure nothrow unittest
{
import std.range.primitives;
wchar[] s = ['ă', 'î'];
auto rc = s.byUTF!char;
static assert(isBidirectionalRange!(typeof(rc)));
assert(rc.back == 0xae);
rc.popBack;
assert(rc.back == 0xc3);
rc.popBack;
assert(rc.back == 0x83);
rc.popBack;
assert(rc.back == 0xc4);
auto rw = s.byUTF!wchar;
static assert(isBidirectionalRange!(typeof(rw)));
assert(rw.back == 'î');
rw.popBack;
assert(rw.back == 'ă');
auto rd = s.byUTF!dchar;
static assert(isBidirectionalRange!(typeof(rd)));
assert(rd.back == 'î');
rd.popBack;
assert(rd.back == 'ă');
}