
/* Compiler implementation of the D programming language
 * Copyright (C) 1999-2021 by The D Language Foundation, All Rights Reserved
 * written by Walter Bright
 * http://www.digitalmars.com
 * Distributed under the Boost Software License, Version 1.0.
 * http://www.boost.org/LICENSE_1_0.txt
 * https://github.com/D-Programming-Language/dmd/blob/master/src/lexer.c
 */

/* Lexical Analyzer */

#include "root/dsystem.h" // for time() and ctime()
#include "root/rmem.h"

#include "mars.h"
#include "lexer.h"
#include "utf.h"
#include "identifier.h"
#include "id.h"

extern int HtmlNamedEntity(const utf8_t *p, size_t length);

#define LS 0x2028       // UTF line separator
#define PS 0x2029       // UTF paragraph separator

/********************************************
 * Do our own char maps
 */

static unsigned char cmtable[256];

const int CMoctal =     0x1;
const int CMhex =       0x2;
const int CMidchar =    0x4;

inline bool isoctal (utf8_t c) { return (cmtable[c] & CMoctal) != 0; }
inline bool ishex   (utf8_t c) { return (cmtable[c] & CMhex) != 0; }
inline bool isidchar(utf8_t c) { return (cmtable[c] & CMidchar) != 0; }

struct CMTableInitializer
{
    CMTableInitializer();
};

static CMTableInitializer cmtableinitializer;

CMTableInitializer::CMTableInitializer()
{
    for (unsigned c = 0; c < 256; c++)
    {
        if ('0' <= c && c <= '7')
            cmtable[c] |= CMoctal;
        if (isxdigit(c))
            cmtable[c] |= CMhex;
        if (isalnum(c) || c == '_')
            cmtable[c] |= CMidchar;
    }
}

/*************************** Lexer ********************************************/

OutBuffer Lexer::stringbuffer;

Lexer::Lexer(const char *filename,
        const utf8_t *base, size_t begoffset, size_t endoffset,
        bool doDocComment, bool commentToken)
{
    scanloc = Loc(filename, 1, 1);
    //printf("Lexer::Lexer(%p,%d)\n",base,length);
    //printf("lexer.filename = %s\n", filename);
    this->token = Token();
    this->token.ptr = NULL;
    this->token.value = TOKreserved;
    this->token.blockComment = NULL;
    this->token.lineComment = NULL;
    this->base = base;
    this->end  = base + endoffset;
    p = base + begoffset;
    line = p;
    this->doDocComment = doDocComment;
    this->anyToken = 0;
    this->commentToken = commentToken;
    this->errors = false;
    //initKeywords();

    /* If first line starts with '#!', ignore the line
     */

    if (p[0] == '#' && p[1] =='!')
    {
        p += 2;
        while (1)
        {
            utf8_t c = *p++;
            switch (c)
            {
                case 0:
                case 0x1A:
                    p--;
                    /* fall through */

                case '\n':
                    break;

                default:
                    continue;
            }
            break;
        }
        endOfLine();
    }
}


void Lexer::endOfLine()
{
    scanloc.linnum++;
    line = p;
}


void Lexer::error(const char *format, ...)
{
    va_list ap;
    va_start(ap, format);
    ::verror(token.loc, format, ap);
    va_end(ap);
    errors = true;
}

void Lexer::error(Loc loc, const char *format, ...)
{
    va_list ap;
    va_start(ap, format);
    ::verror(loc, format, ap);
    va_end(ap);
    errors = true;
}

void Lexer::deprecation(const char *format, ...)
{
    va_list ap;
    va_start(ap, format);
    ::vdeprecation(token.loc, format, ap);
    va_end(ap);
    if (global.params.useDeprecated == DIAGNOSTICerror)
        errors = true;
}

TOK Lexer::nextToken()
{
    if (token.next)
    {
        Token *t = token.next;
        memcpy(&token,t,sizeof(Token));
        t->free();
    }
    else
    {
        scan(&token);
    }
    //token.print();
    return token.value;
}

Token *Lexer::peek(Token *ct)
{
    Token *t;
    if (ct->next)
        t = ct->next;
    else
    {
        t = Token::alloc();
        scan(t);
        ct->next = t;
    }
    return t;
}

/***********************
 * Look ahead at next token's value.
 */

TOK Lexer::peekNext()
{
    return peek(&token)->value;
}

/***********************
 * Look 2 tokens ahead at value.
 */

TOK Lexer::peekNext2()
{
    Token *t = peek(&token);
    return peek(t)->value;
}

/*********************************
 * tk is on the opening (.
 * Look ahead and return token that is past the closing ).
 */

Token *Lexer::peekPastParen(Token *tk)
{
    //printf("peekPastParen()\n");
    int parens = 1;
    int curlynest = 0;
    while (1)
    {
        tk = peek(tk);
        //tk->print();
        switch (tk->value)
        {
            case TOKlparen:
                parens++;
                continue;

            case TOKrparen:
                --parens;
                if (parens)
                    continue;
                tk = peek(tk);
                break;

            case TOKlcurly:
                curlynest++;
                continue;

            case TOKrcurly:
                if (--curlynest >= 0)
                    continue;
                break;

            case TOKsemicolon:
                if (curlynest)
                    continue;
                break;

            case TOKeof:
                break;

            default:
                continue;
        }
        return tk;
    }
}

/****************************
 * Turn next token in buffer into a token.
 */

void Lexer::scan(Token *t)
{
    unsigned lastLine = scanloc.linnum;
    Loc startLoc;

    t->blockComment = NULL;
    t->lineComment = NULL;
    while (1)
    {
        t->ptr = p;
        //printf("p = %p, *p = '%c'\n",p,*p);
        t->loc = loc();
        switch (*p)
        {
            case 0:
            case 0x1A:
                t->value = TOKeof;                      // end of file
                return;

            case ' ':
            case '\t':
            case '\v':
            case '\f':
                p++;
                continue;                       // skip white space

            case '\r':
                p++;
                if (*p != '\n')                 // if CR stands by itself
                    endOfLine();
                continue;                       // skip white space

            case '\n':
                p++;
                endOfLine();
                continue;                       // skip white space

            case '0':   case '1':   case '2':   case '3':   case '4':
            case '5':   case '6':   case '7':   case '8':   case '9':
                t->value = number(t);
                return;

            case '\'':
                t->value = charConstant(t);
                return;

            case 'r':
                if (p[1] != '"')
                    goto case_ident;
                p++;
                /* fall through */
            case '`':
                t->value = wysiwygStringConstant(t, *p);
                return;

            case 'x':
                if (p[1] != '"')
                    goto case_ident;
                p++;
                t->value = hexStringConstant(t);
                return;

            case 'q':
                if (p[1] == '"')
                {
                    p++;
                    t->value = delimitedStringConstant(t);
                    return;
                }
                else if (p[1] == '{')
                {
                    p++;
                    t->value = tokenStringConstant(t);
                    return;
                }
                else
                    goto case_ident;

            case '"':
                t->value = escapeStringConstant(t);
                return;

            case 'a':   case 'b':   case 'c':   case 'd':   case 'e':
            case 'f':   case 'g':   case 'h':   case 'i':   case 'j':
            case 'k':   case 'l':   case 'm':   case 'n':   case 'o':
            case 'p':   /*case 'q': case 'r':*/ case 's':   case 't':
            case 'u':   case 'v':   case 'w': /*case 'x':*/ case 'y':
            case 'z':
            case 'A':   case 'B':   case 'C':   case 'D':   case 'E':
            case 'F':   case 'G':   case 'H':   case 'I':   case 'J':
            case 'K':   case 'L':   case 'M':   case 'N':   case 'O':
            case 'P':   case 'Q':   case 'R':   case 'S':   case 'T':
            case 'U':   case 'V':   case 'W':   case 'X':   case 'Y':
            case 'Z':
            case '_':
            case_ident:
            {   utf8_t c;

                while (1)
                {
                    c = *++p;
                    if (isidchar(c))
                        continue;
                    else if (c & 0x80)
                    {   const utf8_t *s = p;
                        unsigned u = decodeUTF();
                        if (isUniAlpha(u))
                            continue;
                        error("char 0x%04x not allowed in identifier", u);
                        p = s;
                    }
                    break;
                }

                Identifier *id = Identifier::idPool((const char *)t->ptr, p - t->ptr);
                t->ident = id;
                t->value = (TOK) id->getValue();
                anyToken = 1;
                if (*t->ptr == '_')     // if special identifier token
                {
                    static bool initdone = false;
                    static char date[11+1];
                    static char time[8+1];
                    static char timestamp[24+1];

                    if (!initdone)       // lazy evaluation
                    {
                        initdone = true;
                        time_t ct;
                        ::time(&ct);
                        char *p = ctime(&ct);
                        assert(p);
                        sprintf(&date[0], "%.6s %.4s", p + 4, p + 20);
                        sprintf(&time[0], "%.8s", p + 11);
                        sprintf(&timestamp[0], "%.24s", p);
                    }

                    if (id == Id::DATE)
                    {
                        t->ustring = (utf8_t *)date;
                        goto Lstr;
                    }
                    else if (id == Id::TIME)
                    {
                        t->ustring = (utf8_t *)time;
                        goto Lstr;
                    }
                    else if (id == Id::VENDOR)
                    {
                        t->ustring = (utf8_t *)const_cast<char *>(global.vendor.ptr);
                        goto Lstr;
                    }
                    else if (id == Id::TIMESTAMP)
                    {
                        t->ustring = (utf8_t *)timestamp;
                     Lstr:
                        t->value = TOKstring;
                        t->postfix = 0;
                        t->len = (unsigned)strlen((char *)t->ustring);
                    }
                    else if (id == Id::VERSIONX)
                    {   unsigned major = 0;
                        unsigned minor = 0;
                        bool point = false;

                        for (const char *p = global.version.ptr + 1; 1; p++)
                        {
                            c = *p;
                            if (isdigit((utf8_t)c))
                                minor = minor * 10 + c - '0';
                            else if (c == '.')
                            {
                                if (point)
                                    break;      // ignore everything after second '.'
                                point = true;
                                major = minor;
                                minor = 0;
                            }
                            else
                                break;
                        }
                        t->value = TOKint64v;
                        t->uns64value = major * 1000 + minor;
                    }
                    else if (id == Id::EOFX)
                    {
                        t->value = TOKeof;
                        // Advance scanner to end of file
                        while (!(*p == 0 || *p == 0x1A))
                            p++;
                    }
                }
                //printf("t->value = %d\n",t->value);
                return;
            }

            case '/':
                p++;
                switch (*p)
                {
                    case '=':
                        p++;
                        t->value = TOKdivass;
                        return;

                    case '*':
                        p++;
                        startLoc = loc();
                        while (1)
                        {
                            while (1)
                            {   utf8_t c = *p;
                                switch (c)
                                {
                                    case '/':
                                        break;

                                    case '\n':
                                        endOfLine();
                                        p++;
                                        continue;

                                    case '\r':
                                        p++;
                                        if (*p != '\n')
                                            endOfLine();
                                        continue;

                                    case 0:
                                    case 0x1A:
                                        error("unterminated /* */ comment");
                                        p = end;
                                        t->loc = loc();
                                        t->value = TOKeof;
                                        return;

                                    default:
                                        if (c & 0x80)
                                        {   unsigned u = decodeUTF();
                                            if (u == PS || u == LS)
                                                endOfLine();
                                        }
                                        p++;
                                        continue;
                                }
                                break;
                            }
                            p++;
                            if (p[-2] == '*' && p - 3 != t->ptr)
                                break;
                        }
                        if (commentToken)
                        {
                            t->loc = startLoc;
                            t->value = TOKcomment;
                            return;
                        }
                        else if (doDocComment && t->ptr[2] == '*' && p - 4 != t->ptr)
                        {   // if /** but not /**/
                            getDocComment(t, lastLine == startLoc.linnum);
                        }
                        continue;

                    case '/':           // do // style comments
                        startLoc = loc();
                        while (1)
                        {   utf8_t c = *++p;
                            switch (c)
                            {
                                case '\n':
                                    break;

                                case '\r':
                                    if (p[1] == '\n')
                                        p++;
                                    break;

                                case 0:
                                case 0x1A:
                                    if (commentToken)
                                    {
                                        p = end;
                                        t->loc = startLoc;
                                        t->value = TOKcomment;
                                        return;
                                    }
                                    if (doDocComment && t->ptr[2] == '/')
                                        getDocComment(t, lastLine == startLoc.linnum);
                                    p = end;
                                    t->loc = loc();
                                    t->value = TOKeof;
                                    return;

                                default:
                                    if (c & 0x80)
                                    {   unsigned u = decodeUTF();
                                        if (u == PS || u == LS)
                                            break;
                                    }
                                    continue;
                            }
                            break;
                        }

                        if (commentToken)
                        {
                            p++;
                            endOfLine();
                            t->loc = startLoc;
                            t->value = TOKcomment;
                            return;
                        }
                        if (doDocComment && t->ptr[2] == '/')
                            getDocComment(t, lastLine == startLoc.linnum);

                        p++;
                        endOfLine();
                        continue;

                    case '+':
                    {   int nest;

                        startLoc = loc();
                        p++;
                        nest = 1;
                        while (1)
                        {   utf8_t c = *p;
                            switch (c)
                            {
                                case '/':
                                    p++;
                                    if (*p == '+')
                                    {
                                        p++;
                                        nest++;
                                    }
                                    continue;

                                case '+':
                                    p++;
                                    if (*p == '/')
                                    {
                                        p++;
                                        if (--nest == 0)
                                            break;
                                    }
                                    continue;

                                case '\r':
                                    p++;
                                    if (*p != '\n')
                                        endOfLine();
                                    continue;

                                case '\n':
                                    endOfLine();
                                    p++;
                                    continue;

                                case 0:
                                case 0x1A:
                                    error("unterminated /+ +/ comment");
                                    p = end;
                                    t->loc = loc();
                                    t->value = TOKeof;
                                    return;

                                default:
                                    if (c & 0x80)
                                    {   unsigned u = decodeUTF();
                                        if (u == PS || u == LS)
                                            endOfLine();
                                    }
                                    p++;
                                    continue;
                            }
                            break;
                        }
                        if (commentToken)
                        {
                            t->loc = startLoc;
                            t->value = TOKcomment;
                            return;
                        }
                        if (doDocComment && t->ptr[2] == '+' && p - 4 != t->ptr)
                        {   // if /++ but not /++/
                            getDocComment(t, lastLine == startLoc.linnum);
                        }
                        continue;
                    }
                    default:
                        break;
                }
                t->value = TOKdiv;
                return;

            case '.':
                p++;
                if (isdigit(*p))
                {   /* Note that we don't allow ._1 and ._ as being
                     * valid floating point numbers.
                     */
                    p--;
                    t->value = inreal(t);
                }
                else if (p[0] == '.')
                {
                    if (p[1] == '.')
                    {   p += 2;
                        t->value = TOKdotdotdot;
                    }
                    else
                    {   p++;
                        t->value = TOKslice;
                    }
                }
                else
                    t->value = TOKdot;
                return;

            case '&':
                p++;
                if (*p == '=')
                {   p++;
                    t->value = TOKandass;
                }
                else if (*p == '&')
                {   p++;
                    t->value = TOKandand;
                }
                else
                    t->value = TOKand;
                return;

            case '|':
                p++;
                if (*p == '=')
                {   p++;
                    t->value = TOKorass;
                }
                else if (*p == '|')
                {   p++;
                    t->value = TOKoror;
                }
                else
                    t->value = TOKor;
                return;

            case '-':
                p++;
                if (*p == '=')
                {   p++;
                    t->value = TOKminass;
                }
                else if (*p == '-')
                {   p++;
                    t->value = TOKminusminus;
                }
                else
                    t->value = TOKmin;
                return;

            case '+':
                p++;
                if (*p == '=')
                {   p++;
                    t->value = TOKaddass;
                }
                else if (*p == '+')
                {   p++;
                    t->value = TOKplusplus;
                }
                else
                    t->value = TOKadd;
                return;

            case '<':
                p++;
                if (*p == '=')
                {   p++;
                    t->value = TOKle;                   // <=
                }
                else if (*p == '<')
                {   p++;
                    if (*p == '=')
                    {   p++;
                        t->value = TOKshlass;           // <<=
                    }
                    else
                        t->value = TOKshl;              // <<
                }
                else if (*p == '>')
                {   p++;
                    if (*p == '=')
                    {   p++;
                        t->value = TOKleg;              // <>=
                    }
                    else
                        t->value = TOKlg;               // <>
                }
                else
                    t->value = TOKlt;                   // <
                return;

            case '>':
                p++;
                if (*p == '=')
                {   p++;
                    t->value = TOKge;                   // >=
                }
                else if (*p == '>')
                {   p++;
                    if (*p == '=')
                    {   p++;
                        t->value = TOKshrass;           // >>=
                    }
                    else if (*p == '>')
                    {   p++;
                        if (*p == '=')
                        {   p++;
                            t->value = TOKushrass;      // >>>=
                        }
                        else
                            t->value = TOKushr;         // >>>
                    }
                    else
                        t->value = TOKshr;              // >>
                }
                else
                    t->value = TOKgt;                   // >
                return;

            case '!':
                p++;
                if (*p == '=')
                {   p++;
                    t->value = TOKnotequal;         // !=
                }
                else if (*p == '<')
                {   p++;
                    if (*p == '>')
                    {   p++;
                        if (*p == '=')
                        {   p++;
                            t->value = TOKunord; // !<>=
                        }
                        else
                            t->value = TOKue;   // !<>
                    }
                    else if (*p == '=')
                    {   p++;
                        t->value = TOKug;       // !<=
                    }
                    else
                        t->value = TOKuge;      // !<
                }
                else if (*p == '>')
                {   p++;
                    if (*p == '=')
                    {   p++;
                        t->value = TOKul;       // !>=
                    }
                    else
                        t->value = TOKule;      // !>
                }
                else
                    t->value = TOKnot;          // !
                return;

            case '=':
                p++;
                if (*p == '=')
                {   p++;
                    t->value = TOKequal;            // ==
                }
                else if (*p == '>')
                {   p++;
                    t->value = TOKgoesto;               // =>
                }
                else
                    t->value = TOKassign;               // =
                return;

            case '~':
                p++;
                if (*p == '=')
                {   p++;
                    t->value = TOKcatass;               // ~=
                }
                else
                    t->value = TOKtilde;                // ~
                return;

            case '^':
                p++;
                if (*p == '^')
                {   p++;
                    if (*p == '=')
                    {   p++;
                        t->value = TOKpowass;  // ^^=
                    }
                    else
                        t->value = TOKpow;     // ^^
                }
                else if (*p == '=')
                {   p++;
                    t->value = TOKxorass;    // ^=
                }
                else
                    t->value = TOKxor;       // ^
                return;

            case '(': p++; t->value = TOKlparen; return;
            case ')': p++; t->value = TOKrparen; return;
            case '[': p++; t->value = TOKlbracket; return;
            case ']': p++; t->value = TOKrbracket; return;
            case '{': p++; t->value = TOKlcurly; return;
            case '}': p++; t->value = TOKrcurly; return;
            case '?': p++; t->value = TOKquestion; return;
            case ',': p++; t->value = TOKcomma; return;
            case ';': p++; t->value = TOKsemicolon; return;
            case ':': p++; t->value = TOKcolon; return;
            case '$': p++; t->value = TOKdollar; return;
            case '@': p++; t->value = TOKat; return;

            case '*':
                p++;
                if (*p == '=')
                {   p++;
                    t->value = TOKmulass;
                }
                else
                    t->value = TOKmul;
                return;
            case '%':
                p++;
                if (*p == '=')
                {   p++;
                    t->value = TOKmodass;
                }
                else
                    t->value = TOKmod;
                return;

            case '#':
            {
                p++;
                Token n;
                scan(&n);
                if (n.value == TOKidentifier)
                {
                   if (n.ident == Id::line)
                   {
                       poundLine();
                       continue;
                   }
                   else
                   {
                       const Loc locx = loc();
                       warning(locx, "C preprocessor directive `#%s` is not supported", n.ident->toChars());
                   }
                }
                else if (n.value == TOKif)
                {
                    error("C preprocessor directive `#if` is not supported, use `version` or `static if`");
                }
                t->value = TOKpound;
                return;
            }

            default:
            {   unsigned c = *p;

                if (c & 0x80)
                {   c = decodeUTF();

                    // Check for start of unicode identifier
                    if (isUniAlpha(c))
                        goto case_ident;

                    if (c == PS || c == LS)
                    {
                        endOfLine();
                        p++;
                        continue;
                    }
                }
                if (c < 0x80 && isprint(c))
                    error("character '%c' is not a valid token", c);
                else
                    error("character 0x%02x is not a valid token", c);
                p++;
                continue;
            }
        }
    }
}

/*******************************************
 * Parse escape sequence.
 */

unsigned Lexer::escapeSequence()
{   unsigned c = *p;

    int n;
    int ndigits;

    switch (c)
    {
        case '\'':
        case '"':
        case '?':
        case '\\':
        Lconsume:
                p++;
                break;

        case 'a':       c = 7;          goto Lconsume;
        case 'b':       c = 8;          goto Lconsume;
        case 'f':       c = 12;         goto Lconsume;
        case 'n':       c = 10;         goto Lconsume;
        case 'r':       c = 13;         goto Lconsume;
        case 't':       c = 9;          goto Lconsume;
        case 'v':       c = 11;         goto Lconsume;

        case 'u':
                ndigits = 4;
                goto Lhex;
        case 'U':
                ndigits = 8;
                goto Lhex;
        case 'x':
                ndigits = 2;
        Lhex:
                p++;
                c = *p;
                if (ishex((utf8_t)c))
                {   unsigned v;

                    n = 0;
                    v = 0;
                    while (1)
                    {
                        if (isdigit((utf8_t)c))
                            c -= '0';
                        else if (islower(c))
                            c -= 'a' - 10;
                        else
                            c -= 'A' - 10;
                        v = v * 16 + c;
                        c = *++p;
                        if (++n == ndigits)
                            break;
                        if (!ishex((utf8_t)c))
                        {   error("escape hex sequence has %d hex digits instead of %d", n, ndigits);
                            break;
                        }
                    }
                    if (ndigits != 2 && !utf_isValidDchar(v))
                    {   error("invalid UTF character \\U%08x", v);
                        v = '?';        // recover with valid UTF character
                    }
                    c = v;
                }
                else
                    error("undefined escape hex sequence \\%c",c);
                break;

        case '&':                       // named character entity
                for (const utf8_t *idstart = ++p; 1; p++)
                {
                    switch (*p)
                    {
                        case ';':
                            c = HtmlNamedEntity(idstart, p - idstart);
                            if (c == ~0U)
                            {   error("unnamed character entity &%.*s;", (int)(p - idstart), idstart);
                                c = ' ';
                            }
                            p++;
                            break;

                        default:
                            if (isalpha(*p) ||
                                (p != idstart && isdigit(*p)))
                                continue;
                            error("unterminated named entity &%.*s;", (int)(p - idstart + 1), idstart);
                            break;
                    }
                    break;
                }
                break;

        case 0:
        case 0x1A:                      // end of file
                c = '\\';
                break;

        default:
                if (isoctal((utf8_t)c))
                {   unsigned v;

                    n = 0;
                    v = 0;
                    do
                    {
                        v = v * 8 + (c - '0');
                        c = *++p;
                    } while (++n < 3 && isoctal((utf8_t)c));
                    c = v;
                    if (c > 0xFF)
                        error("escape octal sequence \\%03o is larger than \\377", c);
                }
                else
                    error("undefined escape sequence \\%c",c);
                break;
    }
    return c;
}

/**************************************
 */

TOK Lexer::wysiwygStringConstant(Token *t, int tc)
{
    int c;
    Loc start = loc();

    p++;
    stringbuffer.reset();
    while (1)
    {
        c = *p++;
        switch (c)
        {
            case '\n':
                endOfLine();
                break;

            case '\r':
                if (*p == '\n')
                    continue;   // ignore
                c = '\n';       // treat EndOfLine as \n character
                endOfLine();
                break;

            case 0:
            case 0x1A:
                error("unterminated string constant starting at %s", start.toChars());
                t->ustring = (utf8_t *)const_cast<char *>("");
                t->len = 0;
                t->postfix = 0;
                return TOKstring;

            case '"':
            case '`':
                if (c == tc)
                {
                    t->len = (unsigned)stringbuffer.length();
                    stringbuffer.writeByte(0);
                    t->ustring = (utf8_t *)mem.xmalloc(stringbuffer.length());
                    memcpy(t->ustring, stringbuffer.slice().ptr, stringbuffer.length());
                    stringPostfix(t);
                    return TOKstring;
                }
                break;

            default:
                if (c & 0x80)
                {   p--;
                    unsigned u = decodeUTF();
                    p++;
                    if (u == PS || u == LS)
                        endOfLine();
                    stringbuffer.writeUTF8(u);
                    continue;
                }
                break;
        }
        stringbuffer.writeByte(c);
    }
}

/**************************************
 * Lex hex strings:
 *      x"0A ae 34FE BD"
 */

TOK Lexer::hexStringConstant(Token *t)
{
    unsigned c;
    Loc start = loc();
    unsigned n = 0;
    unsigned v = ~0; // dead assignment, needed to suppress warning

    p++;
    stringbuffer.reset();
    while (1)
    {
        c = *p++;
        switch (c)
        {
            case ' ':
            case '\t':
            case '\v':
            case '\f':
                continue;                       // skip white space

            case '\r':
                if (*p == '\n')
                    continue;                   // ignore
                // Treat isolated '\r' as if it were a '\n'
                /* fall through */
            case '\n':
                endOfLine();
                continue;

            case 0:
            case 0x1A:
                error("unterminated string constant starting at %s", start.toChars());
                t->ustring = (utf8_t *)const_cast<char *>("");
                t->len = 0;
                t->postfix = 0;
                return TOKxstring;

            case '"':
                if (n & 1)
                {   error("odd number (%d) of hex characters in hex string", n);
                    stringbuffer.writeByte(v);
                }
                t->len = (unsigned)stringbuffer.length();
                stringbuffer.writeByte(0);
                t->ustring = (utf8_t *)mem.xmalloc(stringbuffer.length());
                memcpy(t->ustring, stringbuffer.slice().ptr, stringbuffer.length());
                stringPostfix(t);
                return TOKxstring;

            default:
                if (c >= '0' && c <= '9')
                    c -= '0';
                else if (c >= 'a' && c <= 'f')
                    c -= 'a' - 10;
                else if (c >= 'A' && c <= 'F')
                    c -= 'A' - 10;
                else if (c & 0x80)
                {   p--;
                    unsigned u = decodeUTF();
                    p++;
                    if (u == PS || u == LS)
                        endOfLine();
                    else
                        error("non-hex character \\u%04x in hex string", u);
                }
                else
                    error("non-hex character '%c' in hex string", c);
                if (n & 1)
                {   v = (v << 4) | c;
                    stringbuffer.writeByte(v);
                }
                else
                    v = c;
                n++;
                break;
        }
    }
}


/**************************************
 * Lex delimited strings:
 *      q"(foo(xxx))"   // "foo(xxx)"
 *      q"[foo(]"       // "foo("
 *      q"/foo]/"       // "foo]"
 *      q"HERE
 *      foo
 *      HERE"           // "foo\n"
 * Input:
 *      p is on the "
 */

TOK Lexer::delimitedStringConstant(Token *t)
{
    unsigned c;
    Loc start = loc();
    unsigned delimleft = 0;
    unsigned delimright = 0;
    unsigned nest = 1;
    unsigned nestcount = ~0; // dead assignment, needed to suppress warning
    Identifier *hereid = NULL;
    unsigned blankrol = 0;
    unsigned startline = 0;

    p++;
    stringbuffer.reset();
    while (1)
    {
        c = *p++;
        //printf("c = '%c'\n", c);
        switch (c)
        {
            case '\n':
            Lnextline:
                endOfLine();
                startline = 1;
                if (blankrol)
                {   blankrol = 0;
                    continue;
                }
                if (hereid)
                {
                    stringbuffer.writeUTF8(c);
                    continue;
                }
                break;

            case '\r':
                if (*p == '\n')
                    continue;   // ignore
                c = '\n';       // treat EndOfLine as \n character
                goto Lnextline;

            case 0:
            case 0x1A:
                error("unterminated delimited string constant starting at %s", start.toChars());
                t->ustring = (utf8_t *)const_cast<char *>("");
                t->len = 0;
                t->postfix = 0;
                return TOKstring;

            default:
                if (c & 0x80)
                {   p--;
                    c = decodeUTF();
                    p++;
                    if (c == PS || c == LS)
                        goto Lnextline;
                }
                break;
        }
        if (delimleft == 0)
        {   delimleft = c;
            nest = 1;
            nestcount = 1;
            if (c == '(')
                delimright = ')';
            else if (c == '{')
                delimright = '}';
            else if (c == '[')
                delimright = ']';
            else if (c == '<')
                delimright = '>';
            else if (isalpha(c) || c == '_' || (c >= 0x80 && isUniAlpha(c)))
            {   // Start of identifier; must be a heredoc
                Token tok;
                p--;
                scan(&tok);               // read in heredoc identifier
                if (tok.value != TOKidentifier)
                {   error("identifier expected for heredoc, not %s", tok.toChars());
                    delimright = c;
                }
                else
                {   hereid = tok.ident;
                    //printf("hereid = '%s'\n", hereid->toChars());
                    blankrol = 1;
                }
                nest = 0;
            }
            else
            {   delimright = c;
                nest = 0;
                if (isspace(c))
                    error("delimiter cannot be whitespace");
            }
        }
        else
        {
            if (blankrol)
            {   error("heredoc rest of line should be blank");
                blankrol = 0;
                continue;
            }
            if (nest == 1)
            {
                if (c == delimleft)
                    nestcount++;
                else if (c == delimright)
                {   nestcount--;
                    if (nestcount == 0)
                        goto Ldone;
                }
            }
            else if (c == delimright)
                goto Ldone;
            if (startline && isalpha(c) && hereid)
            {   Token tok;
                const utf8_t *psave = p;
                p--;
                scan(&tok);               // read in possible heredoc identifier
                //printf("endid = '%s'\n", tok.ident->toChars());
                if (tok.value == TOKidentifier && tok.ident->equals(hereid))
                {   /* should check that rest of line is blank
                     */
                    goto Ldone;
                }
                p = psave;
            }
            stringbuffer.writeUTF8(c);
            startline = 0;
        }
    }

Ldone:
    if (*p == '"')
        p++;
    else if (hereid)
        error("delimited string must end in %s\"", hereid->toChars());
    else
        error("delimited string must end in %c\"", delimright);
    t->len = (unsigned)stringbuffer.length();
    stringbuffer.writeByte(0);
    t->ustring = (utf8_t *)mem.xmalloc(stringbuffer.length());
    memcpy(t->ustring, stringbuffer.slice().ptr, stringbuffer.length());
    stringPostfix(t);
    return TOKstring;
}

/**************************************
 * Lex delimited strings:
 *      q{ foo(xxx) } // " foo(xxx) "
 *      q{foo(}       // "foo("
 *      q{{foo}"}"}   // "{foo}"}""
 * Input:
 *      p is on the q
 */

TOK Lexer::tokenStringConstant(Token *t)
{
    unsigned nest = 1;
    Loc start = loc();
    const utf8_t *pstart = ++p;

    while (1)
    {   Token tok;

        scan(&tok);
        switch (tok.value)
        {
            case TOKlcurly:
                nest++;
                continue;

            case TOKrcurly:
                if (--nest == 0)
                {
                    t->len = (unsigned)(p - 1 - pstart);
                    t->ustring = (utf8_t *)mem.xmalloc(t->len + 1);
                    memcpy(t->ustring, pstart, t->len);
                    t->ustring[t->len] = 0;
                    stringPostfix(t);
                    return TOKstring;
                }
                continue;

            case TOKeof:
                error("unterminated token string constant starting at %s", start.toChars());
                t->ustring = (utf8_t *)const_cast<char *>("");
                t->len = 0;
                t->postfix = 0;
                return TOKstring;

            default:
                continue;
        }
    }
}


/**************************************
 */

TOK Lexer::escapeStringConstant(Token *t)
{
    unsigned c;
    Loc start = loc();

    p++;
    stringbuffer.reset();
    while (1)
    {
        c = *p++;
        switch (c)
        {
            case '\\':
                switch (*p)
                {
                    case 'u':
                    case 'U':
                    case '&':
                        c = escapeSequence();
                        stringbuffer.writeUTF8(c);
                        continue;

                    default:
                        c = escapeSequence();
                        break;
                }
                break;
            case '\n':
                endOfLine();
                break;

            case '\r':
                if (*p == '\n')
                    continue;   // ignore
                c = '\n';       // treat EndOfLine as \n character
                endOfLine();
                break;

            case '"':
                t->len = (unsigned)stringbuffer.length();
                stringbuffer.writeByte(0);
                t->ustring = (utf8_t *)mem.xmalloc(stringbuffer.length());
                memcpy(t->ustring, stringbuffer.slice().ptr, stringbuffer.length());
                stringPostfix(t);
                return TOKstring;

            case 0:
            case 0x1A:
                p--;
                error("unterminated string constant starting at %s", start.toChars());
                t->ustring = (utf8_t *)const_cast<char *>("");
                t->len = 0;
                t->postfix = 0;
                return TOKstring;

            default:
                if (c & 0x80)
                {
                    p--;
                    c = decodeUTF();
                    if (c == LS || c == PS)
                    {   c = '\n';
                        endOfLine();
                    }
                    p++;
                    stringbuffer.writeUTF8(c);
                    continue;
                }
                break;
        }
        stringbuffer.writeByte(c);
    }
}

/**************************************
 */

TOK Lexer::charConstant(Token *t)
{
    unsigned c;
    TOK tk = TOKcharv;

    //printf("Lexer::charConstant\n");
    p++;
    c = *p++;
    switch (c)
    {
        case '\\':
            switch (*p)
            {
                case 'u':
                    t->uns64value = escapeSequence();
                    tk = TOKwcharv;
                    break;

                case 'U':
                case '&':
                    t->uns64value = escapeSequence();
                    tk = TOKdcharv;
                    break;

                default:
                    t->uns64value = escapeSequence();
                    break;
            }
            break;
        case '\n':
        L1:
            endOfLine();
            /* fall through */
        case '\r':
        case 0:
        case 0x1A:
        case '\'':
            error("unterminated character constant");
            t->uns64value = '?';
            return tk;

        default:
            if (c & 0x80)
            {
                p--;
                c = decodeUTF();
                p++;
                if (c == LS || c == PS)
                    goto L1;
                if (c < 0xD800 || (c >= 0xE000 && c < 0xFFFE))
                    tk = TOKwcharv;
                else
                    tk = TOKdcharv;
            }
            t->uns64value = c;
            break;
    }

    if (*p != '\'')
    {
        error("unterminated character constant");
        t->uns64value = '?';
        return tk;
    }
    p++;
    return tk;
}

/***************************************
 * Get postfix of string literal.
 */

void Lexer::stringPostfix(Token *t)
{
    switch (*p)
    {
        case 'c':
        case 'w':
        case 'd':
            t->postfix = *p;
            p++;
            break;

        default:
            t->postfix = 0;
            break;
    }
}

/**************************************
 * Read in a number.
 * If it's an integer, store it in tok.TKutok.Vlong.
 *      integers can be decimal, octal or hex
 *      Handle the suffixes U, UL, LU, L, etc.
 * If it's double, store it in tok.TKutok.Vdouble.
 * Returns:
 *      TKnum
 *      TKdouble,...
 */

TOK Lexer::number(Token *t)
{
    int base = 10;
    const utf8_t *start = p;
    unsigned c;
    uinteger_t n = 0;                       // unsigned >=64 bit integer type
    int d;
    bool err = false;
    bool overflow = false;

    c = *p;
    if (c == '0')
    {
        ++p;
        c = *p;
        switch (c)
        {
            case '0': case '1': case '2': case '3':
            case '4': case '5': case '6': case '7':
                n = c - '0';
                ++p;
                base = 8;
                break;

            case 'x':
            case 'X':
                ++p;
                base = 16;
                break;

            case 'b':
            case 'B':
                ++p;
                base = 2;
                break;

            case '.':
                if (p[1] == '.')
                    goto Ldone; // if ".."
                if (isalpha(p[1]) || p[1] == '_' || p[1] & 0x80)
                    goto Ldone; // if ".identifier" or ".unicode"
                goto Lreal; // '.' is part of current token

            case 'i':
            case 'f':
            case 'F':
                goto Lreal;

            case '_':
                ++p;
                base = 8;
                break;

            case 'L':
                if (p[1] == 'i')
                    goto Lreal;
                break;

            default:
                break;
        }
    }

    while (1)
    {
        c = *p;
        switch (c)
        {
            case '0': case '1':
                ++p;
                d = c - '0';
                break;

            case '2': case '3':
            case '4': case '5': case '6': case '7':
                if (base == 2 && !err)
                {
                    error("binary digit expected");
                    err = true;
                }
                ++p;
                d = c - '0';
                break;

            case '8': case '9':
                ++p;
                if (base < 10 && !err)
                {
                    error("radix %d digit expected, not `%c`", base, c);
                    err = true;
                }
                d = c - '0';
                break;

            case 'a': case 'b': case 'c': case 'd': case 'e': case 'f':
            case 'A': case 'B': case 'C': case 'D': case 'E': case 'F':
                ++p;
                if (base != 16)
                {
                    if (c == 'e' || c == 'E' || c == 'f' || c == 'F')
                        goto Lreal;
                    if (!err)
                    {
                        error("radix %d digit expected, not `%c`", base, c);
                        err = true;
                    }
                }
                if (c >= 'a')
                    d = c + 10 - 'a';
                else
                    d = c + 10 - 'A';
                break;

            case 'L':
                if (p[1] == 'i')
                    goto Lreal;
                goto Ldone;

            case '.':
                if (p[1] == '.')
                    goto Ldone; // if ".."
                if (base == 10 && (isalpha(p[1]) || p[1] == '_' || p[1] & 0x80))
                    goto Ldone; // if ".identifier" or ".unicode"
                goto Lreal; // otherwise as part of a floating point literal

            case 'p':
            case 'P':
            case 'i':
            Lreal:
                p = start;
                return inreal(t);

            case '_':
                ++p;
                continue;

            default:
                goto Ldone;
        }

        uinteger_t n2 = n * base;
        if ((n2 / base != n || n2 + d < n))
        {
            overflow = true;
        }
        n = n2 + d;

        // if n needs more than 64 bits
        if (sizeof(n) > 8 &&
            n > 0xFFFFFFFFFFFFFFFFULL)
        {
            overflow = true;
        }
    }

Ldone:

    if (overflow && !err)
    {
        error("integer overflow");
        err = true;
    }

    enum FLAGS
    {
        FLAGS_none     = 0,
        FLAGS_decimal  = 1,             // decimal
        FLAGS_unsigned = 2,             // u or U suffix
        FLAGS_long     = 4,             // L suffix
    };

    unsigned flags = (base == 10) ? FLAGS_decimal : FLAGS_none;

    // Parse trailing 'u', 'U', 'l' or 'L' in any combination
    const utf8_t *psuffix = p;
    while (1)
    {
        utf8_t f;
        switch (*p)
        {
            case 'U':
            case 'u':
                f = FLAGS_unsigned;
                goto L1;

            case 'l':
                f = FLAGS_long;
                error("lower case integer suffix 'l' is not allowed. Please use 'L' instead");
                goto L1;

            case 'L':
                f = FLAGS_long;
            L1:
                p++;
                if ((flags & f) && !err)
                {
                    error("unrecognized token");
                    err = true;
                }
                flags = (FLAGS) (flags | f);
                continue;
            default:
                break;
        }
        break;
    }

    if (base == 8 && n >= 8)
        error("octal literals 0%llo%.*s are no longer supported, use std.conv.octal!%llo%.*s instead",
                n, p - psuffix, psuffix, n, p - psuffix, psuffix);

    TOK result;
    switch (flags)
    {
        case FLAGS_none:
            /* Octal or Hexadecimal constant.
             * First that fits: int, uint, long, ulong
             */
            if (n & 0x8000000000000000LL)
                result = TOKuns64v;
            else if (n & 0xFFFFFFFF00000000LL)
                result = TOKint64v;
            else if (n & 0x80000000)
                result = TOKuns32v;
            else
                result = TOKint32v;
            break;

        case FLAGS_decimal:
            /* First that fits: int, long, long long
             */
            if (n & 0x8000000000000000LL)
            {
                if (!err)
                {
                    error("signed integer overflow");
                    err = true;
                }
                result = TOKuns64v;
            }
            else if (n & 0xFFFFFFFF80000000LL)
                result = TOKint64v;
            else
                result = TOKint32v;
            break;

        case FLAGS_unsigned:
        case FLAGS_decimal | FLAGS_unsigned:
            /* First that fits: uint, ulong
             */
            if (n & 0xFFFFFFFF00000000LL)
                result = TOKuns64v;
            else
                result = TOKuns32v;
            break;

        case FLAGS_decimal | FLAGS_long:
            if (n & 0x8000000000000000LL)
            {
                if (!err)
                {
                    error("signed integer overflow");
                    err = true;
                }
                result = TOKuns64v;
            }
            else
                result = TOKint64v;
            break;

        case FLAGS_long:
            if (n & 0x8000000000000000LL)
                result = TOKuns64v;
            else
                result = TOKint64v;
            break;

        case FLAGS_unsigned | FLAGS_long:
        case FLAGS_decimal | FLAGS_unsigned | FLAGS_long:
            result = TOKuns64v;
            break;

        default:
            assert(0);
    }
    t->uns64value = n;
    return result;
}

/**************************************
 * Read in characters, converting them to real.
 * Bugs:
 *      Exponent overflow not detected.
 *      Too much requested precision is not detected.
 */

TOK Lexer::inreal(Token *t)
{
    //printf("Lexer::inreal()\n");
    bool isWellformedString = true;
    stringbuffer.reset();
    const utf8_t *pstart = p;
    char hex = 0;
    unsigned c = *p++;

    // Leading '0x'
    if (c == '0')
    {
        c = *p++;
        if (c == 'x' || c == 'X')
        {
            hex = true;
            c = *p++;
        }
    }

    // Digits to left of '.'
    while (1)
    {
        if (c == '.')
        {
            c = *p++;
            break;
        }
        if (isdigit(c) || (hex && isxdigit(c)) || c == '_')
        {
            c = *p++;
            continue;
        }
        break;
    }

    // Digits to right of '.'
    while (1)
    {
        if (isdigit(c) || (hex && isxdigit(c)) || c == '_')
        {
            c = *p++;
            continue;
        }
        break;
    }

    if (c == 'e' || c == 'E' || (hex && (c == 'p' || c == 'P')))
    {
        c = *p++;
        if (c == '-' || c == '+')
        {
            c = *p++;
        }
        bool anyexp = false;
        while (1)
        {
            if (isdigit(c))
            {
                anyexp = true;
                c = *p++;
                continue;
            }
            if (c == '_')
            {
                c = *p++;
                continue;
            }
            if (!anyexp)
            {
                error("missing exponent");
                isWellformedString = false;
            }
            break;
        }
    }
    else if (hex)
    {
        error("exponent required for hex float");
        isWellformedString = false;
    }
    --p;
    while (pstart < p)
    {
        if (*pstart != '_')
            stringbuffer.writeByte(*pstart);
        ++pstart;
    }

    stringbuffer.writeByte(0);
    const char *sbufptr = (char *)stringbuffer.slice().ptr;
    TOK result;
    bool isOutOfRange = false;
    t->floatvalue = (isWellformedString ? CTFloat::parse(sbufptr, &isOutOfRange) : CTFloat::zero);
    errno = 0;
    switch (*p)
    {
        case 'F':
        case 'f':
            if (isWellformedString && !isOutOfRange)
                isOutOfRange = Port::isFloat32LiteralOutOfRange(sbufptr);
            result = TOKfloat32v;
            p++;
            break;

        default:
            if (isWellformedString && !isOutOfRange)
                isOutOfRange = Port::isFloat64LiteralOutOfRange(sbufptr);
            result = TOKfloat64v;
            break;

        case 'l':
            error("use 'L' suffix instead of 'l'");
            /* fall through */
        case 'L':
            result = TOKfloat80v;
            p++;
            break;
    }
    if (*p == 'i' || *p == 'I')
    {
        if (*p == 'I')
            error("use 'i' suffix instead of 'I'");
        p++;
        switch (result)
        {
            case TOKfloat32v:
                result = TOKimaginary32v;
                break;
            case TOKfloat64v:
                result = TOKimaginary64v;
                break;
            case TOKfloat80v:
                result = TOKimaginary80v;
                break;
            default: break;
        }
    }
    const bool isLong = (result == TOKfloat80v || result == TOKimaginary80v);
    if (isOutOfRange && !isLong)
    {
        const char *suffix = (result == TOKfloat32v || result == TOKimaginary32v) ? "f" : "";
        error(scanloc, "number `%s%s` is not representable", (char *)stringbuffer.slice().ptr, suffix);
    }
    return result;
}

/*********************************************
 * parse:
 *      #line linnum [filespec]
 * also allow __LINE__ for linnum, and __FILE__ for filespec
 */

void Lexer::poundLine()
{
    Token tok;
    int linnum = this->scanloc.linnum;
    char *filespec = NULL;
    Loc loc = this->loc();

    scan(&tok);
    if (tok.value == TOKint32v || tok.value == TOKint64v)
    {
        int lin = (int)(tok.uns64value - 1);
        if ((unsigned)lin != tok.uns64value - 1)
            error("line number %lld out of range", (unsigned long long)tok.uns64value);
        else
            linnum = lin;
    }
    else if (tok.value == TOKline)
    {
    }
    else
        goto Lerr;

    while (1)
    {
        switch (*p)
        {
            case 0:
            case 0x1A:
            case '\n':
            Lnewline:
                this->scanloc.linnum = linnum;
                if (filespec)
                    this->scanloc.filename = filespec;
                return;

            case '\r':
                p++;
                if (*p != '\n')
                {   p--;
                    goto Lnewline;
                }
                continue;

            case ' ':
            case '\t':
            case '\v':
            case '\f':
                p++;
                continue;                       // skip white space

            case '_':
                if (memcmp(p, "__FILE__", 8) == 0)
                {
                    p += 8;
                    filespec = mem.xstrdup(scanloc.filename);
                    continue;
                }
                goto Lerr;

            case '"':
                if (filespec)
                    goto Lerr;
                stringbuffer.reset();
                p++;
                while (1)
                {   unsigned c;

                    c = *p;
                    switch (c)
                    {
                        case '\n':
                        case '\r':
                        case 0:
                        case 0x1A:
                            goto Lerr;

                        case '"':
                            stringbuffer.writeByte(0);
                            filespec = mem.xstrdup((char *)stringbuffer.slice().ptr);
                            p++;
                            break;

                        default:
                            if (c & 0x80)
                            {   unsigned u = decodeUTF();
                                if (u == PS || u == LS)
                                    goto Lerr;
                            }
                            stringbuffer.writeByte(c);
                            p++;
                            continue;
                    }
                    break;
                }
                continue;

            default:
                if (*p & 0x80)
                {   unsigned u = decodeUTF();
                    if (u == PS || u == LS)
                        goto Lnewline;
                }
                goto Lerr;
        }
    }

Lerr:
    error(loc, "#line integer [\"filespec\"]\\n expected");
}


/********************************************
 * Decode UTF character.
 * Issue error messages for invalid sequences.
 * Return decoded character, advance p to last character in UTF sequence.
 */

unsigned Lexer::decodeUTF()
{
    dchar_t u;
    utf8_t c;
    const utf8_t *s = p;
    size_t len;
    size_t idx;
    const char *msg;

    c = *s;
    assert(c & 0x80);

    // Check length of remaining string up to 6 UTF-8 characters
    for (len = 1; len < 6 && s[len]; len++)
        ;

    idx = 0;
    msg = utf_decodeChar(s, len, &idx, &u);
    p += idx - 1;
    if (msg)
    {
        error("%s", msg);
    }
    return u;
}

static void trimTrailingWhitespace(OutBuffer &buf)
{
    const unsigned char *s = buf.slice().ptr;
    size_t len = buf.length();
    while (len && (s[len - 1] == ' ' || s[len - 1] == '\t'))
        --len;
    buf.setsize(len);
}

/***************************************************
 * Parse doc comment embedded between t->ptr and p.
 * Remove trailing blanks and tabs from lines.
 * Replace all newlines with \n.
 * Remove leading comment character from each line.
 * Decide if it's a lineComment or a blockComment.
 * Append to previous one for this token.
 */

void Lexer::getDocComment(Token *t, unsigned lineComment)
{
    /* ct tells us which kind of comment it is: '/', '*', or '+'
     */
    utf8_t ct = t->ptr[2];

    /* Start of comment text skips over / * *, / + +, or / / /
     */
    const utf8_t *q = t->ptr + 3;      // start of comment text

    const utf8_t *qend = p;
    if (ct == '*' || ct == '+')
        qend -= 2;

    /* Scan over initial row of ****'s or ++++'s or ////'s
     */
    for (; q < qend; q++)
    {
        if (*q != ct)
            break;
    }

    /* Remove leading spaces until start of the comment
     */
    int linestart = 0;
    if (ct == '/')
    {
        while (q < qend && (*q == ' ' || *q == '\t'))
            ++q;
    }
    else if (q < qend)
    {
        if (*q == '\r')
        {
            ++q;
            if (q < qend && *q == '\n')
                ++q;
            linestart = 1;
        }
        else if (*q == '\n')
        {
            ++q;
            linestart = 1;
        }
    }

    /* Remove trailing row of ****'s or ++++'s
     */
    if (ct != '/')
    {
        for (; q < qend; qend--)
        {
            if (qend[-1] != ct)
                break;
        }
    }

    /* Comment is now [q .. qend].
     * Canonicalize it into buf[].
     */
    OutBuffer buf;

    for (; q < qend; q++)
    {
        utf8_t c = *q;

        switch (c)
        {
            case '*':
            case '+':
                if (linestart && c == ct)
                {   linestart = 0;
                    /* Trim preceding whitespace up to preceding \n
                     */
                    trimTrailingWhitespace(buf);
                    continue;
                }
                break;

            case ' ':
            case '\t':
                break;

            case '\r':
                if (q[1] == '\n')
                    continue;           // skip the \r
                goto Lnewline;

            default:
                if (c == 226)
                {
                    // If LS or PS
                    if (q[1] == 128 &&
                        (q[2] == 168 || q[2] == 169))
                    {
                        q += 2;
                        goto Lnewline;
                    }
                }
                linestart = 0;
                break;

            Lnewline:
                c = '\n';               // replace all newlines with \n
                /* fall through */
            case '\n':
                linestart = 1;

                /* Trim trailing whitespace
                 */
                trimTrailingWhitespace(buf);
                break;
        }
        buf.writeByte(c);
    }

    /* Trim trailing whitespace (if the last line does not have newline)
     */
    if (buf.length() && (buf.slice().ptr[buf.length() - 1] == ' ' || buf.slice().ptr[buf.length() - 1] == '\t'))
    {
        trimTrailingWhitespace(buf);
    }

    // Always end with a newline
    if (!buf.length() || buf.slice().ptr[buf.length() - 1] != '\n')
        buf.writeByte('\n');

    buf.writeByte(0);

    // It's a line comment if the start of the doc comment comes
    // after other non-whitespace on the same line.
    const utf8_t** dc = (lineComment && anyToken)
                         ? &t->lineComment
                         : &t->blockComment;

    // Combine with previous doc comment, if any
    if (*dc)
        *dc = combineComments(*dc, (utf8_t *)buf.slice().ptr);
    else
        *dc = (utf8_t *)buf.extractData();
}

/********************************************
 * Combine two document comments into one,
 * separated by a newline.
 */

const utf8_t *Lexer::combineComments(const utf8_t *c1, const utf8_t *c2)
{
    //printf("Lexer::combineComments('%s', '%s')\n", c1, c2);

    const utf8_t *c = c2;

    if (c1)
    {
        c = c1;
        if (c2)
        {
            size_t len1 = strlen((const char *)c1);
            size_t len2 = strlen((const char *)c2);

            int insertNewLine = 0;
            if (len1 && c1[len1 - 1] != '\n')
            {
                ++len1;
                insertNewLine = 1;
            }

            utf8_t *p = (utf8_t *)mem.xmalloc(len1 + 1 + len2 + 1);
            memcpy(p, c1, len1 - insertNewLine);
            if (insertNewLine)
                p[len1 - 1] = '\n';

            p[len1] = '\n';

            memcpy(p + len1 + 1, c2, len2);
            p[len1 + 1 + len2] = 0;
            c = p;
        }
    }
    return c;
}