blob: c9c7ab45878ee1e77da67123eeb7ee86c27c1d4d [file] [log] [blame]
/* Compiler implementation of the D programming language
* Copyright (C) 1999-2019 by The D Language Foundation, All Rights Reserved
* written by Walter Bright
* http://www.digitalmars.com
* Distributed under the Boost Software License, Version 1.0.
* http://www.boost.org/LICENSE_1_0.txt
* https://github.com/D-Programming-Language/dmd/blob/master/src/lexer.c
*/
#include "root/dsystem.h"
#include "tokens.h"
#include "root/rmem.h"
#include "root/outbuffer.h"
#include "id.h"
#include "identifier.h"
#include "utf.h"
/************************* Token **********************************************/
Token *Token::freelist = NULL;
const char *Token::tochars[TOKMAX];
Token *Token::alloc()
{
if (Token::freelist)
{
Token *t = freelist;
freelist = t->next;
t->next = NULL;
return t;
}
return new Token();
}
void Token::free()
{
next = freelist;
freelist = this;
}
const char *Token::toChars() const
{
static char buffer[3 + 3 * sizeof(floatvalue) + 1];
const char *p = &buffer[0];
switch (value)
{
case TOKint32v:
sprintf(&buffer[0],"%d",(d_int32)int64value);
break;
case TOKuns32v:
case TOKcharv:
case TOKwcharv:
case TOKdcharv:
sprintf(&buffer[0],"%uU",(d_uns32)uns64value);
break;
case TOKint64v:
sprintf(&buffer[0],"%lldL",(longlong)int64value);
break;
case TOKuns64v:
sprintf(&buffer[0],"%lluUL",(ulonglong)uns64value);
break;
case TOKfloat32v:
CTFloat::sprint(&buffer[0], 'g', floatvalue);
strcat(&buffer[0], "f");
break;
case TOKfloat64v:
CTFloat::sprint(&buffer[0], 'g', floatvalue);
break;
case TOKfloat80v:
CTFloat::sprint(&buffer[0], 'g', floatvalue);
strcat(&buffer[0], "L");
break;
case TOKimaginary32v:
CTFloat::sprint(&buffer[0], 'g', floatvalue);
strcat(&buffer[0], "fi");
break;
case TOKimaginary64v:
CTFloat::sprint(&buffer[0], 'g', floatvalue);
strcat(&buffer[0], "i");
break;
case TOKimaginary80v:
CTFloat::sprint(&buffer[0], 'g', floatvalue);
strcat(&buffer[0], "Li");
break;
case TOKstring:
{
OutBuffer buf;
buf.writeByte('"');
for (size_t i = 0; i < len; )
{
unsigned c;
utf_decodeChar((utf8_t *)ustring, len, &i, &c);
switch (c)
{
case 0:
break;
case '"':
case '\\':
buf.writeByte('\\');
/* fall through */
default:
if (c <= 0x7F)
{
if (isprint(c))
buf.writeByte(c);
else
buf.printf("\\x%02x", c);
}
else if (c <= 0xFFFF)
buf.printf("\\u%04x", c);
else
buf.printf("\\U%08x", c);
continue;
}
break;
}
buf.writeByte('"');
if (postfix)
buf.writeByte(postfix);
p = buf.extractString();
}
break;
case TOKxstring:
{
OutBuffer buf;
buf.writeByte('x');
buf.writeByte('"');
for (size_t i = 0; i < len; i++)
{
if (i)
buf.writeByte(' ');
buf.printf("%02x", ustring[i]);
}
buf.writeByte('"');
if (postfix)
buf.writeByte(postfix);
buf.writeByte(0);
p = (char *)buf.extractData();
break;
}
case TOKidentifier:
case TOKenum:
case TOKstruct:
case TOKimport:
case TOKwchar: case TOKdchar:
case TOKbool: case TOKchar:
case TOKint8: case TOKuns8:
case TOKint16: case TOKuns16:
case TOKint32: case TOKuns32:
case TOKint64: case TOKuns64:
case TOKint128: case TOKuns128:
case TOKfloat32: case TOKfloat64: case TOKfloat80:
case TOKimaginary32: case TOKimaginary64: case TOKimaginary80:
case TOKcomplex32: case TOKcomplex64: case TOKcomplex80:
case TOKvoid:
p = ident->toChars();
break;
default:
p = toChars(value);
break;
}
return p;
}
const char *Token::toChars(TOK value)
{
static char buffer[3 + 3 * sizeof(value) + 1];
const char *p = tochars[value];
if (!p)
{
sprintf(&buffer[0],"TOK%d",value);
p = &buffer[0];
}
return p;
}
/****************************************
*/
struct Keyword
{
const char *name;
TOK value;
};
static size_t nkeywords;
static Keyword keywords[] =
{
{ "this", TOKthis },
{ "super", TOKsuper },
{ "assert", TOKassert },
{ "null", TOKnull },
{ "true", TOKtrue },
{ "false", TOKfalse },
{ "cast", TOKcast },
{ "new", TOKnew },
{ "delete", TOKdelete },
{ "throw", TOKthrow },
{ "module", TOKmodule },
{ "pragma", TOKpragma },
{ "typeof", TOKtypeof },
{ "typeid", TOKtypeid },
{ "template", TOKtemplate },
{ "void", TOKvoid },
{ "byte", TOKint8 },
{ "ubyte", TOKuns8 },
{ "short", TOKint16 },
{ "ushort", TOKuns16 },
{ "int", TOKint32 },
{ "uint", TOKuns32 },
{ "long", TOKint64 },
{ "ulong", TOKuns64 },
{ "cent", TOKint128, },
{ "ucent", TOKuns128, },
{ "float", TOKfloat32 },
{ "double", TOKfloat64 },
{ "real", TOKfloat80 },
{ "bool", TOKbool },
{ "char", TOKchar },
{ "wchar", TOKwchar },
{ "dchar", TOKdchar },
{ "ifloat", TOKimaginary32 },
{ "idouble", TOKimaginary64 },
{ "ireal", TOKimaginary80 },
{ "cfloat", TOKcomplex32 },
{ "cdouble", TOKcomplex64 },
{ "creal", TOKcomplex80 },
{ "delegate", TOKdelegate },
{ "function", TOKfunction },
{ "is", TOKis },
{ "if", TOKif },
{ "else", TOKelse },
{ "while", TOKwhile },
{ "for", TOKfor },
{ "do", TOKdo },
{ "switch", TOKswitch },
{ "case", TOKcase },
{ "default", TOKdefault },
{ "break", TOKbreak },
{ "continue", TOKcontinue },
{ "synchronized", TOKsynchronized },
{ "return", TOKreturn },
{ "goto", TOKgoto },
{ "try", TOKtry },
{ "catch", TOKcatch },
{ "finally", TOKfinally },
{ "with", TOKwith },
{ "asm", TOKasm },
{ "foreach", TOKforeach },
{ "foreach_reverse", TOKforeach_reverse },
{ "scope", TOKscope },
{ "struct", TOKstruct },
{ "class", TOKclass },
{ "interface", TOKinterface },
{ "union", TOKunion },
{ "enum", TOKenum },
{ "import", TOKimport },
{ "mixin", TOKmixin },
{ "static", TOKstatic },
{ "final", TOKfinal },
{ "const", TOKconst },
{ "alias", TOKalias },
{ "override", TOKoverride },
{ "abstract", TOKabstract },
{ "debug", TOKdebug },
{ "deprecated", TOKdeprecated },
{ "in", TOKin },
{ "out", TOKout },
{ "inout", TOKinout },
{ "lazy", TOKlazy },
{ "auto", TOKauto },
{ "align", TOKalign },
{ "extern", TOKextern },
{ "private", TOKprivate },
{ "package", TOKpackage },
{ "protected", TOKprotected },
{ "public", TOKpublic },
{ "export", TOKexport },
{ "invariant", TOKinvariant },
{ "unittest", TOKunittest },
{ "version", TOKversion },
{ "__argTypes", TOKargTypes },
{ "__parameters", TOKparameters },
{ "ref", TOKref },
{ "macro", TOKmacro },
{ "pure", TOKpure },
{ "nothrow", TOKnothrow },
{ "__gshared", TOKgshared },
{ "__traits", TOKtraits },
{ "__vector", TOKvector },
{ "__overloadset", TOKoverloadset },
{ "__FILE__", TOKfile },
{ "__FILE_FULL_PATH__", TOKfilefullpath },
{ "__LINE__", TOKline },
{ "__MODULE__", TOKmodulestring },
{ "__FUNCTION__", TOKfuncstring },
{ "__PRETTY_FUNCTION__", TOKprettyfunc },
{ "shared", TOKshared },
{ "immutable", TOKimmutable },
{ NULL, TOKreserved }
};
int Token::isKeyword()
{
for (size_t u = 0; u < nkeywords; u++)
{
if (keywords[u].value == value)
return 1;
}
return 0;
}
struct TokenInitializer
{
TokenInitializer();
};
static TokenInitializer tokeninitializer;
TokenInitializer::TokenInitializer()
{
Identifier::initTable();
for (nkeywords = 0; keywords[nkeywords].name; nkeywords++)
{
//printf("keyword[%d] = '%s'\n",u, keywords[u].name);
const char *s = keywords[nkeywords].name;
size_t len = strlen(s);
TOK v = keywords[nkeywords].value;
Identifier::idPool(s, len, v);
//printf("tochars[%d] = '%s'\n",v, s);
Token::tochars[v] = s;
}
Token::tochars[TOKeof] = "EOF";
Token::tochars[TOKlcurly] = "{";
Token::tochars[TOKrcurly] = "}";
Token::tochars[TOKlparen] = "(";
Token::tochars[TOKrparen] = ")";
Token::tochars[TOKlbracket] = "[";
Token::tochars[TOKrbracket] = "]";
Token::tochars[TOKsemicolon] = ";";
Token::tochars[TOKcolon] = ":";
Token::tochars[TOKcomma] = ",";
Token::tochars[TOKdot] = ".";
Token::tochars[TOKxor] = "^";
Token::tochars[TOKxorass] = "^=";
Token::tochars[TOKassign] = "=";
Token::tochars[TOKconstruct] = "=";
Token::tochars[TOKblit] = "=";
Token::tochars[TOKlt] = "<";
Token::tochars[TOKgt] = ">";
Token::tochars[TOKle] = "<=";
Token::tochars[TOKge] = ">=";
Token::tochars[TOKequal] = "==";
Token::tochars[TOKnotequal] = "!=";
Token::tochars[TOKnotidentity] = "!is";
Token::tochars[TOKunord] = "!<>=";
Token::tochars[TOKue] = "!<>";
Token::tochars[TOKlg] = "<>";
Token::tochars[TOKleg] = "<>=";
Token::tochars[TOKule] = "!>";
Token::tochars[TOKul] = "!>=";
Token::tochars[TOKuge] = "!<";
Token::tochars[TOKug] = "!<=";
Token::tochars[TOKnot] = "!";
Token::tochars[TOKshl] = "<<";
Token::tochars[TOKshr] = ">>";
Token::tochars[TOKushr] = ">>>";
Token::tochars[TOKadd] = "+";
Token::tochars[TOKmin] = "-";
Token::tochars[TOKmul] = "*";
Token::tochars[TOKdiv] = "/";
Token::tochars[TOKmod] = "%";
Token::tochars[TOKslice] = "..";
Token::tochars[TOKdotdotdot] = "...";
Token::tochars[TOKand] = "&";
Token::tochars[TOKandand] = "&&";
Token::tochars[TOKor] = "|";
Token::tochars[TOKoror] = "||";
Token::tochars[TOKarray] = "[]";
Token::tochars[TOKindex] = "[i]";
Token::tochars[TOKaddress] = "&";
Token::tochars[TOKstar] = "*";
Token::tochars[TOKtilde] = "~";
Token::tochars[TOKdollar] = "$";
Token::tochars[TOKcast] = "cast";
Token::tochars[TOKplusplus] = "++";
Token::tochars[TOKminusminus] = "--";
Token::tochars[TOKpreplusplus] = "++";
Token::tochars[TOKpreminusminus] = "--";
Token::tochars[TOKtype] = "type";
Token::tochars[TOKquestion] = "?";
Token::tochars[TOKneg] = "-";
Token::tochars[TOKuadd] = "+";
Token::tochars[TOKvar] = "var";
Token::tochars[TOKaddass] = "+=";
Token::tochars[TOKminass] = "-=";
Token::tochars[TOKmulass] = "*=";
Token::tochars[TOKdivass] = "/=";
Token::tochars[TOKmodass] = "%=";
Token::tochars[TOKshlass] = "<<=";
Token::tochars[TOKshrass] = ">>=";
Token::tochars[TOKushrass] = ">>>=";
Token::tochars[TOKandass] = "&=";
Token::tochars[TOKorass] = "|=";
Token::tochars[TOKcatass] = "~=";
Token::tochars[TOKcat] = "~";
Token::tochars[TOKcall] = "call";
Token::tochars[TOKidentity] = "is";
Token::tochars[TOKnotidentity] = "!is";
Token::tochars[TOKorass] = "|=";
Token::tochars[TOKidentifier] = "identifier";
Token::tochars[TOKat] = "@";
Token::tochars[TOKpow] = "^^";
Token::tochars[TOKpowass] = "^^=";
Token::tochars[TOKgoesto] = "=>";
Token::tochars[TOKpound] = "#";
// For debugging
Token::tochars[TOKerror] = "error";
Token::tochars[TOKdotid] = "dotid";
Token::tochars[TOKdottd] = "dottd";
Token::tochars[TOKdotti] = "dotti";
Token::tochars[TOKdotvar] = "dotvar";
Token::tochars[TOKdottype] = "dottype";
Token::tochars[TOKsymoff] = "symoff";
Token::tochars[TOKarraylength] = "arraylength";
Token::tochars[TOKarrayliteral] = "arrayliteral";
Token::tochars[TOKassocarrayliteral] = "assocarrayliteral";
Token::tochars[TOKstructliteral] = "structliteral";
Token::tochars[TOKstring] = "string";
Token::tochars[TOKdsymbol] = "symbol";
Token::tochars[TOKtuple] = "tuple";
Token::tochars[TOKdeclaration] = "declaration";
Token::tochars[TOKon_scope_exit] = "scope(exit)";
Token::tochars[TOKon_scope_success] = "scope(success)";
Token::tochars[TOKon_scope_failure] = "scope(failure)";
Token::tochars[TOKdelegateptr] = "delegateptr";
Token::tochars[TOKvectorarray] = "vectorarray";
}