"Fossies" - the Fresh Open Source Software Archive

Member "dmd2/src/dmd/dmd/tokens.d" (20 Nov 2020, 21248 Bytes) of package /linux/misc/dmd.2.094.2.linux.tar.xz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) D source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file.

    1 /**
    2  * Defines lexical tokens.
    3  *
    4  * Specification: $(LINK2 https://dlang.org/spec/lex.html#tokens, Tokens)
    5  *
    6  * Copyright:   Copyright (C) 1999-2020 by The D Language Foundation, All Rights Reserved
    7  * Authors:     $(LINK2 http://www.digitalmars.com, Walter Bright)
    8  * License:     $(LINK2 http://www.boost.org/LICENSE_1_0.txt, Boost License 1.0)
    9  * Source:      $(LINK2 https://github.com/dlang/dmd/blob/master/src/dmd/tokens.d, _tokens.d)
   10  * Documentation:  https://dlang.org/phobos/dmd_tokens.html
   11  * Coverage:    https://codecov.io/gh/dlang/dmd/src/master/src/dmd/tokens.d
   12  */
   13 
   14 module dmd.tokens;
   15 
   16 import core.stdc.ctype;
   17 import core.stdc.stdio;
   18 import core.stdc.string;
   19 import dmd.globals;
   20 import dmd.identifier;
   21 import dmd.root.ctfloat;
   22 import dmd.root.outbuffer;
   23 import dmd.root.rmem;
   24 import dmd.utf;
   25 
   26 enum TOK : ubyte
   27 {
   28     reserved,
   29 
   30     // Other
   31     leftParentheses,
   32     rightParentheses,
   33     leftBracket,
   34     rightBracket,
   35     leftCurly,
   36     rightCurly,
   37     colon,
   38     negate,
   39     semicolon,
   40     dotDotDot,
   41     endOfFile,
   42     cast_,
   43     null_,
   44     assert_,
   45     true_,
   46     false_,
   47     array,
   48     call,
   49     address,
   50     type,
   51     throw_,
   52     new_,
   53     delete_,
   54     star,
   55     symbolOffset,
   56     variable,
   57     dotVariable,
   58     dotIdentifier,
   59     dotTemplateInstance,
   60     dotType,
   61     slice,
   62     arrayLength,
   63     version_,
   64     module_,
   65     dollar,
   66     template_,
   67     dotTemplateDeclaration,
   68     declaration,
   69     typeof_,
   70     pragma_,
   71     dSymbol,
   72     typeid_,
   73     uadd,
   74     remove,
   75     newAnonymousClass,
   76     comment,
   77     arrayLiteral,
   78     assocArrayLiteral,
   79     structLiteral,
   80     classReference,
   81     thrownException,
   82     delegatePointer,
   83     delegateFunctionPointer,
   84 
   85     // Operators
   86     lessThan = 54,
   87     greaterThan,
   88     lessOrEqual,
   89     greaterOrEqual,
   90     equal,
   91     notEqual,
   92     identity,
   93     notIdentity,
   94     index,
   95     is_,
   96 
   97     leftShift = 64,
   98     rightShift,
   99     leftShiftAssign,
  100     rightShiftAssign,
  101     unsignedRightShift,
  102     unsignedRightShiftAssign,
  103     concatenate,
  104     concatenateAssign, // ~=
  105     concatenateElemAssign,
  106     concatenateDcharAssign,
  107     add,
  108     min,
  109     addAssign,
  110     minAssign,
  111     mul,
  112     div,
  113     mod,
  114     mulAssign,
  115     divAssign,
  116     modAssign,
  117     and,
  118     or,
  119     xor,
  120     andAssign,
  121     orAssign,
  122     xorAssign,
  123     assign,
  124     not,
  125     tilde,
  126     plusPlus,
  127     minusMinus,
  128     construct,
  129     blit,
  130     dot,
  131     arrow,
  132     comma,
  133     question,
  134     andAnd,
  135     orOr,
  136     prePlusPlus,
  137     preMinusMinus,
  138 
  139     // Numeric literals
  140     int32Literal = 105,
  141     uns32Literal,
  142     int64Literal,
  143     uns64Literal,
  144     int128Literal,
  145     uns128Literal,
  146     float32Literal,
  147     float64Literal,
  148     float80Literal,
  149     imaginary32Literal,
  150     imaginary64Literal,
  151     imaginary80Literal,
  152 
  153     // Char constants
  154     charLiteral = 117,
  155     wcharLiteral,
  156     dcharLiteral,
  157 
  158     // Leaf operators
  159     identifier = 120,
  160     string_,
  161     hexadecimalString,
  162     this_,
  163     super_,
  164     halt,
  165     tuple,
  166     error,
  167 
  168     // Basic types
  169     void_ = 128,
  170     int8,
  171     uns8,
  172     int16,
  173     uns16,
  174     int32,
  175     uns32,
  176     int64,
  177     uns64,
  178     int128,
  179     uns128,
  180     float32,
  181     float64,
  182     float80,
  183     imaginary32,
  184     imaginary64,
  185     imaginary80,
  186     complex32,
  187     complex64,
  188     complex80,
  189     char_,
  190     wchar_,
  191     dchar_,
  192     bool_,
  193 
  194     // Aggregates
  195     struct_ = 152,
  196     class_,
  197     interface_,
  198     union_,
  199     enum_,
  200     import_,
  201     alias_,
  202     override_,
  203     delegate_,
  204     function_,
  205     mixin_,
  206     align_,
  207     extern_,
  208     private_,
  209     protected_,
  210     public_,
  211     export_,
  212     static_,
  213     final_,
  214     const_,
  215     abstract_,
  216     debug_,
  217     deprecated_,
  218     in_,
  219     out_,
  220     inout_,
  221     lazy_,
  222     auto_,
  223     package_,
  224     immutable_,
  225 
  226     // Statements
  227     if_ = 182,
  228     else_,
  229     while_,
  230     for_,
  231     do_,
  232     switch_,
  233     case_,
  234     default_,
  235     break_,
  236     continue_,
  237     with_,
  238     synchronized_,
  239     return_,
  240     goto_,
  241     try_,
  242     catch_,
  243     finally_,
  244     asm_,
  245     foreach_,
  246     foreach_reverse_,
  247     scope_,
  248     onScopeExit,
  249     onScopeFailure,
  250     onScopeSuccess,
  251 
  252     // Contracts
  253     invariant_ = 206,
  254 
  255     // Testing
  256     unittest_,
  257 
  258     // Added after 1.0
  259     argumentTypes,
  260     ref_,
  261     macro_,
  262 
  263     parameters = 211,
  264     traits,
  265     overloadSet,
  266     pure_,
  267     nothrow_,
  268     gshared,
  269     line,
  270     file,
  271     fileFullPath,
  272     moduleString,
  273     functionString,
  274     prettyFunction,
  275     shared_,
  276     at,
  277     pow,
  278     powAssign,
  279     goesTo,
  280     vector,
  281     pound,
  282 
  283     interval = 230,
  284     voidExpression,
  285     cantExpression,
  286     showCtfeContext,
  287 
  288     objcClassReference,
  289     vectorArray,
  290 
  291     max_,
  292 }
  293 
  294 // Assert that all token enum members have consecutive values and
  295 // that none of them overlap
  296 static assert(() {
  297     foreach (idx, enumName; __traits(allMembers, TOK)) {
  298        static if (idx != __traits(getMember, TOK, enumName)) {
  299            pragma(msg, "Error: Expected TOK.", enumName, " to be ", idx, " but is ", __traits(getMember, TOK, enumName));
  300            static assert(0);
  301        }
  302     }
  303     return true;
  304 }());
  305 
  306 
  307 /****************************************
  308  */
  309 
  310 private immutable TOK[] keywords =
  311 [
  312     TOK.this_,
  313     TOK.super_,
  314     TOK.assert_,
  315     TOK.null_,
  316     TOK.true_,
  317     TOK.false_,
  318     TOK.cast_,
  319     TOK.new_,
  320     TOK.delete_,
  321     TOK.throw_,
  322     TOK.module_,
  323     TOK.pragma_,
  324     TOK.typeof_,
  325     TOK.typeid_,
  326     TOK.template_,
  327     TOK.void_,
  328     TOK.int8,
  329     TOK.uns8,
  330     TOK.int16,
  331     TOK.uns16,
  332     TOK.int32,
  333     TOK.uns32,
  334     TOK.int64,
  335     TOK.uns64,
  336     TOK.int128,
  337     TOK.uns128,
  338     TOK.float32,
  339     TOK.float64,
  340     TOK.float80,
  341     TOK.bool_,
  342     TOK.char_,
  343     TOK.wchar_,
  344     TOK.dchar_,
  345     TOK.imaginary32,
  346     TOK.imaginary64,
  347     TOK.imaginary80,
  348     TOK.complex32,
  349     TOK.complex64,
  350     TOK.complex80,
  351     TOK.delegate_,
  352     TOK.function_,
  353     TOK.is_,
  354     TOK.if_,
  355     TOK.else_,
  356     TOK.while_,
  357     TOK.for_,
  358     TOK.do_,
  359     TOK.switch_,
  360     TOK.case_,
  361     TOK.default_,
  362     TOK.break_,
  363     TOK.continue_,
  364     TOK.synchronized_,
  365     TOK.return_,
  366     TOK.goto_,
  367     TOK.try_,
  368     TOK.catch_,
  369     TOK.finally_,
  370     TOK.with_,
  371     TOK.asm_,
  372     TOK.foreach_,
  373     TOK.foreach_reverse_,
  374     TOK.scope_,
  375     TOK.struct_,
  376     TOK.class_,
  377     TOK.interface_,
  378     TOK.union_,
  379     TOK.enum_,
  380     TOK.import_,
  381     TOK.mixin_,
  382     TOK.static_,
  383     TOK.final_,
  384     TOK.const_,
  385     TOK.alias_,
  386     TOK.override_,
  387     TOK.abstract_,
  388     TOK.debug_,
  389     TOK.deprecated_,
  390     TOK.in_,
  391     TOK.out_,
  392     TOK.inout_,
  393     TOK.lazy_,
  394     TOK.auto_,
  395     TOK.align_,
  396     TOK.extern_,
  397     TOK.private_,
  398     TOK.package_,
  399     TOK.protected_,
  400     TOK.public_,
  401     TOK.export_,
  402     TOK.invariant_,
  403     TOK.unittest_,
  404     TOK.version_,
  405     TOK.argumentTypes,
  406     TOK.parameters,
  407     TOK.ref_,
  408     TOK.macro_,
  409     TOK.pure_,
  410     TOK.nothrow_,
  411     TOK.gshared,
  412     TOK.traits,
  413     TOK.vector,
  414     TOK.overloadSet,
  415     TOK.file,
  416     TOK.fileFullPath,
  417     TOK.line,
  418     TOK.moduleString,
  419     TOK.functionString,
  420     TOK.prettyFunction,
  421     TOK.shared_,
  422     TOK.immutable_,
  423 ];
  424 
  425 // Initialize the identifier pool
  426 shared static this() nothrow
  427 {
  428     Identifier.initTable();
  429     foreach (kw; keywords)
  430     {
  431         //printf("keyword[%d] = '%s'\n",kw, tochars[kw].ptr);
  432         Identifier.idPool(Token.tochars[kw].ptr, Token.tochars[kw].length, cast(uint)kw);
  433     }
  434 }
  435 
  436 /***********************************************************
  437  */
  438 extern (C++) struct Token
  439 {
  440     Token* next;
  441     Loc loc;
  442     const(char)* ptr; // pointer to first character of this token within buffer
  443     TOK value;
  444     const(char)[] blockComment; // doc comment string prior to this token
  445     const(char)[] lineComment; // doc comment for previous token
  446 
  447     union
  448     {
  449         // Integers
  450         sinteger_t intvalue;
  451         uinteger_t unsvalue;
  452         // Floats
  453         real_t floatvalue;
  454 
  455         struct
  456         {
  457             const(char)* ustring; // UTF8 string
  458             uint len;
  459             ubyte postfix; // 'c', 'w', 'd'
  460         }
  461 
  462         Identifier ident;
  463     }
  464 
  465     extern (D) private static immutable string[TOK.max_] tochars =
  466     [
  467         // Keywords
  468         TOK.this_: "this",
  469         TOK.super_: "super",
  470         TOK.assert_: "assert",
  471         TOK.null_: "null",
  472         TOK.true_: "true",
  473         TOK.false_: "false",
  474         TOK.cast_: "cast",
  475         TOK.new_: "new",
  476         TOK.delete_: "delete",
  477         TOK.throw_: "throw",
  478         TOK.module_: "module",
  479         TOK.pragma_: "pragma",
  480         TOK.typeof_: "typeof",
  481         TOK.typeid_: "typeid",
  482         TOK.template_: "template",
  483         TOK.void_: "void",
  484         TOK.int8: "byte",
  485         TOK.uns8: "ubyte",
  486         TOK.int16: "short",
  487         TOK.uns16: "ushort",
  488         TOK.int32: "int",
  489         TOK.uns32: "uint",
  490         TOK.int64: "long",
  491         TOK.uns64: "ulong",
  492         TOK.int128: "cent",
  493         TOK.uns128: "ucent",
  494         TOK.float32: "float",
  495         TOK.float64: "double",
  496         TOK.float80: "real",
  497         TOK.bool_: "bool",
  498         TOK.char_: "char",
  499         TOK.wchar_: "wchar",
  500         TOK.dchar_: "dchar",
  501         TOK.imaginary32: "ifloat",
  502         TOK.imaginary64: "idouble",
  503         TOK.imaginary80: "ireal",
  504         TOK.complex32: "cfloat",
  505         TOK.complex64: "cdouble",
  506         TOK.complex80: "creal",
  507         TOK.delegate_: "delegate",
  508         TOK.function_: "function",
  509         TOK.is_: "is",
  510         TOK.if_: "if",
  511         TOK.else_: "else",
  512         TOK.while_: "while",
  513         TOK.for_: "for",
  514         TOK.do_: "do",
  515         TOK.switch_: "switch",
  516         TOK.case_: "case",
  517         TOK.default_: "default",
  518         TOK.break_: "break",
  519         TOK.continue_: "continue",
  520         TOK.synchronized_: "synchronized",
  521         TOK.return_: "return",
  522         TOK.goto_: "goto",
  523         TOK.try_: "try",
  524         TOK.catch_: "catch",
  525         TOK.finally_: "finally",
  526         TOK.with_: "with",
  527         TOK.asm_: "asm",
  528         TOK.foreach_: "foreach",
  529         TOK.foreach_reverse_: "foreach_reverse",
  530         TOK.scope_: "scope",
  531         TOK.struct_: "struct",
  532         TOK.class_: "class",
  533         TOK.interface_: "interface",
  534         TOK.union_: "union",
  535         TOK.enum_: "enum",
  536         TOK.import_: "import",
  537         TOK.mixin_: "mixin",
  538         TOK.static_: "static",
  539         TOK.final_: "final",
  540         TOK.const_: "const",
  541         TOK.alias_: "alias",
  542         TOK.override_: "override",
  543         TOK.abstract_: "abstract",
  544         TOK.debug_: "debug",
  545         TOK.deprecated_: "deprecated",
  546         TOK.in_: "in",
  547         TOK.out_: "out",
  548         TOK.inout_: "inout",
  549         TOK.lazy_: "lazy",
  550         TOK.auto_: "auto",
  551         TOK.align_: "align",
  552         TOK.extern_: "extern",
  553         TOK.private_: "private",
  554         TOK.package_: "package",
  555         TOK.protected_: "protected",
  556         TOK.public_: "public",
  557         TOK.export_: "export",
  558         TOK.invariant_: "invariant",
  559         TOK.unittest_: "unittest",
  560         TOK.version_: "version",
  561         TOK.argumentTypes: "__argTypes",
  562         TOK.parameters: "__parameters",
  563         TOK.ref_: "ref",
  564         TOK.macro_: "macro",
  565         TOK.pure_: "pure",
  566         TOK.nothrow_: "nothrow",
  567         TOK.gshared: "__gshared",
  568         TOK.traits: "__traits",
  569         TOK.vector: "__vector",
  570         TOK.overloadSet: "__overloadset",
  571         TOK.file: "__FILE__",
  572         TOK.fileFullPath: "__FILE_FULL_PATH__",
  573         TOK.line: "__LINE__",
  574         TOK.moduleString: "__MODULE__",
  575         TOK.functionString: "__FUNCTION__",
  576         TOK.prettyFunction: "__PRETTY_FUNCTION__",
  577         TOK.shared_: "shared",
  578         TOK.immutable_: "immutable",
  579 
  580         TOK.endOfFile: "End of File",
  581         TOK.leftCurly: "{",
  582         TOK.rightCurly: "}",
  583         TOK.leftParentheses: "(",
  584         TOK.rightParentheses: ")",
  585         TOK.leftBracket: "[",
  586         TOK.rightBracket: "]",
  587         TOK.semicolon: ";",
  588         TOK.colon: ":",
  589         TOK.comma: ",",
  590         TOK.dot: ".",
  591         TOK.xor: "^",
  592         TOK.xorAssign: "^=",
  593         TOK.assign: "=",
  594         TOK.construct: "=",
  595         TOK.blit: "=",
  596         TOK.lessThan: "<",
  597         TOK.greaterThan: ">",
  598         TOK.lessOrEqual: "<=",
  599         TOK.greaterOrEqual: ">=",
  600         TOK.equal: "==",
  601         TOK.notEqual: "!=",
  602         TOK.not: "!",
  603         TOK.leftShift: "<<",
  604         TOK.rightShift: ">>",
  605         TOK.unsignedRightShift: ">>>",
  606         TOK.add: "+",
  607         TOK.min: "-",
  608         TOK.mul: "*",
  609         TOK.div: "/",
  610         TOK.mod: "%",
  611         TOK.slice: "..",
  612         TOK.dotDotDot: "...",
  613         TOK.and: "&",
  614         TOK.andAnd: "&&",
  615         TOK.or: "|",
  616         TOK.orOr: "||",
  617         TOK.array: "[]",
  618         TOK.index: "[i]",
  619         TOK.address: "&",
  620         TOK.star: "*",
  621         TOK.tilde: "~",
  622         TOK.dollar: "$",
  623         TOK.plusPlus: "++",
  624         TOK.minusMinus: "--",
  625         TOK.prePlusPlus: "++",
  626         TOK.preMinusMinus: "--",
  627         TOK.type: "type",
  628         TOK.question: "?",
  629         TOK.negate: "-",
  630         TOK.uadd: "+",
  631         TOK.variable: "var",
  632         TOK.addAssign: "+=",
  633         TOK.minAssign: "-=",
  634         TOK.mulAssign: "*=",
  635         TOK.divAssign: "/=",
  636         TOK.modAssign: "%=",
  637         TOK.leftShiftAssign: "<<=",
  638         TOK.rightShiftAssign: ">>=",
  639         TOK.unsignedRightShiftAssign: ">>>=",
  640         TOK.andAssign: "&=",
  641         TOK.orAssign: "|=",
  642         TOK.concatenateAssign: "~=",
  643         TOK.concatenateElemAssign: "~=",
  644         TOK.concatenateDcharAssign: "~=",
  645         TOK.concatenate: "~",
  646         TOK.call: "call",
  647         TOK.identity: "is",
  648         TOK.notIdentity: "!is",
  649         TOK.identifier: "identifier",
  650         TOK.at: "@",
  651         TOK.pow: "^^",
  652         TOK.powAssign: "^^=",
  653         TOK.goesTo: "=>",
  654         TOK.pound: "#",
  655 
  656         // For debugging
  657         TOK.error: "error",
  658         TOK.dotIdentifier: "dotid",
  659         TOK.dotTemplateDeclaration: "dottd",
  660         TOK.dotTemplateInstance: "dotti",
  661         TOK.dotVariable: "dotvar",
  662         TOK.dotType: "dottype",
  663         TOK.symbolOffset: "symoff",
  664         TOK.arrayLength: "arraylength",
  665         TOK.arrayLiteral: "arrayliteral",
  666         TOK.assocArrayLiteral: "assocarrayliteral",
  667         TOK.structLiteral: "structliteral",
  668         TOK.string_: "string",
  669         TOK.dSymbol: "symbol",
  670         TOK.tuple: "tuple",
  671         TOK.declaration: "declaration",
  672         TOK.onScopeExit: "scope(exit)",
  673         TOK.onScopeSuccess: "scope(success)",
  674         TOK.onScopeFailure: "scope(failure)",
  675         TOK.delegatePointer: "delegateptr",
  676 
  677         // Finish up
  678         TOK.reserved: "reserved",
  679         TOK.remove: "remove",
  680         TOK.newAnonymousClass: "newanonclass",
  681         TOK.comment: "comment",
  682         TOK.classReference: "classreference",
  683         TOK.thrownException: "thrownexception",
  684         TOK.delegateFunctionPointer: "delegatefuncptr",
  685         TOK.arrow: "arrow",
  686         TOK.int32Literal: "int32v",
  687         TOK.uns32Literal: "uns32v",
  688         TOK.int64Literal: "int64v",
  689         TOK.uns64Literal: "uns64v",
  690         TOK.int128Literal: "int128v",
  691         TOK.uns128Literal: "uns128v",
  692         TOK.float32Literal: "float32v",
  693         TOK.float64Literal: "float64v",
  694         TOK.float80Literal: "float80v",
  695         TOK.imaginary32Literal: "imaginary32v",
  696         TOK.imaginary64Literal: "imaginary64v",
  697         TOK.imaginary80Literal: "imaginary80v",
  698         TOK.charLiteral: "charv",
  699         TOK.wcharLiteral: "wcharv",
  700         TOK.dcharLiteral: "dcharv",
  701 
  702         TOK.halt: "halt",
  703         TOK.hexadecimalString: "xstring",
  704 
  705         TOK.interval: "interval",
  706         TOK.voidExpression: "voidexp",
  707         TOK.cantExpression: "cantexp",
  708         TOK.showCtfeContext : "showCtfeContext",
  709 
  710         TOK.objcClassReference: "class",
  711         TOK.vectorArray: "vectorarray",
  712     ];
  713 
  714     static assert(() {
  715         foreach (s; tochars)
  716             assert(s.length);
  717         return true;
  718     }());
  719 
  720 nothrow:
  721 
  722     int isKeyword() const
  723     {
  724         foreach (kw; keywords)
  725         {
  726             if (kw == value)
  727                 return 1;
  728         }
  729         return 0;
  730     }
  731 
  732     /****
  733      * Set to contents of ptr[0..length]
  734      * Params:
  735      *  ptr = pointer to string
  736      *  length = length of string
  737      */
  738     void setString(const(char)* ptr, size_t length)
  739     {
  740         auto s = cast(char*)mem.xmalloc_noscan(length + 1);
  741         memcpy(s, ptr, length);
  742         s[length] = 0;
  743         ustring = s;
  744         len = cast(uint)length;
  745         postfix = 0;
  746     }
  747 
  748     /****
  749      * Set to contents of buf
  750      * Params:
  751      *  buf = string (not zero terminated)
  752      */
  753     void setString(const ref OutBuffer buf)
  754     {
  755         setString(cast(const(char)*)buf[].ptr, buf.length);
  756     }
  757 
  758     /****
  759      * Set to empty string
  760      */
  761     void setString()
  762     {
  763         ustring = "";
  764         len = 0;
  765         postfix = 0;
  766     }
  767 
  768     extern (C++) const(char)* toChars() const
  769     {
  770         __gshared char[3 + 3 * floatvalue.sizeof + 1] buffer;
  771         const(char)* p = &buffer[0];
  772         switch (value)
  773         {
  774         case TOK.int32Literal:
  775             sprintf(&buffer[0], "%d", cast(d_int32)intvalue);
  776             break;
  777         case TOK.uns32Literal:
  778         case TOK.charLiteral:
  779         case TOK.wcharLiteral:
  780         case TOK.dcharLiteral:
  781             sprintf(&buffer[0], "%uU", cast(d_uns32)unsvalue);
  782             break;
  783         case TOK.int64Literal:
  784             sprintf(&buffer[0], "%lldL", cast(long)intvalue);
  785             break;
  786         case TOK.uns64Literal:
  787             sprintf(&buffer[0], "%lluUL", cast(ulong)unsvalue);
  788             break;
  789         case TOK.float32Literal:
  790             CTFloat.sprint(&buffer[0], 'g', floatvalue);
  791             strcat(&buffer[0], "f");
  792             break;
  793         case TOK.float64Literal:
  794             CTFloat.sprint(&buffer[0], 'g', floatvalue);
  795             break;
  796         case TOK.float80Literal:
  797             CTFloat.sprint(&buffer[0], 'g', floatvalue);
  798             strcat(&buffer[0], "L");
  799             break;
  800         case TOK.imaginary32Literal:
  801             CTFloat.sprint(&buffer[0], 'g', floatvalue);
  802             strcat(&buffer[0], "fi");
  803             break;
  804         case TOK.imaginary64Literal:
  805             CTFloat.sprint(&buffer[0], 'g', floatvalue);
  806             strcat(&buffer[0], "i");
  807             break;
  808         case TOK.imaginary80Literal:
  809             CTFloat.sprint(&buffer[0], 'g', floatvalue);
  810             strcat(&buffer[0], "Li");
  811             break;
  812         case TOK.string_:
  813             {
  814                 OutBuffer buf;
  815                 buf.writeByte('"');
  816                 for (size_t i = 0; i < len;)
  817                 {
  818                     dchar c;
  819                     utf_decodeChar(ustring[0 .. len], i, c);
  820                     switch (c)
  821                     {
  822                     case 0:
  823                         break;
  824                     case '"':
  825                     case '\\':
  826                         buf.writeByte('\\');
  827                         goto default;
  828                     default:
  829                         if (c <= 0x7F)
  830                         {
  831                             if (isprint(c))
  832                                 buf.writeByte(c);
  833                             else
  834                                 buf.printf("\\x%02x", c);
  835                         }
  836                         else if (c <= 0xFFFF)
  837                             buf.printf("\\u%04x", c);
  838                         else
  839                             buf.printf("\\U%08x", c);
  840                         continue;
  841                     }
  842                     break;
  843                 }
  844                 buf.writeByte('"');
  845                 if (postfix)
  846                     buf.writeByte(postfix);
  847                 buf.writeByte(0);
  848                 p = buf.extractSlice().ptr;
  849             }
  850             break;
  851         case TOK.hexadecimalString:
  852             {
  853                 OutBuffer buf;
  854                 buf.writeByte('x');
  855                 buf.writeByte('"');
  856                 foreach (size_t i; 0 .. len)
  857                 {
  858                     if (i)
  859                         buf.writeByte(' ');
  860                     buf.printf("%02x", ustring[i]);
  861                 }
  862                 buf.writeByte('"');
  863                 if (postfix)
  864                     buf.writeByte(postfix);
  865                 buf.writeByte(0);
  866                 p = buf.extractSlice().ptr;
  867                 break;
  868             }
  869         case TOK.identifier:
  870         case TOK.enum_:
  871         case TOK.struct_:
  872         case TOK.import_:
  873         case TOK.wchar_:
  874         case TOK.dchar_:
  875         case TOK.bool_:
  876         case TOK.char_:
  877         case TOK.int8:
  878         case TOK.uns8:
  879         case TOK.int16:
  880         case TOK.uns16:
  881         case TOK.int32:
  882         case TOK.uns32:
  883         case TOK.int64:
  884         case TOK.uns64:
  885         case TOK.int128:
  886         case TOK.uns128:
  887         case TOK.float32:
  888         case TOK.float64:
  889         case TOK.float80:
  890         case TOK.imaginary32:
  891         case TOK.imaginary64:
  892         case TOK.imaginary80:
  893         case TOK.complex32:
  894         case TOK.complex64:
  895         case TOK.complex80:
  896         case TOK.void_:
  897             p = ident.toChars();
  898             break;
  899         default:
  900             p = toChars(value);
  901             break;
  902         }
  903         return p;
  904     }
  905 
  906     static const(char)* toChars(ubyte value)
  907     {
  908         return toString(value).ptr;
  909     }
  910 
  911     extern (D) static string toString(ubyte value) pure nothrow @nogc @safe
  912     {
  913         return tochars[value];
  914     }
  915 }