"Fossies" - the Fresh Open Source Software Archive

Member "highlight-3.57/src/core/codegenerator.cpp" (14 May 2020, 83224 Bytes) of package /linux/www/highlight-3.57.tar.bz2:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) C and C++ source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file. For more information about "codegenerator.cpp" see the Fossies "Dox" file reference documentation and the latest Fossies "Diffs" side-by-side code changes report: 3.56_vs_3.57.

    1 /***************************************************************************
    2                           codegenerator.cpp  -  description
    3                              -------------------
    4     begin                : Die Jul 9 2002
    5     copyright            : (C) 2002-2020 by Andre Simon
    6     email                : a.simon@mailbox.org
    7  ***************************************************************************/
    8 
    9 
   10 /*
   11 This file is part of Highlight.
   12 
   13 Highlight is free software: you can redistribute it and/or modify
   14 it under the terms of the GNU General Public License as published by
   15 the Free Software Foundation, either version 3 of the License, or
   16 (at your option) any later version.
   17 
   18 Highlight is distributed in the hope that it will be useful,
   19 but WITHOUT ANY WARRANTY; without even the implied warranty of
   20 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
   21 GNU General Public License for more details.
   22 
   23 You should have received a copy of the GNU General Public License
   24 along with Highlight.  If not, see <http://www.gnu.org/licenses/>.
   25 */
   26 
   27 
   28 #include <climits>
   29 #include <memory>
   30 #include <boost/xpressive/xpressive_dynamic.hpp>
   31 
   32 #include "codegenerator.h"
   33 
   34 #include "htmlgenerator.h"
   35 #include "xhtmlgenerator.h"
   36 #include "rtfgenerator.h"
   37 #include "latexgenerator.h"
   38 #include "texgenerator.h"
   39 #include "svggenerator.h"
   40 #include "bbcodegenerator.h"
   41 #include "pangogenerator.h"
   42 #include "odtgenerator.h"
   43 #include "astyle/astyle.h"
   44 #include "astyle/ASStreamIterator.h"
   45 
   46 #if !defined (QT)
   47 #include "ansigenerator.h"
   48 #include "xterm256generator.h"
   49 #endif
   50 
   51 namespace highlight
   52 {
   53 const unsigned int CodeGenerator::NUMBER_BUILTIN_STATES = highlight::KEYWORD;
   54 
   55 const string CodeGenerator::STY_NAME_STD="std";
   56 const string CodeGenerator::STY_NAME_STR="str";
   57 const string CodeGenerator::STY_NAME_NUM="num";
   58 const string CodeGenerator::STY_NAME_SLC="slc";
   59 const string CodeGenerator::STY_NAME_COM="com";
   60 const string CodeGenerator::STY_NAME_ESC="esc";
   61 const string CodeGenerator::STY_NAME_DIR="ppc"; //preprocessor
   62 const string CodeGenerator::STY_NAME_DST="pps"; //preprocessor string
   63 const string CodeGenerator::STY_NAME_LIN="lin";
   64 const string CodeGenerator::STY_NAME_SYM="opt"; //operator
   65 const string CodeGenerator::STY_NAME_IPL="ipl"; //interpolation
   66 
   67 vector<Diluculum::LuaFunction*> CodeGenerator::pluginChunks;
   68 
   69 
   70 CodeGenerator * CodeGenerator::getInstance ( OutputType type )
   71 {
   72     CodeGenerator* generator=NULL;
   73     switch ( type ) {
   74     case HTML:
   75         generator = new HtmlGenerator();
   76         break;
   77     case XHTML:
   78         generator = new XHtmlGenerator();
   79         break;
   80     case TEX:
   81         generator = new TexGenerator ();
   82         break;
   83     case LATEX:
   84         generator = new LatexGenerator();
   85         break;
   86     case RTF:
   87         generator = new RtfGenerator ();
   88         break;
   89     case SVG:
   90         generator = new SVGGenerator();
   91         break;
   92     case BBCODE:
   93         generator = new BBCodeGenerator();
   94         break;
   95     case PANGO:
   96         generator = new PangoGenerator();
   97         break;
   98     case ODTFLAT:
   99         generator = new ODTGenerator();
  100         break;
  101     case ESC_ANSI:
  102         generator = new AnsiGenerator();
  103         break;
  104     case ESC_XTERM256:
  105     case ESC_TRUECOLOR:
  106         generator = new Xterm256Generator();
  107         generator->setESCTrueColor(type==ESC_TRUECOLOR);
  108         break;
  109     default:
  110         break;
  111     }
  112     return generator;
  113 }
  114 
  115 
  116 CodeGenerator::CodeGenerator ( highlight::OutputType type )
  117     :currentSyntax(NULL),
  118      in ( NULL ),
  119      out ( NULL ),
  120      encoding ( "none" ),
  121      docTitle ( "Source file" ),
  122      maskWs ( false ),
  123      excludeWs ( false ),
  124      fragmentOutput ( false ),
  125      keepInjections( false ),
  126      showLineNumbers ( false ),
  127      lineNumberFillZeroes ( false ),
  128      printNewLines(true),
  129      omitVersionComment(false),
  130      isolateTags(false),
  131      disableStyleCache(false),
  132      baseFontSize("10"),
  133      lineNumber ( 0 ),
  134      lineNumberOffset ( 0 ),
  135      currentState ( _UNKNOWN ),
  136      currentKeywordClass ( 0 ),
  137      includeStyleDef ( false ),
  138      numberCurrentLine ( false ),
  139      lineIndex ( 0 ),
  140      lastLineLength( 0 ),
  141      syntaxChangeIndex(UINT_MAX),
  142      syntaxChangeLineNo(UINT_MAX),
  143      lineNumberWidth ( 5 ),
  144      startLineCnt( 1 ),
  145      startLineCntCurFile( 1 ),
  146      maxLineCnt ( UINT_MAX ),
  147      inputFilesCnt (0),
  148      processedFilesCnt (0),
  149      noTrailingNewLine(0),
  150 
  151      terminatingChar ( '\0' ),
  152      formatter ( NULL ),
  153      formattingEnabled ( false ),
  154      formattingPossible ( false ),
  155      validateInput ( false ),
  156      numberWrappedLines ( true ),
  157      resultOfHook(false),
  158      lineContainedTestCase(false),
  159      applySyntaxTestCase(false),
  160      toggleDynRawString(false),
  161      
  162      keywordCase ( StringTools::CASE_UNCHANGED ),
  163      eolDelimiter ('\n'),
  164      outputType ( type )
  165 {
  166 }
  167 
  168 
  169 CodeGenerator::~CodeGenerator()
  170 {
  171     delete formatter;
  172 
  173     resetSyntaxReaders();
  174     
  175     for (unsigned int i=0; i<pluginChunks.size(); i++) {
  176         delete pluginChunks[i];
  177     }
  178     pluginChunks.clear();
  179 }
  180 
  181 
  182 bool CodeGenerator::initTheme ( const string& themePath )
  183 {
  184     this->themePath=themePath;
  185     bool loadOK = docStyle.load ( themePath, outputType );
  186     initOutputTags();
  187     return loadOK;
  188 }
  189 
  190 const string& CodeGenerator::getStyleName()
  191 {
  192     return themePath;
  193 }
  194 
  195 void CodeGenerator::setLineNumberWidth ( int w )
  196 {
  197     lineNumberWidth=w;
  198 }
  199 
  200 int CodeGenerator::getLineNumberWidth()
  201 {
  202     return lineNumberWidth;
  203 }
  204 
  205 void CodeGenerator::setPrintLineNumbers ( bool flag, unsigned int startCnt )
  206 {
  207     showLineNumbers=flag;
  208     lineNumberOffset = startCnt-1;
  209 }
  210 
  211 bool CodeGenerator::getPrintLineNumbers()
  212 {
  213     return showLineNumbers;
  214 }
  215 
  216 void CodeGenerator::setPrintZeroes ( bool flag )
  217 {
  218     lineNumberFillZeroes=flag;
  219 }
  220 
  221 bool CodeGenerator::getPrintZeroes()
  222 {
  223     return lineNumberFillZeroes;
  224 }
  225 
  226 void CodeGenerator::setIncludeStyle ( bool flag )
  227 {
  228     includeStyleDef = flag;
  229 }
  230 
  231 void CodeGenerator::disableTrailingNL ( int flag )
  232 {
  233     noTrailingNewLine = flag;
  234 }
  235 
  236 void CodeGenerator::setStyleInputPath ( const string& path )
  237 {
  238     styleInputPath = path;
  239 }
  240 
  241 void CodeGenerator::setStyleOutputPath ( const string& path )
  242 {
  243     styleOutputPath = path;
  244 }
  245 
  246 void CodeGenerator::setPluginParameter ( const string& param )
  247 {
  248     pluginParameter = param;
  249 }
  250 
  251 const string&  CodeGenerator::getStyleInputPath()
  252 {
  253     return styleInputPath;
  254 }
  255 
  256 const string&  CodeGenerator::getStyleOutputPath()
  257 {
  258     return styleOutputPath;
  259 }
  260 
  261 void CodeGenerator::setFragmentCode ( bool flag )
  262 {
  263     fragmentOutput=flag;
  264 }
  265 
  266 bool CodeGenerator::getFragmentCode()
  267 {
  268     return fragmentOutput;
  269 }
  270 void CodeGenerator::setKeepInjections ( bool flag )
  271 {
  272     keepInjections=flag;
  273 }
  274 
  275 bool CodeGenerator::getKeepInjections()
  276 {
  277     return keepInjections;
  278 }
  279 void CodeGenerator::setValidateInput ( bool flag )
  280 {
  281     validateInput=flag;
  282 }
  283 
  284 bool CodeGenerator::getValidateInput()
  285 {
  286     return validateInput;
  287 }
  288 
  289 
  290 void CodeGenerator::setNumberWrappedLines ( bool flag )
  291 {
  292     numberWrappedLines=flag;
  293 }
  294 
  295 bool CodeGenerator::getNumberWrappedLines()
  296 {
  297     return numberWrappedLines;
  298 }
  299 
  300 void CodeGenerator::setOmitVersionComment ( bool flag )
  301 {
  302     omitVersionComment=flag;
  303 }
  304 
  305 bool CodeGenerator::getOmitVersionComment ()
  306 {
  307     return omitVersionComment;
  308 }
  309 
  310 void CodeGenerator::setIsolateTags ( bool flag )
  311 {
  312     isolateTags=flag;
  313 }
  314 
  315 bool CodeGenerator::getIsolateTags ()
  316 {
  317     return isolateTags;
  318 }
  319 
  320 void CodeGenerator::setBaseFont ( const string& fontName )
  321 {
  322     baseFont = fontName;
  323 }
  324 
  325 void CodeGenerator::setBaseFontSize ( const string& fontSize)
  326 {
  327     baseFontSize = fontSize;
  328 }
  329 
  330 void CodeGenerator::setStartingNestedLang(const string &langName)
  331 {
  332     embedLangStart = langName;
  333 }
  334 
  335 const string CodeGenerator::getBaseFont() const
  336 {
  337     if ( !baseFont.empty() ) return baseFont;
  338     switch ( outputType ) {
  339     case HTML:
  340     case XHTML:
  341     case SVG:
  342         return "'Courier New',monospace";
  343         break;
  344     case LATEX:
  345         return "ttfamily";
  346         break;
  347     case TEX:
  348         return "tt";
  349         break;
  350     default:
  351         return "Courier New";
  352     }
  353 }
  354 
  355 const string CodeGenerator::getBaseFontSize()
  356 {
  357     return baseFontSize;
  358 }
  359 
  360 void CodeGenerator::setTitle ( const string & title )
  361 {
  362     if ( !title.empty() ) docTitle= title;
  363 }
  364 
  365 string CodeGenerator::getTitle()
  366 {
  367     return docTitle;
  368 }
  369 
  370 void CodeGenerator::setEncoding ( const string& encodingName )
  371 {
  372     encoding = encodingName;
  373 }
  374 
  375 bool CodeGenerator::formattingDisabled()
  376 {
  377     return !formattingEnabled;
  378 }
  379 
  380 void CodeGenerator::setStartingInputLine ( unsigned int begin )
  381 {
  382     startLineCnt = startLineCntCurFile = begin;
  383 }
  384 
  385 void CodeGenerator::setMaxInputLineCnt ( unsigned int cnt )
  386 {
  387     maxLineCnt = cnt;
  388 }
  389 
  390 void CodeGenerator::setFilesCnt ( unsigned int cnt )
  391 {
  392     inputFilesCnt = cnt;
  393     processedFilesCnt = 0;
  394 }
  395 
  396 bool CodeGenerator::formattingIsPossible()
  397 {
  398     return formattingPossible;
  399 }
  400 
  401 void CodeGenerator::setPreformatting ( WrapMode lineWrappingStyle,
  402                                        unsigned int lineLength,
  403                                        int numberSpaces )
  404 {
  405     bool enableWrap = lineWrappingStyle!=WRAP_DISABLED;
  406     bool replaceTabs = numberSpaces > 0;
  407 
  408     if ( enableWrap || replaceTabs ) {
  409         preFormatter.setWrap ( enableWrap );
  410         preFormatter.setWrapIndentBraces ( lineWrappingStyle==WRAP_DEFAULT );
  411         preFormatter.setWrapLineLength ( lineLength );
  412         preFormatter.setReplaceTabs ( replaceTabs );
  413         preFormatter.setNumberSpaces ( numberSpaces );
  414     }
  415 }
  416 
  417 void CodeGenerator::setKeyWordCase ( StringTools::KeywordCase keyCase )
  418 {
  419     keywordCase = keyCase;
  420 }
  421 
  422 void CodeGenerator::setEOLDelimiter(char delim)
  423 {
  424     eolDelimiter = delim;
  425 }
  426 
  427 void CodeGenerator::reset()
  428 {
  429     lineIndex = 0;
  430     lineNumber = 0;
  431     line.clear();
  432     preFormatter.reset();
  433     inFile.clear();
  434     outFile.clear();
  435     embedLangDefPath.clear();
  436     printNewLines=true;
  437     syntaxChangeIndex = syntaxChangeLineNo = UINT_MAX;
  438     startLineCntCurFile = startLineCnt;
  439     applySyntaxTestCase=lineContainedTestCase=false;
  440     if (currentSyntax){
  441         vector<int> overrideStyleAttrs=currentSyntax->getOverrideStyleAttributes();
  442         docStyle.overrideAttributes(overrideStyleAttrs);
  443         if (overrideStyleAttrs.size())
  444             disableStyleCache = true;
  445     }
  446 }
  447 
  448 string CodeGenerator::getThemeInitError()
  449 {
  450     return docStyle.getErrorMessage();
  451 }
  452 
  453 string CodeGenerator::getPluginScriptError()
  454 {
  455     return userScriptError;
  456 }
  457 
  458 string CodeGenerator::getSyntaxRegexError()
  459 {
  460     return (currentSyntax)? currentSyntax->getFailedRegex(): "syntax undef";
  461 }
  462 string CodeGenerator::getSyntaxLuaError()
  463 {
  464     return (currentSyntax)? currentSyntax->getLuaErrorText(): "syntax undef";
  465 
  466 }
  467 string CodeGenerator::getSyntaxDescription()
  468 {
  469     return (currentSyntax)? currentSyntax->getDescription(): "syntax undef";
  470 
  471 }
  472 string CodeGenerator::getSyntaxEncodingHint()
  473 {
  474     return (currentSyntax)? currentSyntax->getEncodingHint(): "";
  475 
  476 }
  477 string CodeGenerator::getThemeDescription()
  478 {
  479     return docStyle.getDescription();
  480 }
  481 
  482 string CodeGenerator::getSyntaxCatDescription(){
  483     return (currentSyntax)? currentSyntax->getCategoryDescription(): "";
  484 }
  485 
  486 string CodeGenerator::getThemeCatDescription(){
  487     return docStyle.getCategoryDescription();
  488 }
  489 
  490 unsigned int CodeGenerator::getLineNumber()
  491 {
  492     return lineNumber;
  493 }
  494 
  495 bool CodeGenerator::readNewLine ( string &newLine )
  496 {
  497     bool eof=false;
  498     
  499     if ( lineIndex ) terminatingChar=newLine[lineIndex-1];
  500     
  501     while (!eof && startLineCntCurFile>0) {
  502         if ( formattingPossible && formattingEnabled ) {
  503             eof=!formatter->hasMoreLines();
  504             if ( !eof ) {
  505                 newLine = formatter->nextLine();
  506             }
  507         } else {
  508             eof = ! getline ( *in, newLine, eolDelimiter );
  509         }
  510         --startLineCntCurFile;
  511     }
  512     startLineCntCurFile=1;
  513 #ifndef _WIN32
  514     // drop CR of CRLF files
  515     if (!newLine.empty() && newLine[newLine.size() - 1] == '\r')
  516         newLine.erase(newLine.size() - 1);
  517 #endif
  518 
  519     return eof || ( lineNumber == maxLineCnt );
  520 }
  521 
  522 void CodeGenerator::matchRegex ( const string &line, State skipState)
  523 {
  524     regexGroups.clear();
  525     int matchBegin=0;
  526     int groupID=0;
  527 
  528     // cycle through all regex, save the start and ending indices of matches to report them later
  529     for ( unsigned int i=0; i<currentSyntax->getRegexElements().size(); i++ ) {
  530         RegexElement *regexElem = currentSyntax->getRegexElements() [i];
  531 
  532         if (regexElem->open == skipState) continue;
  533 
  534         if (regexElem->constraintLineNum && regexElem->constraintLineNum != lineNumber) {
  535             continue;
  536         }
  537         
  538         if (regexElem->constraintFilename.size() && regexElem->constraintFilename != inFile) {
  539             continue;
  540         }
  541 
  542         boost::xpressive::sregex_iterator cur( line.begin(), line.end(), regexElem->rex );
  543         boost::xpressive::sregex_iterator end;
  544 
  545         for( ; cur != end; ++cur )  {
  546             groupID = ( regexElem->capturingGroup<0 ) ? cur->size()-1 : regexElem->capturingGroup;
  547             matchBegin = cur->position(groupID);
  548         
  549             regexGroups.insert (
  550                 make_pair ( matchBegin + 1, ReGroup ( regexElem->open, cur->length(groupID), regexElem->kwClass, regexElem->langName ) ) );
  551 
  552             // priority regex (match required)
  553             if (regexElem->priority) {
  554                 return;
  555             }
  556         }
  557     }
  558 }
  559 
  560 unsigned char CodeGenerator::getInputChar()
  561 {
  562     // end of line?
  563     if ( lineIndex == line.length() ) {
  564         
  565         //more testing required:
  566         if (outputType==ESC_TRUECOLOR || outputType==ESC_XTERM256)
  567             lastLineLength=StringTools::utf8_strlen(line);
  568         
  569         bool eof=false;
  570         if ( preFormatter.isEnabled() ) {
  571             if ( !preFormatter.hasMoreLines() ) {
  572                 eof=readNewLine ( line );
  573                 preFormatter.setLine ( line );
  574                 ++lineNumber;
  575                 numberCurrentLine = true;
  576             } else {
  577                 if (numberWrappedLines)
  578                     ++lineNumber;
  579                 numberCurrentLine = numberWrappedLines;
  580             }
  581 
  582             line = preFormatter.getNextLine();
  583         } else {
  584             eof=readNewLine ( line );
  585             ++lineNumber;
  586 
  587             numberCurrentLine = true;
  588         }
  589         lineIndex=0;
  590         
  591         if (!lineContainedTestCase && applySyntaxTestCase){
  592             stateTraceTest = stateTraceCurrent;
  593             stateTraceCurrent.clear();
  594         } 
  595         
  596         lineContainedTestCase=false;
  597             
  598         matchRegex ( line );
  599         stateTrace.clear();
  600         return ( eof ) ?'\0':'\n';
  601     }
  602 
  603     return line[lineIndex++];
  604 }
  605 
  606 /** changing this method requires regression testing with nested syntax files (HTML+PHP+JS+CSS, Coffeescript with block regex, Pas + ASM) 
  607     especially nested syntax in one line
  608  */
  609 State CodeGenerator::getCurrentState (State oldState)
  610 {
  611     unsigned char c='\0';
  612 
  613     if ( token.length() ==0 ) {
  614         c=getInputChar();
  615     } else {
  616         lineIndex-= ( token.length()-1 );
  617         c=token[0];
  618     }
  619     if ( c=='\n' ) {
  620         return _EOL;   // End of line
  621     }
  622 
  623     if ( c=='\0' ) {
  624         return _EOF;   // End of file
  625     }
  626 
  627     if ( c==' ' || c=='\t' ) {
  628         token= c;
  629         return _WS;    // White space
  630     }
  631 
  632     if ( applySyntaxTestCase && ( c=='^' || c=='<') && (oldState == ML_COMMENT || oldState==SL_COMMENT)  ) {
  633         token= c;
  634         return _TESTPOS;
  635     }
  636         
  637     // at this position the syntax change takes place
  638     if (lineIndex >= syntaxChangeIndex-1 || syntaxChangeLineNo < lineNumber){
  639         loadEmbeddedLang(embedLangDefPath);  // load new syntax                     
  640         matchRegex(line);                    // recognize new patterns in the (remaining) line
  641         syntaxChangeIndex = syntaxChangeLineNo = UINT_MAX;
  642     }
  643 
  644 SKIP_EMBEDDED:
  645     
  646     // Test if a regular expression was found at the current position
  647     if ( !regexGroups.empty() ) {
  648         if ( regexGroups.count ( lineIndex ) ) {
  649             token = line.substr ( lineIndex-1, regexGroups[lineIndex].length );
  650 
  651             unsigned int oldIndex= lineIndex;
  652             if ( regexGroups[oldIndex].length>1 ) lineIndex+= regexGroups[oldIndex].length-1;
  653 
  654             if ( regexGroups[oldIndex].state==EMBEDDED_CODE_BEGIN ) {
  655                 //do not handle a nested section if the syntax is marked as "sealed" 
  656                 if (embedLangDefPath.length()==0 || currentSyntax->allowsInnerSection(embedLangDefPath) ) {
  657                     embedLangDefPath = currentSyntax->getNewPath(regexGroups[oldIndex].name);
  658                     //remember position 
  659                     syntaxChangeIndex = lineIndex+2;
  660                     syntaxChangeLineNo = lineNumber;
  661                 }
  662                 
  663                 // repeat parsing of this line without nested state recognition to highlight opening delimiter in the host syntax
  664                 matchRegex(line, EMBEDDED_CODE_BEGIN);
  665                 lineIndex = oldIndex;
  666                 goto SKIP_EMBEDDED; // this is how it should be done
  667             }
  668 
  669             if ( regexGroups[oldIndex].state==IDENTIFIER_BEGIN || regexGroups[oldIndex].state==KEYWORD ) {
  670                 string reservedWord= ( currentSyntax->isIgnoreCase() ) ? StringTools::change_case ( token ) :token;
  671                 currentKeywordClass=currentSyntax->getKeywordListGroup ( reservedWord ); //check in lists (no regex)
  672                 
  673                 if ( !currentKeywordClass && regexGroups[oldIndex].state==KEYWORD ){
  674                     currentKeywordClass = regexGroups[oldIndex].kwClass;
  675                 }
  676                 return validateState(( currentKeywordClass ) ? KEYWORD : STANDARD, oldState );
  677             } else {
  678                 return validateState(regexGroups[oldIndex].state, oldState);
  679             }
  680         }
  681     }
  682 
  683     // Character not referring to any state
  684     token = c;
  685     return STANDARD;
  686 }
  687 
  688 State CodeGenerator::validateState(State newState, State oldState)
  689 {
  690 
  691     if (currentSyntax->getValidateStateChangeFct()) {
  692         Diluculum::LuaValueList params;
  693         params.push_back(Diluculum::LuaValue(oldState));
  694         params.push_back(Diluculum::LuaValue(newState));
  695         params.push_back(Diluculum::LuaValue(token));
  696         params.push_back(Diluculum::LuaValue(getCurrentKeywordClassId()) );
  697         params.push_back(Diluculum::LuaValue(lineNumber) );
  698         params.push_back(Diluculum::LuaValue(lineIndex-(unsigned int)token.length()) );
  699 
  700         Diluculum::LuaValueList res=
  701             currentSyntax->getLuaState()->call ( *currentSyntax->getValidateStateChangeFct(),
  702                     params,"getValidateStateChangeFct call")  ;
  703 
  704         resultOfHook = res.size()>=1;
  705         if (resultOfHook) {
  706             
  707             // test balloon
  708             if (currentSyntax->requiresParamUpdate()) {
  709                 
  710                  if ( currentSyntax->getOverrideConfigVal("state.string.raw")=="true"){
  711                      toggleDynRawString=true; // reset to false in string state fct
  712                  }
  713                  if ( currentSyntax->getOverrideConfigVal("format.maskws")=="true") {
  714                      maskWs=true;
  715                  }
  716                  if ( currentSyntax->getOverrideConfigVal("format.spacer").size()) {
  717                      spacer=currentSyntax->getOverrideConfigVal("format.spacer");
  718                  }
  719             }
  720             
  721             State validatedState = (State)res[0].asInteger();
  722             if ( validatedState== _REJECT) {
  723                 // proceed using only the first character of the token
  724                 // TODO evaluate if token clear would be better
  725                 if (res.size()==1) { 
  726                     lineIndex -= (token.length() -1);
  727                     token=token.substr(0, 1);
  728                 }
  729                 
  730                 //experimental for slim.lang: evaluate second return arg after _REJECT
  731                 if (res.size()>=2) {
  732                     lineIndex -= (token.length() );
  733                     token.clear();
  734                     return (State)res[1].asInteger();
  735                 }
  736                 return oldState;
  737             }
  738             
  739             stateTrace.push_back(validatedState);
  740             if (stateTrace.size()>200) stateTrace.erase(stateTrace.begin(), stateTrace.begin() + 100 );
  741             return validatedState;
  742         }
  743     }
  744     resultOfHook  = false;
  745     stateTrace.push_back(newState);
  746     if (stateTrace.size()>200) stateTrace.erase(stateTrace.begin(), stateTrace.begin() + 100 );    
  747     return newState;
  748 }
  749 
  750 
  751 unsigned int CodeGenerator::getCurrentKeywordClassId(){
  752     unsigned int kwClassId=0;
  753 
  754     // this vector contains the defined keyword classes, and currentKeywordClass is its index:
  755     vector<string> kwClasses=currentSyntax->getKeywordClasses();
  756     if (currentKeywordClass && currentKeywordClass<=kwClasses.size()) {
  757         string kwClassName=kwClasses[currentKeywordClass-1];
  758         if (kwClassName.size()==3)
  759             kwClassId = kwClassName[2] - 'a' + 1;
  760     }
  761     return kwClassId;
  762 }
  763 
  764 //it is faster to pass ostream reference
  765 void CodeGenerator::maskString ( ostream& ss, const string & s )
  766 {
  767     for ( unsigned int i=0; i< s.length(); i++ ) {
  768         ss << maskCharacter ( s[i] );
  769     }
  770     
  771     // The test markers position should also be deternmined by calculating the code points
  772     if ( applySyntaxTestCase ) {
  773             
  774         PositionState ps(currentState, getCurrentKeywordClassId(), false);
  775         
  776         int slen = encoding=="utf-8" ? StringTools::utf8_strlen(s) : s.length();  
  777         for (int i=0; i< slen; i++ ) {
  778             stateTraceCurrent.push_back(ps);
  779         }
  780         if (stateTraceCurrent.size()>200) 
  781             stateTraceCurrent.erase(stateTraceCurrent.begin(), stateTraceCurrent.begin() + 100 ); 
  782     }
  783 }
  784 
  785 
  786 Diluculum::LuaValueList CodeGenerator::callDecorateFct(const string&token)
  787 {
  788     Diluculum::LuaValueList params;
  789     params.push_back(Diluculum::LuaValue(token));
  790     params.push_back(Diluculum::LuaValue(currentState));
  791     params.push_back(Diluculum::LuaValue(currentKeywordClass));
  792     string trace(";");
  793     if (stateTrace.size()>1){
  794         for (size_t i=0; i<stateTrace.size()-1;i++){
  795             trace += std::to_string (stateTrace[i]);
  796             trace += ";";
  797         }
  798     }
  799     
  800     //std::cerr <<"TRC1: "<<trace<<"\n";
  801     
  802     params.push_back(Diluculum::LuaValue(trace));
  803 
  804     return currentSyntax->getLuaState()->call ( *currentSyntax->getDecorateFct(),
  805             params,"getDecorateFct call")  ;
  806 }
  807 
  808 void CodeGenerator::printMaskedToken (bool flushWhiteSpace, StringTools::KeywordCase tcase )
  809 {
  810     if ( flushWhiteSpace )
  811         flushWs(1);
  812     string caseToken = StringTools::change_case ( token, tcase );
  813     if (currentSyntax->getDecorateFct()) {
  814 
  815         Diluculum::LuaValueList res=callDecorateFct(caseToken);
  816         if (res.size()==1) {
  817             *out<<res[0].asString();
  818         } else {
  819             maskString ( *out, caseToken );
  820         }
  821     } else {
  822         maskString ( *out, caseToken );
  823     }
  824 
  825     token.clear();
  826 }
  827 
  828 bool CodeGenerator::styleFound()
  829 {
  830     return docStyle.found();
  831 }
  832 
  833 bool CodeGenerator::printIndexFile ( const vector<string> &fileList,
  834                                      const string &outPath )
  835 {
  836     return true;
  837 }
  838 
  839 bool CodeGenerator::initIndentationScheme ( const string &indentScheme )
  840 {
  841 
  842     if ( formatter!=NULL ) {
  843         return true;
  844     }
  845 
  846     if ( !indentScheme.size() ) return false;
  847 
  848     formatter=new astyle::ASFormatter();
  849 
  850     if ( indentScheme=="allman" || indentScheme=="bsd" || indentScheme=="ansi" ) {
  851         formatter->setFormattingStyle ( astyle::STYLE_ALLMAN );
  852     } else if ( indentScheme=="kr"||indentScheme=="k&r"||indentScheme=="k/r" ) {
  853         formatter->setFormattingStyle ( astyle::STYLE_KR );
  854     } else if ( indentScheme=="java" ) {
  855         formatter->setFormattingStyle ( astyle::STYLE_JAVA );
  856     } else if ( indentScheme=="stroustrup" ) {
  857         formatter->setFormattingStyle ( astyle::STYLE_STROUSTRUP );
  858     } else if ( indentScheme=="whitesmith" ) {
  859         formatter->setFormattingStyle ( astyle::STYLE_WHITESMITH );
  860     } else if ( indentScheme=="banner" || indentScheme=="ratliff") {
  861         formatter->setFormattingStyle ( astyle::STYLE_RATLIFF );
  862     } else if ( indentScheme=="gnu" ) {
  863         formatter->setFormattingStyle ( astyle::STYLE_GNU );
  864     } else if ( indentScheme=="linux" ) {
  865         formatter->setFormattingStyle ( astyle::STYLE_LINUX );
  866     } else if ( indentScheme=="horstmann" ) {
  867         formatter->setFormattingStyle ( astyle::STYLE_HORSTMANN );
  868     } else if ( indentScheme=="otbs" ||  indentScheme=="1tbs") {
  869         formatter->setFormattingStyle ( astyle::STYLE_1TBS );
  870     } else if ( indentScheme=="google") {
  871         formatter->setFormattingStyle ( astyle::STYLE_GOOGLE );
  872     } else if ( indentScheme=="pico" ||  indentScheme=="a11") {
  873         formatter->setFormattingStyle ( astyle::STYLE_PICO );
  874     } else if ( indentScheme=="lisp" ||  indentScheme=="python"||  indentScheme=="a12") {
  875         formatter->setFormattingStyle ( astyle::STYLE_LISP );
  876     } else if ( indentScheme=="vtk") {
  877         formatter->setFormattingStyle ( astyle::STYLE_VTK );
  878     } else if ( indentScheme=="mozilla") {
  879         formatter->setFormattingStyle ( astyle::STYLE_MOZILLA );
  880     } else if ( indentScheme=="webkit") {
  881         formatter->setFormattingStyle ( astyle::STYLE_WEBKIT );
  882     } else if ( indentScheme!="user" ){
  883         return false;
  884     }
  885     return formattingEnabled=true;
  886 }
  887 
  888 
  889 /*Helper functions for astyle option parsing*/
  890 string CodeGenerator::getParam(const string& arg, const char* op)
  891 {
  892     return arg.substr(strlen(op));
  893 }
  894 
  895 string CodeGenerator::getParam(const string& arg, const char* op1, const char* op2)
  896 {
  897     return isParamOption(arg, op1) ? getParam(arg, op1) : getParam(arg, op2);
  898 }
  899 
  900 bool CodeGenerator::isOption(const string& arg, const char* op)
  901 {
  902     return arg.compare(op) == 0;
  903 }
  904 
  905 bool CodeGenerator::isOption(const string& arg, const char* op1, const char* op2)
  906 {
  907     return (isOption(arg, op1) || isOption(arg, op2));
  908 }
  909 
  910 bool CodeGenerator::isParamOption(const string& arg, const char* option)
  911 {
  912     bool retVal = arg.compare(0, strlen(option), option) == 0;
  913     // if comparing for short option, 2nd char of arg must be numeric
  914     if (retVal && strlen(option) == 1 && arg.length() > 1)
  915         if (!isdigit((unsigned char) arg[1]))
  916             retVal = false;
  917     return retVal;
  918 }
  919 
  920 bool CodeGenerator::isParamOption(const string& arg, const char* option1, const char* option2)
  921 {
  922     return isParamOption(arg, option1) || isParamOption(arg, option2);
  923 }
  924 
  925 //apply the same options as astyle
  926 void CodeGenerator::setIndentationOptions (const vector<string>& options){
  927     if (formatter) {
  928         string arg;
  929         for (unsigned int i=0; i<options.size(); i++) {
  930             arg=options[i];
  931             
  932             if (isOption(arg, "mode=cs"))
  933             {
  934                 formatter->setSharpStyle();
  935                 formatter->setModeManuallySet(true);
  936             }
  937             else if (isOption(arg, "mode=c"))
  938             {
  939                 formatter->setCStyle();
  940                 formatter->setModeManuallySet(true);
  941             }
  942             else if (isOption(arg, "mode=java"))
  943             {
  944                 formatter->setJavaStyle();
  945                 formatter->setModeManuallySet(true);
  946             }
  947             else if (isParamOption(arg, "t", "indent=tab="))
  948             {
  949                 int spaceNum = 4;
  950                 string spaceNumParam = getParam(arg, "t", "indent=tab=");
  951                 if (spaceNumParam.length() > 0)
  952                     spaceNum = atoi(spaceNumParam.c_str());
  953                 if (spaceNum >= 2 && spaceNum <= 20)
  954                     formatter->setTabIndentation(spaceNum, false);
  955             }
  956             else if (isOption(arg, "indent=tab"))
  957             {
  958                 formatter->setTabIndentation(4);
  959             }
  960             else if (isParamOption(arg, "T", "indent=force-tab="))
  961             {
  962                 int spaceNum = 4;
  963                 string spaceNumParam = getParam(arg, "T", "indent=force-tab=");
  964                 if (spaceNumParam.length() > 0)
  965                     spaceNum = atoi(spaceNumParam.c_str());
  966                 if (spaceNum >= 2 && spaceNum <= 20)
  967                     formatter->setTabIndentation(spaceNum, true);
  968             }
  969             else if (isOption(arg, "indent=force-tab"))
  970             {
  971                 formatter->setTabIndentation(4, true);
  972             }
  973             else if (isParamOption(arg, "xT", "indent=force-tab-x="))
  974             {
  975                 int tabNum = 8;
  976                 string tabNumParam = getParam(arg, "xT", "indent=force-tab-x=");
  977                 if (tabNumParam.length() > 0)
  978                     tabNum = atoi(tabNumParam.c_str());
  979                 if (tabNum >= 2 && tabNum <= 20)
  980                     formatter->setForceTabXIndentation(tabNum);
  981                 
  982             }
  983             else if (isOption(arg, "indent=force-tab-x"))
  984             {
  985                 formatter->setForceTabXIndentation(8);
  986             }
  987             else if (isParamOption(arg, "s", "indent=spaces="))
  988             {
  989                 int spaceNum = 4;
  990                 string spaceNumParam = getParam(arg, "s", "indent=spaces=");
  991                 if (spaceNumParam.length() > 0)
  992                     spaceNum = atoi(spaceNumParam.c_str());
  993                 if (spaceNum >= 2 && spaceNum <= 20)
  994                     formatter->setSpaceIndentation(spaceNum);
  995             }
  996             else if (isOption(arg, "indent=spaces"))
  997             {
  998                 formatter->setSpaceIndentation(4);
  999             }
 1000             else if (isParamOption(arg, "xt", "indent-continuation="))
 1001             {
 1002                 int contIndent = 1;
 1003                 string contIndentParam = getParam(arg, "xt", "indent-continuation=");
 1004                 if (contIndentParam.length() > 0)
 1005                     contIndent = atoi(contIndentParam.c_str());
 1006                 if (contIndent > 0 && contIndent < 5)
 1007                     formatter->setContinuationIndentation(contIndent);
 1008             }
 1009             else if (isParamOption(arg, "m", "min-conditional-indent="))
 1010             {
 1011                 int minIndent = astyle::MINCOND_TWO;
 1012                 string minIndentParam = getParam(arg, "m", "min-conditional-indent=");
 1013                 if (minIndentParam.length() > 0)
 1014                     minIndent = atoi(minIndentParam.c_str());
 1015                 if (minIndent < astyle::MINCOND_END)
 1016                     formatter->setMinConditionalIndentOption(minIndent);
 1017             }
 1018             else if (isParamOption(arg, "M", "max-continuation-indent="))
 1019             {
 1020                 int maxIndent = 40;
 1021                 string maxIndentParam = getParam(arg, "M", "max-continuation-indent=");
 1022                 if (maxIndentParam.length() > 0)
 1023                     maxIndent = atoi(maxIndentParam.c_str());
 1024                 if (maxIndent >= 40 && maxIndent <= 120)
 1025                     formatter->setMaxContinuationIndentLength(maxIndent);
 1026             }
 1027             else if (isOption(arg, "N", "indent-namespaces"))
 1028             {
 1029                 formatter->setNamespaceIndent(true);
 1030             }
 1031             else if (isOption(arg, "C", "indent-classes"))
 1032             {
 1033                 formatter->setClassIndent(true);
 1034             }
 1035             else if (isOption(arg, "xG", "indent-modifiers"))
 1036             {
 1037                 formatter->setModifierIndent(true);
 1038             }
 1039             else if (isOption(arg, "S", "indent-switches"))
 1040             {
 1041                 formatter->setSwitchIndent(true);
 1042             }
 1043             else if (isOption(arg, "K", "indent-cases"))
 1044             {
 1045                 formatter->setCaseIndent(true);
 1046             }
 1047             else if (isOption(arg, "xU", "indent-after-parens"))
 1048             {
 1049                 formatter->setAfterParenIndent(true);
 1050             }
 1051             else if (isOption(arg, "L", "indent-labels"))
 1052             {
 1053                 formatter->setLabelIndent(true);
 1054             }
 1055             else if (isOption(arg, "xW", "indent-preproc-block"))
 1056             {
 1057                 formatter->setPreprocBlockIndent(true);
 1058             }
 1059             else if (isOption(arg, "w", "indent-preproc-define"))
 1060             {
 1061                 formatter->setPreprocDefineIndent(true);
 1062             }
 1063             else if (isOption(arg, "xw", "indent-preproc-cond"))
 1064             {
 1065                 formatter->setPreprocConditionalIndent(true);
 1066             }
 1067             else if (isOption(arg, "y", "break-closing-braces"))
 1068             {
 1069                 formatter->setBreakClosingHeaderBracesMode(true);
 1070             }
 1071             else if (isOption(arg, "O", "keep-one-line-blocks"))
 1072             {
 1073                 formatter->setBreakOneLineBlocksMode(false);
 1074             }
 1075             else if (isOption(arg, "o", "keep-one-line-statements"))
 1076             {
 1077                 formatter->setBreakOneLineStatementsMode(false);
 1078             }
 1079             else if (isOption(arg, "P", "pad-paren"))
 1080             {
 1081                 formatter->setParensOutsidePaddingMode(true);
 1082                 formatter->setParensInsidePaddingMode(true);
 1083             }
 1084             else if (isOption(arg, "d", "pad-paren-out"))
 1085             {
 1086                 formatter->setParensOutsidePaddingMode(true);
 1087             }
 1088             else if (isOption(arg, "xd", "pad-first-paren-out"))
 1089             {
 1090                 formatter->setParensFirstPaddingMode(true);
 1091             }
 1092             else if (isOption(arg, "D", "pad-paren-in"))
 1093             {
 1094                 formatter->setParensInsidePaddingMode(true);
 1095             }
 1096             else if (isOption(arg, "H", "pad-header"))
 1097             {
 1098                 formatter->setParensHeaderPaddingMode(true);
 1099             }
 1100             else if (isOption(arg, "U", "unpad-paren"))
 1101             {
 1102                 formatter->setParensUnPaddingMode(true);
 1103             }
 1104             else if (isOption(arg, "p", "pad-oper"))
 1105             {
 1106                 formatter->setOperatorPaddingMode(true);
 1107             }
 1108             else if (isOption(arg, "xg", "pad-comma"))
 1109             {
 1110                 formatter->setCommaPaddingMode(true);
 1111             }
 1112             else if (isOption(arg, "xe", "delete-empty-lines"))
 1113             {
 1114                 formatter->setDeleteEmptyLinesMode(true);
 1115             }
 1116             else if (isOption(arg, "E", "fill-empty-lines"))
 1117             {
 1118                 formatter->setEmptyLineFill(true);
 1119             }
 1120             else if (isOption(arg, "c", "convert-tabs"))
 1121             {
 1122                 formatter->setTabSpaceConversionMode(true);
 1123             }
 1124             else if (isOption(arg, "xy", "close-templates"))
 1125             {
 1126                 formatter->setCloseTemplatesMode(true);
 1127             }
 1128             else if (isOption(arg, "F", "break-blocks=all"))
 1129             {
 1130                 formatter->setBreakBlocksMode(true);
 1131                 formatter->setBreakClosingHeaderBlocksMode(true);
 1132             }
 1133             else if (isOption(arg, "f", "break-blocks"))
 1134             {
 1135                 formatter->setBreakBlocksMode(true);
 1136             }
 1137             else if (isOption(arg, "e", "break-elseifs"))
 1138             {
 1139                 formatter->setBreakElseIfsMode(true);
 1140             }
 1141             else if (isOption(arg, "xb", "break-one-line-headers"))
 1142             {
 1143                 formatter->setBreakOneLineHeadersMode(true);
 1144             }
 1145             else if (isOption(arg, "j", "add-braces"))
 1146             {
 1147                 formatter->setAddBracesMode(true);
 1148             }
 1149             else if (isOption(arg, "J", "add-one-line-braces"))
 1150             {
 1151                 formatter->setAddOneLineBracesMode(true);
 1152             }
 1153             else if (isOption(arg, "xj", "remove-braces"))
 1154             {
 1155                 formatter->setRemoveBracesMode(true);
 1156             }
 1157             else if (isOption(arg, "Y", "indent-col1-comments"))
 1158             {
 1159                 formatter->setIndentCol1CommentsMode(true);
 1160             }
 1161             else if (isOption(arg, "align-pointer=type"))
 1162             {
 1163                 formatter->setPointerAlignment(astyle::PTR_ALIGN_TYPE);
 1164             }
 1165             else if (isOption(arg, "align-pointer=middle"))
 1166             {
 1167                 formatter->setPointerAlignment(astyle::PTR_ALIGN_MIDDLE);
 1168             }
 1169             else if (isOption(arg, "align-pointer=name"))
 1170             {
 1171                 formatter->setPointerAlignment(astyle::PTR_ALIGN_NAME);
 1172             }
 1173             else if (isParamOption(arg, "k"))
 1174             {
 1175                 int align = 0;
 1176                 string styleParam = getParam(arg, "k");
 1177                 if (styleParam.length() > 0)
 1178                     align = atoi(styleParam.c_str());
 1179                 if (align == 1)
 1180                     formatter->setPointerAlignment(astyle::PTR_ALIGN_TYPE);
 1181                 else if (align == 2)
 1182                     formatter->setPointerAlignment(astyle::PTR_ALIGN_MIDDLE);
 1183                 else if (align == 3)
 1184                     formatter->setPointerAlignment(astyle::PTR_ALIGN_NAME);
 1185             }
 1186             else if (isOption(arg, "align-reference=none"))
 1187             {
 1188                 formatter->setReferenceAlignment(astyle::REF_ALIGN_NONE);
 1189             }
 1190             else if (isOption(arg, "align-reference=type"))
 1191             {
 1192                 formatter->setReferenceAlignment(astyle::REF_ALIGN_TYPE);
 1193             }
 1194             else if (isOption(arg, "align-reference=middle"))
 1195             {
 1196                 formatter->setReferenceAlignment(astyle::REF_ALIGN_MIDDLE);
 1197             }
 1198             else if (isOption(arg, "align-reference=name"))
 1199             {
 1200                 formatter->setReferenceAlignment(astyle::REF_ALIGN_NAME);
 1201             }
 1202             else if (isParamOption(arg, "W"))
 1203             {
 1204                 int align = 0;
 1205                 string styleParam = getParam(arg, "W");
 1206                 if (styleParam.length() > 0)
 1207                     align = atoi(styleParam.c_str());
 1208                 if (align == 0)
 1209                     formatter->setReferenceAlignment(astyle::REF_ALIGN_NONE);
 1210                 else if (align == 1)
 1211                     formatter->setReferenceAlignment(astyle::REF_ALIGN_TYPE);
 1212                 else if (align == 2)
 1213                     formatter->setReferenceAlignment(astyle::REF_ALIGN_MIDDLE);
 1214                 else if (align == 3)
 1215                     formatter->setReferenceAlignment(astyle::REF_ALIGN_NAME);
 1216             }
 1217             else if (isParamOption(arg, "max-code-length="))
 1218             {
 1219                 int maxLength = 50;
 1220                 string maxLengthParam = getParam(arg, "max-code-length=");
 1221                 if (maxLengthParam.length() > 0)
 1222                     maxLength = atoi(maxLengthParam.c_str());
 1223                 if (maxLength >= 50 && maxLength<= 200)
 1224                     formatter->setMaxCodeLength(maxLength);
 1225             }
 1226             else if (isParamOption(arg, "xC"))
 1227             {
 1228                 int maxLength = 50;
 1229                 string maxLengthParam = getParam(arg, "xC");
 1230                 if (maxLengthParam.length() > 0)
 1231                     maxLength = atoi(maxLengthParam.c_str());
 1232                 if (maxLength > 0 && maxLength<= 200)
 1233                     formatter->setMaxCodeLength(maxLength);
 1234             }
 1235             else if (isOption(arg, "xL", "break-after-logical"))
 1236             {
 1237                 formatter->setBreakAfterMode(true);
 1238             }
 1239             else if (isOption(arg, "xc", "attach-classes"))
 1240             {
 1241                 formatter->setAttachClass(true);
 1242             }
 1243             else if (isOption(arg, "xV", "attach-closing-while"))
 1244             {
 1245                 formatter->setAttachClosingWhile(true);
 1246             }
 1247             else if (isOption(arg, "xk", "attach-extern-c"))
 1248             {
 1249                 formatter->setAttachExternC(true);
 1250             }
 1251             else if (isOption(arg, "xn", "attach-namespaces"))
 1252             {
 1253                 formatter->setAttachNamespace(true);
 1254             }
 1255             else if (isOption(arg, "xl", "attach-inlines"))
 1256             {
 1257                 formatter->setAttachInline(true);
 1258             }
 1259             else if (isOption(arg, "xp", "remove-comment-prefix"))
 1260             {
 1261                 formatter->setStripCommentPrefix(true);
 1262             }
 1263             else if (isOption(arg, "xB", "break-return-type"))
 1264             {
 1265                 formatter->setBreakReturnType(true);
 1266             }
 1267             else if (isOption(arg, "xD", "break-return-type-decl"))
 1268             {
 1269                 formatter->setBreakReturnTypeDecl(true);
 1270             }
 1271             else if (isOption(arg, "xf", "attach-return-type"))
 1272             {
 1273                 formatter->setAttachReturnType(true);
 1274             }
 1275             else if (isOption(arg, "xh", "attach-return-type-decl"))
 1276             {
 1277                 formatter->setAttachReturnTypeDecl(true);
 1278             }
 1279             // Objective-C options
 1280             else if (isOption(arg, "xQ", "pad-method-prefix"))
 1281             {
 1282                 formatter->setMethodPrefixPaddingMode(true);
 1283             }
 1284             else if (isOption(arg, "xR", "unpad-method-prefix"))
 1285             {
 1286                 formatter->setMethodPrefixUnPaddingMode(true);
 1287             }
 1288             else if (isOption(arg, "xq", "pad-return-type"))
 1289             {
 1290                 formatter->setReturnTypePaddingMode(true);
 1291             }
 1292             else if (isOption(arg, "xr", "unpad-return-type"))
 1293             {
 1294                 formatter->setReturnTypeUnPaddingMode(true);
 1295             }
 1296             else if (isOption(arg, "xS", "pad-param-type"))
 1297             {
 1298                 formatter->setParamTypePaddingMode(true);
 1299             }
 1300             else if (isOption(arg, "xs", "unpad-param-type"))
 1301             {
 1302                 formatter->setParamTypeUnPaddingMode(true);
 1303             }
 1304             else if (isOption(arg, "xM", "align-method-colon"))
 1305             {
 1306                 formatter->setAlignMethodColon(true);
 1307             }
 1308             else if (isOption(arg, "xP0", "pad-method-colon=none"))
 1309             {
 1310                 formatter->setObjCColonPaddingMode(astyle::COLON_PAD_NONE);
 1311             }
 1312             else if (isOption(arg, "xP1", "pad-method-colon=all"))
 1313             {
 1314                 formatter->setObjCColonPaddingMode(astyle::COLON_PAD_ALL);
 1315             }
 1316             else if (isOption(arg, "xP2", "pad-method-colon=after"))
 1317             {
 1318                 formatter->setObjCColonPaddingMode(astyle::COLON_PAD_AFTER);
 1319             }
 1320             else if (isOption(arg, "xP3", "pad-method-colon=before"))
 1321             {
 1322                 formatter->setObjCColonPaddingMode(astyle::COLON_PAD_BEFORE);
 1323             }
 1324         }
 1325     }
 1326 }
 1327 
 1328 LoadResult CodeGenerator::loadLanguage ( const string& langDefPath, bool embedded )
 1329 {
 1330 
 1331     if (!embedded) {
 1332         while (!nestedLangs.empty()) {
 1333             nestedLangs.pop();
 1334         }   
 1335     }
 1336     
 1337     bool reloadNecessary= currentSyntax ? currentSyntax->needsReload ( langDefPath ): true;
 1338     LoadResult result=LOAD_OK;
 1339     if ( reloadNecessary ) {
 1340         if (syntaxReaders.count(langDefPath)) {
 1341             currentSyntax=syntaxReaders[langDefPath];
 1342             result=LOAD_OK;
 1343         } else {
 1344             currentSyntax=new SyntaxReader();
 1345             result=currentSyntax->load(langDefPath, pluginParameter, outputType);
 1346             syntaxReaders[langDefPath]=currentSyntax;
 1347         }
 1348 
 1349         if ( result==LOAD_OK ) {
 1350             formattingPossible=currentSyntax->enableReformatting();
 1351 
 1352             if ( openTags.size() >NUMBER_BUILTIN_STATES ) {
 1353                 // remove dynamic keyword tag delimiters of the old language definition
 1354                 vector<string>::iterator keyStyleOpenBegin =
 1355                     openTags.begin() + NUMBER_BUILTIN_STATES;
 1356                 vector<string>::iterator keyStyleCloseBegin =
 1357                     closeTags.begin() + NUMBER_BUILTIN_STATES;
 1358                 openTags.erase ( keyStyleOpenBegin, openTags.end() );
 1359                 closeTags.erase ( keyStyleCloseBegin, closeTags.end() );
 1360             }
 1361             // add new keyword tag delimiters
 1362             for ( unsigned int i=0; i< currentSyntax->getKeywordClasses().size(); i++ ) {
 1363                 openTags.push_back ( getKeywordOpenTag ( i ) );
 1364                 closeTags.push_back ( getKeywordCloseTag ( i ) );
 1365             }
 1366             
 1367             //test balloon
 1368             string overrideSpacer(currentSyntax->getOverrideConfigVal("spacer"));
 1369             if (!overrideSpacer.empty()) {
 1370                 spacer = overrideSpacer;
 1371             }
 1372             string overrideMaskWS(currentSyntax->getOverrideConfigVal("maskws"));
 1373             if (!overrideMaskWS.empty()) {
 1374                 maskWs = overrideMaskWS=="true";
 1375             }
 1376             
 1377         }
 1378     }
 1379     return result;
 1380 }
 1381 
 1382 bool CodeGenerator::validateInputStream()
 1383 {
 1384     if ( !in ) return false;
 1385 
 1386     // it is not possible to move stream pointer back with stdin
 1387     if ( ( int ) in->tellg() == -1 ) // -1 : stdin
 1388         return true;
 1389 
 1390     // Sources: http://en.wikipedia.org/wiki/Magic_number_(programming)
 1391     // Magic configuration of "file"
 1392     // This is intended for web plugins - only check filetypes often found in the net
 1393     char magic_gif[]    = {'G','I','F','8', 0};
 1394     char magic_png[]    = {'\x89','P','N','G', 0};
 1395     char magic_java[]   = {'\xCA','\xFE','\xBA','\xBE', 0};
 1396     char magic_jpeg[]   = {'\xFF','\xD8','\xFF', 0};
 1397     char magic_bmp[]    = {'B','M', 0};
 1398     char magic_pdf[]    = {'%','P','D','F', 0};
 1399     char magic_utf8[]   = {'\xEF','\xBB','\xBF',0};
 1400     char magic_rar[]    = {'R','a','r','!', 0};
 1401     char magic_zip[]    = {'P','K','\x03','\x04', 0};
 1402     char magic_ace[]    = {'*','*','A','C','E','*','*', 0};
 1403     char magic_tgz[]    = {'\x8b','\x1f', '\x00', '\x08', 0};
 1404     char magic_bzip[]   = {'B','Z', 0};
 1405 
 1406     char* magic_table[] = {magic_utf8,
 1407                            magic_gif, magic_png, magic_jpeg, magic_bmp, magic_pdf,
 1408                            magic_java,
 1409                            magic_rar, magic_zip, magic_ace, magic_tgz, magic_bzip,
 1410                            0
 1411                           };
 1412 
 1413     char buffer [10]= {0};
 1414     in->read ( buffer,8 );  //only read the first 8 bytes of input stream
 1415 
 1416     int magic_index=0;
 1417     while ( magic_table[magic_index] ) {
 1418         if ( !strncmp ( buffer, magic_table[magic_index], strlen ( magic_table[magic_index] ) ) ) {
 1419             break;
 1420         }
 1421         magic_index++;
 1422     }
 1423     int streamReadPos=0;
 1424     if ( magic_table[magic_index] == magic_utf8 ) {
 1425         //setEncoding("utf-8");
 1426         streamReadPos=3; // remove UTF-8 magic number from output
 1427     }
 1428 
 1429     in -> seekg ( streamReadPos, ios::beg );
 1430     in-> clear();  // clear fail bit to continue reading
 1431 
 1432     return !magic_table[magic_index] // points to 0 if no pattern was found
 1433            || magic_table[magic_index] == magic_utf8;
 1434 }
 1435 
 1436 void CodeGenerator::applyPluginChunk(const string& fctName, string *result, bool *keepDefault) {
 1437     
 1438     if (currentSyntax && pluginChunks.size()) {
 1439     
 1440         Diluculum::LuaState luaState;
 1441 
 1442         Diluculum::LuaValueList chunkParams;
 1443         chunkParams.push_back(currentSyntax->getDescription());
 1444         for (unsigned int i=0; i<pluginChunks.size(); i++) {
 1445             luaState.call(*pluginChunks[i], chunkParams, "format user function");
 1446         }
 1447         
 1448         if (luaState.globals().count(fctName)) {
 1449             Diluculum::LuaFunction* documentFct=new Diluculum::LuaFunction(luaState[fctName].value().asFunction());
 1450         
 1451             luaState["HL_INPUT_FILE"] = luaState["HL_PLUGIN_PARAM"] = pluginParameter;
 1452             luaState["HL_OUTPUT"] = outputType;
 1453             luaState["HL_FORMAT_HTML"]=HTML;
 1454             luaState["HL_FORMAT_XHTML"]=XHTML;
 1455             luaState["HL_FORMAT_TEX"]=TEX;
 1456             luaState["HL_FORMAT_LATEX"]=LATEX;
 1457             luaState["HL_FORMAT_RTF"]=RTF;
 1458             luaState["HL_FORMAT_ANSI"]=ESC_ANSI;
 1459             luaState["HL_FORMAT_XTERM256"]=ESC_XTERM256;
 1460             luaState["HL_FORMAT_TRUECOLOR"]=ESC_TRUECOLOR;
 1461             luaState["HL_FORMAT_SVG"]=SVG;
 1462             luaState["HL_FORMAT_BBCODE"]=BBCODE;
 1463             luaState["HL_FORMAT_PANGO"]=PANGO;
 1464             luaState["HL_FORMAT_ODT"]=ODTFLAT;
 1465                         
 1466             Diluculum::LuaValueList params;
 1467             Diluculum::LuaValueMap options;
 1468             options[Diluculum::LuaValue("title")] =  Diluculum::LuaValue( docTitle );   
 1469             options[Diluculum::LuaValue("encoding")] =  Diluculum::LuaValue(encoding);   
 1470             options[Diluculum::LuaValue("fragment")] =  Diluculum::LuaValue(fragmentOutput);   
 1471             options[Diluculum::LuaValue("font")] =  Diluculum::LuaValue(getBaseFont());   
 1472             options[Diluculum::LuaValue("fontsize")] =  Diluculum::LuaValue(getBaseFontSize());   
 1473 
 1474             params.push_back(inputFilesCnt);
 1475             params.push_back(processedFilesCnt);
 1476             params.push_back(options);
 1477             
 1478             Diluculum::LuaValueList res=luaState.call ( *documentFct, params, fctName+" call")  ;
 1479             if (res.size()>=1) {
 1480                 *keepDefault=false;
 1481                 *result = res[0].asString();
 1482                 if (res.size()==2)
 1483                     *keepDefault = res[1].asBoolean();
 1484             }
 1485             delete documentFct;
 1486         }
 1487     }
 1488 }
 1489 
 1490 void CodeGenerator::printHeader()
 1491 {
 1492     bool keepDefaultHeader=true;
 1493     string pluginHeader;
 1494     
 1495     processedFilesCnt++;
 1496     
 1497     applyPluginChunk("DocumentHeader", &pluginHeader, &keepDefaultHeader);
 1498 
 1499     if ( ! fragmentOutput && keepDefaultHeader)
 1500         *out << getHeader();
 1501     
 1502     *out << pluginHeader; 
 1503    
 1504     if ( !fragmentOutput || keepInjections)
 1505         *out << currentSyntax->getHeaderInjection();
 1506 }
 1507 
 1508 void CodeGenerator::printFooter()
 1509 {
 1510     
 1511     bool keepDefaultFooter=true;
 1512     string pluginFooter;
 1513     
 1514     applyPluginChunk("DocumentFooter", &pluginFooter, &keepDefaultFooter);
 1515     
 1516     if ( !fragmentOutput || keepInjections)
 1517         *out << currentSyntax->getFooterInjection();
 1518 
 1519     *out << pluginFooter; 
 1520     
 1521     if ( ! fragmentOutput && keepDefaultFooter )
 1522         *out << getFooter();
 1523 }
 1524 
 1525 ParseError CodeGenerator::generateFile ( const string &inFileName,
 1526         const string &outFileName )
 1527 {
 1528     if ( !docStyle.found() ) {
 1529         return BAD_STYLE;
 1530     }
 1531 
 1532     reset();
 1533 
 1534     ParseError error=PARSE_OK;
 1535 
 1536     inFile=inFileName;
 1537     outFile=outFileName;
 1538         
 1539     in = ( inFileName.empty() ? &cin :new ifstream ( inFileName.c_str() ) );
 1540 
 1541     if ( validateInput )
 1542         if ( !validateInputStream() ) error= BAD_INPUT;
 1543 
 1544     if ( !in->fail() && error==PARSE_OK ) {
 1545         out = ( outFileName.empty() ? &cout :new ofstream ( outFileName.c_str() ) );
 1546         if ( out->fail() ) {
 1547             error=BAD_OUTPUT;
 1548         }
 1549     }
 1550 
 1551     if ( in->fail() ) {
 1552         error=BAD_INPUT;
 1553     }
 1554 
 1555     if ( error==PARSE_OK ) {
 1556         if ( formatter != NULL ) {
 1557             formatter->init ( new astyle::ASStreamIterator ( in ) );
 1558         }
 1559         currentSyntax->setInputFileName(inFile);
 1560         printHeader();
 1561         printBody();
 1562         printFooter();
 1563     }
 1564 
 1565     if ( !outFileName.empty() ) {
 1566         delete out;
 1567         out=NULL;
 1568     }
 1569     if ( !inFileName.empty() ) {
 1570         delete in;
 1571         in=NULL;
 1572     }
 1573     return error;
 1574 }
 1575 
 1576 string CodeGenerator::generateString ( const string &input )
 1577 {
 1578 
 1579     if ( !docStyle.found() ) {
 1580         return "";
 1581     }
 1582 
 1583     reset();
 1584 
 1585     in = new istringstream ( input );
 1586     out = new ostringstream ();
 1587 
 1588     if ( in->fail() || out->fail() ) {
 1589         return "";
 1590     }
 1591 
 1592     if ( formatter != NULL ) {
 1593         formatter->init ( new astyle::ASStreamIterator ( in ) );
 1594     }
 1595     printHeader();
 1596     printBody();
 1597     printFooter();
 1598 
 1599     string result = static_cast<ostringstream*> ( out )->str();
 1600 
 1601     delete out;
 1602     out=NULL;
 1603     delete in;
 1604     in=NULL;
 1605 
 1606     return result;
 1607 }
 1608 
 1609 string CodeGenerator::generateStringFromFile ( const string &inFileName )
 1610 {
 1611 
 1612     if ( !docStyle.found() ) {
 1613         return "";
 1614     }
 1615 
 1616     reset();
 1617 
 1618     inFile = inFileName;
 1619     
 1620     in = new ifstream ( inFileName.c_str() );
 1621     out = new ostringstream ();
 1622 
 1623     if ( in->fail() || out->fail() ) {
 1624         return "";
 1625     }
 1626 
 1627     if ( validateInput && !validateInputStream() ) {
 1628         return "ERROR: detected binary input";
 1629     }
 1630 
 1631     if ( formatter != NULL ) {
 1632         formatter->init ( new astyle::ASStreamIterator ( in ) );
 1633     }
 1634     currentSyntax->setInputFileName(inFile);
 1635     
 1636     printHeader();
 1637     printBody();
 1638     printFooter();
 1639 
 1640     string result = static_cast<ostringstream*> ( out )->str();
 1641 
 1642     delete out;
 1643     out=NULL;
 1644     delete in;
 1645     in=NULL;
 1646 
 1647     return result;
 1648 }
 1649 
 1650 unsigned int CodeGenerator::getStyleID ( State s, unsigned int kwClassID )
 1651 {
 1652     if ( s==KEYWORD && kwClassID ) {
 1653         return NUMBER_BUILTIN_STATES + kwClassID-1;
 1654     }
 1655     return ( unsigned int ) s ;
 1656 }
 1657 
 1658 void CodeGenerator::openTag ( State s )
 1659 {
 1660     *out << openTags[ ( unsigned int ) s];
 1661     currentState=s;
 1662 
 1663 }
 1664 
 1665 void CodeGenerator::closeTag ( State s )
 1666 {
 1667     *out << closeTags[ ( unsigned int ) s];
 1668     flushWs(2);
 1669     currentState=_UNKNOWN;
 1670 }
 1671 
 1672 void CodeGenerator::openKWTag ( unsigned int kwClassID )
 1673 {
 1674     *out << openTags.at(getStyleID ( KEYWORD, kwClassID ) );
 1675     currentState=KEYWORD;
 1676 }
 1677 
 1678 void CodeGenerator::closeKWTag ( unsigned int kwClassID )
 1679 {
 1680     *out << closeTags.at(getStyleID ( KEYWORD, kwClassID ) );
 1681     flushWs(3);
 1682     currentState=_UNKNOWN;
 1683 }
 1684 
 1685 bool CodeGenerator::loadEmbeddedLang(const string&embedLangDefPath)
 1686 {
 1687     if (nestedLangs.empty()) {
 1688         nestedLangs.push(currentSyntax->getCurrentPath() );
 1689     }
 1690     if (nestedLangs.top() != embedLangDefPath) {
 1691         nestedLangs.push(embedLangDefPath);
 1692     }
 1693     LoadResult res = loadLanguage(embedLangDefPath, true);
 1694     //pass end delimiter regex to syntax description
 1695     currentSyntax->restoreLangEndDelim(embedLangDefPath);
 1696     return res == LOAD_OK;
 1697 }
 1698 
 1699 ///////////////////////////////////////////////////////////////////////////////
 1700 
 1701 void CodeGenerator::processRootState()
 1702 {
 1703     bool eof=false,
 1704          firstLine=true; // avoid newline before printing the first output line
 1705          
 1706     applySyntaxTestCase = inFile.find("syntax_test_")!=string::npos;
 1707     
 1708     if ( currentSyntax->highlightingDisabled() ) {
 1709         string line;
 1710         while ( getline ( *in, line ) && lineNumber < maxLineCnt ) {
 1711             ++lineNumber;
 1712             insertLineNumber ( !firstLine );
 1713             flushWs(4);
 1714             firstLine=false;
 1715             if (lineNumber>=startLineCntCurFile && lineNumber <=maxLineCnt)
 1716                 maskString ( *out, line );
 1717         }
 1718         *out << flush;
 1719         return;
 1720     }
 1721 
 1722     if (!embedLangStart.empty()) {
 1723         if (!loadEmbeddedLang(currentSyntax->getNewPath(embedLangStart))) return;
 1724     }
 1725 
 1726     State state=STANDARD;
 1727 
 1728     //if (outputType!=ESC_TRUECOLOR && outputType!=ESC_XTERM256)
 1729     openTag ( STANDARD );
 1730     
 1731     do {
 1732         // determine next state
 1733         state= getCurrentState(STANDARD);
 1734 
 1735         // handle current state
 1736         switch ( state ) {
 1737         case KEYWORD:
 1738             closeTag ( STANDARD );
 1739             eof=processKeywordState ( state );
 1740             openTag ( STANDARD );
 1741             break;
 1742         case NUMBER:
 1743             closeTag ( STANDARD );
 1744             eof=processNumberState();
 1745             openTag ( STANDARD );
 1746             break;
 1747         case ML_COMMENT:
 1748             closeTag ( STANDARD );
 1749             eof=processMultiLineCommentState();
 1750             openTag ( STANDARD );
 1751             break;
 1752         case SL_COMMENT:
 1753             closeTag ( STANDARD );
 1754             eof=processSingleLineCommentState();
 1755             openTag ( STANDARD );
 1756             break;
 1757         case STRING:
 1758             closeTag ( STANDARD );
 1759             eof=processStringState ( STANDARD );
 1760             openTag ( STANDARD );
 1761             break;
 1762         case DIRECTIVE:
 1763             closeTag ( STANDARD );
 1764             eof=processDirectiveState();
 1765             openTag ( STANDARD );
 1766             break;
 1767         case ESC_CHAR:
 1768             closeTag ( STANDARD );
 1769             eof=processEscapeCharState();
 1770             openTag ( STANDARD );
 1771             break;
 1772         case SYMBOL:
 1773             closeTag ( STANDARD );
 1774             eof=processSymbolState();
 1775             openTag ( STANDARD );
 1776             break;
 1777 
 1778         case EMBEDDED_CODE_END:
 1779             closeTag ( STANDARD );
 1780             eof=processSyntaxChangeState(state);
 1781             openTag ( STANDARD );
 1782             break;
 1783         case _EOL:
 1784 
 1785             // XTERM256 fix (issue with less cmd)
 1786             if  (!firstLine || showLineNumbers) {
 1787                 closeTag ( STANDARD );
 1788             }
 1789             insertLineNumber ( !firstLine );
 1790             if (!firstLine || showLineNumbers) {
 1791                 flushWs(5);
 1792                 stateTraceCurrent.clear();
 1793                 openTag ( STANDARD );
 1794             }
 1795             firstLine=false;
 1796             break;
 1797         case _EOF:
 1798             eof=true;
 1799             break;
 1800         case _WS:
 1801             processWsState();
 1802             break;
 1803         default:
 1804            
 1805             // see https://gitlab.com/saalen/highlight/-/issues/152
 1806           //  if (lineNumber==1 && lineIndex==1 && (outputType==ESC_TRUECOLOR || outputType==ESC_XTERM256) && token.size())
 1807           //      openTag ( STANDARD );
 1808 
 1809             printMaskedToken ();
 1810             break;
 1811         }
 1812     } while ( !eof );
 1813     
 1814     if (token.size() || lineNumber>1 || (outputType!=ESC_TRUECOLOR && outputType!=ESC_XTERM256))
 1815         closeTag ( STANDARD );
 1816 
 1817     if (currentSyntax->getDecorateLineEndFct()) {
 1818         Diluculum::LuaValueList res=callDecorateLineFct(false);
 1819         if (res.size()==1) {
 1820             *out << res[0].asString();
 1821         }
 1822     }
 1823 
 1824     printNewLines = noTrailingNewLine==0 || ( noTrailingNewLine==2 && ( token.size() || lineNumber>1) );
 1825     *out << getNewLine();
 1826     *out << flush;
 1827 }
 1828 
 1829 bool CodeGenerator::processSyntaxChangeState(State myState)
 1830 {
 1831     State newState=STANDARD;
 1832     bool eof=false,
 1833          exitState=false;
 1834 
 1835     openTag ( KEYWORD );
 1836     do {
 1837 
 1838         if (myState==EMBEDDED_CODE_END) {
 1839             if (!nestedLangs.empty()) {
 1840                 nestedLangs.pop();
 1841             }
 1842             // load host language syntax
 1843             if (!nestedLangs.empty()) {
 1844                 loadLanguage(nestedLangs.top(), true);
 1845             }
 1846             matchRegex(line, EMBEDDED_CODE_BEGIN); // match remaining line using the host syntax
 1847         }
 1848         
 1849         printMaskedToken ( newState!=_WS );
 1850 
 1851         newState= getCurrentState(myState);
 1852 
 1853         switch ( newState ) {
 1854         case _WS:
 1855             processWsState();
 1856             break;
 1857         case _EOL:
 1858             insertLineNumber();
 1859             exitState=true;
 1860             break;
 1861         case _EOF:
 1862             eof = true;
 1863             break;
 1864         default:
 1865             exitState=true;
 1866             break;
 1867         }
 1868     } while (  !exitState  &&  !eof );
 1869     closeTag ( KEYWORD );
 1870 
 1871     return eof;
 1872 }
 1873 
 1874 
 1875 bool CodeGenerator::processKeywordState ( State myState )
 1876 {
 1877     State newState=STANDARD;
 1878     unsigned int myClassID=currentKeywordClass;
 1879     bool eof=false,
 1880          exitState=false;
 1881 
 1882     openKWTag ( myClassID );
 1883     do {
 1884         printMaskedToken ( newState!=_WS,
 1885                            ( currentSyntax->isIgnoreCase() ) ? keywordCase : StringTools::CASE_UNCHANGED );
 1886         newState= getCurrentState(myState);
 1887         switch ( newState ) {
 1888         case _WS:
 1889             processWsState();
 1890             exitState=isolateTags;
 1891             break;
 1892         case _EOL:
 1893             insertLineNumber();
 1894             exitState=true;
 1895             
 1896             break;
 1897         case _EOF:
 1898             eof = true;
 1899             break;
 1900         case KEYWORD_END:
 1901             exitState=true;
 1902             break;
 1903         default:
 1904             exitState= ( myClassID!=currentKeywordClass ) || ( myState!=newState );
 1905             break;
 1906         }
 1907     } while ( !exitState  &&  !eof );
 1908 
 1909     closeKWTag ( myClassID );
 1910 
 1911     currentKeywordClass=0;
 1912     return eof;
 1913 }
 1914 
 1915 bool CodeGenerator::processNumberState()
 1916 {
 1917     State newState=STANDARD;
 1918     bool eof=false,
 1919          exitState=false;
 1920     openTag ( NUMBER );
 1921     do {
 1922         printMaskedToken ( newState!=_WS );
 1923         newState= getCurrentState(NUMBER);
 1924         switch ( newState ) {
 1925         case _WS:
 1926             processWsState();
 1927             exitState=isolateTags;
 1928             break;
 1929         case _EOL:
 1930             insertLineNumber();
 1931             exitState=true;
 1932             break;
 1933         case _EOF:
 1934             eof = true;
 1935             break;
 1936         default:
 1937             exitState=newState!=NUMBER;
 1938             break;
 1939         }
 1940     } while ( !exitState && !eof );
 1941 
 1942     closeTag ( NUMBER );
 1943     return eof;
 1944 }
 1945 
 1946 
 1947 bool CodeGenerator::processMultiLineCommentState()
 1948 {
 1949     int commentCount=1;
 1950     int openDelimID=currentSyntax->getOpenDelimiterID ( token, ML_COMMENT);
 1951     State newState=STANDARD;
 1952     bool eof=false, exitState=false, containedTestCase=false;
 1953     unsigned int startColumn=lineIndex - token.size() ;
 1954     openTag ( ML_COMMENT );
 1955     do {
 1956         printMaskedToken (newState!=_WS );
 1957         newState= getCurrentState(ML_COMMENT);
 1958 
 1959         switch ( newState ) {
 1960         case _WS:
 1961             processWsState();
 1962             break;
 1963         case _EOL:
 1964             wsBuffer += closeTags[ML_COMMENT];
 1965             insertLineNumber();
 1966             wsBuffer += openTags[ML_COMMENT];
 1967             startColumn=0;
 1968             break;
 1969         case _EOF:
 1970             eof = true;
 1971             break;
 1972         case _TESTPOS:
 1973             runSyntaxTestcases(token=="<" ? startColumn : lineIndex - 1 );
 1974             printMaskedToken();
 1975             containedTestCase=true;
 1976             break;
 1977         case ML_COMMENT:
 1978 
 1979             if ( currentSyntax->allowNestedMLComments() ) {
 1980                 ++commentCount;
 1981             }
 1982             // if delimiters are equal, close the comment by continuing to
 1983             // ML_COMMENT_END section
 1984             if (currentSyntax->delimiterIsDistinct(currentSyntax->getOpenDelimiterID ( token, ML_COMMENT  ))) break;
 1985 
 1986         case ML_COMMENT_END:
 1987 
 1988             if (!currentSyntax->matchesOpenDelimiter (token,  ML_COMMENT_END, openDelimID)) {
 1989                 break;
 1990             }
 1991             commentCount--;
 1992             if ( !commentCount ) {
 1993                 printMaskedToken();
 1994                 exitState=true;
 1995             }
 1996             break;
 1997         default:
 1998             break;
 1999         }
 2000     } while ( !exitState  &&  !eof );
 2001 
 2002     closeTag ( ML_COMMENT );
 2003    
 2004     if (containedTestCase){
 2005         stateTraceCurrent.clear();
 2006     }
 2007     return eof;
 2008 }
 2009 
 2010 
 2011 bool CodeGenerator::processSingleLineCommentState()
 2012 {
 2013     State newState=STANDARD;
 2014     bool eof=false, exitState=false, containedTestCase=false;
 2015     unsigned int startColumn = lineIndex - token.size() ;
 2016 
 2017     openTag ( SL_COMMENT );
 2018     do {
 2019         printMaskedToken ( newState!=_WS );
 2020         newState= getCurrentState(SL_COMMENT);
 2021 
 2022         switch ( newState ) {
 2023         case _WS:
 2024             processWsState();
 2025             break;
 2026         case _EOL:
 2027             printMaskedToken();
 2028             if ( preFormatter.isEnabled() && preFormatter.isWrappedLine ( lineNumber-1 ) ) {
 2029                 exitState=false;
 2030             } else {
 2031                 exitState=true;
 2032             }
 2033             if ( !exitState ) wsBuffer += closeTags[SL_COMMENT];
 2034             insertLineNumber();
 2035             if ( !exitState ) wsBuffer += openTags[SL_COMMENT];
 2036 
 2037             break;
 2038         case _EOF:
 2039             eof = true;
 2040             break;
 2041         case _TESTPOS:
 2042             runSyntaxTestcases(token=="<" ? startColumn : lineIndex - 1 );
 2043             printMaskedToken();
 2044             containedTestCase=true;
 2045             break;
 2046      
 2047         default:
 2048             break;
 2049         }
 2050     } while ( !exitState  &&  !eof );
 2051 
 2052     closeTag ( SL_COMMENT );
 2053     
 2054     if (containedTestCase) {
 2055         stateTraceCurrent.clear();
 2056     }
 2057     
 2058     return eof;
 2059 }
 2060 
 2061 bool CodeGenerator::processDirectiveState()
 2062 {
 2063     State  newState=STANDARD;
 2064     bool eof=false, exitState=false;
 2065 
 2066     openTag ( DIRECTIVE );
 2067     do {
 2068         printMaskedToken ( newState!=_WS );
 2069         newState= getCurrentState(DIRECTIVE);
 2070         switch ( newState ) {
 2071         case _WS:
 2072             processWsState();
 2073             break;
 2074         case DIRECTIVE_END:
 2075             printMaskedToken();
 2076             exitState=true;
 2077             break;
 2078         case _EOL:
 2079             printMaskedToken();
 2080             
 2081             if ( preFormatter.isEnabled() && preFormatter.isWrappedLine ( lineNumber-1 ) ) {
 2082                 exitState=false;
 2083             } else {
 2084                 if (currentSyntax->getContinuationChar()!=0x13){
 2085                     exitState= ( terminatingChar!=currentSyntax->getContinuationChar() );
 2086                 } 
 2087             }
 2088             if ( !exitState ) wsBuffer += closeTags[DIRECTIVE];
 2089             insertLineNumber();
 2090             if ( !exitState ) wsBuffer += openTags[DIRECTIVE];
 2091             break;
 2092         case ML_COMMENT:
 2093             closeTag ( DIRECTIVE );
 2094             eof= processMultiLineCommentState();
 2095             openTag ( DIRECTIVE );
 2096             break;
 2097         case SL_COMMENT:
 2098             closeTag ( DIRECTIVE );
 2099             eof= processSingleLineCommentState();
 2100             openTag ( DIRECTIVE );
 2101             exitState=true;
 2102             break;
 2103         case STRING:
 2104             closeTag ( DIRECTIVE );
 2105             eof=processStringState ( DIRECTIVE );
 2106             openTag ( DIRECTIVE );
 2107             break;
 2108         case _EOF:
 2109             eof = true;
 2110             break;
 2111         default:
 2112             break;
 2113         }
 2114     } while ( !exitState && !eof );
 2115 
 2116     closeTag ( DIRECTIVE );
 2117     return eof;
 2118 }
 2119 
 2120 bool CodeGenerator::processStringState ( State oldState )
 2121 {
 2122     State newState=STANDARD;
 2123     bool eof=false, exitState=false;
 2124     bool returnedFromOtherState=false;
 2125 
 2126     State myState= ( oldState==DIRECTIVE ) ? DIRECTIVE_STRING : STRING;
 2127 
 2128     int openDelimID=currentSyntax->getOpenDelimiterID ( token, myState);
 2129     string openDelim=token;
 2130 
 2131     //Raw String by definition:
 2132     bool isRawString=currentSyntax->delimiterIsRawString(openDelimID) || toggleDynRawString;
 2133 
 2134     // Test if character before string open delimiter token equals to the
 2135     // raw string prefix (Example: r" ", r""" """ in Python)
 2136 
 2137     //Raw String Prefix:
 2138     if ( lineIndex>token.length() &&line[lineIndex-token.length()-1]==currentSyntax->getRawStringPrefix() ) {
 2139         isRawString=true;
 2140     }
 2141 
 2142     openTag ( myState );
 2143     do {
 2144         // true if last token was an escape char
 2145         if ( !returnedFromOtherState ) {
 2146             printMaskedToken (newState!=_WS );
 2147         }
 2148         returnedFromOtherState=false;
 2149         newState= getCurrentState(myState);
 2150 
 2151         switch ( newState ) {
 2152         case _WS:
 2153             processWsState();
 2154             break;
 2155         case _EOL:
 2156             wsBuffer += closeTags[myState];
 2157             insertLineNumber();
 2158             wsBuffer += openTags[myState];
 2159             break;
 2160         case STRING_END:
 2161             if (resultOfHook || currentSyntax->matchesOpenDelimiter (token,  STRING_END, openDelimID)) {
 2162                 if (currentSyntax->assertDelimEqualLength()) {
 2163                     exitState= openDelim.length()==token.length();
 2164                 } else {
 2165                     exitState= true;
 2166                 }
 2167                 printMaskedToken();
 2168             }
 2169             break;
 2170         case STRING:
 2171             // if there exist multiple string delimiters, close string if
 2172             // current delimiter is equal to the opening delimiter
 2173             exitState=currentSyntax->delimiterIsDistinct(currentSyntax->getOpenDelimiterID ( token, STRING  ))&&token==openDelim;
 2174             printMaskedToken();
 2175             break;
 2176         case ESC_CHAR:
 2177             if ( !isRawString ) {
 2178                 closeTag ( myState );
 2179                 eof=processEscapeCharState();
 2180                 openTag ( myState );
 2181                 returnedFromOtherState=true;
 2182             } else {
 2183                 // FIXME not a fix for Python r"""\"""
 2184                 exitState=token.size()>1 && token[1] == openDelim[0];
 2185                 printMaskedToken();
 2186             }
 2187             break;
 2188         case STRING_INTERPOLATION:
 2189             closeTag ( myState );
 2190             eof=processInterpolationState();
 2191             openTag ( myState );
 2192             returnedFromOtherState=true;
 2193             break;
 2194     
 2195         case _EOF:
 2196             eof = true;
 2197             break;
 2198         default:
 2199             printMaskedToken();
 2200             break;
 2201         }
 2202     } while ( !exitState && !eof );
 2203 
 2204     closeTag ( myState );
 2205     
 2206     toggleDynRawString = false;
 2207     
 2208     return eof;
 2209 }
 2210 
 2211 bool CodeGenerator::processSymbolState()
 2212 {
 2213 
 2214     State newState=STANDARD;
 2215     bool eof=false,
 2216          exitState=false;
 2217 
 2218     openTag ( SYMBOL );
 2219     do {
 2220         printMaskedToken ( newState!=_WS );
 2221         newState= getCurrentState(SYMBOL);
 2222         switch ( newState ) {
 2223         case _WS:
 2224             processWsState();
 2225             exitState=isolateTags;
 2226             break;
 2227         case _EOL:
 2228             insertLineNumber();
 2229             exitState=true;
 2230             break;
 2231         case _EOF:
 2232             eof = true;
 2233             break;
 2234         default:
 2235             exitState=newState!=SYMBOL;
 2236             break;
 2237         }
 2238     } while ( !exitState && !eof );
 2239 
 2240     closeTag ( SYMBOL );
 2241     return eof;
 2242 }
 2243 
 2244 bool CodeGenerator::processEscapeCharState()
 2245 {
 2246     State newState=STANDARD;
 2247     bool eof=false, exitState=false;
 2248     openTag ( ESC_CHAR );
 2249     do {
 2250         printMaskedToken (newState!=_WS );
 2251         newState= getCurrentState(ESC_CHAR);
 2252         switch ( newState ) {
 2253         case _EOL:
 2254             insertLineNumber();
 2255             exitState=true;
 2256             break;
 2257         case _WS:
 2258             processWsState();
 2259             exitState=isolateTags;
 2260             break;
 2261         case _EOF:
 2262             eof = true;
 2263             break;
 2264         default:
 2265             exitState=newState!=ESC_CHAR;
 2266             break;
 2267         }
 2268     } while ( !exitState && !eof );
 2269 
 2270     closeTag ( ESC_CHAR );
 2271     return eof;
 2272 }
 2273 
 2274 bool CodeGenerator::processInterpolationState()
 2275 {
 2276     State newState=STANDARD;
 2277     bool eof=false, exitState=false;
 2278     openTag ( STRING_INTERPOLATION );
 2279     do {
 2280         printMaskedToken (newState!=_WS );
 2281         newState= getCurrentState(STRING_INTERPOLATION);
 2282         switch ( newState ) {
 2283         case _EOL:
 2284             insertLineNumber();
 2285             exitState=true;
 2286             break;
 2287         case _WS:
 2288             processWsState();
 2289             exitState=isolateTags;
 2290             break;
 2291         case _EOF:
 2292             eof = true;
 2293             break;
 2294         default:
 2295             exitState=newState!=STRING_INTERPOLATION;
 2296             break;
 2297         }
 2298     } while ( !exitState && !eof );
 2299 
 2300     closeTag ( STRING_INTERPOLATION );
 2301     return eof;
 2302 }
 2303 
 2304 void CodeGenerator::processWsState()
 2305 {
 2306     if ( !maskWs ) {
 2307         wsBuffer += token;
 2308         token.clear();
 2309         return;
 2310     }
 2311 
 2312     flushWs(6);
 2313 
 2314     int cntWs=0;
 2315     lineIndex--;
 2316     PositionState ps(currentState, 0, true);
 2317             
 2318     while ( line[lineIndex]==' ' || line[lineIndex]=='\t' ) {
 2319         ++cntWs;
 2320         ++lineIndex;
 2321     }
 2322     if ( cntWs>1 ) {
 2323 
 2324         unsigned int styleID=getStyleID ( currentState, currentKeywordClass );
 2325         if ( excludeWs && styleID!=_UNKNOWN ) {
 2326             *out << closeTags[styleID];
 2327         }
 2328         *out << maskWsBegin ;
 2329         for ( int i=0; i<cntWs; i++ ) {
 2330             *out <<  spacer;
 2331             if (applySyntaxTestCase){
 2332                 stateTraceCurrent.push_back(ps);
 2333             }
 2334         }
 2335         *out << maskWsEnd;
 2336         if ( excludeWs && styleID!=_UNKNOWN ) {
 2337             *out << openTags[styleID];
 2338         }
 2339     } else {
 2340     
 2341         *out << spacer; //Bugfix fehlender Space nach Strings
 2342         if (applySyntaxTestCase){
 2343             stateTraceCurrent.push_back(ps);            
 2344         }
 2345     }
 2346     token.clear();
 2347 }
 2348 
 2349 void CodeGenerator::flushWs(int arg)
 2350 {
 2351     PositionState ps(currentState, 0, true);
 2352     //workaround condition
 2353     for ( size_t i=0; i<wsBuffer.size() && ((arg > 3) || ( (arg<4) && lineIndex>1)) && applySyntaxTestCase ; i++ ) {
 2354         stateTraceCurrent.push_back(ps);
 2355         //std::cerr <<"\nflush >"<<wsBuffer<<"< arg:"<<arg;           
 2356     }
 2357      
 2358     //fix canvas whitespace
 2359     if (wsBuffer.length() && (outputType==ESC_XTERM256 || outputType==ESC_TRUECOLOR) ){
 2360         *out<< maskWsBegin;
 2361     }
 2362      
 2363     *out<<wsBuffer;
 2364     wsBuffer.clear();
 2365 }
 2366 
 2367 string CodeGenerator::getTestcaseName(State s, unsigned int kwClass) {
 2368     switch (s) {
 2369         
 2370         case STANDARD:
 2371             return STY_NAME_STD;
 2372         case STRING:
 2373             return STY_NAME_STR;
 2374         case NUMBER:
 2375             return STY_NAME_NUM;
 2376         case SL_COMMENT:
 2377             return STY_NAME_SLC;
 2378         case ML_COMMENT:
 2379             return STY_NAME_COM;
 2380         case ESC_CHAR:
 2381             return STY_NAME_ESC;
 2382         case DIRECTIVE:
 2383             return STY_NAME_DIR;
 2384         case DIRECTIVE_STRING:
 2385             return STY_NAME_DST;
 2386         case SYMBOL:
 2387             return STY_NAME_SYM;
 2388         case STRING_INTERPOLATION:
 2389             return STY_NAME_IPL;
 2390         case _WS:
 2391             return "ws";
 2392         case KEYWORD: {
 2393             
 2394             if (!kwClass)
 2395                 return "ws";
 2396             
 2397             char kwName[5] = {0};
 2398             snprintf(kwName, sizeof(kwName), "kw%c", ('a'+kwClass-1));
 2399             return string(kwName);
 2400         }
 2401         default:
 2402             return "unknown_test";
 2403     }
 2404 }
 2405 
 2406 void CodeGenerator::printTrace(const string &s){
 2407     std::cout<<"\n curr "<<lineNumber<<" "<<s<<": ";
 2408     for (unsigned int i=0; i< stateTraceCurrent.size(); i++) {
 2409         std::cout<<" "<<stateTraceCurrent[i].state;
 2410     }
 2411     std::cout<<"\n test "<<lineNumber<<" "<<s<<": ";
 2412     for (unsigned int i=0; i< stateTraceTest.size(); i++) {
 2413         std::cout<<" "<<stateTraceTest[i].state;
 2414     }
 2415     /*
 2416     for (unsigned int i=0; i< stateTrace.size(); i++) {
 2417         std::cout<<" "<<stateTrace[i];
 2418     }
 2419    */
 2420     std::cout<<"\n";
 2421 }
 2422 
 2423 //column: lineIndex (not a UTF-8 validated string position)
 2424 void CodeGenerator::runSyntaxTestcases(unsigned int column){
 2425     
 2426     if (encoding=="utf-8")
 2427         column = StringTools::utf8_strlen(line.substr(0, column));
 2428     
 2429     unsigned int assertGroup=0;
 2430     size_t typeDescPos=line.find_first_not_of("\t ^", lineIndex);
 2431     State assertState=_UNKNOWN;
 2432     bool negation=false;
 2433     bool testFailed=false;
 2434    
 2435     ostringstream errMsg;
 2436     string prefix;
 2437     //printTrace("trace 2");
 2438     
 2439     if (typeDescPos!=string::npos) {
 2440     
 2441         if (line[typeDescPos]=='~') {
 2442         
 2443             negation=true;
 2444             prefix="~";
 2445             ++typeDescPos;
 2446         }
 2447         
 2448         if (line.find(STY_NAME_NUM, typeDescPos)==typeDescPos)
 2449             assertState=NUMBER;
 2450         else if (line.find(STY_NAME_STR, typeDescPos)==typeDescPos)
 2451             assertState=STRING;
 2452         else if (line.find(STY_NAME_ESC, typeDescPos)==typeDescPos)
 2453             assertState=ESC_CHAR;
 2454         else if (line.find(STY_NAME_IPL, typeDescPos)==typeDescPos)
 2455             assertState=STRING_INTERPOLATION;
 2456         else if (line.find(STY_NAME_SYM, typeDescPos)==typeDescPos)
 2457             assertState=SYMBOL;
 2458         else if (line.find(STY_NAME_DIR, typeDescPos)==typeDescPos)
 2459             assertState=DIRECTIVE;
 2460         else if (line.find(STY_NAME_SLC, typeDescPos)==typeDescPos)
 2461             assertState=SL_COMMENT;
 2462         else if (line.find(STY_NAME_COM, typeDescPos)==typeDescPos)
 2463             assertState=ML_COMMENT;
 2464         else if (line.find("ws", typeDescPos)==typeDescPos)
 2465             assertState=_WS;
 2466         else if (line.find(STY_NAME_STD, typeDescPos)==typeDescPos)
 2467             assertState=STANDARD;
 2468         else if (line.find(STY_NAME_DST, typeDescPos)==typeDescPos)
 2469             assertState=DIRECTIVE_STRING;
 2470         
 2471         else if (line.find("kw", typeDescPos)==typeDescPos) {
 2472             assertState=KEYWORD;
 2473             if (isalpha(line[typeDescPos+2]))
 2474                 assertGroup=line[typeDescPos+2] - 'a' +1;
 2475         }
 2476     
 2477        if (   (assertState!=_WS && stateTraceTest[column].state != assertState && !stateTraceTest[column].isWhiteSpace )
 2478             || (assertState==_WS && !stateTraceTest[column].isWhiteSpace)
 2479             || assertGroup != stateTraceTest[column].kwClass) {
 2480             
 2481             testFailed=!negation;
 2482        
 2483         } else if (negation ) {
 2484             
 2485             //TODO Fix ~ws 
 2486             if (assertState!=_WS  && !stateTraceTest[column].isWhiteSpace )
 2487                 testFailed=true;
 2488             
 2489         }
 2490 
 2491         if (testFailed) {
 2492             errMsg << inFile << " line " << lineNumber << ", column "<< column 
 2493                     << ": got " << getTestcaseName(stateTraceTest[column].state, stateTraceTest[column].kwClass)  
 2494                     << " instead of " << prefix << getTestcaseName(assertState, assertGroup);
 2495         
 2496             failedPosTests.push_back(errMsg.str());    
 2497         }
 2498         
 2499     }
 2500     
 2501     lineContainedTestCase=true; 
 2502 }
 2503 
 2504 
 2505 string CodeGenerator::getNewLine()
 2506 {
 2507     return (printNewLines) ? newLineTag : "";
 2508 }
 2509 
 2510 Diluculum::LuaValueList CodeGenerator::callDecorateLineFct(bool isLineStart)
 2511 {
 2512 
 2513     Diluculum::LuaValueList params;
 2514     params.push_back(Diluculum::LuaValue(lineNumber));
 2515 
 2516     return currentSyntax->getLuaState()->call ( isLineStart ?
 2517             *currentSyntax->getDecorateLineBeginFct(): *currentSyntax->getDecorateLineEndFct(),
 2518             params,"getDecorateLineFct call")  ;
 2519 
 2520 }
 2521 
 2522 void CodeGenerator::insertLineNumber ( bool insertNewLine )
 2523 {
 2524     if ( insertNewLine ) {
 2525         if (currentSyntax->getDecorateLineEndFct()) {
 2526             Diluculum::LuaValueList res=callDecorateLineFct(false);
 2527             if (res.size()==1) {
 2528                 wsBuffer +=res[0].asString();
 2529             }
 2530         }
 2531 
 2532         wsBuffer += getNewLine();
 2533     }
 2534 
 2535     if (currentSyntax->getDecorateLineBeginFct()) {
 2536         Diluculum::LuaValueList res=callDecorateLineFct(true);
 2537         if (res.size()==1) {
 2538             wsBuffer +=res[0].asString();
 2539         }
 2540     }
 2541 
 2542     if ( showLineNumbers ) {
 2543         ostringstream os;
 2544         ostringstream numberPrefix;
 2545 
 2546         os << setw ( getLineNumberWidth() ) << right;
 2547         if( numberCurrentLine ) {
 2548             if ( lineNumberFillZeroes ) {
 2549                 os.fill ( '0' );
 2550             }
 2551             os << lineNumber+lineNumberOffset;
 2552         } else {
 2553             os << "";
 2554         }
 2555 
 2556         numberPrefix << openTags[LINENUMBER];
 2557         maskString ( numberPrefix, os.str() );
 2558         numberPrefix << spacer << closeTags[LINENUMBER];
 2559 
 2560         wsBuffer += numberPrefix.str();
 2561     }
 2562 }
 2563 
 2564 unsigned int CodeGenerator::getLineIndex()
 2565 {
 2566     return lineIndex;
 2567 }
 2568 unsigned int CodeGenerator::getLastLineLength()
 2569 {
 2570     return lastLineLength;
 2571 }
 2572 
 2573 bool CodeGenerator::requiresTwoPassParsing() const {
 2574     if (!currentSyntax) return false;
 2575     return currentSyntax->getPersistentSnippetsNum()>0;
 2576 }
 2577 
 2578 
 2579 bool CodeGenerator::printExternalStyle ( const string &outFile )
 2580 {
 2581     if ( !includeStyleDef ) {
 2582         ostream *cssOutFile = ( outFile.empty() ? &cout :new ofstream ( outFile.c_str() ) );
 2583         if ( !cssOutFile->fail() ) {
 2584             if (!omitVersionComment) {
 2585                 *cssOutFile << styleCommentOpen
 2586                             <<" Style definition file generated by highlight "
 2587                             << HIGHLIGHT_VERSION << ", " << HIGHLIGHT_URL
 2588                             << " " << styleCommentClose << "\n";
 2589             }
 2590             *cssOutFile << getStyleDefinition()
 2591                         << "\n";
 2592             *cssOutFile << readUserStyleDef();
 2593             if ( !outFile.empty() ) delete cssOutFile;
 2594         } else {
 2595             return false;
 2596         }
 2597     }
 2598     return true;
 2599 }
 2600 
 2601 bool CodeGenerator::printPersistentState ( const string &outFile )
 2602 {
 2603     if (!currentSyntax) return false;
 2604 
 2605     ofstream pluginOutFile( outFile.c_str());
 2606     if ( !pluginOutFile.fail() ) {
 2607         
 2608         pluginOutFile   <<"Description=\"Plugin generated by highlight using the --two-pass option\"\n\n"
 2609                         <<"Categories = {\"two-pass\" }\n\n"
 2610                         <<"function syntaxUpdate(desc)\n\n";
 2611                         
 2612         pluginOutFile << currentSyntax->getPersistentHookConditions();
 2613      
 2614         for (auto snippet: currentSyntax->getPersistentSnippets())
 2615         {   
 2616             pluginOutFile << snippet <<"\n\n";
 2617         }
 2618         
 2619         pluginOutFile<<"end\n\n"
 2620                      <<"Plugins={\n"
 2621                      <<"  { Type=\"lang\", Chunk=syntaxUpdate }\n"
 2622                      <<"}\n";
 2623     } else {
 2624         return false;
 2625     }
 2626     
 2627     return true;
 2628 }
 2629 
 2630 string CodeGenerator::readUserStyleDef()
 2631 {
 2632     ostringstream ostr;
 2633     if ( !styleInputPath.empty() ) {
 2634         ifstream userStyleDef ( styleInputPath.c_str() );
 2635         if ( userStyleDef ) {
 2636             ostr    << "\n" << styleCommentOpen
 2637                     << " Content of " << styleInputPath
 2638                     << ": " <<styleCommentClose << "\n";
 2639             string line;
 2640             while ( getline ( userStyleDef, line ) ) {
 2641                 ostr << line << "\n";
 2642             }
 2643             userStyleDef.close();
 2644         } else {
 2645             ostr    << styleCommentOpen
 2646                     << " ERROR: Could not include " << styleInputPath
 2647                     << "." << styleCommentClose << "\n";
 2648         }
 2649     }
 2650 
 2651     string injections=docStyle.getInjections();
 2652     if (!injections.empty()) {
 2653         ostr    << "\n" << styleCommentOpen
 2654                 << " Plug-in theme injections: " <<styleCommentClose << "\n";
 2655         ostr << injections<<"\n";
 2656     }
 2657     return ostr.str();
 2658 }
 2659 
 2660 bool CodeGenerator::initPluginScript(const string& script)
 2661 {
 2662 
 2663     if (script.empty()) return true;
 2664 
 2665     try {
 2666 
 2667         userScriptError="";
 2668         Diluculum::LuaState ls;
 2669         
 2670         ls.doFile (script);
 2671         int listIdx=1;
 2672 
 2673         while (ls["Plugins"][listIdx].value() !=Diluculum::Nil) {
 2674 
 2675             // Theme plugins
 2676             if (ls["Plugins"][listIdx]["Type"].value().asString()=="theme") {
 2677                 if (ls["Plugins"][listIdx]["Chunk"].value().type()==LUA_TFUNCTION) {
 2678                     docStyle.addUserChunk(ls["Plugins"][listIdx]["Chunk"].value().asFunction());
 2679                 }
 2680             }
 2681             // Syntax plugins
 2682             else if (ls["Plugins"][listIdx]["Type"].value().asString()=="lang") {
 2683                 if (ls["Plugins"][listIdx]["Chunk"].value().type()==LUA_TFUNCTION) {
 2684                     currentSyntax->addUserChunk(ls["Plugins"][listIdx]["Chunk"].value().asFunction());
 2685                 }
 2686             }
 2687             // Format plugins
 2688             else if (ls["Plugins"][listIdx]["Type"].value().asString()=="format") {
 2689                 if (ls["Plugins"][listIdx]["Chunk"].value().type()==LUA_TFUNCTION) {
 2690                     addUserChunk(ls["Plugins"][listIdx]["Chunk"].value().asFunction());
 2691                 }
 2692             }
 2693             
 2694             listIdx++;
 2695         }
 2696     }  catch (Diluculum::LuaError &err) {
 2697         userScriptError=err.what();
 2698         return false;
 2699     }
 2700     return true;
 2701 }
 2702 
 2703 void CodeGenerator::resetSyntaxReaders() {
 2704     for ( map<string, SyntaxReader*>::iterator it=syntaxReaders.begin(); it!=syntaxReaders.end(); it++ ) {
 2705         delete it->second;
 2706     }
 2707     currentSyntax=NULL;
 2708     syntaxReaders.clear();
 2709 }
 2710 
 2711 bool CodeGenerator::syntaxRequiresTwoPassRun() {
 2712     if (!currentSyntax) return false;
 2713     return currentSyntax->requiresTwoPassRun();
 2714 }
 2715 
 2716 void CodeGenerator::clearPersistentSnippets(){
 2717     if (currentSyntax) {
 2718         currentSyntax->clearPersistentSnippets();
 2719     }
 2720 }
 2721 
 2722 
 2723 }