"Fossies" - the Fresh Open Source Software Archive

Member "highlight-3.57-x64/src/core/codegenerator.cpp" (12 May 2020, 83224 Bytes) of package /windows/www/highlight-3.57-x64.zip:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) C and C++ source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file. See also the last Fossies "Diffs" side-by-side code changes report for "codegenerator.cpp": 3.56_vs_3.57.

    1 /***************************************************************************
    2                           codegenerator.cpp  -  description
    3                              -------------------
    4     begin                : Die Jul 9 2002
    5     copyright            : (C) 2002-2020 by Andre Simon
    6     email                : a.simon@mailbox.org
    7  ***************************************************************************/
    8 
    9 
   10 /*
   11 This file is part of Highlight.
   12 
   13 Highlight is free software: you can redistribute it and/or modify
   14 it under the terms of the GNU General Public License as published by
   15 the Free Software Foundation, either version 3 of the License, or
   16 (at your option) any later version.
   17 
   18 Highlight is distributed in the hope that it will be useful,
   19 but WITHOUT ANY WARRANTY; without even the implied warranty of
   20 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
   21 GNU General Public License for more details.
   22 
   23 You should have received a copy of the GNU General Public License
   24 along with Highlight.  If not, see <http://www.gnu.org/licenses/>.
   25 */
   26 
   27 
   28 #include <climits>
   29 #include <memory>
   30 #include <boost/xpressive/xpressive_dynamic.hpp>
   31 
   32 #include "codegenerator.h"
   33 
   34 #include "htmlgenerator.h"
   35 #include "xhtmlgenerator.h"
   36 #include "rtfgenerator.h"
   37 #include "latexgenerator.h"
   38 #include "texgenerator.h"
   39 #include "svggenerator.h"
   40 #include "bbcodegenerator.h"
   41 #include "pangogenerator.h"
   42 #include "odtgenerator.h"
   43 #include "astyle/astyle.h"
   44 #include "astyle/ASStreamIterator.h"
   45 
   46 #if !defined (QT)
   47 #include "ansigenerator.h"
   48 #include "xterm256generator.h"
   49 #endif
   50 
   51 namespace highlight
   52 {
   53 const unsigned int CodeGenerator::NUMBER_BUILTIN_STATES = highlight::KEYWORD;
   54 
   55 const string CodeGenerator::STY_NAME_STD="std";
   56 const string CodeGenerator::STY_NAME_STR="str";
   57 const string CodeGenerator::STY_NAME_NUM="num";
   58 const string CodeGenerator::STY_NAME_SLC="slc";
   59 const string CodeGenerator::STY_NAME_COM="com";
   60 const string CodeGenerator::STY_NAME_ESC="esc";
   61 const string CodeGenerator::STY_NAME_DIR="ppc"; //preprocessor
   62 const string CodeGenerator::STY_NAME_DST="pps"; //preprocessor string
   63 const string CodeGenerator::STY_NAME_LIN="lin";
   64 const string CodeGenerator::STY_NAME_SYM="opt"; //operator
   65 const string CodeGenerator::STY_NAME_IPL="ipl"; //interpolation
   66 
   67 vector<Diluculum::LuaFunction*> CodeGenerator::pluginChunks;
   68 
   69 
   70 CodeGenerator * CodeGenerator::getInstance ( OutputType type )
   71 {
   72     CodeGenerator* generator=NULL;
   73     switch ( type ) {
   74     case HTML:
   75         generator = new HtmlGenerator();
   76         break;
   77     case XHTML:
   78         generator = new XHtmlGenerator();
   79         break;
   80     case TEX:
   81         generator = new TexGenerator ();
   82         break;
   83     case LATEX:
   84         generator = new LatexGenerator();
   85         break;
   86     case RTF:
   87         generator = new RtfGenerator ();
   88         break;
   89     case SVG:
   90         generator = new SVGGenerator();
   91         break;
   92     case BBCODE:
   93         generator = new BBCodeGenerator();
   94         break;
   95     case PANGO:
   96         generator = new PangoGenerator();
   97         break;
   98     case ODTFLAT:
   99         generator = new ODTGenerator();
  100         break;
  101     case ESC_ANSI:
  102         generator = new AnsiGenerator();
  103         break;
  104     case ESC_XTERM256:
  105     case ESC_TRUECOLOR:
  106         generator = new Xterm256Generator();
  107         generator->setESCTrueColor(type==ESC_TRUECOLOR);
  108         break;
  109     default:
  110         break;
  111     }
  112     return generator;
  113 }
  114 
  115 
  116 CodeGenerator::CodeGenerator ( highlight::OutputType type )
  117     :currentSyntax(NULL),
  118      in ( NULL ),
  119      out ( NULL ),
  120      encoding ( "none" ),
  121      docTitle ( "Source file" ),
  122      maskWs ( false ),
  123      excludeWs ( false ),
  124      fragmentOutput ( false ),
  125      keepInjections( false ),
  126      showLineNumbers ( false ),
  127      lineNumberFillZeroes ( false ),
  128      printNewLines(true),
  129      omitVersionComment(false),
  130      isolateTags(false),
  131      disableStyleCache(false),
  132      baseFontSize("10"),
  133      lineNumber ( 0 ),
  134      lineNumberOffset ( 0 ),
  135      currentState ( _UNKNOWN ),
  136      currentKeywordClass ( 0 ),
  137      includeStyleDef ( false ),
  138      numberCurrentLine ( false ),
  139      lineIndex ( 0 ),
  140      lastLineLength( 0 ),
  141      syntaxChangeIndex(UINT_MAX),
  142      syntaxChangeLineNo(UINT_MAX),
  143      lineNumberWidth ( 5 ),
  144      startLineCnt( 1 ),
  145      startLineCntCurFile( 1 ),
  146      maxLineCnt ( UINT_MAX ),
  147      inputFilesCnt (0),
  148      processedFilesCnt (0),
  149      noTrailingNewLine(0),
  150 
  151      terminatingChar ( '\0' ),
  152      formatter ( NULL ),
  153      formattingEnabled ( false ),
  154      formattingPossible ( false ),
  155      validateInput ( false ),
  156      numberWrappedLines ( true ),
  157      resultOfHook(false),
  158      lineContainedTestCase(false),
  159      applySyntaxTestCase(false),
  160      toggleDynRawString(false),
  161      
  162      keywordCase ( StringTools::CASE_UNCHANGED ),
  163      eolDelimiter ('\n'),
  164      outputType ( type )
  165 {
  166 }
  167 
  168 
  169 CodeGenerator::~CodeGenerator()
  170 {
  171     delete formatter;
  172 
  173     resetSyntaxReaders();
  174     
  175     for (unsigned int i=0; i<pluginChunks.size(); i++) {
  176         delete pluginChunks[i];
  177     }
  178     pluginChunks.clear();
  179 }
  180 
  181 
  182 bool CodeGenerator::initTheme ( const string& themePath )
  183 {
  184     this->themePath=themePath;
  185     bool loadOK = docStyle.load ( themePath, outputType );
  186     initOutputTags();
  187     return loadOK;
  188 }
  189 
  190 const string& CodeGenerator::getStyleName()
  191 {
  192     return themePath;
  193 }
  194 
  195 void CodeGenerator::setLineNumberWidth ( int w )
  196 {
  197     lineNumberWidth=w;
  198 }
  199 
  200 int CodeGenerator::getLineNumberWidth()
  201 {
  202     return lineNumberWidth;
  203 }
  204 
  205 void CodeGenerator::setPrintLineNumbers ( bool flag, unsigned int startCnt )
  206 {
  207     showLineNumbers=flag;
  208     lineNumberOffset = startCnt-1;
  209 }
  210 
  211 bool CodeGenerator::getPrintLineNumbers()
  212 {
  213     return showLineNumbers;
  214 }
  215 
  216 void CodeGenerator::setPrintZeroes ( bool flag )
  217 {
  218     lineNumberFillZeroes=flag;
  219 }
  220 
  221 bool CodeGenerator::getPrintZeroes()
  222 {
  223     return lineNumberFillZeroes;
  224 }
  225 
  226 void CodeGenerator::setIncludeStyle ( bool flag )
  227 {
  228     includeStyleDef = flag;
  229 }
  230 
  231 void CodeGenerator::disableTrailingNL ( int flag )
  232 {
  233     noTrailingNewLine = flag;
  234 }
  235 
  236 void CodeGenerator::setStyleInputPath ( const string& path )
  237 {
  238     styleInputPath = path;
  239 }
  240 
  241 void CodeGenerator::setStyleOutputPath ( const string& path )
  242 {
  243     styleOutputPath = path;
  244 }
  245 
  246 void CodeGenerator::setPluginParameter ( const string& param )
  247 {
  248     pluginParameter = param;
  249 }
  250 
  251 const string&  CodeGenerator::getStyleInputPath()
  252 {
  253     return styleInputPath;
  254 }
  255 
  256 const string&  CodeGenerator::getStyleOutputPath()
  257 {
  258     return styleOutputPath;
  259 }
  260 
  261 void CodeGenerator::setFragmentCode ( bool flag )
  262 {
  263     fragmentOutput=flag;
  264 }
  265 
  266 bool CodeGenerator::getFragmentCode()
  267 {
  268     return fragmentOutput;
  269 }
  270 void CodeGenerator::setKeepInjections ( bool flag )
  271 {
  272     keepInjections=flag;
  273 }
  274 
  275 bool CodeGenerator::getKeepInjections()
  276 {
  277     return keepInjections;
  278 }
  279 void CodeGenerator::setValidateInput ( bool flag )
  280 {
  281     validateInput=flag;
  282 }
  283 
  284 bool CodeGenerator::getValidateInput()
  285 {
  286     return validateInput;
  287 }
  288 
  289 
  290 void CodeGenerator::setNumberWrappedLines ( bool flag )
  291 {
  292     numberWrappedLines=flag;
  293 }
  294 
  295 bool CodeGenerator::getNumberWrappedLines()
  296 {
  297     return numberWrappedLines;
  298 }
  299 
  300 void CodeGenerator::setOmitVersionComment ( bool flag )
  301 {
  302     omitVersionComment=flag;
  303 }
  304 
  305 bool CodeGenerator::getOmitVersionComment ()
  306 {
  307     return omitVersionComment;
  308 }
  309 
  310 void CodeGenerator::setIsolateTags ( bool flag )
  311 {
  312     isolateTags=flag;
  313 }
  314 
  315 bool CodeGenerator::getIsolateTags ()
  316 {
  317     return isolateTags;
  318 }
  319 
  320 void CodeGenerator::setBaseFont ( const string& fontName )
  321 {
  322     baseFont = fontName;
  323 }
  324 
  325 void CodeGenerator::setBaseFontSize ( const string& fontSize)
  326 {
  327     baseFontSize = fontSize;
  328 }
  329 
  330 void CodeGenerator::setStartingNestedLang(const string &langName)
  331 {
  332     embedLangStart = langName;
  333 }
  334 
  335 const string CodeGenerator::getBaseFont() const
  336 {
  337     if ( !baseFont.empty() ) return baseFont;
  338     switch ( outputType ) {
  339     case HTML:
  340     case XHTML:
  341     case SVG:
  342         return "'Courier New',monospace";
  343         break;
  344     case LATEX:
  345         return "ttfamily";
  346         break;
  347     case TEX:
  348         return "tt";
  349         break;
  350     default:
  351         return "Courier New";
  352     }
  353 }
  354 
  355 const string CodeGenerator::getBaseFontSize()
  356 {
  357     return baseFontSize;
  358 }
  359 
  360 void CodeGenerator::setTitle ( const string & title )
  361 {
  362     if ( !title.empty() ) docTitle= title;
  363 }
  364 
  365 string CodeGenerator::getTitle()
  366 {
  367     return docTitle;
  368 }
  369 
  370 void CodeGenerator::setEncoding ( const string& encodingName )
  371 {
  372     encoding = encodingName;
  373 }
  374 
  375 bool CodeGenerator::formattingDisabled()
  376 {
  377     return !formattingEnabled;
  378 }
  379 
  380 void CodeGenerator::setStartingInputLine ( unsigned int begin )
  381 {
  382     startLineCnt = startLineCntCurFile = begin;
  383 }
  384 
  385 void CodeGenerator::setMaxInputLineCnt ( unsigned int cnt )
  386 {
  387     maxLineCnt = cnt;
  388 }
  389 
  390 void CodeGenerator::setFilesCnt ( unsigned int cnt )
  391 {
  392     inputFilesCnt = cnt;
  393     processedFilesCnt = 0;
  394 }
  395 
  396 bool CodeGenerator::formattingIsPossible()
  397 {
  398     return formattingPossible;
  399 }
  400 
  401 void CodeGenerator::setPreformatting ( WrapMode lineWrappingStyle,
  402                                        unsigned int lineLength,
  403                                        int numberSpaces )
  404 {
  405     bool enableWrap = lineWrappingStyle!=WRAP_DISABLED;
  406     bool replaceTabs = numberSpaces > 0;
  407 
  408     if ( enableWrap || replaceTabs ) {
  409         preFormatter.setWrap ( enableWrap );
  410         preFormatter.setWrapIndentBraces ( lineWrappingStyle==WRAP_DEFAULT );
  411         preFormatter.setWrapLineLength ( lineLength );
  412         preFormatter.setReplaceTabs ( replaceTabs );
  413         preFormatter.setNumberSpaces ( numberSpaces );
  414     }
  415 }
  416 
  417 void CodeGenerator::setKeyWordCase ( StringTools::KeywordCase keyCase )
  418 {
  419     keywordCase = keyCase;
  420 }
  421 
  422 void CodeGenerator::setEOLDelimiter(char delim)
  423 {
  424     eolDelimiter = delim;
  425 }
  426 
  427 void CodeGenerator::reset()
  428 {
  429     lineIndex = 0;
  430     lineNumber = 0;
  431     line.clear();
  432     preFormatter.reset();
  433     inFile.clear();
  434     outFile.clear();
  435     embedLangDefPath.clear();
  436     printNewLines=true;
  437     syntaxChangeIndex = syntaxChangeLineNo = UINT_MAX;
  438     startLineCntCurFile = startLineCnt;
  439     applySyntaxTestCase=lineContainedTestCase=false;
  440     if (currentSyntax){
  441         vector<int> overrideStyleAttrs=currentSyntax->getOverrideStyleAttributes();
  442         docStyle.overrideAttributes(overrideStyleAttrs);
  443         if (overrideStyleAttrs.size())
  444             disableStyleCache = true;
  445     }
  446 }
  447 
  448 string CodeGenerator::getThemeInitError()
  449 {
  450     return docStyle.getErrorMessage();
  451 }
  452 
  453 string CodeGenerator::getPluginScriptError()
  454 {
  455     return userScriptError;
  456 }
  457 
  458 string CodeGenerator::getSyntaxRegexError()
  459 {
  460     return (currentSyntax)? currentSyntax->getFailedRegex(): "syntax undef";
  461 }
  462 string CodeGenerator::getSyntaxLuaError()
  463 {
  464     return (currentSyntax)? currentSyntax->getLuaErrorText(): "syntax undef";
  465 
  466 }
  467 string CodeGenerator::getSyntaxDescription()
  468 {
  469     return (currentSyntax)? currentSyntax->getDescription(): "syntax undef";
  470 
  471 }
  472 string CodeGenerator::getSyntaxEncodingHint()
  473 {
  474     return (currentSyntax)? currentSyntax->getEncodingHint(): "";
  475 
  476 }
  477 string CodeGenerator::getThemeDescription()
  478 {
  479     return docStyle.getDescription();
  480 }
  481 
  482 string CodeGenerator::getSyntaxCatDescription(){
  483     return (currentSyntax)? currentSyntax->getCategoryDescription(): "";
  484 }
  485 
  486 string CodeGenerator::getThemeCatDescription(){
  487     return docStyle.getCategoryDescription();
  488 }
  489 
  490 unsigned int CodeGenerator::getLineNumber()
  491 {
  492     return lineNumber;
  493 }
  494 
  495 bool CodeGenerator::readNewLine ( string &newLine )
  496 {
  497     bool eof=false;
  498     
  499     if ( lineIndex ) terminatingChar=newLine[lineIndex-1];
  500     
  501     while (!eof && startLineCntCurFile>0) {
  502         if ( formattingPossible && formattingEnabled ) {
  503             eof=!formatter->hasMoreLines();
  504             if ( !eof ) {
  505                 newLine = formatter->nextLine();
  506             }
  507         } else {
  508             eof = ! getline ( *in, newLine, eolDelimiter );
  509         }
  510         --startLineCntCurFile;
  511     }
  512     startLineCntCurFile=1;
  513 #ifndef _WIN32
  514     // drop CR of CRLF files
  515     if (!newLine.empty() && newLine[newLine.size() - 1] == '\r')
  516         newLine.erase(newLine.size() - 1);
  517 #endif
  518 
  519     return eof || ( lineNumber == maxLineCnt );
  520 }
  521 
  522 void CodeGenerator::matchRegex ( const string &line, State skipState)
  523 {
  524     regexGroups.clear();
  525     int matchBegin=0;
  526     int groupID=0;
  527 
  528     // cycle through all regex, save the start and ending indices of matches to report them later
  529     for ( unsigned int i=0; i<currentSyntax->getRegexElements().size(); i++ ) {
  530         RegexElement *regexElem = currentSyntax->getRegexElements() [i];
  531 
  532         if (regexElem->open == skipState) continue;
  533 
  534         if (regexElem->constraintLineNum && regexElem->constraintLineNum != lineNumber) {
  535             continue;
  536         }
  537         
  538         if (regexElem->constraintFilename.size() && regexElem->constraintFilename != inFile) {
  539             continue;
  540         }
  541 
  542         boost::xpressive::sregex_iterator cur( line.begin(), line.end(), regexElem->rex );
  543         boost::xpressive::sregex_iterator end;
  544 
  545         for( ; cur != end; ++cur )  {
  546             groupID = ( regexElem->capturingGroup<0 ) ? cur->size()-1 : regexElem->capturingGroup;
  547             matchBegin = cur->position(groupID);
  548         
  549             regexGroups.insert (
  550                 make_pair ( matchBegin + 1, ReGroup ( regexElem->open, cur->length(groupID), regexElem->kwClass, regexElem->langName ) ) );
  551 
  552             // priority regex (match required)
  553             if (regexElem->priority) {
  554                 return;
  555             }
  556         }
  557     }
  558 }
  559 
  560 unsigned char CodeGenerator::getInputChar()
  561 {
  562     // end of line?
  563     if ( lineIndex == line.length() ) {
  564         
  565         //more testing required:
  566         if (outputType==ESC_TRUECOLOR || outputType==ESC_XTERM256)
  567             lastLineLength=StringTools::utf8_strlen(line);
  568         
  569         bool eof=false;
  570         if ( preFormatter.isEnabled() ) {
  571             if ( !preFormatter.hasMoreLines() ) {
  572                 eof=readNewLine ( line );
  573                 preFormatter.setLine ( line );
  574                 ++lineNumber;
  575                 numberCurrentLine = true;
  576             } else {
  577                 if (numberWrappedLines)
  578                     ++lineNumber;
  579                 numberCurrentLine = numberWrappedLines;
  580             }
  581 
  582             line = preFormatter.getNextLine();
  583         } else {
  584             eof=readNewLine ( line );
  585             ++lineNumber;
  586 
  587             numberCurrentLine = true;
  588         }
  589         lineIndex=0;
  590         
  591         if (!lineContainedTestCase && applySyntaxTestCase){
  592             stateTraceTest = stateTraceCurrent;
  593             stateTraceCurrent.clear();
  594         } 
  595         
  596         lineContainedTestCase=false;
  597             
  598         matchRegex ( line );
  599         stateTrace.clear();
  600         return ( eof ) ?'\0':'\n';
  601     }
  602 
  603     return line[lineIndex++];
  604 }
  605 
  606 /** changing this method requires regression testing with nested syntax files (HTML+PHP+JS+CSS, Coffeescript with block regex, Pas + ASM) 
  607     especially nested syntax in one line
  608  */
  609 State CodeGenerator::getCurrentState (State oldState)
  610 {
  611     unsigned char c='\0';
  612 
  613     if ( token.length() ==0 ) {
  614         c=getInputChar();
  615     } else {
  616         lineIndex-= ( token.length()-1 );
  617         c=token[0];
  618     }
  619     if ( c=='\n' ) {
  620         return _EOL;   // End of line
  621     }
  622 
  623     if ( c=='\0' ) {
  624         return _EOF;   // End of file
  625     }
  626 
  627     if ( c==' ' || c=='\t' ) {
  628         token= c;
  629         return _WS;    // White space
  630     }
  631 
  632     if ( applySyntaxTestCase && ( c=='^' || c=='<') && (oldState == ML_COMMENT || oldState==SL_COMMENT)  ) {
  633         token= c;
  634         return _TESTPOS;
  635     }
  636         
  637     // at this position the syntax change takes place
  638     if (lineIndex >= syntaxChangeIndex-1 || syntaxChangeLineNo < lineNumber){
  639         loadEmbeddedLang(embedLangDefPath);  // load new syntax                     
  640         matchRegex(line);                    // recognize new patterns in the (remaining) line
  641         syntaxChangeIndex = syntaxChangeLineNo = UINT_MAX;
  642     }
  643 
  644 SKIP_EMBEDDED:
  645     
  646     // Test if a regular expression was found at the current position
  647     if ( !regexGroups.empty() ) {
  648         if ( regexGroups.count ( lineIndex ) ) {
  649             token = line.substr ( lineIndex-1, regexGroups[lineIndex].length );
  650 
  651             unsigned int oldIndex= lineIndex;
  652             if ( regexGroups[oldIndex].length>1 ) lineIndex+= regexGroups[oldIndex].length-1;
  653 
  654             if ( regexGroups[oldIndex].state==EMBEDDED_CODE_BEGIN ) {
  655                 //do not handle a nested section if the syntax is marked as "sealed" 
  656                 if (embedLangDefPath.length()==0 || currentSyntax->allowsInnerSection(embedLangDefPath) ) {
  657                     embedLangDefPath = currentSyntax->getNewPath(regexGroups[oldIndex].name);
  658                     //remember position 
  659                     syntaxChangeIndex = lineIndex+2;
  660                     syntaxChangeLineNo = lineNumber;
  661                 }
  662                 
  663                 // repeat parsing of this line without nested state recognition to highlight opening delimiter in the host syntax
  664                 matchRegex(line, EMBEDDED_CODE_BEGIN);
  665                 lineIndex = oldIndex;
  666                 goto SKIP_EMBEDDED; // this is how it should be done
  667             }
  668 
  669             if ( regexGroups[oldIndex].state==IDENTIFIER_BEGIN || regexGroups[oldIndex].state==KEYWORD ) {
  670                 string reservedWord= ( currentSyntax->isIgnoreCase() ) ? StringTools::change_case ( token ) :token;
  671                 currentKeywordClass=currentSyntax->getKeywordListGroup ( reservedWord ); //check in lists (no regex)
  672                 
  673                 if ( !currentKeywordClass && regexGroups[oldIndex].state==KEYWORD ){
  674                     currentKeywordClass = regexGroups[oldIndex].kwClass;
  675                 }
  676                 return validateState(( currentKeywordClass ) ? KEYWORD : STANDARD, oldState );
  677             } else {
  678                 return validateState(regexGroups[oldIndex].state, oldState);
  679             }
  680         }
  681     }
  682 
  683     // Character not referring to any state
  684     token = c;
  685     return STANDARD;
  686 }
  687 
  688 State CodeGenerator::validateState(State newState, State oldState)
  689 {
  690 
  691     if (currentSyntax->getValidateStateChangeFct()) {
  692         Diluculum::LuaValueList params;
  693         params.push_back(Diluculum::LuaValue(oldState));
  694         params.push_back(Diluculum::LuaValue(newState));
  695         params.push_back(Diluculum::LuaValue(token));
  696         params.push_back(Diluculum::LuaValue(getCurrentKeywordClassId()) );
  697         params.push_back(Diluculum::LuaValue(lineNumber) );
  698         params.push_back(Diluculum::LuaValue(lineIndex-(unsigned int)token.length()) );
  699 
  700         Diluculum::LuaValueList res=
  701             currentSyntax->getLuaState()->call ( *currentSyntax->getValidateStateChangeFct(),
  702                     params,"getValidateStateChangeFct call")  ;
  703 
  704         resultOfHook = res.size()>=1;
  705         if (resultOfHook) {
  706             
  707             // test balloon
  708             if (currentSyntax->requiresParamUpdate()) {
  709                 
  710                  if ( currentSyntax->getOverrideConfigVal("state.string.raw")=="true"){
  711                      toggleDynRawString=true; // reset to false in string state fct
  712                  }
  713                  if ( currentSyntax->getOverrideConfigVal("format.maskws")=="true") {
  714                      maskWs=true;
  715                  }
  716                  if ( currentSyntax->getOverrideConfigVal("format.spacer").size()) {
  717                      spacer=currentSyntax->getOverrideConfigVal("format.spacer");
  718                  }
  719             }
  720             
  721             State validatedState = (State)res[0].asInteger();
  722             if ( validatedState== _REJECT) {
  723                 // proceed using only the first character of the token
  724                 // TODO evaluate if token clear would be better
  725                 if (res.size()==1) { 
  726                     lineIndex -= (token.length() -1);
  727                     token=token.substr(0, 1);
  728                 }
  729                 
  730                 //experimental for slim.lang: evaluate second return arg after _REJECT
  731                 if (res.size()>=2) {
  732                     lineIndex -= (token.length() );
  733                     token.clear();
  734                     return (State)res[1].asInteger();
  735                 }
  736                 return oldState;
  737             }
  738             
  739             stateTrace.push_back(validatedState);
  740             if (stateTrace.size()>200) stateTrace.erase(stateTrace.begin(), stateTrace.begin() + 100 );
  741             return validatedState;
  742         }
  743     }
  744     resultOfHook  = false;
  745     stateTrace.push_back(newState);
  746     if (stateTrace.size()>200) stateTrace.erase(stateTrace.begin(), stateTrace.begin() + 100 );    
  747     return newState;
  748 }
  749 
  750 
  751 unsigned int CodeGenerator::getCurrentKeywordClassId(){
  752     unsigned int kwClassId=0;
  753 
  754     // this vector contains the defined keyword classes, and currentKeywordClass is its index:
  755     vector<string> kwClasses=currentSyntax->getKeywordClasses();
  756     if (currentKeywordClass && currentKeywordClass<=kwClasses.size()) {
  757         string kwClassName=kwClasses[currentKeywordClass-1];
  758         if (kwClassName.size()==3)
  759             kwClassId = kwClassName[2] - 'a' + 1;
  760     }
  761     return kwClassId;
  762 }
  763 
  764 //it is faster to pass ostream reference
  765 void CodeGenerator::maskString ( ostream& ss, const string & s )
  766 {
  767     for ( unsigned int i=0; i< s.length(); i++ ) {
  768         ss << maskCharacter ( s[i] );
  769     }
  770     
  771     // The test markers position should also be deternmined by calculating the code points
  772     if ( applySyntaxTestCase ) {
  773             
  774         PositionState ps(currentState, getCurrentKeywordClassId(), false);
  775         
  776         int slen = encoding=="utf-8" ? StringTools::utf8_strlen(s) : s.length();  
  777         for (int i=0; i< slen; i++ ) {
  778             stateTraceCurrent.push_back(ps);
  779         }
  780         if (stateTraceCurrent.size()>200) 
  781             stateTraceCurrent.erase(stateTraceCurrent.begin(), stateTraceCurrent.begin() + 100 ); 
  782     }
  783 }
  784 
  785 
  786 Diluculum::LuaValueList CodeGenerator::callDecorateFct(const string&token)
  787 {
  788     Diluculum::LuaValueList params;
  789     params.push_back(Diluculum::LuaValue(token));
  790     params.push_back(Diluculum::LuaValue(currentState));
  791     params.push_back(Diluculum::LuaValue(currentKeywordClass));
  792     string trace(";");
  793     if (stateTrace.size()>1){
  794         for (size_t i=0; i<stateTrace.size()-1;i++){
  795             trace += std::to_string (stateTrace[i]);
  796             trace += ";";
  797         }
  798     }
  799     
  800     //std::cerr <<"TRC1: "<<trace<<"\n";
  801     
  802     params.push_back(Diluculum::LuaValue(trace));
  803 
  804     return currentSyntax->getLuaState()->call ( *currentSyntax->getDecorateFct(),
  805             params,"getDecorateFct call")  ;
  806 }
  807 
  808 void CodeGenerator::printMaskedToken (bool flushWhiteSpace, StringTools::KeywordCase tcase )
  809 {
  810     if ( flushWhiteSpace )
  811         flushWs(1);
  812     string caseToken = StringTools::change_case ( token, tcase );
  813     if (currentSyntax->getDecorateFct()) {
  814 
  815         Diluculum::LuaValueList res=callDecorateFct(caseToken);
  816         if (res.size()==1) {
  817             *out<<res[0].asString();
  818         } else {
  819             maskString ( *out, caseToken );
  820         }
  821     } else {
  822         maskString ( *out, caseToken );
  823     }
  824 
  825     token.clear();
  826 }
  827 
  828 bool CodeGenerator::styleFound()
  829 {
  830     return docStyle.found();
  831 }
  832 
  833 bool CodeGenerator::printIndexFile ( const vector<string> &fileList,
  834                                      const string &outPath )
  835 {
  836     return true;
  837 }
  838 
  839 bool CodeGenerator::initIndentationScheme ( const string &indentScheme )
  840 {
  841 
  842     if ( formatter!=NULL ) {
  843         return true;
  844     }
  845 
  846     if ( !indentScheme.size() ) return false;
  847 
  848     formatter=new astyle::ASFormatter();
  849 
  850     if ( indentScheme=="allman" || indentScheme=="bsd" || indentScheme=="ansi" ) {
  851         formatter->setFormattingStyle ( astyle::STYLE_ALLMAN );
  852     } else if ( indentScheme=="kr"||indentScheme=="k&r"||indentScheme=="k/r" ) {
  853         formatter->setFormattingStyle ( astyle::STYLE_KR );
  854     } else if ( indentScheme=="java" ) {
  855         formatter->setFormattingStyle ( astyle::STYLE_JAVA );
  856     } else if ( indentScheme=="stroustrup" ) {
  857         formatter->setFormattingStyle ( astyle::STYLE_STROUSTRUP );
  858     } else if ( indentScheme=="whitesmith" ) {
  859         formatter->setFormattingStyle ( astyle::STYLE_WHITESMITH );
  860     } else if ( indentScheme=="banner" || indentScheme=="ratliff") {
  861         formatter->setFormattingStyle ( astyle::STYLE_RATLIFF );
  862     } else if ( indentScheme=="gnu" ) {
  863         formatter->setFormattingStyle ( astyle::STYLE_GNU );
  864     } else if ( indentScheme=="linux" ) {
  865         formatter->setFormattingStyle ( astyle::STYLE_LINUX );
  866     } else if ( indentScheme=="horstmann" ) {
  867         formatter->setFormattingStyle ( astyle::STYLE_HORSTMANN );
  868     } else if ( indentScheme=="otbs" ||  indentScheme=="1tbs") {
  869         formatter->setFormattingStyle ( astyle::STYLE_1TBS );
  870     } else if ( indentScheme=="google") {
  871         formatter->setFormattingStyle ( astyle::STYLE_GOOGLE );
  872     } else if ( indentScheme=="pico" ||  indentScheme=="a11") {
  873         formatter->setFormattingStyle ( astyle::STYLE_PICO );
  874     } else if ( indentScheme=="lisp" ||  indentScheme=="python"||  indentScheme=="a12") {
  875         formatter->setFormattingStyle ( astyle::STYLE_LISP );
  876     } else if ( indentScheme=="vtk") {
  877         formatter->setFormattingStyle ( astyle::STYLE_VTK );
  878     } else if ( indentScheme=="mozilla") {
  879         formatter->setFormattingStyle ( astyle::STYLE_MOZILLA );
  880     } else if ( indentScheme=="webkit") {
  881         formatter->setFormattingStyle ( astyle::STYLE_WEBKIT );
  882     } else if ( indentScheme!="user" ){
  883         return false;
  884     }
  885     return formattingEnabled=true;
  886 }
  887 
  888 
  889 /*Helper functions for astyle option parsing*/
  890 string CodeGenerator::getParam(const string& arg, const char* op)
  891 {
  892     return arg.substr(strlen(op));
  893 }
  894 
  895 string CodeGenerator::getParam(const string& arg, const char* op1, const char* op2)
  896 {
  897     return isParamOption(arg, op1) ? getParam(arg, op1) : getParam(arg, op2);
  898 }
  899 
  900 bool CodeGenerator::isOption(const string& arg, const char* op)
  901 {
  902     return arg.compare(op) == 0;
  903 }
  904 
  905 bool CodeGenerator::isOption(const string& arg, const char* op1, const char* op2)
  906 {
  907     return (isOption(arg, op1) || isOption(arg, op2));
  908 }
  909 
  910 bool CodeGenerator::isParamOption(const string& arg, const char* option)
  911 {
  912     bool retVal = arg.compare(0, strlen(option), option) == 0;
  913     // if comparing for short option, 2nd char of arg must be numeric
  914     if (retVal && strlen(option) == 1 && arg.length() > 1)
  915         if (!isdigit((unsigned char) arg[1]))
  916             retVal = false;
  917     return retVal;
  918 }
  919 
  920 bool CodeGenerator::isParamOption(const string& arg, const char* option1, const char* option2)
  921 {
  922     return isParamOption(arg, option1) || isParamOption(arg, option2);
  923 }
  924 
  925 //apply the same options as astyle
  926 void CodeGenerator::setIndentationOptions (const vector<string>& options){
  927     if (formatter) {
  928         string arg;
  929         for (unsigned int i=0; i<options.size(); i++) {
  930             arg=options[i];
  931             
  932             if (isOption(arg, "mode=cs"))
  933             {
  934                 formatter->setSharpStyle();
  935                 formatter->setModeManuallySet(true);
  936             }
  937             else if (isOption(arg, "mode=c"))
  938             {
  939                 formatter->setCStyle();
  940                 formatter->setModeManuallySet(true);
  941             }
  942             else if (isOption(arg, "mode=java"))
  943             {
  944                 formatter->setJavaStyle();
  945                 formatter->setModeManuallySet(true);
  946             }
  947             else if (isParamOption(arg, "t", "indent=tab="))
  948             {
  949                 int spaceNum = 4;
  950                 string spaceNumParam = getParam(arg, "t", "indent=tab=");
  951                 if (spaceNumParam.length() > 0)
  952                     spaceNum = atoi(spaceNumParam.c_str());
  953                 if (spaceNum >= 2 && spaceNum <= 20)
  954                     formatter->setTabIndentation(spaceNum, false);
  955             }
  956             else if (isOption(arg, "indent=tab"))
  957             {
  958                 formatter->setTabIndentation(4);
  959             }
  960             else if (isParamOption(arg, "T", "indent=force-tab="))
  961             {
  962                 int spaceNum = 4;
  963                 string spaceNumParam = getParam(arg, "T", "indent=force-tab=");
  964                 if (spaceNumParam.length() > 0)
  965                     spaceNum = atoi(spaceNumParam.c_str());
  966                 if (spaceNum >= 2 && spaceNum <= 20)
  967                     formatter->setTabIndentation(spaceNum, true);
  968             }
  969             else if (isOption(arg, "indent=force-tab"))
  970             {
  971                 formatter->setTabIndentation(4, true);
  972             }
  973             else if (isParamOption(arg, "xT", "indent=force-tab-x="))
  974             {
  975                 int tabNum = 8;
  976                 string tabNumParam = getParam(arg, "xT", "indent=force-tab-x=");
  977                 if (tabNumParam.length() > 0)
  978                     tabNum = atoi(tabNumParam.c_str());
  979                 if (tabNum >= 2 && tabNum <= 20)
  980                     formatter->setForceTabXIndentation(tabNum);
  981                 
  982             }
  983             else if (isOption(arg, "indent=force-tab-x"))
  984             {
  985                 formatter->setForceTabXIndentation(8);
  986             }
  987             else if (isParamOption(arg, "s", "indent=spaces="))
  988             {
  989                 int spaceNum = 4;
  990                 string spaceNumParam = getParam(arg, "s", "indent=spaces=");
  991                 if (spaceNumParam.length() > 0)
  992                     spaceNum = atoi(spaceNumParam.c_str());
  993                 if (spaceNum >= 2 && spaceNum <= 20)
  994                     formatter->setSpaceIndentation(spaceNum);
  995             }
  996             else if (isOption(arg, "indent=spaces"))
  997             {
  998                 formatter->setSpaceIndentation(4);
  999             }
 1000             else if (isParamOption(arg, "xt", "indent-continuation="))
 1001             {
 1002                 int contIndent = 1;
 1003                 string contIndentParam = getParam(arg, "xt", "indent-continuation=");
 1004                 if (contIndentParam.length() > 0)
 1005                     contIndent = atoi(contIndentParam.c_str());
 1006                 if (contIndent > 0 && contIndent < 5)
 1007                     formatter->setContinuationIndentation(contIndent);
 1008             }
 1009             else if (isParamOption(arg, "m", "min-conditional-indent="))
 1010             {
 1011                 int minIndent = astyle::MINCOND_TWO;
 1012                 string minIndentParam = getParam(arg, "m", "min-conditional-indent=");
 1013                 if (minIndentParam.length() > 0)
 1014                     minIndent = atoi(minIndentParam.c_str());
 1015                 if (minIndent < astyle::MINCOND_END)
 1016                     formatter->setMinConditionalIndentOption(minIndent);
 1017             }
 1018             else if (isParamOption(arg, "M", "max-continuation-indent="))
 1019             {
 1020                 int maxIndent = 40;
 1021                 string maxIndentParam = getParam(arg, "M", "max-continuation-indent=");
 1022                 if (maxIndentParam.length() > 0)
 1023                     maxIndent = atoi(maxIndentParam.c_str());
 1024                 if (maxIndent >= 40 && maxIndent <= 120)
 1025                     formatter->setMaxContinuationIndentLength(maxIndent);
 1026             }
 1027             else if (isOption(arg, "N", "indent-namespaces"))
 1028             {
 1029                 formatter->setNamespaceIndent(true);
 1030             }
 1031             else if (isOption(arg, "C", "indent-classes"))
 1032             {
 1033                 formatter->setClassIndent(true);
 1034             }
 1035             else if (isOption(arg, "xG", "indent-modifiers"))
 1036             {
 1037                 formatter->setModifierIndent(true);
 1038             }
 1039             else if (isOption(arg, "S", "indent-switches"))
 1040             {
 1041                 formatter->setSwitchIndent(true);
 1042             }
 1043             else if (isOption(arg, "K", "indent-cases"))
 1044             {
 1045                 formatter->setCaseIndent(true);
 1046             }
 1047             else if (isOption(arg, "xU", "indent-after-parens"))
 1048             {
 1049                 formatter->setAfterParenIndent(true);
 1050             }
 1051             else if (isOption(arg, "L", "indent-labels"))
 1052             {
 1053                 formatter->setLabelIndent(true);
 1054             }
 1055             else if (isOption(arg, "xW", "indent-preproc-block"))
 1056             {
 1057                 formatter->setPreprocBlockIndent(true);
 1058             }
 1059             else if (isOption(arg, "w", "indent-preproc-define"))
 1060             {
 1061                 formatter->setPreprocDefineIndent(true);
 1062             }
 1063             else if (isOption(arg, "xw", "indent-preproc-cond"))
 1064             {
 1065                 formatter->setPreprocConditionalIndent(true);
 1066             }
 1067             else if (isOption(arg, "y", "break-closing-braces"))
 1068             {
 1069                 formatter->setBreakClosingHeaderBracesMode(true);
 1070             }
 1071             else if (isOption(arg, "O", "keep-one-line-blocks"))
 1072             {
 1073                 formatter->setBreakOneLineBlocksMode(false);
 1074             }
 1075             else if (isOption(arg, "o", "keep-one-line-statements"))
 1076             {
 1077                 formatter->setBreakOneLineStatementsMode(false);
 1078             }
 1079             else if (isOption(arg, "P", "pad-paren"))
 1080             {
 1081                 formatter->setParensOutsidePaddingMode(true);
 1082                 formatter->setParensInsidePaddingMode(true);
 1083             }
 1084             else if (isOption(arg, "d", "pad-paren-out"))
 1085             {
 1086                 formatter->setParensOutsidePaddingMode(true);
 1087             }
 1088             else if (isOption(arg, "xd", "pad-first-paren-out"))
 1089             {
 1090                 formatter->setParensFirstPaddingMode(true);
 1091             }
 1092             else if (isOption(arg, "D", "pad-paren-in"))
 1093             {
 1094                 formatter->setParensInsidePaddingMode(true);
 1095             }
 1096             else if (isOption(arg, "H", "pad-header"))
 1097             {
 1098                 formatter->setParensHeaderPaddingMode(true);
 1099             }
 1100             else if (isOption(arg, "U", "unpad-paren"))
 1101             {
 1102                 formatter->setParensUnPaddingMode(true);
 1103             }
 1104             else if (isOption(arg, "p", "pad-oper"))
 1105             {
 1106                 formatter->setOperatorPaddingMode(true);
 1107             }
 1108             else if (isOption(arg, "xg", "pad-comma"))
 1109             {
 1110                 formatter->setCommaPaddingMode(true);
 1111             }
 1112             else if (isOption(arg, "xe", "delete-empty-lines"))
 1113             {
 1114                 formatter->setDeleteEmptyLinesMode(true);
 1115             }
 1116             else if (isOption(arg, "E", "fill-empty-lines"))
 1117             {
 1118                 formatter->setEmptyLineFill(true);
 1119             }
 1120             else if (isOption(arg, "c", "convert-tabs"))
 1121             {
 1122                 formatter->setTabSpaceConversionMode(true);
 1123             }
 1124             else if (isOption(arg, "xy", "close-templates"))
 1125             {
 1126                 formatter->setCloseTemplatesMode(true);
 1127             }
 1128             else if (isOption(arg, "F", "break-blocks=all"))
 1129             {
 1130                 formatter->setBreakBlocksMode(true);
 1131                 formatter->setBreakClosingHeaderBlocksMode(true);
 1132             }
 1133             else if (isOption(arg, "f", "break-blocks"))
 1134             {
 1135                 formatter->setBreakBlocksMode(true);
 1136             }
 1137             else if (isOption(arg, "e", "break-elseifs"))
 1138             {
 1139                 formatter->setBreakElseIfsMode(true);
 1140             }
 1141             else if (isOption(arg, "xb", "break-one-line-headers"))
 1142             {
 1143                 formatter->setBreakOneLineHeadersMode(true);
 1144             }
 1145             else if (isOption(arg, "j", "add-braces"))
 1146             {
 1147                 formatter->setAddBracesMode(true);
 1148             }
 1149             else if (isOption(arg, "J", "add-one-line-braces"))
 1150             {
 1151                 formatter->setAddOneLineBracesMode(true);
 1152             }
 1153             else if (isOption(arg, "xj", "remove-braces"))
 1154             {
 1155                 formatter->setRemoveBracesMode(true);
 1156             }
 1157             else if (isOption(arg, "Y", "indent-col1-comments"))
 1158             {
 1159                 formatter->setIndentCol1CommentsMode(true);
 1160             }
 1161             else if (isOption(arg, "align-pointer=type"))
 1162             {
 1163                 formatter->setPointerAlignment(astyle::PTR_ALIGN_TYPE);
 1164             }
 1165             else if (isOption(arg, "align-pointer=middle"))
 1166             {
 1167                 formatter->setPointerAlignment(astyle::PTR_ALIGN_MIDDLE);
 1168             }
 1169             else if (isOption(arg, "align-pointer=name"))
 1170             {
 1171                 formatter->setPointerAlignment(astyle::PTR_ALIGN_NAME);
 1172             }
 1173             else if (isParamOption(arg, "k"))
 1174             {
 1175                 int align = 0;
 1176                 string styleParam = getParam(arg, "k");
 1177                 if (styleParam.length() > 0)
 1178                     align = atoi(styleParam.c_str());
 1179                 if (align == 1)
 1180                     formatter->setPointerAlignment(astyle::PTR_ALIGN_TYPE);
 1181                 else if (align == 2)
 1182                     formatter->setPointerAlignment(astyle::PTR_ALIGN_MIDDLE);
 1183                 else if (align == 3)
 1184                     formatter->setPointerAlignment(astyle::PTR_ALIGN_NAME);
 1185             }
 1186             else if (isOption(arg, "align-reference=none"))
 1187             {
 1188                 formatter->setReferenceAlignment(astyle::REF_ALIGN_NONE);
 1189             }
 1190             else if (isOption(arg, "align-reference=type"))
 1191             {
 1192                 formatter->setReferenceAlignment(astyle::REF_ALIGN_TYPE);
 1193             }
 1194             else if (isOption(arg, "align-reference=middle"))
 1195             {
 1196                 formatter->setReferenceAlignment(astyle::REF_ALIGN_MIDDLE);
 1197             }
 1198             else if (isOption(arg, "align-reference=name"))
 1199             {
 1200                 formatter->setReferenceAlignment(astyle::REF_ALIGN_NAME);
 1201             }
 1202             else if (isParamOption(arg, "W"))
 1203             {
 1204                 int align = 0;
 1205                 string styleParam = getParam(arg, "W");
 1206                 if (styleParam.length() > 0)
 1207                     align = atoi(styleParam.c_str());
 1208                 if (align == 0)
 1209                     formatter->setReferenceAlignment(astyle::REF_ALIGN_NONE);
 1210                 else if (align == 1)
 1211                     formatter->setReferenceAlignment(astyle::REF_ALIGN_TYPE);
 1212                 else if (align == 2)
 1213                     formatter->setReferenceAlignment(astyle::REF_ALIGN_MIDDLE);
 1214                 else if (align == 3)
 1215                     formatter->setReferenceAlignment(astyle::REF_ALIGN_NAME);
 1216             }
 1217             else if (isParamOption(arg, "max-code-length="))
 1218             {
 1219                 int maxLength = 50;
 1220                 string maxLengthParam = getParam(arg, "max-code-length=");
 1221                 if (maxLengthParam.length() > 0)
 1222                     maxLength = atoi(maxLengthParam.c_str());
 1223                 if (maxLength >= 50 && maxLength<= 200)
 1224                     formatter->setMaxCodeLength(maxLength);
 1225             }
 1226             else if (isParamOption(arg, "xC"))
 1227             {
 1228                 int maxLength = 50;
 1229                 string maxLengthParam = getParam(arg, "xC");
 1230                 if (maxLengthParam.length() > 0)
 1231                     maxLength = atoi(maxLengthParam.c_str());
 1232                 if (maxLength > 0 && maxLength<= 200)
 1233                     formatter->setMaxCodeLength(maxLength);
 1234             }
 1235             else if (isOption(arg, "xL", "break-after-logical"))
 1236             {
 1237                 formatter->setBreakAfterMode(true);
 1238             }
 1239             else if (isOption(arg, "xc", "attach-classes"))
 1240             {
 1241                 formatter->setAttachClass(true);
 1242             }
 1243             else if (isOption(arg, "xV", "attach-closing-while"))
 1244             {
 1245                 formatter->setAttachClosingWhile(true);
 1246             }
 1247             else if (isOption(arg, "xk", "attach-extern-c"))
 1248             {
 1249                 formatter->setAttachExternC(true);
 1250             }
 1251             else if (isOption(arg, "xn", "attach-namespaces"))
 1252             {
 1253                 formatter->setAttachNamespace(true);
 1254             }
 1255             else if (isOption(arg, "xl", "attach-inlines"))
 1256             {
 1257                 formatter->setAttachInline(true);
 1258             }
 1259             else if (isOption(arg, "xp", "remove-comment-prefix"))
 1260             {
 1261                 formatter->setStripCommentPrefix(true);
 1262             }
 1263             else if (isOption(arg, "xB", "break-return-type"))
 1264             {
 1265                 formatter->setBreakReturnType(true);
 1266             }
 1267             else if (isOption(arg, "xD", "break-return-type-decl"))
 1268             {
 1269                 formatter->setBreakReturnTypeDecl(true);
 1270             }
 1271             else if (isOption(arg, "xf", "attach-return-type"))
 1272             {
 1273                 formatter->setAttachReturnType(true);
 1274             }
 1275             else if (isOption(arg, "xh", "attach-return-type-decl"))
 1276             {
 1277                 formatter->setAttachReturnTypeDecl(true);
 1278             }
 1279             // Objective-C options
 1280             else if (isOption(arg, "xQ", "pad-method-prefix"))
 1281             {
 1282                 formatter->setMethodPrefixPaddingMode(true);
 1283             }
 1284             else if (isOption(arg, "xR", "unpad-method-prefix"))
 1285             {
 1286                 formatter->setMethodPrefixUnPaddingMode(true);
 1287             }
 1288             else if (isOption(arg, "xq", "pad-return-type"))
 1289             {
 1290                 formatter->setReturnTypePaddingMode(true);
 1291             }
 1292             else if (isOption(arg, "xr", "unpad-return-type"))
 1293             {
 1294                 formatter->setReturnTypeUnPaddingMode(true);
 1295             }
 1296             else if (isOption(arg, "xS", "pad-param-type"))
 1297             {
 1298                 formatter->setParamTypePaddingMode(true);
 1299             }
 1300             else if (isOption(arg, "xs", "unpad-param-type"))
 1301             {
 1302                 formatter->setParamTypeUnPaddingMode(true);
 1303             }
 1304             else if (isOption(arg, "xM", "align-method-colon"))
 1305             {
 1306                 formatter->setAlignMethodColon(true);
 1307             }
 1308             else if (isOption(arg, "xP0", "pad-method-colon=none"))
 1309             {
 1310                 formatter->setObjCColonPaddingMode(astyle::COLON_PAD_NONE);
 1311             }
 1312             else if (isOption(arg, "xP1", "pad-method-colon=all"))
 1313             {
 1314                 formatter->setObjCColonPaddingMode(astyle::COLON_PAD_ALL);
 1315             }
 1316             else if (isOption(arg, "xP2", "pad-method-colon=after"))
 1317             {
 1318                 formatter->setObjCColonPaddingMode(astyle::COLON_PAD_AFTER);
 1319             }
 1320             else if (isOption(arg, "xP3", "pad-method-colon=before"))
 1321             {
 1322                 formatter->setObjCColonPaddingMode(astyle::COLON_PAD_BEFORE);
 1323             }
 1324         }
 1325     }
 1326 }
 1327 
 1328 LoadResult CodeGenerator::loadLanguage ( const string& langDefPath, bool embedded )
 1329 {
 1330 
 1331     if (!embedded) {
 1332         while (!nestedLangs.empty()) {
 1333             nestedLangs.pop();
 1334         }   
 1335     }
 1336     
 1337     bool reloadNecessary= currentSyntax ? currentSyntax->needsReload ( langDefPath ): true;
 1338     LoadResult result=LOAD_OK;
 1339     if ( reloadNecessary ) {
 1340         if (syntaxReaders.count(langDefPath)) {
 1341             currentSyntax=syntaxReaders[langDefPath];
 1342             result=LOAD_OK;
 1343         } else {
 1344             currentSyntax=new SyntaxReader();
 1345             result=currentSyntax->load(langDefPath, pluginParameter, outputType);
 1346             syntaxReaders[langDefPath]=currentSyntax;
 1347         }
 1348 
 1349         if ( result==LOAD_OK ) {
 1350             formattingPossible=currentSyntax->enableReformatting();
 1351 
 1352             if ( openTags.size() >NUMBER_BUILTIN_STATES ) {
 1353                 // remove dynamic keyword tag delimiters of the old language definition
 1354                 vector<string>::iterator keyStyleOpenBegin =
 1355                     openTags.begin() + NUMBER_BUILTIN_STATES;
 1356                 vector<string>::iterator keyStyleCloseBegin =
 1357                     closeTags.begin() + NUMBER_BUILTIN_STATES;
 1358                 openTags.erase ( keyStyleOpenBegin, openTags.end() );
 1359                 closeTags.erase ( keyStyleCloseBegin, closeTags.end() );
 1360             }
 1361             // add new keyword tag delimiters
 1362             for ( unsigned int i=0; i< currentSyntax->getKeywordClasses().size(); i++ ) {
 1363                 openTags.push_back ( getKeywordOpenTag ( i ) );
 1364                 closeTags.push_back ( getKeywordCloseTag ( i ) );
 1365             }
 1366             
 1367             //test balloon
 1368             string overrideSpacer(currentSyntax->getOverrideConfigVal("spacer"));
 1369             if (!overrideSpacer.empty()) {
 1370                 spacer = overrideSpacer;
 1371             }
 1372             string overrideMaskWS(currentSyntax->getOverrideConfigVal("maskws"));
 1373             if (!overrideMaskWS.empty()) {
 1374                 maskWs = overrideMaskWS=="true";
 1375             }
 1376             
 1377         }
 1378     }
 1379     return result;
 1380 }
 1381 
 1382 bool CodeGenerator::validateInputStream()
 1383 {
 1384     if ( !in ) return false;
 1385 
 1386     // it is not possible to move stream pointer back with stdin
 1387     if ( ( int ) in->tellg() == -1 ) // -1 : stdin
 1388         return true;
 1389 
 1390     // Sources: http://en.wikipedia.org/wiki/Magic_number_(programming)
 1391     // Magic configuration of "file"
 1392     // This is intended for web plugins - only check filetypes often found in the net
 1393     char magic_gif[]    = {'G','I','F','8', 0};
 1394     char magic_png[]    = {'\x89','P','N','G', 0};
 1395     char magic_java[]   = {'\xCA','\xFE','\xBA','\xBE', 0};
 1396     char magic_jpeg[]   = {'\xFF','\xD8','\xFF', 0};
 1397     char magic_bmp[]    = {'B','M', 0};
 1398     char magic_pdf[]    = {'%','P','D','F', 0};
 1399     char magic_utf8[]   = {'\xEF','\xBB','\xBF',0};
 1400     char magic_rar[]    = {'R','a','r','!', 0};
 1401     char magic_zip[]    = {'P','K','\x03','\x04', 0};
 1402     char magic_ace[]    = {'*','*','A','C','E','*','*', 0};
 1403     char magic_tgz[]    = {'\x8b','\x1f', '\x00', '\x08', 0};
 1404     char magic_bzip[]   = {'B','Z', 0};
 1405 
 1406     char* magic_table[] = {magic_utf8,
 1407                            magic_gif, magic_png, magic_jpeg, magic_bmp, magic_pdf,
 1408                            magic_java,
 1409                            magic_rar, magic_zip, magic_ace, magic_tgz, magic_bzip,
 1410                            0
 1411                           };
 1412 
 1413     char buffer [10]= {0};
 1414     in->read ( buffer,8 );  //only read the first 8 bytes of input stream
 1415 
 1416     int magic_index=0;
 1417     while ( magic_table[magic_index] ) {
 1418         if ( !strncmp ( buffer, magic_table[magic_index], strlen ( magic_table[magic_index] ) ) ) {
 1419             break;
 1420         }
 1421         magic_index++;
 1422     }
 1423     int streamReadPos=0;
 1424     if ( magic_table[magic_index] == magic_utf8 ) {
 1425         //setEncoding("utf-8");
 1426         streamReadPos=3; // remove UTF-8 magic number from output
 1427     }
 1428 
 1429     in -> seekg ( streamReadPos, ios::beg );
 1430     in-> clear();  // clear fail bit to continue reading
 1431 
 1432     return !magic_table[magic_index] // points to 0 if no pattern was found
 1433            || magic_table[magic_index] == magic_utf8;
 1434 }
 1435 
 1436 void CodeGenerator::applyPluginChunk(const string& fctName, string *result, bool *keepDefault) {
 1437     
 1438     if (currentSyntax && pluginChunks.size()) {
 1439     
 1440         Diluculum::LuaState luaState;
 1441 
 1442         Diluculum::LuaValueList chunkParams;
 1443         chunkParams.push_back(currentSyntax->getDescription());
 1444         for (unsigned int i=0; i<pluginChunks.size(); i++) {
 1445             luaState.call(*pluginChunks[i], chunkParams, "format user function");
 1446         }
 1447         
 1448         if (luaState.globals().count(fctName)) {
 1449             Diluculum::LuaFunction* documentFct=new Diluculum::LuaFunction(luaState[fctName].value().asFunction());
 1450         
 1451             luaState["HL_INPUT_FILE"] = luaState["HL_PLUGIN_PARAM"] = pluginParameter;
 1452             luaState["HL_OUTPUT"] = outputType;
 1453             luaState["HL_FORMAT_HTML"]=HTML;
 1454             luaState["HL_FORMAT_XHTML"]=XHTML;
 1455             luaState["HL_FORMAT_TEX"]=TEX;
 1456             luaState["HL_FORMAT_LATEX"]=LATEX;
 1457             luaState["HL_FORMAT_RTF"]=RTF;
 1458             luaState["HL_FORMAT_ANSI"]=ESC_ANSI;
 1459             luaState["HL_FORMAT_XTERM256"]=ESC_XTERM256;
 1460             luaState["HL_FORMAT_TRUECOLOR"]=ESC_TRUECOLOR;
 1461             luaState["HL_FORMAT_SVG"]=SVG;
 1462             luaState["HL_FORMAT_BBCODE"]=BBCODE;
 1463             luaState["HL_FORMAT_PANGO"]=PANGO;
 1464             luaState["HL_FORMAT_ODT"]=ODTFLAT;
 1465                         
 1466             Diluculum::LuaValueList params;
 1467             Diluculum::LuaValueMap options;
 1468             options[Diluculum::LuaValue("title")] =  Diluculum::LuaValue( docTitle );   
 1469             options[Diluculum::LuaValue("encoding")] =  Diluculum::LuaValue(encoding);   
 1470             options[Diluculum::LuaValue("fragment")] =  Diluculum::LuaValue(fragmentOutput);   
 1471             options[Diluculum::LuaValue("font")] =  Diluculum::LuaValue(getBaseFont());   
 1472             options[Diluculum::LuaValue("fontsize")] =  Diluculum::LuaValue(getBaseFontSize());   
 1473 
 1474             params.push_back(inputFilesCnt);
 1475             params.push_back(processedFilesCnt);
 1476             params.push_back(options);
 1477             
 1478             Diluculum::LuaValueList res=luaState.call ( *documentFct, params, fctName+" call")  ;
 1479             if (res.size()>=1) {
 1480                 *keepDefault=false;
 1481                 *result = res[0].asString();
 1482                 if (res.size()==2)
 1483                     *keepDefault = res[1].asBoolean();
 1484             }
 1485             delete documentFct;
 1486         }
 1487     }
 1488 }
 1489 
 1490 void CodeGenerator::printHeader()
 1491 {
 1492     bool keepDefaultHeader=true;
 1493     string pluginHeader;
 1494     
 1495     processedFilesCnt++;
 1496     
 1497     applyPluginChunk("DocumentHeader", &pluginHeader, &keepDefaultHeader);
 1498 
 1499     if ( ! fragmentOutput && keepDefaultHeader)
 1500         *out << getHeader();
 1501     
 1502     *out << pluginHeader; 
 1503    
 1504     if ( !fragmentOutput || keepInjections)
 1505         *out << currentSyntax->getHeaderInjection();
 1506 }
 1507 
 1508 void CodeGenerator::printFooter()
 1509 {
 1510     
 1511     bool keepDefaultFooter=true;
 1512     string pluginFooter;
 1513     
 1514     applyPluginChunk("DocumentFooter", &pluginFooter, &keepDefaultFooter);
 1515     
 1516     if ( !fragmentOutput || keepInjections)
 1517         *out << currentSyntax->getFooterInjection();
 1518 
 1519     *out << pluginFooter; 
 1520     
 1521     if ( ! fragmentOutput && keepDefaultFooter )
 1522         *out << getFooter();
 1523 }
 1524 
 1525 ParseError CodeGenerator::generateFile ( const string &inFileName,
 1526         const string &outFileName )
 1527 {
 1528     if ( !docStyle.found() ) {
 1529         return BAD_STYLE;
 1530     }
 1531 
 1532     reset();
 1533 
 1534     ParseError error=PARSE_OK;
 1535 
 1536     inFile=inFileName;
 1537     outFile=outFileName;
 1538         
 1539     in = ( inFileName.empty() ? &cin :new ifstream ( inFileName.c_str() ) );
 1540 
 1541     if ( validateInput )
 1542         if ( !validateInputStream() ) error= BAD_INPUT;
 1543 
 1544     if ( !in->fail() && error==PARSE_OK ) {
 1545         out = ( outFileName.empty() ? &cout :new ofstream ( outFileName.c_str() ) );
 1546         if ( out->fail() ) {
 1547             error=BAD_OUTPUT;
 1548         }
 1549     }
 1550 
 1551     if ( in->fail() ) {
 1552         error=BAD_INPUT;
 1553     }
 1554 
 1555     if ( error==PARSE_OK ) {
 1556         if ( formatter != NULL ) {
 1557             formatter->init ( new astyle::ASStreamIterator ( in ) );
 1558         }
 1559         currentSyntax->setInputFileName(inFile);
 1560         printHeader();
 1561         printBody();
 1562         printFooter();
 1563     }
 1564 
 1565     if ( !outFileName.empty() ) {
 1566         delete out;
 1567         out=NULL;
 1568     }
 1569     if ( !inFileName.empty() ) {
 1570         delete in;
 1571         in=NULL;
 1572     }
 1573     return error;
 1574 }
 1575 
 1576 string CodeGenerator::generateString ( const string &input )
 1577 {
 1578 
 1579     if ( !docStyle.found() ) {
 1580         return "";
 1581     }
 1582 
 1583     reset();
 1584 
 1585     in = new istringstream ( input );
 1586     out = new ostringstream ();
 1587 
 1588     if ( in->fail() || out->fail() ) {
 1589         return "";
 1590     }
 1591 
 1592     if ( formatter != NULL ) {
 1593         formatter->init ( new astyle::ASStreamIterator ( in ) );
 1594     }
 1595     printHeader();
 1596     printBody();
 1597     printFooter();
 1598 
 1599     string result = static_cast<ostringstream*> ( out )->str();
 1600 
 1601     delete out;
 1602     out=NULL;
 1603     delete in;
 1604     in=NULL;
 1605 
 1606     return result;
 1607 }
 1608 
 1609 string CodeGenerator::generateStringFromFile ( const string &inFileName )
 1610 {
 1611 
 1612     if ( !docStyle.found() ) {
 1613         return "";
 1614     }
 1615 
 1616     reset();
 1617 
 1618     inFile = inFileName;
 1619     
 1620     in = new ifstream ( inFileName.c_str() );
 1621     out = new ostringstream ();
 1622 
 1623     if ( in->fail() || out->fail() ) {
 1624         return "";
 1625     }
 1626 
 1627     if ( validateInput && !validateInputStream() ) {
 1628         return "ERROR: detected binary input";
 1629     }
 1630 
 1631     if ( formatter != NULL ) {
 1632         formatter->init ( new astyle::ASStreamIterator ( in ) );
 1633     }
 1634     currentSyntax->setInputFileName(inFile);
 1635     
 1636     printHeader();
 1637     printBody();
 1638     printFooter();
 1639 
 1640     string result = static_cast<ostringstream*> ( out )->str();
 1641 
 1642     delete out;
 1643     out=NULL;
 1644     delete in;
 1645     in=NULL;
 1646 
 1647     return result;
 1648 }
 1649 
 1650 unsigned int CodeGenerator::getStyleID ( State s, unsigned int kwClassID )
 1651 {
 1652     if ( s==KEYWORD && kwClassID ) {
 1653         return NUMBER_BUILTIN_STATES + kwClassID-1;
 1654     }
 1655     return ( unsigned int ) s ;
 1656 }
 1657 
 1658 void CodeGenerator::openTag ( State s )
 1659 {
 1660     *out << openTags[ ( unsigned int ) s];
 1661     currentState=s;
 1662 
 1663 }
 1664 
 1665 void CodeGenerator::closeTag ( State s )
 1666 {
 1667     *out << closeTags[ ( unsigned int ) s];
 1668     flushWs(2);
 1669     currentState=_UNKNOWN;
 1670 }
 1671 
 1672 void CodeGenerator::openKWTag ( unsigned int kwClassID )
 1673 {
 1674     *out << openTags.at(getStyleID ( KEYWORD, kwClassID ) );
 1675     currentState=KEYWORD;
 1676 }
 1677 
 1678 void CodeGenerator::closeKWTag ( unsigned int kwClassID )
 1679 {
 1680     *out << closeTags.at(getStyleID ( KEYWORD, kwClassID ) );
 1681     flushWs(3);
 1682     currentState=_UNKNOWN;
 1683 }
 1684 
 1685 bool CodeGenerator::loadEmbeddedLang(const string&embedLangDefPath)
 1686 {
 1687     if (nestedLangs.empty()) {
 1688         nestedLangs.push(currentSyntax->getCurrentPath() );
 1689     }
 1690     if (nestedLangs.top() != embedLangDefPath) {
 1691         nestedLangs.push(embedLangDefPath);
 1692     }
 1693     LoadResult res = loadLanguage(embedLangDefPath, true);
 1694     //pass end delimiter regex to syntax description
 1695     currentSyntax->restoreLangEndDelim(embedLangDefPath);
 1696     return res == LOAD_OK;
 1697 }
 1698 
 1699 ///////////////////////////////////////////////////////////////////////////////
 1700 
 1701 void CodeGenerator::processRootState()
 1702 {
 1703     bool eof=false,
 1704          firstLine=true; // avoid newline before printing the first output line
 1705          
 1706     applySyntaxTestCase = inFile.find("syntax_test_")!=string::npos;
 1707     
 1708     if ( currentSyntax->highlightingDisabled() ) {
 1709         string line;
 1710         while ( getline ( *in, line ) && lineNumber < maxLineCnt ) {
 1711             ++lineNumber;
 1712             insertLineNumber ( !firstLine );
 1713             flushWs(4);
 1714             firstLine=false;
 1715             if (lineNumber>=startLineCntCurFile && lineNumber <=maxLineCnt)
 1716                 maskString ( *out, line );
 1717         }
 1718         *out << flush;
 1719         return;
 1720     }
 1721 
 1722     if (!embedLangStart.empty()) {
 1723         if (!loadEmbeddedLang(currentSyntax->getNewPath(embedLangStart))) return;
 1724     }
 1725 
 1726     State state=STANDARD;
 1727 
 1728     //if (outputType!=ESC_TRUECOLOR && outputType!=ESC_XTERM256)
 1729     openTag ( STANDARD );
 1730     
 1731     do {
 1732         // determine next state
 1733         state= getCurrentState(STANDARD);
 1734 
 1735         // handle current state
 1736         switch ( state ) {
 1737         case KEYWORD:
 1738             closeTag ( STANDARD );
 1739             eof=processKeywordState ( state );
 1740             openTag ( STANDARD );
 1741             break;
 1742         case NUMBER:
 1743             closeTag ( STANDARD );
 1744             eof=processNumberState();
 1745             openTag ( STANDARD );
 1746             break;
 1747         case ML_COMMENT:
 1748             closeTag ( STANDARD );
 1749             eof=processMultiLineCommentState();
 1750             openTag ( STANDARD );
 1751             break;
 1752         case SL_COMMENT:
 1753             closeTag ( STANDARD );
 1754             eof=processSingleLineCommentState();
 1755             openTag ( STANDARD );
 1756             break;
 1757         case STRING:
 1758             closeTag ( STANDARD );
 1759             eof=processStringState ( STANDARD );
 1760             openTag ( STANDARD );
 1761             break;
 1762         case DIRECTIVE:
 1763             closeTag ( STANDARD );
 1764             eof=processDirectiveState();
 1765             openTag ( STANDARD );
 1766             break;
 1767         case ESC_CHAR:
 1768             closeTag ( STANDARD );
 1769             eof=processEscapeCharState();
 1770             openTag ( STANDARD );
 1771             break;
 1772         case SYMBOL:
 1773             closeTag ( STANDARD );
 1774             eof=processSymbolState();
 1775             openTag ( STANDARD );
 1776             break;
 1777 
 1778         case EMBEDDED_CODE_END:
 1779             closeTag ( STANDARD );
 1780             eof=processSyntaxChangeState(state);
 1781             openTag ( STANDARD );
 1782             break;
 1783         case _EOL:
 1784 
 1785             // XTERM256 fix (issue with less cmd)
 1786             if  (!firstLine || showLineNumbers) {
 1787                 closeTag ( STANDARD );
 1788             }
 1789             insertLineNumber ( !firstLine );
 1790             if (!firstLine || showLineNumbers) {
 1791                 flushWs(5);
 1792                 stateTraceCurrent.clear();
 1793                 openTag ( STANDARD );
 1794             }
 1795             firstLine=false;
 1796             break;
 1797         case _EOF:
 1798             eof=true;
 1799             break;
 1800         case _WS:
 1801             processWsState();
 1802             break;
 1803         default:
 1804            
 1805             // see https://gitlab.com/saalen/highlight/-/issues/152
 1806           //  if (lineNumber==1 && lineIndex==1 && (outputType==ESC_TRUECOLOR || outputType==ESC_XTERM256) && token.size())
 1807           //      openTag ( STANDARD );
 1808 
 1809             printMaskedToken ();
 1810             break;
 1811         }
 1812     } while ( !eof );
 1813     
 1814     if (token.size() || lineNumber>1 || (outputType!=ESC_TRUECOLOR && outputType!=ESC_XTERM256))
 1815         closeTag ( STANDARD );
 1816 
 1817     if (currentSyntax->getDecorateLineEndFct()) {
 1818         Diluculum::LuaValueList res=callDecorateLineFct(false);
 1819         if (res.size()==1) {
 1820             *out << res[0].asString();
 1821         }
 1822     }
 1823 
 1824     printNewLines = noTrailingNewLine==0 || ( noTrailingNewLine==2 && ( token.size() || lineNumber>1) );
 1825     *out << getNewLine();
 1826     *out << flush;
 1827 }
 1828 
 1829 bool CodeGenerator::processSyntaxChangeState(State myState)
 1830 {
 1831     State newState=STANDARD;
 1832     bool eof=false,
 1833          exitState=false;
 1834 
 1835     openTag ( KEYWORD );
 1836     do {
 1837 
 1838         if (myState==EMBEDDED_CODE_END) {
 1839             if (!nestedLangs.empty()) {
 1840                 nestedLangs.pop();
 1841             }
 1842             // load host language syntax
 1843             if (!nestedLangs.empty()) {
 1844                 loadLanguage(nestedLangs.top(), true);
 1845             }
 1846             matchRegex(line, EMBEDDED_CODE_BEGIN); // match remaining line using the host syntax
 1847         }
 1848         
 1849         printMaskedToken ( newState!=_WS );
 1850 
 1851         newState= getCurrentState(myState);
 1852 
 1853         switch ( newState ) {
 1854         case _WS:
 1855             processWsState();
 1856             break;
 1857         case _EOL:
 1858             insertLineNumber();
 1859             exitState=true;
 1860             break;
 1861         case _EOF:
 1862             eof = true;
 1863             break;
 1864         default:
 1865             exitState=true;
 1866             break;
 1867         }
 1868     } while (  !exitState  &&  !eof );
 1869     closeTag ( KEYWORD );
 1870 
 1871     return eof;
 1872 }
 1873 
 1874 
 1875 bool CodeGenerator::processKeywordState ( State myState )
 1876 {
 1877     State newState=STANDARD;
 1878     unsigned int myClassID=currentKeywordClass;
 1879     bool eof=false,
 1880          exitState=false;
 1881 
 1882     openKWTag ( myClassID );
 1883     do {
 1884         printMaskedToken ( newState!=_WS,
 1885                            ( currentSyntax->isIgnoreCase() ) ? keywordCase : StringTools::CASE_UNCHANGED );
 1886         newState= getCurrentState(myState);
 1887         switch ( newState ) {
 1888         case _WS:
 1889             processWsState();
 1890             exitState=isolateTags;
 1891             break;
 1892         case _EOL:
 1893             insertLineNumber();
 1894             exitState=true;
 1895             
 1896             break;
 1897         case _EOF:
 1898             eof = true;
 1899             break;
 1900         case KEYWORD_END:
 1901             exitState=true;
 1902             break;
 1903         default:
 1904             exitState= ( myClassID!=currentKeywordClass ) || ( myState!=newState );
 1905             break;
 1906         }
 1907     } while ( !exitState  &&  !eof );
 1908 
 1909     closeKWTag ( myClassID );
 1910 
 1911     currentKeywordClass=0;
 1912     return eof;
 1913 }
 1914 
 1915 bool CodeGenerator::processNumberState()
 1916 {
 1917     State newState=STANDARD;
 1918     bool eof=false,
 1919          exitState=false;
 1920     openTag ( NUMBER );
 1921     do {
 1922         printMaskedToken ( newState!=_WS );
 1923         newState= getCurrentState(NUMBER);
 1924         switch ( newState ) {
 1925         case _WS:
 1926             processWsState();
 1927             exitState=isolateTags;
 1928             break;
 1929         case _EOL:
 1930             insertLineNumber();
 1931             exitState=true;
 1932             break;
 1933         case _EOF:
 1934             eof = true;
 1935             break;
 1936         default:
 1937             exitState=newState!=NUMBER;
 1938             break;
 1939         }
 1940     } while ( !exitState && !eof );
 1941 
 1942     closeTag ( NUMBER );
 1943     return eof;
 1944 }
 1945 
 1946 
 1947 bool CodeGenerator::processMultiLineCommentState()
 1948 {
 1949     int commentCount=1;
 1950     int openDelimID=currentSyntax->getOpenDelimiterID ( token, ML_COMMENT);
 1951     State newState=STANDARD;
 1952     bool eof=false, exitState=false, containedTestCase=false;
 1953     unsigned int startColumn=lineIndex - token.size() ;
 1954     openTag ( ML_COMMENT );
 1955     do {
 1956         printMaskedToken (newState!=_WS );
 1957         newState= getCurrentState(ML_COMMENT);
 1958 
 1959         switch ( newState ) {
 1960         case _WS:
 1961             processWsState();
 1962             break;
 1963         case _EOL:
 1964             wsBuffer += closeTags[ML_COMMENT];
 1965             insertLineNumber();
 1966             wsBuffer += openTags[ML_COMMENT];
 1967             startColumn=0;
 1968             break;
 1969         case _EOF:
 1970             eof = true;
 1971             break;
 1972         case _TESTPOS:
 1973             runSyntaxTestcases(token=="<" ? startColumn : lineIndex - 1 );
 1974             printMaskedToken();
 1975             containedTestCase=true;
 1976             break;
 1977         case ML_COMMENT:
 1978 
 1979             if ( currentSyntax->allowNestedMLComments() ) {
 1980                 ++commentCount;
 1981             }
 1982             // if delimiters are equal, close the comment by continuing to
 1983             // ML_COMMENT_END section
 1984             if (currentSyntax->delimiterIsDistinct(currentSyntax->getOpenDelimiterID ( token, ML_COMMENT  ))) break;
 1985 
 1986         case ML_COMMENT_END:
 1987 
 1988             if (!currentSyntax->matchesOpenDelimiter (token,  ML_COMMENT_END, openDelimID)) {
 1989                 break;
 1990             }
 1991             commentCount--;
 1992             if ( !commentCount ) {
 1993                 printMaskedToken();
 1994                 exitState=true;
 1995             }
 1996             break;
 1997         default:
 1998             break;
 1999         }
 2000     } while ( !exitState  &&  !eof );
 2001 
 2002     closeTag ( ML_COMMENT );
 2003    
 2004     if (containedTestCase){
 2005         stateTraceCurrent.clear();
 2006     }
 2007     return eof;
 2008 }
 2009 
 2010 
 2011 bool CodeGenerator::processSingleLineCommentState()
 2012 {
 2013     State newState=STANDARD;
 2014     bool eof=false, exitState=false, containedTestCase=false;
 2015     unsigned int startColumn = lineIndex - token.size() ;
 2016 
 2017     openTag ( SL_COMMENT );
 2018     do {
 2019         printMaskedToken ( newState!=_WS );
 2020         newState= getCurrentState(SL_COMMENT);
 2021 
 2022         switch ( newState ) {
 2023         case _WS:
 2024             processWsState();
 2025             break;
 2026         case _EOL:
 2027             printMaskedToken();
 2028             if ( preFormatter.isEnabled() && preFormatter.isWrappedLine ( lineNumber-1 ) ) {
 2029                 exitState=false;
 2030             } else {
 2031                 exitState=true;
 2032             }
 2033             if ( !exitState ) wsBuffer += closeTags[SL_COMMENT];
 2034             insertLineNumber();
 2035             if ( !exitState ) wsBuffer += openTags[SL_COMMENT];
 2036 
 2037             break;
 2038         case _EOF:
 2039             eof = true;
 2040             break;
 2041         case _TESTPOS:
 2042             runSyntaxTestcases(token=="<" ? startColumn : lineIndex - 1 );
 2043             printMaskedToken();
 2044             containedTestCase=true;
 2045             break;
 2046      
 2047         default:
 2048             break;
 2049         }
 2050     } while ( !exitState  &&  !eof );
 2051 
 2052     closeTag ( SL_COMMENT );
 2053     
 2054     if (containedTestCase) {
 2055         stateTraceCurrent.clear();
 2056     }
 2057     
 2058     return eof;
 2059 }
 2060 
 2061 bool CodeGenerator::processDirectiveState()
 2062 {
 2063     State  newState=STANDARD;
 2064     bool eof=false, exitState=false;
 2065 
 2066     openTag ( DIRECTIVE );
 2067     do {
 2068         printMaskedToken ( newState!=_WS );
 2069         newState= getCurrentState(DIRECTIVE);
 2070         switch ( newState ) {
 2071         case _WS:
 2072             processWsState();
 2073             break;
 2074         case DIRECTIVE_END:
 2075             printMaskedToken();
 2076             exitState=true;
 2077             break;
 2078         case _EOL:
 2079             printMaskedToken();
 2080             
 2081             if ( preFormatter.isEnabled() && preFormatter.isWrappedLine ( lineNumber-1 ) ) {
 2082                 exitState=false;
 2083             } else {
 2084                 if (currentSyntax->getContinuationChar()!=0x13){
 2085                     exitState= ( terminatingChar!=currentSyntax->getContinuationChar() );
 2086                 } 
 2087             }
 2088             if ( !exitState ) wsBuffer += closeTags[DIRECTIVE];
 2089             insertLineNumber();
 2090             if ( !exitState ) wsBuffer += openTags[DIRECTIVE];
 2091             break;
 2092         case ML_COMMENT:
 2093             closeTag ( DIRECTIVE );
 2094             eof= processMultiLineCommentState();
 2095             openTag ( DIRECTIVE );
 2096             break;
 2097         case SL_COMMENT:
 2098             closeTag ( DIRECTIVE );
 2099             eof= processSingleLineCommentState();
 2100             openTag ( DIRECTIVE );
 2101             exitState=true;
 2102             break;
 2103         case STRING:
 2104             closeTag ( DIRECTIVE );
 2105             eof=processStringState ( DIRECTIVE );
 2106             openTag ( DIRECTIVE );
 2107             break;
 2108         case _EOF:
 2109             eof = true;
 2110             break;
 2111         default:
 2112             break;
 2113         }
 2114     } while ( !exitState && !eof );
 2115 
 2116     closeTag ( DIRECTIVE );
 2117     return eof;
 2118 }
 2119 
 2120 bool CodeGenerator::processStringState ( State oldState )
 2121 {
 2122     State newState=STANDARD;
 2123     bool eof=false, exitState=false;
 2124     bool returnedFromOtherState=false;
 2125 
 2126     State myState= ( oldState==DIRECTIVE ) ? DIRECTIVE_STRING : STRING;
 2127 
 2128     int openDelimID=currentSyntax->getOpenDelimiterID ( token, myState);
 2129     string openDelim=token;
 2130 
 2131     //Raw String by definition:
 2132     bool isRawString=currentSyntax->delimiterIsRawString(openDelimID) || toggleDynRawString;
 2133 
 2134     // Test if character before string open delimiter token equals to the
 2135     // raw string prefix (Example: r" ", r""" """ in Python)
 2136 
 2137     //Raw String Prefix:
 2138     if ( lineIndex>token.length() &&line[lineIndex-token.length()-1]==currentSyntax->getRawStringPrefix() ) {
 2139         isRawString=true;
 2140     }
 2141 
 2142     openTag ( myState );
 2143     do {
 2144         // true if last token was an escape char
 2145         if ( !returnedFromOtherState ) {
 2146             printMaskedToken (newState!=_WS );
 2147         }
 2148         returnedFromOtherState=false;
 2149         newState= getCurrentState(myState);
 2150 
 2151         switch ( newState ) {
 2152         case _WS:
 2153             processWsState();
 2154             break;
 2155         case _EOL:
 2156             wsBuffer += closeTags[myState];
 2157             insertLineNumber();
 2158             wsBuffer += openTags[myState];
 2159             break;
 2160         case STRING_END:
 2161             if (resultOfHook || currentSyntax->matchesOpenDelimiter (token,  STRING_END, openDelimID)) {
 2162                 if (currentSyntax->assertDelimEqualLength()) {
 2163                     exitState= openDelim.length()==token.length();
 2164                 } else {
 2165                     exitState= true;
 2166                 }
 2167                 printMaskedToken();
 2168             }
 2169             break;
 2170         case STRING:
 2171             // if there exist multiple string delimiters, close string if
 2172             // current delimiter is equal to the opening delimiter
 2173             exitState=currentSyntax->delimiterIsDistinct(currentSyntax->getOpenDelimiterID ( token, STRING  ))&&token==openDelim;
 2174             printMaskedToken();
 2175             break;
 2176         case ESC_CHAR:
 2177             if ( !isRawString ) {
 2178                 closeTag ( myState );
 2179                 eof=processEscapeCharState();
 2180                 openTag ( myState );
 2181                 returnedFromOtherState=true;
 2182             } else {
 2183                 // FIXME not a fix for Python r"""\"""
 2184                 exitState=token.size()>1 && token[1] == openDelim[0];
 2185                 printMaskedToken();
 2186             }
 2187             break;
 2188         case STRING_INTERPOLATION:
 2189             closeTag ( myState );
 2190             eof=processInterpolationState();
 2191             openTag ( myState );
 2192             returnedFromOtherState=true;
 2193             break;
 2194     
 2195         case _EOF:
 2196             eof = true;
 2197             break;
 2198         default:
 2199             printMaskedToken();
 2200             break;
 2201         }
 2202     } while ( !exitState && !eof );
 2203 
 2204     closeTag ( myState );
 2205     
 2206     toggleDynRawString = false;
 2207     
 2208     return eof;
 2209 }
 2210 
 2211 bool CodeGenerator::processSymbolState()
 2212 {
 2213 
 2214     State newState=STANDARD;
 2215     bool eof=false,
 2216          exitState=false;
 2217 
 2218     openTag ( SYMBOL );
 2219     do {
 2220         printMaskedToken ( newState!=_WS );
 2221         newState= getCurrentState(SYMBOL);
 2222         switch ( newState ) {
 2223         case _WS:
 2224             processWsState();
 2225             exitState=isolateTags;
 2226             break;
 2227         case _EOL:
 2228             insertLineNumber();
 2229             exitState=true;
 2230             break;
 2231         case _EOF:
 2232             eof = true;
 2233             break;
 2234         default:
 2235             exitState=newState!=SYMBOL;
 2236             break;
 2237         }
 2238     } while ( !exitState && !eof );
 2239 
 2240     closeTag ( SYMBOL );
 2241     return eof;
 2242 }
 2243 
 2244 bool CodeGenerator::processEscapeCharState()
 2245 {
 2246     State newState=STANDARD;
 2247     bool eof=false, exitState=false;
 2248     openTag ( ESC_CHAR );
 2249     do {
 2250         printMaskedToken (newState!=_WS );
 2251         newState= getCurrentState(ESC_CHAR);
 2252         switch ( newState ) {
 2253         case _EOL:
 2254             insertLineNumber();
 2255             exitState=true;
 2256             break;
 2257         case _WS:
 2258             processWsState();
 2259             exitState=isolateTags;
 2260             break;
 2261         case _EOF:
 2262             eof = true;
 2263             break;
 2264         default:
 2265             exitState=newState!=ESC_CHAR;
 2266             break;
 2267         }
 2268     } while ( !exitState && !eof );
 2269 
 2270     closeTag ( ESC_CHAR );
 2271     return eof;
 2272 }
 2273 
 2274 bool CodeGenerator::processInterpolationState()
 2275 {
 2276     State newState=STANDARD;
 2277     bool eof=false, exitState=false;
 2278     openTag ( STRING_INTERPOLATION );
 2279     do {
 2280         printMaskedToken (newState!=_WS );
 2281         newState= getCurrentState(STRING_INTERPOLATION);
 2282         switch ( newState ) {
 2283         case _EOL:
 2284             insertLineNumber();
 2285             exitState=true;
 2286             break;
 2287         case _WS:
 2288             processWsState();
 2289             exitState=isolateTags;
 2290             break;
 2291         case _EOF:
 2292             eof = true;
 2293             break;
 2294         default:
 2295             exitState=newState!=STRING_INTERPOLATION;
 2296             break;
 2297         }
 2298     } while ( !exitState && !eof );
 2299 
 2300     closeTag ( STRING_INTERPOLATION );
 2301     return eof;
 2302 }
 2303 
 2304 void CodeGenerator::processWsState()
 2305 {
 2306     if ( !maskWs ) {
 2307         wsBuffer += token;
 2308         token.clear();
 2309         return;
 2310     }
 2311 
 2312     flushWs(6);
 2313 
 2314     int cntWs=0;
 2315     lineIndex--;
 2316     PositionState ps(currentState, 0, true);
 2317             
 2318     while ( line[lineIndex]==' ' || line[lineIndex]=='\t' ) {
 2319         ++cntWs;
 2320         ++lineIndex;
 2321     }
 2322     if ( cntWs>1 ) {
 2323 
 2324         unsigned int styleID=getStyleID ( currentState, currentKeywordClass );
 2325         if ( excludeWs && styleID!=_UNKNOWN ) {
 2326             *out << closeTags[styleID];
 2327         }
 2328         *out << maskWsBegin ;
 2329         for ( int i=0; i<cntWs; i++ ) {
 2330             *out <<  spacer;
 2331             if (applySyntaxTestCase){
 2332                 stateTraceCurrent.push_back(ps);
 2333             }
 2334         }
 2335         *out << maskWsEnd;
 2336         if ( excludeWs && styleID!=_UNKNOWN ) {
 2337             *out << openTags[styleID];
 2338         }
 2339     } else {
 2340     
 2341         *out << spacer; //Bugfix fehlender Space nach Strings
 2342         if (applySyntaxTestCase){
 2343             stateTraceCurrent.push_back(ps);            
 2344         }
 2345     }
 2346     token.clear();
 2347 }
 2348 
 2349 void CodeGenerator::flushWs(int arg)
 2350 {
 2351     PositionState ps(currentState, 0, true);
 2352     //workaround condition
 2353     for ( size_t i=0; i<wsBuffer.size() && ((arg > 3) || ( (arg<4) && lineIndex>1)) && applySyntaxTestCase ; i++ ) {
 2354         stateTraceCurrent.push_back(ps);
 2355         //std::cerr <<"\nflush >"<<wsBuffer<<"< arg:"<<arg;           
 2356     }
 2357      
 2358     //fix canvas whitespace
 2359     if (wsBuffer.length() && (outputType==ESC_XTERM256 || outputType==ESC_TRUECOLOR) ){
 2360         *out<< maskWsBegin;
 2361     }
 2362      
 2363     *out<<wsBuffer;
 2364     wsBuffer.clear();
 2365 }
 2366 
 2367 string CodeGenerator::getTestcaseName(State s, unsigned int kwClass) {
 2368     switch (s) {
 2369         
 2370         case STANDARD:
 2371             return STY_NAME_STD;
 2372         case STRING:
 2373             return STY_NAME_STR;
 2374         case NUMBER:
 2375             return STY_NAME_NUM;
 2376         case SL_COMMENT:
 2377             return STY_NAME_SLC;
 2378         case ML_COMMENT:
 2379             return STY_NAME_COM;
 2380         case ESC_CHAR:
 2381             return STY_NAME_ESC;
 2382         case DIRECTIVE:
 2383             return STY_NAME_DIR;
 2384         case DIRECTIVE_STRING:
 2385             return STY_NAME_DST;
 2386         case SYMBOL:
 2387             return STY_NAME_SYM;
 2388         case STRING_INTERPOLATION:
 2389             return STY_NAME_IPL;
 2390         case _WS:
 2391             return "ws";
 2392         case KEYWORD: {
 2393             
 2394             if (!kwClass)
 2395                 return "ws";
 2396             
 2397             char kwName[5] = {0};
 2398             snprintf(kwName, sizeof(kwName), "kw%c", ('a'+kwClass-1));
 2399             return string(kwName);
 2400         }
 2401         default:
 2402             return "unknown_test";
 2403     }
 2404 }
 2405 
 2406 void CodeGenerator::printTrace(const string &s){
 2407     std::cout<<"\n curr "<<lineNumber<<" "<<s<<": ";
 2408     for (unsigned int i=0; i< stateTraceCurrent.size(); i++) {
 2409         std::cout<<" "<<stateTraceCurrent[i].state;
 2410     }
 2411     std::cout<<"\n test "<<lineNumber<<" "<<s<<": ";
 2412     for (unsigned int i=0; i< stateTraceTest.size(); i++) {
 2413         std::cout<<" "<<stateTraceTest[i].state;
 2414     }
 2415     /*
 2416     for (unsigned int i=0; i< stateTrace.size(); i++) {
 2417         std::cout<<" "<<stateTrace[i];
 2418     }
 2419    */
 2420     std::cout<<"\n";
 2421 }
 2422 
 2423 //column: lineIndex (not a UTF-8 validated string position)
 2424 void CodeGenerator::runSyntaxTestcases(unsigned int column){
 2425     
 2426     if (encoding=="utf-8")
 2427         column = StringTools::utf8_strlen(line.substr(0, column));
 2428     
 2429     unsigned int assertGroup=0;
 2430     size_t typeDescPos=line.find_first_not_of("\t ^", lineIndex);
 2431     State assertState=_UNKNOWN;
 2432     bool negation=false;
 2433     bool testFailed=false;
 2434    
 2435     ostringstream errMsg;
 2436     string prefix;
 2437     //printTrace("trace 2");
 2438     
 2439     if (typeDescPos!=string::npos) {
 2440     
 2441         if (line[typeDescPos]=='~') {
 2442         
 2443             negation=true;
 2444             prefix="~";
 2445             ++typeDescPos;
 2446         }
 2447         
 2448         if (line.find(STY_NAME_NUM, typeDescPos)==typeDescPos)
 2449             assertState=NUMBER;
 2450         else if (line.find(STY_NAME_STR, typeDescPos)==typeDescPos)
 2451             assertState=STRING;
 2452         else if (line.find(STY_NAME_ESC, typeDescPos)==typeDescPos)
 2453             assertState=ESC_CHAR;
 2454         else if (line.find(STY_NAME_IPL, typeDescPos)==typeDescPos)
 2455             assertState=STRING_INTERPOLATION;
 2456         else if (line.find(STY_NAME_SYM, typeDescPos)==typeDescPos)
 2457             assertState=SYMBOL;
 2458         else if (line.find(STY_NAME_DIR, typeDescPos)==typeDescPos)
 2459             assertState=DIRECTIVE;
 2460         else if (line.find(STY_NAME_SLC, typeDescPos)==typeDescPos)
 2461             assertState=SL_COMMENT;
 2462         else if (line.find(STY_NAME_COM, typeDescPos)==typeDescPos)
 2463             assertState=ML_COMMENT;
 2464         else if (line.find("ws", typeDescPos)==typeDescPos)
 2465             assertState=_WS;
 2466         else if (line.find(STY_NAME_STD, typeDescPos)==typeDescPos)
 2467             assertState=STANDARD;
 2468         else if (line.find(STY_NAME_DST, typeDescPos)==typeDescPos)
 2469             assertState=DIRECTIVE_STRING;
 2470         
 2471         else if (line.find("kw", typeDescPos)==typeDescPos) {
 2472             assertState=KEYWORD;
 2473             if (isalpha(line[typeDescPos+2]))
 2474                 assertGroup=line[typeDescPos+2] - 'a' +1;
 2475         }
 2476     
 2477        if (   (assertState!=_WS && stateTraceTest[column].state != assertState && !stateTraceTest[column].isWhiteSpace )
 2478             || (assertState==_WS && !stateTraceTest[column].isWhiteSpace)
 2479             || assertGroup != stateTraceTest[column].kwClass) {
 2480             
 2481             testFailed=!negation;
 2482        
 2483         } else if (negation ) {
 2484             
 2485             //TODO Fix ~ws 
 2486             if (assertState!=_WS  && !stateTraceTest[column].isWhiteSpace )
 2487                 testFailed=true;
 2488             
 2489         }
 2490 
 2491         if (testFailed) {
 2492             errMsg << inFile << " line " << lineNumber << ", column "<< column 
 2493                     << ": got " << getTestcaseName(stateTraceTest[column].state, stateTraceTest[column].kwClass)  
 2494                     << " instead of " << prefix << getTestcaseName(assertState, assertGroup);
 2495         
 2496             failedPosTests.push_back(errMsg.str());    
 2497         }
 2498         
 2499     }
 2500     
 2501     lineContainedTestCase=true; 
 2502 }
 2503 
 2504 
 2505 string CodeGenerator::getNewLine()
 2506 {
 2507     return (printNewLines) ? newLineTag : "";
 2508 }
 2509 
 2510 Diluculum::LuaValueList CodeGenerator::callDecorateLineFct(bool isLineStart)
 2511 {
 2512 
 2513     Diluculum::LuaValueList params;
 2514     params.push_back(Diluculum::LuaValue(lineNumber));
 2515 
 2516     return currentSyntax->getLuaState()->call ( isLineStart ?
 2517             *currentSyntax->getDecorateLineBeginFct(): *currentSyntax->getDecorateLineEndFct(),
 2518             params,"getDecorateLineFct call")  ;
 2519 
 2520 }
 2521 
 2522 void CodeGenerator::insertLineNumber ( bool insertNewLine )
 2523 {
 2524     if ( insertNewLine ) {
 2525         if (currentSyntax->getDecorateLineEndFct()) {
 2526             Diluculum::LuaValueList res=callDecorateLineFct(false);
 2527             if (res.size()==1) {
 2528                 wsBuffer +=res[0].asString();
 2529             }
 2530         }
 2531 
 2532         wsBuffer += getNewLine();
 2533     }
 2534 
 2535     if (currentSyntax->getDecorateLineBeginFct()) {
 2536         Diluculum::LuaValueList res=callDecorateLineFct(true);
 2537         if (res.size()==1) {
 2538             wsBuffer +=res[0].asString();
 2539         }
 2540     }
 2541 
 2542     if ( showLineNumbers ) {
 2543         ostringstream os;
 2544         ostringstream numberPrefix;
 2545 
 2546         os << setw ( getLineNumberWidth() ) << right;
 2547         if( numberCurrentLine ) {
 2548             if ( lineNumberFillZeroes ) {
 2549                 os.fill ( '0' );
 2550             }
 2551             os << lineNumber+lineNumberOffset;
 2552         } else {
 2553             os << "";
 2554         }
 2555 
 2556         numberPrefix << openTags[LINENUMBER];
 2557         maskString ( numberPrefix, os.str() );
 2558         numberPrefix << spacer << closeTags[LINENUMBER];
 2559 
 2560         wsBuffer += numberPrefix.str();
 2561     }
 2562 }
 2563 
 2564 unsigned int CodeGenerator::getLineIndex()
 2565 {
 2566     return lineIndex;
 2567 }
 2568 unsigned int CodeGenerator::getLastLineLength()
 2569 {
 2570     return lastLineLength;
 2571 }
 2572 
 2573 bool CodeGenerator::requiresTwoPassParsing() const {
 2574     if (!currentSyntax) return false;
 2575     return currentSyntax->getPersistentSnippetsNum()>0;
 2576 }
 2577 
 2578 
 2579 bool CodeGenerator::printExternalStyle ( const string &outFile )
 2580 {
 2581     if ( !includeStyleDef ) {
 2582         ostream *cssOutFile = ( outFile.empty() ? &cout :new ofstream ( outFile.c_str() ) );
 2583         if ( !cssOutFile->fail() ) {
 2584             if (!omitVersionComment) {
 2585                 *cssOutFile << styleCommentOpen
 2586                             <<" Style definition file generated by highlight "
 2587                             << HIGHLIGHT_VERSION << ", " << HIGHLIGHT_URL
 2588                             << " " << styleCommentClose << "\n";
 2589             }
 2590             *cssOutFile << getStyleDefinition()
 2591                         << "\n";
 2592             *cssOutFile << readUserStyleDef();
 2593             if ( !outFile.empty() ) delete cssOutFile;
 2594         } else {
 2595             return false;
 2596         }
 2597     }
 2598     return true;
 2599 }
 2600 
 2601 bool CodeGenerator::printPersistentState ( const string &outFile )
 2602 {
 2603     if (!currentSyntax) return false;
 2604 
 2605     ofstream pluginOutFile( outFile.c_str());
 2606     if ( !pluginOutFile.fail() ) {
 2607         
 2608         pluginOutFile   <<"Description=\"Plugin generated by highlight using the --two-pass option\"\n\n"
 2609                         <<"Categories = {\"two-pass\" }\n\n"
 2610                         <<"function syntaxUpdate(desc)\n\n";
 2611                         
 2612         pluginOutFile << currentSyntax->getPersistentHookConditions();
 2613      
 2614         for (auto snippet: currentSyntax->getPersistentSnippets())
 2615         {   
 2616             pluginOutFile << snippet <<"\n\n";
 2617         }
 2618         
 2619         pluginOutFile<<"end\n\n"
 2620                      <<"Plugins={\n"
 2621                      <<"  { Type=\"lang\", Chunk=syntaxUpdate }\n"
 2622                      <<"}\n";
 2623     } else {
 2624         return false;
 2625     }
 2626     
 2627     return true;
 2628 }
 2629 
 2630 string CodeGenerator::readUserStyleDef()
 2631 {
 2632     ostringstream ostr;
 2633     if ( !styleInputPath.empty() ) {
 2634         ifstream userStyleDef ( styleInputPath.c_str() );
 2635         if ( userStyleDef ) {
 2636             ostr    << "\n" << styleCommentOpen
 2637                     << " Content of " << styleInputPath
 2638                     << ": " <<styleCommentClose << "\n";
 2639             string line;
 2640             while ( getline ( userStyleDef, line ) ) {
 2641                 ostr << line << "\n";
 2642             }
 2643             userStyleDef.close();
 2644         } else {
 2645             ostr    << styleCommentOpen
 2646                     << " ERROR: Could not include " << styleInputPath
 2647                     << "." << styleCommentClose << "\n";
 2648         }
 2649     }
 2650 
 2651     string injections=docStyle.getInjections();
 2652     if (!injections.empty()) {
 2653         ostr    << "\n" << styleCommentOpen
 2654                 << " Plug-in theme injections: " <<styleCommentClose << "\n";
 2655         ostr << injections<<"\n";
 2656     }
 2657     return ostr.str();
 2658 }
 2659 
 2660 bool CodeGenerator::initPluginScript(const string& script)
 2661 {
 2662 
 2663     if (script.empty()) return true;
 2664 
 2665     try {
 2666 
 2667         userScriptError="";
 2668         Diluculum::LuaState ls;
 2669         
 2670         ls.doFile (script);
 2671         int listIdx=1;
 2672 
 2673         while (ls["Plugins"][listIdx].value() !=Diluculum::Nil) {
 2674 
 2675             // Theme plugins
 2676             if (ls["Plugins"][listIdx]["Type"].value().asString()=="theme") {
 2677                 if (ls["Plugins"][listIdx]["Chunk"].value().type()==LUA_TFUNCTION) {
 2678                     docStyle.addUserChunk(ls["Plugins"][listIdx]["Chunk"].value().asFunction());
 2679                 }
 2680             }
 2681             // Syntax plugins
 2682             else if (ls["Plugins"][listIdx]["Type"].value().asString()=="lang") {
 2683                 if (ls["Plugins"][listIdx]["Chunk"].value().type()==LUA_TFUNCTION) {
 2684                     currentSyntax->addUserChunk(ls["Plugins"][listIdx]["Chunk"].value().asFunction());
 2685                 }
 2686             }
 2687             // Format plugins
 2688             else if (ls["Plugins"][listIdx]["Type"].value().asString()=="format") {
 2689                 if (ls["Plugins"][listIdx]["Chunk"].value().type()==LUA_TFUNCTION) {
 2690                     addUserChunk(ls["Plugins"][listIdx]["Chunk"].value().asFunction());
 2691                 }
 2692             }
 2693             
 2694             listIdx++;
 2695         }
 2696     }  catch (Diluculum::LuaError &err) {
 2697         userScriptError=err.what();
 2698         return false;
 2699     }
 2700     return true;
 2701 }
 2702 
 2703 void CodeGenerator::resetSyntaxReaders() {
 2704     for ( map<string, SyntaxReader*>::iterator it=syntaxReaders.begin(); it!=syntaxReaders.end(); it++ ) {
 2705         delete it->second;
 2706     }
 2707     currentSyntax=NULL;
 2708     syntaxReaders.clear();
 2709 }
 2710 
 2711 bool CodeGenerator::syntaxRequiresTwoPassRun() {
 2712     if (!currentSyntax) return false;
 2713     return currentSyntax->requiresTwoPassRun();
 2714 }
 2715 
 2716 void CodeGenerator::clearPersistentSnippets(){
 2717     if (currentSyntax) {
 2718         currentSyntax->clearPersistentSnippets();
 2719     }
 2720 }
 2721 
 2722 
 2723 }