"Fossies" - the Fresh Open Source Software Archive  

Source code changes of the file "src/Text/Pandoc/Readers/LaTeX/Parsing.hs" between
pandoc-2.14.0.3.tar.gz and pandoc-2.14.1.tar.gz

About: Pandoc converts files from one markup format into another.

Parsing.hs  (pandoc-2.14.0.3):Parsing.hs  (pandoc-2.14.1)
{-# LANGUAGE FlexibleInstances #-} {-# LANGUAGE FlexibleInstances #-}
{-# LANGUAGE FlexibleContexts #-}
{-# LANGUAGE LambdaCase #-} {-# LANGUAGE LambdaCase #-}
{-# LANGUAGE MultiParamTypeClasses #-} {-# LANGUAGE MultiParamTypeClasses #-}
{-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE ScopedTypeVariables #-}
{- | {- |
Module : Text.Pandoc.Readers.LaTeX.Parsing Module : Text.Pandoc.Readers.LaTeX.Parsing
Copyright : Copyright (C) 2006-2021 John MacFarlane Copyright : Copyright (C) 2006-2021 John MacFarlane
License : GNU GPL, version 2 or above License : GNU GPL, version 2 or above
Maintainer : John MacFarlane <jgm@berkeley.edu> Maintainer : John MacFarlane <jgm@berkeley.edu>
skipping to change at line 31 skipping to change at line 32
, TheoremSpec(..) , TheoremSpec(..)
, TheoremStyle(..) , TheoremStyle(..)
, LaTeXState(..) , LaTeXState(..)
, defaultLaTeXState , defaultLaTeXState
, LP , LP
, withVerbatimMode , withVerbatimMode
, rawLaTeXParser , rawLaTeXParser
, applyMacros , applyMacros
, tokenize , tokenize
, tokenizeSources , tokenizeSources
, getInputTokens
, untokenize , untokenize
, untoken , untoken
, totoks , totoks
, toksToString , toksToString
, satisfyTok , satisfyTok
, parseFromToks , parseFromToks
, disablingWithRaw , disablingWithRaw
, doMacros , doMacros
, doMacros' , doMacros'
, setpos , setpos
skipping to change at line 249 skipping to change at line 251
withVerbatimMode parser = do withVerbatimMode parser = do
alreadyVerbatimMode <- sVerbatimMode <$> getState alreadyVerbatimMode <- sVerbatimMode <$> getState
if alreadyVerbatimMode if alreadyVerbatimMode
then parser then parser
else do else do
updateState $ \st -> st{ sVerbatimMode = True } updateState $ \st -> st{ sVerbatimMode = True }
result <- parser result <- parser
updateState $ \st -> st{ sVerbatimMode = False } updateState $ \st -> st{ sVerbatimMode = False }
return result return result
rawLaTeXParser :: (PandocMonad m, HasMacros s, HasReaderOptions s) rawLaTeXParser :: (PandocMonad m, HasMacros s, HasReaderOptions s, Show a)
=> [Tok] -> Bool -> LP m a -> LP m a => [Tok] -> Bool -> LP m a -> LP m a
-> ParserT Sources s m (a, Text) -> ParserT Sources s m (a, Text)
rawLaTeXParser toks retokenize parser valParser = do rawLaTeXParser toks retokenize parser valParser = do
pstate <- getState pstate <- getState
let lstate = def{ sOptions = extractReaderOptions pstate } let lstate = def{ sOptions = extractReaderOptions pstate }
let lstate' = lstate { sMacros = extractMacros pstate } let lstate' = lstate { sMacros = extractMacros pstate }
let setStartPos = case toks of
Tok pos _ _ : _ -> setPosition pos
_ -> return ()
let preparser = setStartPos >> parser
let rawparser = (,) <$> withRaw valParser <*> getState let rawparser = (,) <$> withRaw valParser <*> getState
res' <- lift $ runParserT (snd <$> withRaw parser) lstate "chunk" toks res' <- lift $ runParserT (withRaw (preparser >> getPosition))
lstate "chunk" toks
case res' of case res' of
Left _ -> mzero Left _ -> mzero
Right toks' -> do Right (endpos, toks') -> do
res <- lift $ runParserT (do when retokenize $ do res <- lift $ runParserT (do when retokenize $ do
-- retokenize, applying macros -- retokenize, applying macros
ts <- many (satisfyTok (const True)) ts <- many (satisfyTok (const True))
setInput ts setInput ts
rawparser) rawparser)
lstate' "chunk" toks' lstate' "chunk" toks'
case res of case res of
Left _ -> mzero Left _ -> mzero
Right ((val, raw), st) -> do Right ((val, raw), st) -> do
updateState (updateMacros (sMacros st <>)) updateState (updateMacros (sMacros st <>))
void $ count (T.length (untokenize toks')) anyChar let skipTilPos stopPos = do
anyChar
pos <- getPosition
if pos >= stopPos
then return ()
else skipTilPos stopPos
skipTilPos endpos
let result = untokenize raw let result = untokenize raw
-- ensure we end with space if input did, see #4442 -- ensure we end with space if input did, see #4442
let result' = let result' =
case reverse toks' of case reverse toks' of
(Tok _ (CtrlSeq _) t : _) (Tok _ (CtrlSeq _) t : _)
| " " `T.isSuffixOf` t | " " `T.isSuffixOf` t
, not (" " `T.isSuffixOf` result) , not (" " `T.isSuffixOf` result)
-> result <> " " -> result <> " "
_ -> result _ -> result
return (val, result') return (val, result')
skipping to change at line 309 skipping to change at line 322
> tokUntokRoundtrip :: String -> Bool > tokUntokRoundtrip :: String -> Bool
> tokUntokRoundtrip s = > tokUntokRoundtrip s =
> let t = T.pack s in untokenize (tokenize "random" t) == t > let t = T.pack s in untokenize (tokenize "random" t) == t
-} -}
tokenizeSources :: Sources -> [Tok] tokenizeSources :: Sources -> [Tok]
tokenizeSources = concatMap tokenizeSource . unSources tokenizeSources = concatMap tokenizeSource . unSources
where where
tokenizeSource (pos, t) = totoks pos t tokenizeSource (pos, t) = totoks pos t
-- Return tokens from input sources. Ensure that starting position is
-- correct.
getInputTokens :: PandocMonad m => ParserT Sources s m [Tok]
getInputTokens = do
pos <- getPosition
ss <- getInput
return $
case ss of
Sources [] -> []
Sources ((_,t):rest) -> tokenizeSources $ Sources ((pos,t):rest)
tokenize :: SourceName -> Text -> [Tok] tokenize :: SourceName -> Text -> [Tok]
tokenize sourcename = totoks (initialPos sourcename) tokenize sourcename = totoks (initialPos sourcename)
totoks :: SourcePos -> Text -> [Tok] totoks :: SourcePos -> Text -> [Tok]
totoks pos t = totoks pos t =
case T.uncons t of case T.uncons t of
Nothing -> [] Nothing -> []
Just (c, rest) Just (c, rest)
| c == '\n' -> | c == '\n' ->
Tok pos Newline "\n" Tok pos Newline "\n"
skipping to change at line 436 skipping to change at line 460
untoken :: Tok -> Text untoken :: Tok -> Text
untoken t = untokenAccum t mempty untoken t = untokenAccum t mempty
toksToString :: [Tok] -> String toksToString :: [Tok] -> String
toksToString = T.unpack . untokenize toksToString = T.unpack . untokenize
parseFromToks :: PandocMonad m => LP m a -> [Tok] -> LP m a parseFromToks :: PandocMonad m => LP m a -> [Tok] -> LP m a
parseFromToks parser toks = do parseFromToks parser toks = do
oldInput <- getInput oldInput <- getInput
setInput toks setInput toks
oldpos <- getPosition
case toks of
Tok pos _ _ : _ -> setPosition pos
_ -> return ()
result <- disablingWithRaw parser result <- disablingWithRaw parser
setInput oldInput setInput oldInput
setPosition oldpos
return result return result
disablingWithRaw :: PandocMonad m => LP m a -> LP m a disablingWithRaw :: PandocMonad m => LP m a -> LP m a
disablingWithRaw parser = do disablingWithRaw parser = do
oldEnableWithRaw <- sEnableWithRaw <$> getState oldEnableWithRaw <- sEnableWithRaw <$> getState
updateState $ \st -> st{ sEnableWithRaw = False } updateState $ \st -> st{ sEnableWithRaw = False }
result <- parser result <- parser
updateState $ \st -> st{ sEnableWithRaw = oldEnableWithRaw } updateState $ \st -> st{ sEnableWithRaw = oldEnableWithRaw }
return result return result
skipping to change at line 461 skipping to change at line 490
res <- tokenPrim (T.unpack . untoken) updatePos matcher res <- tokenPrim (T.unpack . untoken) updatePos matcher
updateState $ \st -> updateState $ \st ->
if sEnableWithRaw st if sEnableWithRaw st
then st{ sRawTokens = IntMap.map (res:) $ sRawTokens st } then st{ sRawTokens = IntMap.map (res:) $ sRawTokens st }
else st else st
return $! res return $! res
where matcher t | f t = Just t where matcher t | f t = Just t
| otherwise = Nothing | otherwise = Nothing
updatePos :: SourcePos -> Tok -> [Tok] -> SourcePos updatePos :: SourcePos -> Tok -> [Tok] -> SourcePos
updatePos _spos _ (Tok pos _ _ : _) = pos updatePos _spos _ (Tok pos _ _ : _) = pos
updatePos spos _ [] = incSourceColumn spos 1 updatePos spos (Tok _ _ t) [] = incSourceColumn spos (T.length t)
doMacros :: PandocMonad m => LP m () doMacros :: PandocMonad m => LP m ()
doMacros = do doMacros = do
st <- getState st <- getState
unless (sVerbatimMode st) $ unless (sVerbatimMode st) $
getInput >>= doMacros' 1 >>= setInput getInput >>= doMacros' 1 >>= setInput
doMacros' :: PandocMonad m => Int -> [Tok] -> LP m [Tok] doMacros' :: PandocMonad m => Int -> [Tok] -> LP m [Tok]
doMacros' n inp = doMacros' n inp =
case inp of case inp of
 End of changes. 11 change blocks. 
5 lines changed or deleted 34 lines changed or added

Home  |  About  |  Features  |  All  |  Newest  |  Dox  |  Diffs  |  RSS Feeds  |  Screenshots  |  Comments  |  Imprint  |  Privacy  |  HTTP(S)