{-# LANGUAGE BangPatterns, CPP, Rank2Types #-}

-- |
-- Module      : Data.Text.Lazy.Encoding.Fusion
-- Copyright   : (c) 2009, 2010 Bryan O'Sullivan
--
-- License     : BSD-style
-- Maintainer  : bos@serpentine.com
-- Stability   : experimental
-- Portability : portable
--
-- /Warning/: this is an internal module, and does not have a stable
-- API or name. Functions in this module may not check or enforce
-- preconditions expected by public modules. Use at your own risk!
--
-- Fusible 'Stream'-oriented functions for converting between lazy
-- 'Text' and several common encodings.

module Data.Text.Internal.Lazy.Encoding.Fusion
    (
    -- * Streaming
    --  streamASCII
      streamUtf8
    , streamUtf16LE
    , streamUtf16BE
    , streamUtf32LE
    , streamUtf32BE

    -- * Unstreaming
    , unstream

    , module Data.Text.Internal.Encoding.Fusion.Common
    ) where

import Data.ByteString.Lazy.Internal (ByteString(..), defaultChunkSize)
import qualified Data.ByteString as B
import qualified Data.ByteString.Unsafe as B
import Data.Text.Internal.ByteStringCompat
import Data.Text.Internal.Encoding.Fusion.Common
import Data.Text.Encoding.Error
import Data.Text.Internal.Fusion (Step(..), Stream(..))
import Data.Text.Internal.Fusion.Size
import Data.Text.Internal.Unsafe.Char (unsafeChr, unsafeChr8, unsafeChr32)
import Data.Text.Internal.Unsafe.Shift (shiftL)
import Data.Text.Internal.Functions (unsafeWithForeignPtr)
import Data.Word (Word8, Word16, Word32)
import qualified Data.Text.Internal.Encoding.Utf8 as U8
import qualified Data.Text.Internal.Encoding.Utf16 as U16
import qualified Data.Text.Internal.Encoding.Utf32 as U32
import Data.Text.Unsafe (unsafeDupablePerformIO)
import Foreign.ForeignPtr (ForeignPtr)
import Foreign.Storable (pokeByteOff)
import Data.ByteString.Internal (mallocByteString, memcpy)
#if defined(ASSERTS)
import Control.Exception (assert)
#endif

data S = S0
       | S1 {-# UNPACK #-} !Word8
       | S2 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8
       | S3 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8
       | S4 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8 {-# UNPACK #-} !Word8

data T = T !ByteString !S {-# UNPACK #-} !Int

-- | /O(n)/ Convert a lazy 'ByteString' into a 'Stream Char', using
-- UTF-8 encoding.
streamUtf8 :: OnDecodeError -> ByteString -> Stream Char
streamUtf8 :: OnDecodeError -> ByteString -> Stream Char
streamUtf8 OnDecodeError
onErr ByteString
bs0 = forall a s. (s -> Step s a) -> s -> Size -> Stream a
Stream T -> Step T Char
next (ByteString -> S -> Int -> T
T ByteString
bs0 S
S0 Int
0) Size
unknownSize
  where
    next :: T -> Step T Char
next (T bs :: ByteString
bs@(Chunk ByteString
ps ByteString
_) S
S0 Int
i)
      | Int
i forall a. Ord a => a -> a -> Bool
< Int
len Bool -> Bool -> Bool
&& Word8 -> Bool
U8.validate1 Word8
a =
          forall s a. a -> s -> Step s a
Yield (Word8 -> Char
unsafeChr8 Word8
a)    (ByteString -> S -> Int -> T
T ByteString
bs S
S0 (Int
iforall a. Num a => a -> a -> a
+Int
1))
      | Int
i forall a. Num a => a -> a -> a
+ Int
1 forall a. Ord a => a -> a -> Bool
< Int
len Bool -> Bool -> Bool
&& Word8 -> Word8 -> Bool
U8.validate2 Word8
a Word8
b =
          forall s a. a -> s -> Step s a
Yield (Word8 -> Word8 -> Char
U8.chr2 Word8
a Word8
b)     (ByteString -> S -> Int -> T
T ByteString
bs S
S0 (Int
iforall a. Num a => a -> a -> a
+Int
2))
      | Int
i forall a. Num a => a -> a -> a
+ Int
2 forall a. Ord a => a -> a -> Bool
< Int
len Bool -> Bool -> Bool
&& Word8 -> Word8 -> Word8 -> Bool
U8.validate3 Word8
a Word8
b Word8
c =
          forall s a. a -> s -> Step s a
Yield (Word8 -> Word8 -> Word8 -> Char
U8.chr3 Word8
a Word8
b Word8
c)   (ByteString -> S -> Int -> T
T ByteString
bs S
S0 (Int
iforall a. Num a => a -> a -> a
+Int
3))
      | Int
i forall a. Num a => a -> a -> a
+ Int
3 forall a. Ord a => a -> a -> Bool
< Int
len Bool -> Bool -> Bool
&& Word8 -> Word8 -> Word8 -> Word8 -> Bool
U8.validate4 Word8
a Word8
b Word8
c Word8
d =
          forall s a. a -> s -> Step s a
Yield (Word8 -> Word8 -> Word8 -> Word8 -> Char
U8.chr4 Word8
a Word8
b Word8
c Word8
d) (ByteString -> S -> Int -> T
T ByteString
bs S
S0 (Int
iforall a. Num a => a -> a -> a
+Int
4))
      where len :: Int
len = ByteString -> Int
B.length ByteString
ps
            a :: Word8
a = ByteString -> Int -> Word8
B.unsafeIndex ByteString
ps Int
i
            b :: Word8
b = ByteString -> Int -> Word8
B.unsafeIndex ByteString
ps (Int
iforall a. Num a => a -> a -> a
+Int
1)
            c :: Word8
c = ByteString -> Int -> Word8
B.unsafeIndex ByteString
ps (Int
iforall a. Num a => a -> a -> a
+Int
2)
            d :: Word8
d = ByteString -> Int -> Word8
B.unsafeIndex ByteString
ps (Int
iforall a. Num a => a -> a -> a
+Int
3)
    next st :: T
st@(T ByteString
bs S
s Int
i) =
      case S
s of
        S1 Word8
a       | Word8 -> Bool
U8.validate1 Word8
a       -> forall s a. a -> s -> Step s a
Yield (Word8 -> Char
unsafeChr8 Word8
a)    T
es
        S2 Word8
a Word8
b     | Word8 -> Word8 -> Bool
U8.validate2 Word8
a Word8
b     -> forall s a. a -> s -> Step s a
Yield (Word8 -> Word8 -> Char
U8.chr2 Word8
a Word8
b)     T
es
        S3 Word8
a Word8
b Word8
c   | Word8 -> Word8 -> Word8 -> Bool
U8.validate3 Word8
a Word8
b Word8
c   -> forall s a. a -> s -> Step s a
Yield (Word8 -> Word8 -> Word8 -> Char
U8.chr3 Word8
a Word8
b Word8
c)   T
es
        S4 Word8
a Word8
b Word8
c Word8
d | Word8 -> Word8 -> Word8 -> Word8 -> Bool
U8.validate4 Word8
a Word8
b Word8
c Word8
d -> forall s a. a -> s -> Step s a
Yield (Word8 -> Word8 -> Word8 -> Word8 -> Char
U8.chr4 Word8
a Word8
b Word8
c Word8
d) T
es
        S
_ -> T -> Step T Char
consume T
st
       where es :: T
es = ByteString -> S -> Int -> T
T ByteString
bs S
S0 Int
i
    consume :: T -> Step T Char
consume (T bs :: ByteString
bs@(Chunk ByteString
ps ByteString
rest) S
s Int
i)
        | Int
i forall a. Ord a => a -> a -> Bool
>= ByteString -> Int
B.length ByteString
ps = T -> Step T Char
consume (ByteString -> S -> Int -> T
T ByteString
rest S
s Int
0)
        | Bool
otherwise =
      case S
s of
        S
S0         -> T -> Step T Char
next (ByteString -> S -> Int -> T
T ByteString
bs (Word8 -> S
S1 Word8
x)       (Int
iforall a. Num a => a -> a -> a
+Int
1))
        S1 Word8
a       -> T -> Step T Char
next (ByteString -> S -> Int -> T
T ByteString
bs (Word8 -> Word8 -> S
S2 Word8
a Word8
x)     (Int
iforall a. Num a => a -> a -> a
+Int
1))
        S2 Word8
a Word8
b     -> T -> Step T Char
next (ByteString -> S -> Int -> T
T ByteString
bs (Word8 -> Word8 -> Word8 -> S
S3 Word8
a Word8
b Word8
x)   (Int
iforall a. Num a => a -> a -> a
+Int
1))
        S3 Word8
a Word8
b Word8
c   -> T -> Step T Char
next (ByteString -> S -> Int -> T
T ByteString
bs (Word8 -> Word8 -> Word8 -> Word8 -> S
S4 Word8
a Word8
b Word8
c Word8
x) (Int
iforall a. Num a => a -> a -> a
+Int
1))
        S4 Word8
a Word8
b Word8
c Word8
d -> forall s.
String
-> String -> OnDecodeError -> Maybe Word8 -> s -> Step s Char
decodeError String
"streamUtf8" String
"UTF-8" OnDecodeError
onErr (forall a. a -> Maybe a
Just Word8
a)
                           (ByteString -> S -> Int -> T
T ByteString
bs (Word8 -> Word8 -> Word8 -> S
S3 Word8
b Word8
c Word8
d)   (Int
iforall a. Num a => a -> a -> a
+Int
1))
        where x :: Word8
x = ByteString -> Int -> Word8
B.unsafeIndex ByteString
ps Int
i
    consume (T ByteString
Empty S
S0 Int
_) = forall s a. Step s a
Done
    consume T
st             = forall s.
String
-> String -> OnDecodeError -> Maybe Word8 -> s -> Step s Char
decodeError String
"streamUtf8" String
"UTF-8" OnDecodeError
onErr forall a. Maybe a
Nothing T
st
{-# INLINE [0] streamUtf8 #-}

-- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using little
-- endian UTF-16 encoding.
streamUtf16LE :: OnDecodeError -> ByteString -> Stream Char
streamUtf16LE :: OnDecodeError -> ByteString -> Stream Char
streamUtf16LE OnDecodeError
onErr ByteString
bs0 = forall a s. (s -> Step s a) -> s -> Size -> Stream a
Stream T -> Step T Char
next (ByteString -> S -> Int -> T
T ByteString
bs0 S
S0 Int
0) Size
unknownSize
  where
    next :: T -> Step T Char
next (T bs :: ByteString
bs@(Chunk ByteString
ps ByteString
_) S
S0 Int
i)
      | Int
i forall a. Num a => a -> a -> a
+ Int
1 forall a. Ord a => a -> a -> Bool
< Int
len Bool -> Bool -> Bool
&& Word16 -> Bool
U16.validate1 Word16
x1 =
          forall s a. a -> s -> Step s a
Yield (Word16 -> Char
unsafeChr Word16
x1)         (ByteString -> S -> Int -> T
T ByteString
bs S
S0 (Int
iforall a. Num a => a -> a -> a
+Int
2))
      | Int
i forall a. Num a => a -> a -> a
+ Int
3 forall a. Ord a => a -> a -> Bool
< Int
len Bool -> Bool -> Bool
&& Word16 -> Word16 -> Bool
U16.validate2 Word16
x1 Word16
x2 =
          forall s a. a -> s -> Step s a
Yield (Word16 -> Word16 -> Char
U16.chr2 Word16
x1 Word16
x2)       (ByteString -> S -> Int -> T
T ByteString
bs S
S0 (Int
iforall a. Num a => a -> a -> a
+Int
4))
      where len :: Int
len = ByteString -> Int
B.length ByteString
ps
            x1 :: Word16
x1   = forall {a}. (Num a, UnsafeShift a) => a -> a -> a
c (Int -> Word16
idx  Int
i)      (Int -> Word16
idx (Int
i forall a. Num a => a -> a -> a
+ Int
1))
            x2 :: Word16
x2   = forall {a}. (Num a, UnsafeShift a) => a -> a -> a
c (Int -> Word16
idx (Int
i forall a. Num a => a -> a -> a
+ Int
2)) (Int -> Word16
idx (Int
i forall a. Num a => a -> a -> a
+ Int
3))
            c :: a -> a -> a
c a
w1 a
w2 = a
w1 forall a. Num a => a -> a -> a
+ (a
w2 forall a. UnsafeShift a => a -> Int -> a
`shiftL` Int
8)
            idx :: Int -> Word16
idx = Word8 -> Word16
word8ToWord16 forall b c a. (b -> c) -> (a -> b) -> a -> c
. ByteString -> Int -> Word8
B.unsafeIndex ByteString
ps :: Int -> Word16
    next st :: T
st@(T ByteString
bs S
s Int
i) =
      case S
s of
        S2 Word8
w1 Word8
w2       | Word16 -> Bool
U16.validate1 (Word8 -> Word8 -> Word16
c Word8
w1 Word8
w2)           ->
          forall s a. a -> s -> Step s a
Yield (Word16 -> Char
unsafeChr (Word8 -> Word8 -> Word16
c Word8
w1 Word8
w2))   T
es
        S4 Word8
w1 Word8
w2 Word8
w3 Word8
w4 | Word16 -> Word16 -> Bool
U16.validate2 (Word8 -> Word8 -> Word16
c Word8
w1 Word8
w2) (Word8 -> Word8 -> Word16
c Word8
w3 Word8
w4) ->
          forall s a. a -> s -> Step s a
Yield (Word16 -> Word16 -> Char
U16.chr2 (Word8 -> Word8 -> Word16
c Word8
w1 Word8
w2) (Word8 -> Word8 -> Word16
c Word8
w3 Word8
w4)) T
es
        S
_ -> T -> Step T Char
consume T
st
       where es :: T
es = ByteString -> S -> Int -> T
T ByteString
bs S
S0 Int
i
             c :: Word8 -> Word8 -> Word16
             c :: Word8 -> Word8 -> Word16
c Word8
w1 Word8
w2 = Word8 -> Word16
word8ToWord16 Word8
w1 forall a. Num a => a -> a -> a
+ (Word8 -> Word16
word8ToWord16 Word8
w2 forall a. UnsafeShift a => a -> Int -> a
`shiftL` Int
8)
    consume :: T -> Step T Char
consume (T bs :: ByteString
bs@(Chunk ByteString
ps ByteString
rest) S
s Int
i)
        | Int
i forall a. Ord a => a -> a -> Bool
>= ByteString -> Int
B.length ByteString
ps = T -> Step T Char
consume (ByteString -> S -> Int -> T
T ByteString
rest S
s Int
0)
        | Bool
otherwise =
      case S
s of
        S
S0             -> T -> Step T Char
next (ByteString -> S -> Int -> T
T ByteString
bs (Word8 -> S
S1 Word8
x)          (Int
iforall a. Num a => a -> a -> a
+Int
1))
        S1 Word8
w1          -> T -> Step T Char
next (ByteString -> S -> Int -> T
T ByteString
bs (Word8 -> Word8 -> S
S2 Word8
w1 Word8
x)       (Int
iforall a. Num a => a -> a -> a
+Int
1))
        S2 Word8
w1 Word8
w2       -> T -> Step T Char
next (ByteString -> S -> Int -> T
T ByteString
bs (Word8 -> Word8 -> Word8 -> S
S3 Word8
w1 Word8
w2 Word8
x)    (Int
iforall a. Num a => a -> a -> a
+Int
1))
        S3 Word8
w1 Word8
w2 Word8
w3    -> T -> Step T Char
next (ByteString -> S -> Int -> T
T ByteString
bs (Word8 -> Word8 -> Word8 -> Word8 -> S
S4 Word8
w1 Word8
w2 Word8
w3 Word8
x) (Int
iforall a. Num a => a -> a -> a
+Int
1))
        S4 Word8
w1 Word8
w2 Word8
w3 Word8
w4 -> forall s.
String
-> String -> OnDecodeError -> Maybe Word8 -> s -> Step s Char
decodeError String
"streamUtf16LE" String
"UTF-16LE" OnDecodeError
onErr (forall a. a -> Maybe a
Just Word8
w1)
                           (ByteString -> S -> Int -> T
T ByteString
bs (Word8 -> Word8 -> Word8 -> S
S3 Word8
w2 Word8
w3 Word8
w4)       (Int
iforall a. Num a => a -> a -> a
+Int
1))
        where x :: Word8
x = ByteString -> Int -> Word8
B.unsafeIndex ByteString
ps Int
i
    consume (T ByteString
Empty S
S0 Int
_) = forall s a. Step s a
Done
    consume T
st             = forall s.
String
-> String -> OnDecodeError -> Maybe Word8 -> s -> Step s Char
decodeError String
"streamUtf16LE" String
"UTF-16LE" OnDecodeError
onErr forall a. Maybe a
Nothing T
st
{-# INLINE [0] streamUtf16LE #-}

-- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using big
-- endian UTF-16 encoding.
streamUtf16BE :: OnDecodeError -> ByteString -> Stream Char
streamUtf16BE :: OnDecodeError -> ByteString -> Stream Char
streamUtf16BE OnDecodeError
onErr ByteString
bs0 = forall a s. (s -> Step s a) -> s -> Size -> Stream a
Stream T -> Step T Char
next (ByteString -> S -> Int -> T
T ByteString
bs0 S
S0 Int
0) Size
unknownSize
  where
    next :: T -> Step T Char
next (T bs :: ByteString
bs@(Chunk ByteString
ps ByteString
_) S
S0 Int
i)
      | Int
i forall a. Num a => a -> a -> a
+ Int
1 forall a. Ord a => a -> a -> Bool
< Int
len Bool -> Bool -> Bool
&& Word16 -> Bool
U16.validate1 Word16
x1 =
          forall s a. a -> s -> Step s a
Yield (Word16 -> Char
unsafeChr Word16
x1)         (ByteString -> S -> Int -> T
T ByteString
bs S
S0 (Int
iforall a. Num a => a -> a -> a
+Int
2))
      | Int
i forall a. Num a => a -> a -> a
+ Int
3 forall a. Ord a => a -> a -> Bool
< Int
len Bool -> Bool -> Bool
&& Word16 -> Word16 -> Bool
U16.validate2 Word16
x1 Word16
x2 =
          forall s a. a -> s -> Step s a
Yield (Word16 -> Word16 -> Char
U16.chr2 Word16
x1 Word16
x2)       (ByteString -> S -> Int -> T
T ByteString
bs S
S0 (Int
iforall a. Num a => a -> a -> a
+Int
4))
      where len :: Int
len = ByteString -> Int
B.length ByteString
ps
            x1 :: Word16
x1   = forall {a}. (Num a, UnsafeShift a) => a -> a -> a
c (Int -> Word16
idx  Int
i)      (Int -> Word16
idx (Int
i forall a. Num a => a -> a -> a
+ Int
1))
            x2 :: Word16
x2   = forall {a}. (Num a, UnsafeShift a) => a -> a -> a
c (Int -> Word16
idx (Int
i forall a. Num a => a -> a -> a
+ Int
2)) (Int -> Word16
idx (Int
i forall a. Num a => a -> a -> a
+ Int
3))
            c :: a -> a -> a
c a
w1 a
w2 = (a
w1 forall a. UnsafeShift a => a -> Int -> a
`shiftL` Int
8) forall a. Num a => a -> a -> a
+ a
w2
            idx :: Int -> Word16
idx = Word8 -> Word16
word8ToWord16 forall b c a. (b -> c) -> (a -> b) -> a -> c
. ByteString -> Int -> Word8
B.unsafeIndex ByteString
ps :: Int -> Word16
    next st :: T
st@(T ByteString
bs S
s Int
i) =
      case S
s of
        S2 Word8
w1 Word8
w2       | Word16 -> Bool
U16.validate1 (Word8 -> Word8 -> Word16
c Word8
w1 Word8
w2)           ->
          forall s a. a -> s -> Step s a
Yield (Word16 -> Char
unsafeChr (Word8 -> Word8 -> Word16
c Word8
w1 Word8
w2))   T
es
        S4 Word8
w1 Word8
w2 Word8
w3 Word8
w4 | Word16 -> Word16 -> Bool
U16.validate2 (Word8 -> Word8 -> Word16
c Word8
w1 Word8
w2) (Word8 -> Word8 -> Word16
c Word8
w3 Word8
w4) ->
          forall s a. a -> s -> Step s a
Yield (Word16 -> Word16 -> Char
U16.chr2 (Word8 -> Word8 -> Word16
c Word8
w1 Word8
w2) (Word8 -> Word8 -> Word16
c Word8
w3 Word8
w4)) T
es
        S
_ -> T -> Step T Char
consume T
st
       where es :: T
es = ByteString -> S -> Int -> T
T ByteString
bs S
S0 Int
i
             c :: Word8 -> Word8 -> Word16
             c :: Word8 -> Word8 -> Word16
c Word8
w1 Word8
w2 = (Word8 -> Word16
word8ToWord16 Word8
w1 forall a. UnsafeShift a => a -> Int -> a
`shiftL` Int
8) forall a. Num a => a -> a -> a
+ Word8 -> Word16
word8ToWord16 Word8
w2
    consume :: T -> Step T Char
consume (T bs :: ByteString
bs@(Chunk ByteString
ps ByteString
rest) S
s Int
i)
        | Int
i forall a. Ord a => a -> a -> Bool
>= ByteString -> Int
B.length ByteString
ps = T -> Step T Char
consume (ByteString -> S -> Int -> T
T ByteString
rest S
s Int
0)
        | Bool
otherwise =
      case S
s of
        S
S0             -> T -> Step T Char
next (ByteString -> S -> Int -> T
T ByteString
bs (Word8 -> S
S1 Word8
x)          (Int
iforall a. Num a => a -> a -> a
+Int
1))
        S1 Word8
w1          -> T -> Step T Char
next (ByteString -> S -> Int -> T
T ByteString
bs (Word8 -> Word8 -> S
S2 Word8
w1 Word8
x)       (Int
iforall a. Num a => a -> a -> a
+Int
1))
        S2 Word8
w1 Word8
w2       -> T -> Step T Char
next (ByteString -> S -> Int -> T
T ByteString
bs (Word8 -> Word8 -> Word8 -> S
S3 Word8
w1 Word8
w2 Word8
x)    (Int
iforall a. Num a => a -> a -> a
+Int
1))
        S3 Word8
w1 Word8
w2 Word8
w3    -> T -> Step T Char
next (ByteString -> S -> Int -> T
T ByteString
bs (Word8 -> Word8 -> Word8 -> Word8 -> S
S4 Word8
w1 Word8
w2 Word8
w3 Word8
x) (Int
iforall a. Num a => a -> a -> a
+Int
1))
        S4 Word8
w1 Word8
w2 Word8
w3 Word8
w4 -> forall s.
String
-> String -> OnDecodeError -> Maybe Word8 -> s -> Step s Char
decodeError String
"streamUtf16BE" String
"UTF-16BE" OnDecodeError
onErr (forall a. a -> Maybe a
Just Word8
w1)
                           (ByteString -> S -> Int -> T
T ByteString
bs (Word8 -> Word8 -> Word8 -> S
S3 Word8
w2 Word8
w3 Word8
w4)       (Int
iforall a. Num a => a -> a -> a
+Int
1))
        where x :: Word8
x = ByteString -> Int -> Word8
B.unsafeIndex ByteString
ps Int
i
    consume (T ByteString
Empty S
S0 Int
_) = forall s a. Step s a
Done
    consume T
st             = forall s.
String
-> String -> OnDecodeError -> Maybe Word8 -> s -> Step s Char
decodeError String
"streamUtf16BE" String
"UTF-16BE" OnDecodeError
onErr forall a. Maybe a
Nothing T
st
{-# INLINE [0] streamUtf16BE #-}

-- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using big
-- endian UTF-32 encoding.
streamUtf32BE :: OnDecodeError -> ByteString -> Stream Char
streamUtf32BE :: OnDecodeError -> ByteString -> Stream Char
streamUtf32BE OnDecodeError
onErr ByteString
bs0 = forall a s. (s -> Step s a) -> s -> Size -> Stream a
Stream T -> Step T Char
next (ByteString -> S -> Int -> T
T ByteString
bs0 S
S0 Int
0) Size
unknownSize
  where
    next :: T -> Step T Char
next (T bs :: ByteString
bs@(Chunk ByteString
ps ByteString
_) S
S0 Int
i)
      | Int
i forall a. Num a => a -> a -> a
+ Int
3 forall a. Ord a => a -> a -> Bool
< Int
len Bool -> Bool -> Bool
&& Word32 -> Bool
U32.validate Word32
x =
          forall s a. a -> s -> Step s a
Yield (Word32 -> Char
unsafeChr32 Word32
x)       (ByteString -> S -> Int -> T
T ByteString
bs S
S0 (Int
iforall a. Num a => a -> a -> a
+Int
4))
      where len :: Int
len = ByteString -> Int
B.length ByteString
ps
            x :: Word32
x = forall a. UnsafeShift a => a -> Int -> a
shiftL Word32
x1 Int
24 forall a. Num a => a -> a -> a
+ forall a. UnsafeShift a => a -> Int -> a
shiftL Word32
x2 Int
16 forall a. Num a => a -> a -> a
+ forall a. UnsafeShift a => a -> Int -> a
shiftL Word32
x3 Int
8 forall a. Num a => a -> a -> a
+ Word32
x4
            x1 :: Word32
x1    = Int -> Word32
idx Int
i
            x2 :: Word32
x2    = Int -> Word32
idx (Int
iforall a. Num a => a -> a -> a
+Int
1)
            x3 :: Word32
x3    = Int -> Word32
idx (Int
iforall a. Num a => a -> a -> a
+Int
2)
            x4 :: Word32
x4    = Int -> Word32
idx (Int
iforall a. Num a => a -> a -> a
+Int
3)
            idx :: Int -> Word32
idx = Word8 -> Word32
word8ToWord32 forall b c a. (b -> c) -> (a -> b) -> a -> c
. ByteString -> Int -> Word8
B.unsafeIndex ByteString
ps :: Int -> Word32
    next st :: T
st@(T ByteString
bs S
s Int
i) =
      case S
s of
        S4 Word8
w1 Word8
w2 Word8
w3 Word8
w4 | Word32 -> Bool
U32.validate (Word8 -> Word8 -> Word8 -> Word8 -> Word32
c Word8
w1 Word8
w2 Word8
w3 Word8
w4) ->
          forall s a. a -> s -> Step s a
Yield (Word32 -> Char
unsafeChr32 (Word8 -> Word8 -> Word8 -> Word8 -> Word32
c Word8
w1 Word8
w2 Word8
w3 Word8
w4)) T
es
        S
_ -> T -> Step T Char
consume T
st
       where es :: T
es = ByteString -> S -> Int -> T
T ByteString
bs S
S0 Int
i
             c :: Word8 -> Word8 -> Word8 -> Word8 -> Word32
             c :: Word8 -> Word8 -> Word8 -> Word8 -> Word32
c Word8
w1 Word8
w2 Word8
w3 Word8
w4 = Word32
shifted
              where
               shifted :: Word32
shifted = forall a. UnsafeShift a => a -> Int -> a
shiftL Word32
x1 Int
24 forall a. Num a => a -> a -> a
+ forall a. UnsafeShift a => a -> Int -> a
shiftL Word32
x2 Int
16 forall a. Num a => a -> a -> a
+ forall a. UnsafeShift a => a -> Int -> a
shiftL Word32
x3 Int
8 forall a. Num a => a -> a -> a
+ Word32
x4
               x1 :: Word32
x1 = Word8 -> Word32
word8ToWord32 Word8
w1
               x2 :: Word32
x2 = Word8 -> Word32
word8ToWord32 Word8
w2
               x3 :: Word32
x3 = Word8 -> Word32
word8ToWord32 Word8
w3
               x4 :: Word32
x4 = Word8 -> Word32
word8ToWord32 Word8
w4
    consume :: T -> Step T Char
consume (T bs :: ByteString
bs@(Chunk ByteString
ps ByteString
rest) S
s Int
i)
        | Int
i forall a. Ord a => a -> a -> Bool
>= ByteString -> Int
B.length ByteString
ps = T -> Step T Char
consume (ByteString -> S -> Int -> T
T ByteString
rest S
s Int
0)
        | Bool
otherwise =
      case S
s of
        S
S0             -> T -> Step T Char
next (ByteString -> S -> Int -> T
T ByteString
bs (Word8 -> S
S1 Word8
x)          (Int
iforall a. Num a => a -> a -> a
+Int
1))
        S1 Word8
w1          -> T -> Step T Char
next (ByteString -> S -> Int -> T
T ByteString
bs (Word8 -> Word8 -> S
S2 Word8
w1 Word8
x)       (Int
iforall a. Num a => a -> a -> a
+Int
1))
        S2 Word8
w1 Word8
w2       -> T -> Step T Char
next (ByteString -> S -> Int -> T
T ByteString
bs (Word8 -> Word8 -> Word8 -> S
S3 Word8
w1 Word8
w2 Word8
x)    (Int
iforall a. Num a => a -> a -> a
+Int
1))
        S3 Word8
w1 Word8
w2 Word8
w3    -> T -> Step T Char
next (ByteString -> S -> Int -> T
T ByteString
bs (Word8 -> Word8 -> Word8 -> Word8 -> S
S4 Word8
w1 Word8
w2 Word8
w3 Word8
x) (Int
iforall a. Num a => a -> a -> a
+Int
1))
        S4 Word8
w1 Word8
w2 Word8
w3 Word8
w4 -> forall s.
String
-> String -> OnDecodeError -> Maybe Word8 -> s -> Step s Char
decodeError String
"streamUtf32BE" String
"UTF-32BE" OnDecodeError
onErr (forall a. a -> Maybe a
Just Word8
w1)
                           (ByteString -> S -> Int -> T
T ByteString
bs (Word8 -> Word8 -> Word8 -> S
S3 Word8
w2 Word8
w3 Word8
w4)       (Int
iforall a. Num a => a -> a -> a
+Int
1))
        where x :: Word8
x = ByteString -> Int -> Word8
B.unsafeIndex ByteString
ps Int
i
    consume (T ByteString
Empty S
S0 Int
_) = forall s a. Step s a
Done
    consume T
st             = forall s.
String
-> String -> OnDecodeError -> Maybe Word8 -> s -> Step s Char
decodeError String
"streamUtf32BE" String
"UTF-32BE" OnDecodeError
onErr forall a. Maybe a
Nothing T
st
{-# INLINE [0] streamUtf32BE #-}

-- | /O(n)/ Convert a 'ByteString' into a 'Stream Char', using little
-- endian UTF-32 encoding.
streamUtf32LE :: OnDecodeError -> ByteString -> Stream Char
streamUtf32LE :: OnDecodeError -> ByteString -> Stream Char
streamUtf32LE OnDecodeError
onErr ByteString
bs0 = forall a s. (s -> Step s a) -> s -> Size -> Stream a
Stream T -> Step T Char
next (ByteString -> S -> Int -> T
T ByteString
bs0 S
S0 Int
0) Size
unknownSize
  where
    next :: T -> Step T Char
next (T bs :: ByteString
bs@(Chunk ByteString
ps ByteString
_) S
S0 Int
i)
      | Int
i forall a. Num a => a -> a -> a
+ Int
3 forall a. Ord a => a -> a -> Bool
< Int
len Bool -> Bool -> Bool
&& Word32 -> Bool
U32.validate Word32
x =
          forall s a. a -> s -> Step s a
Yield (Word32 -> Char
unsafeChr32 Word32
x)       (ByteString -> S -> Int -> T
T ByteString
bs S
S0 (Int
iforall a. Num a => a -> a -> a
+Int
4))
      where len :: Int
len = ByteString -> Int
B.length ByteString
ps
            x :: Word32
x = forall a. UnsafeShift a => a -> Int -> a
shiftL Word32
x4 Int
24 forall a. Num a => a -> a -> a
+ forall a. UnsafeShift a => a -> Int -> a
shiftL Word32
x3 Int
16 forall a. Num a => a -> a -> a
+ forall a. UnsafeShift a => a -> Int -> a
shiftL Word32
x2 Int
8 forall a. Num a => a -> a -> a
+ Word32
x1
            x1 :: Word32
x1    = Int -> Word32
idx Int
i
            x2 :: Word32
x2    = Int -> Word32
idx (Int
iforall a. Num a => a -> a -> a
+Int
1)
            x3 :: Word32
x3    = Int -> Word32
idx (Int
iforall a. Num a => a -> a -> a
+Int
2)
            x4 :: Word32
x4    = Int -> Word32
idx (Int
iforall a. Num a => a -> a -> a
+Int
3)
            idx :: Int -> Word32
idx = Word8 -> Word32
word8ToWord32 forall b c a. (b -> c) -> (a -> b) -> a -> c
. ByteString -> Int -> Word8
B.unsafeIndex ByteString
ps :: Int -> Word32
    next st :: T
st@(T ByteString
bs S
s Int
i) =
      case S
s of
        S4 Word8
w1 Word8
w2 Word8
w3 Word8
w4 | Word32 -> Bool
U32.validate (Word8 -> Word8 -> Word8 -> Word8 -> Word32
c Word8
w1 Word8
w2 Word8
w3 Word8
w4) ->
          forall s a. a -> s -> Step s a
Yield (Word32 -> Char
unsafeChr32 (Word8 -> Word8 -> Word8 -> Word8 -> Word32
c Word8
w1 Word8
w2 Word8
w3 Word8
w4)) T
es
        S
_ -> T -> Step T Char
consume T
st
       where es :: T
es = ByteString -> S -> Int -> T
T ByteString
bs S
S0 Int
i
             c :: Word8 -> Word8 -> Word8 -> Word8 -> Word32
             c :: Word8 -> Word8 -> Word8 -> Word8 -> Word32
c Word8
w1 Word8
w2 Word8
w3 Word8
w4 = Word32
shifted
              where
               shifted :: Word32
shifted = forall a. UnsafeShift a => a -> Int -> a
shiftL Word32
x4 Int
24 forall a. Num a => a -> a -> a
+ forall a. UnsafeShift a => a -> Int -> a
shiftL Word32
x3 Int
16 forall a. Num a => a -> a -> a
+ forall a. UnsafeShift a => a -> Int -> a
shiftL Word32
x2 Int
8 forall a. Num a => a -> a -> a
+ Word32
x1
               x1 :: Word32
x1 = Word8 -> Word32
word8ToWord32 Word8
w1
               x2 :: Word32
x2 = Word8 -> Word32
word8ToWord32 Word8
w2
               x3 :: Word32
x3 = Word8 -> Word32
word8ToWord32 Word8
w3
               x4 :: Word32
x4 = Word8 -> Word32
word8ToWord32 Word8
w4
    consume :: T -> Step T Char
consume (T bs :: ByteString
bs@(Chunk ByteString
ps ByteString
rest) S
s Int
i)
        | Int
i forall a. Ord a => a -> a -> Bool
>= ByteString -> Int
B.length ByteString
ps = T -> Step T Char
consume (ByteString -> S -> Int -> T
T ByteString
rest S
s Int
0)
        | Bool
otherwise =
      case S
s of
        S
S0             -> T -> Step T Char
next (ByteString -> S -> Int -> T
T ByteString
bs (Word8 -> S
S1 Word8
x)          (Int
iforall a. Num a => a -> a -> a
+Int
1))
        S1 Word8
w1          -> T -> Step T Char
next (ByteString -> S -> Int -> T
T ByteString
bs (Word8 -> Word8 -> S
S2 Word8
w1 Word8
x)       (Int
iforall a. Num a => a -> a -> a
+Int
1))
        S2 Word8
w1 Word8
w2       -> T -> Step T Char
next (ByteString -> S -> Int -> T
T ByteString
bs (Word8 -> Word8 -> Word8 -> S
S3 Word8
w1 Word8
w2 Word8
x)    (Int
iforall a. Num a => a -> a -> a
+Int
1))
        S3 Word8
w1 Word8
w2 Word8
w3    -> T -> Step T Char
next (ByteString -> S -> Int -> T
T ByteString
bs (Word8 -> Word8 -> Word8 -> Word8 -> S
S4 Word8
w1 Word8
w2 Word8
w3 Word8
x) (Int
iforall a. Num a => a -> a -> a
+Int
1))
        S4 Word8
w1 Word8
w2 Word8
w3 Word8
w4 -> forall s.
String
-> String -> OnDecodeError -> Maybe Word8 -> s -> Step s Char
decodeError String
"streamUtf32LE" String
"UTF-32LE" OnDecodeError
onErr (forall a. a -> Maybe a
Just Word8
w1)
                           (ByteString -> S -> Int -> T
T ByteString
bs (Word8 -> Word8 -> Word8 -> S
S3 Word8
w2 Word8
w3 Word8
w4)       (Int
iforall a. Num a => a -> a -> a
+Int
1))
        where x :: Word8
x = ByteString -> Int -> Word8
B.unsafeIndex ByteString
ps Int
i
    consume (T ByteString
Empty S
S0 Int
_) = forall s a. Step s a
Done
    consume T
st             = forall s.
String
-> String -> OnDecodeError -> Maybe Word8 -> s -> Step s Char
decodeError String
"streamUtf32LE" String
"UTF-32LE" OnDecodeError
onErr forall a. Maybe a
Nothing T
st
{-# INLINE [0] streamUtf32LE #-}

-- | /O(n)/ Convert a 'Stream' 'Word8' to a lazy 'ByteString'.
unstreamChunks :: Int -> Stream Word8 -> ByteString
unstreamChunks :: Int -> Stream Word8 -> ByteString
unstreamChunks Int
chunkSize (Stream s -> Step s Word8
next s
s0 Size
len0) = s -> Int -> ByteString
chunk s
s0 (Int -> Size -> Int
upperBound Int
4 Size
len0)
  where chunk :: s -> Int -> ByteString
chunk s
s1 Int
len1 = forall a. IO a -> a
unsafeDupablePerformIO forall a b. (a -> b) -> a -> b
$ do
          let len :: Int
len = forall a. Ord a => a -> a -> a
max Int
4 (forall a. Ord a => a -> a -> a
min Int
len1 Int
chunkSize)
          forall a. Int -> IO (ForeignPtr a)
mallocByteString Int
len forall (m :: * -> *) a b. Monad m => m a -> (a -> m b) -> m b
>>= Int -> Int -> s -> ForeignPtr Word8 -> IO ByteString
loop Int
len Int
0 s
s1
          where
            loop :: Int -> Int -> s -> ForeignPtr Word8 -> IO ByteString
loop !Int
n !Int
off !s
s ForeignPtr Word8
fp = case s -> Step s Word8
next s
s of
                Step s Word8
Done | Int
off forall a. Eq a => a -> a -> Bool
== Int
0 -> forall (m :: * -> *) a. Monad m => a -> m a
return ByteString
Empty
                     | Bool
otherwise -> forall (m :: * -> *) a. Monad m => a -> m a
return forall a b. (a -> b) -> a -> b
$! ByteString -> ByteString -> ByteString
Chunk (ForeignPtr Word8 -> Int -> ByteString
trimUp ForeignPtr Word8
fp Int
off) ByteString
Empty
                Skip s
s' -> Int -> Int -> s -> ForeignPtr Word8 -> IO ByteString
loop Int
n Int
off s
s' ForeignPtr Word8
fp
                Yield Word8
x s
s'
                    | Int
off forall a. Eq a => a -> a -> Bool
== Int
chunkSize -> do
                      let !newLen :: Int
newLen = Int
n forall a. Num a => a -> a -> a
- Int
off
                      forall (m :: * -> *) a. Monad m => a -> m a
return forall a b. (a -> b) -> a -> b
$! ByteString -> ByteString -> ByteString
Chunk (ForeignPtr Word8 -> Int -> ByteString
trimUp ForeignPtr Word8
fp Int
off) (s -> Int -> ByteString
chunk s
s Int
newLen)
                    | Int
off forall a. Eq a => a -> a -> Bool
== Int
n -> ForeignPtr Word8 -> Int -> Int -> s -> Word8 -> IO ByteString
realloc ForeignPtr Word8
fp Int
n Int
off s
s' Word8
x
                    | Bool
otherwise -> do
                      forall a b. ForeignPtr a -> (Ptr a -> IO b) -> IO b
unsafeWithForeignPtr ForeignPtr Word8
fp forall a b. (a -> b) -> a -> b
$ \Ptr Word8
p -> forall a b. Storable a => Ptr b -> Int -> a -> IO ()
pokeByteOff Ptr Word8
p Int
off Word8
x
                      Int -> Int -> s -> ForeignPtr Word8 -> IO ByteString
loop Int
n (Int
offforall a. Num a => a -> a -> a
+Int
1) s
s' ForeignPtr Word8
fp
            {-# NOINLINE realloc #-}
            realloc :: ForeignPtr Word8 -> Int -> Int -> s -> Word8 -> IO ByteString
realloc ForeignPtr Word8
fp Int
n Int
off s
s Word8
x = do
              let n' :: Int
n' = forall a. Ord a => a -> a -> a
min (Int
nforall a. Num a => a -> a -> a
+Int
n) Int
chunkSize
              ForeignPtr Word8
fp' <- ForeignPtr Word8 -> Int -> Int -> IO (ForeignPtr Word8)
copy0 ForeignPtr Word8
fp Int
n Int
n'
              forall a b. ForeignPtr a -> (Ptr a -> IO b) -> IO b
unsafeWithForeignPtr ForeignPtr Word8
fp' forall a b. (a -> b) -> a -> b
$ \Ptr Word8
p -> forall a b. Storable a => Ptr b -> Int -> a -> IO ()
pokeByteOff Ptr Word8
p Int
off Word8
x
              Int -> Int -> s -> ForeignPtr Word8 -> IO ByteString
loop Int
n' (Int
offforall a. Num a => a -> a -> a
+Int
1) s
s ForeignPtr Word8
fp'
            trimUp :: ForeignPtr Word8 -> Int -> ByteString
trimUp ForeignPtr Word8
fp Int
off = ForeignPtr Word8 -> Int -> ByteString
mkBS ForeignPtr Word8
fp Int
off
            copy0 :: ForeignPtr Word8 -> Int -> Int -> IO (ForeignPtr Word8)
            copy0 :: ForeignPtr Word8 -> Int -> Int -> IO (ForeignPtr Word8)
copy0 !ForeignPtr Word8
src !Int
srcLen !Int
destLen =
#if defined(ASSERTS)
              assert (srcLen <= destLen) $
#endif
              do
                ForeignPtr Word8
dest <- forall a. Int -> IO (ForeignPtr a)
mallocByteString Int
destLen
                forall a b. ForeignPtr a -> (Ptr a -> IO b) -> IO b
unsafeWithForeignPtr ForeignPtr Word8
src  forall a b. (a -> b) -> a -> b
$ \Ptr Word8
src'  ->
                    forall a b. ForeignPtr a -> (Ptr a -> IO b) -> IO b
unsafeWithForeignPtr ForeignPtr Word8
dest forall a b. (a -> b) -> a -> b
$ \Ptr Word8
dest' ->
                        Ptr Word8 -> Ptr Word8 -> Int -> IO ()
memcpy Ptr Word8
dest' Ptr Word8
src' Int
srcLen
                forall (m :: * -> *) a. Monad m => a -> m a
return ForeignPtr Word8
dest

-- | /O(n)/ Convert a 'Stream' 'Word8' to a lazy 'ByteString'.
unstream :: Stream Word8 -> ByteString
unstream :: Stream Word8 -> ByteString
unstream = Int -> Stream Word8 -> ByteString
unstreamChunks Int
defaultChunkSize

decodeError :: forall s. String -> String -> OnDecodeError -> Maybe Word8
            -> s -> Step s Char
decodeError :: forall s.
String
-> String -> OnDecodeError -> Maybe Word8 -> s -> Step s Char
decodeError String
func String
kind OnDecodeError
onErr Maybe Word8
mb s
i =
    case OnDecodeError
onErr String
desc Maybe Word8
mb of
      Maybe Char
Nothing -> forall s a. s -> Step s a
Skip s
i
      Just Char
c  -> forall s a. a -> s -> Step s a
Yield Char
c s
i
    where desc :: String
desc = String
"Data.Text.Lazy.Encoding.Fusion." forall a. [a] -> [a] -> [a]
++ String
func forall a. [a] -> [a] -> [a]
++ String
": Invalid " forall a. [a] -> [a] -> [a]
++
                 String
kind forall a. [a] -> [a] -> [a]
++ String
" stream"

word8ToWord16 :: Word8 -> Word16
word8ToWord16 :: Word8 -> Word16
word8ToWord16 = forall a b. (Integral a, Num b) => a -> b
fromIntegral

word8ToWord32 :: Word8 -> Word32
word8ToWord32 :: Word8 -> Word32
word8ToWord32 = forall a b. (Integral a, Num b) => a -> b
fromIntegral