module GHC.StgToCmm.Foreign (
cgForeignCall,
emitPrimCall, emitCCall,
emitForeignCall,
emitSaveThreadState,
saveThreadState,
emitLoadThreadState,
emitSaveRegs,
emitRestoreRegs,
loadThreadState,
emitOpenNursery,
emitCloseNursery,
) where
import GHC.Prelude hiding( succ, (<*>) )
import GHC.Platform
import GHC.Platform.Profile
import GHC.Stg.Syntax
import GHC.StgToCmm.Prof (storeCurCCS, ccsType)
import GHC.StgToCmm.Env
import GHC.StgToCmm.Monad
import GHC.StgToCmm.Utils
import GHC.StgToCmm.Closure
import GHC.StgToCmm.Layout
import GHC.Cmm.BlockId (newBlockId)
import GHC.Cmm
import GHC.Cmm.Utils
import GHC.Cmm.Graph
import GHC.Cmm.CallConv
import GHC.Core.Type
import GHC.Types.RepType
import GHC.Cmm.CLabel
import GHC.Runtime.Heap.Layout
import GHC.Types.ForeignCall
import GHC.Data.Maybe
import GHC.Utils.Panic
import GHC.Types.Unique.Supply
import GHC.Types.Basic
import GHC.Unit.Types
import GHC.Core.TyCo.Rep
import GHC.Builtin.Types.Prim
import GHC.Utils.Misc (zipEqual)
import Control.Monad
cgForeignCall :: ForeignCall
-> Type
-> [StgArg]
-> Type
-> FCode ReturnKind
cgForeignCall (CCall (CCallSpec target cconv safety)) typ stg_args res_ty
= do { platform <- getPlatform
; let
call_size args
| StdCallConv <- cconv = Just (sum (map arg_size args))
| otherwise = Nothing
arg_size (arg, _) = max (widthInBytes $ typeWidth $ cmmExprType platform arg)
(platformWordSizeInBytes platform)
; cmm_args <- getFCallArgs stg_args typ
; (res_regs, res_hints) <- newUnboxedTupleRegs res_ty
; let ((call_args, arg_hints), cmm_target)
= case target of
StaticTarget _ _ _ False ->
panic "cgForeignCall: unexpected FFI value import"
StaticTarget _ lbl mPkgId True
-> let labelSource
= case mPkgId of
Nothing -> ForeignLabelInThisPackage
Just pkgId -> ForeignLabelInPackage (toUnitId pkgId)
size = call_size cmm_args
in ( unzip cmm_args
, CmmLit (CmmLabel
(mkForeignLabel lbl size labelSource IsFunction)))
DynamicTarget -> case cmm_args of
(fn,_):rest -> (unzip rest, fn)
[] -> panic "cgForeignCall []"
fc = ForeignConvention cconv arg_hints res_hints CmmMayReturn
call_target = ForeignTarget cmm_target fc
; sequel <- getSequel
; case sequel of
AssignTo assign_to_these _ ->
emitForeignCall safety assign_to_these call_target call_args
_something_else ->
do { _ <- emitForeignCall safety res_regs call_target call_args
; emitReturn (map (CmmReg . CmmLocal) res_regs)
}
}
emitCCall :: [(CmmFormal,ForeignHint)]
-> CmmExpr
-> [(CmmActual,ForeignHint)]
-> FCode ()
emitCCall hinted_results fn hinted_args
= void $ emitForeignCall PlayRisky results target args
where
(args, arg_hints) = unzip hinted_args
(results, result_hints) = unzip hinted_results
target = ForeignTarget fn fc
fc = ForeignConvention CCallConv arg_hints result_hints CmmMayReturn
emitPrimCall :: [CmmFormal] -> CallishMachOp -> [CmmActual] -> FCode ()
emitPrimCall res op args
= void $ emitForeignCall PlayRisky res (PrimTarget op) args
emitForeignCall
:: Safety
-> [CmmFormal]
-> ForeignTarget
-> [CmmActual]
-> FCode ReturnKind
emitForeignCall safety results target args
| not (playSafe safety) = do
platform <- getPlatform
let (caller_save, caller_load) = callerSaveVolatileRegs platform
emit caller_save
target' <- load_target_into_temp target
args' <- mapM maybe_assign_temp args
emit $ mkUnsafeCall target' results args'
emit caller_load
return AssignedDirectly
| otherwise = do
profile <- getProfile
platform <- getPlatform
updfr_off <- getUpdFrameOff
target' <- load_target_into_temp target
args' <- mapM maybe_assign_temp args
k <- newBlockId
let (off, _, copyout) = copyInOflow profile NativeReturn (Young k) results []
tscope <- getTickScope
emit $
( mkStore (CmmStackSlot (Young k) (widthInBytes (wordWidth platform)))
(CmmLit (CmmBlock k))
<*> mkLast (CmmForeignCall { tgt = target'
, res = results
, args = args'
, succ = k
, ret_args = off
, ret_off = updfr_off
, intrbl = playInterruptible safety })
<*> mkLabel k tscope
<*> copyout
)
return (ReturnedTo k off)
load_target_into_temp :: ForeignTarget -> FCode ForeignTarget
load_target_into_temp (ForeignTarget expr conv) = do
tmp <- maybe_assign_temp expr
return (ForeignTarget tmp conv)
load_target_into_temp other_target@(PrimTarget _) =
return other_target
maybe_assign_temp :: CmmExpr -> FCode CmmExpr
maybe_assign_temp e = do
platform <- getPlatform
reg <- newTemp (cmmExprType platform e)
emitAssign (CmmLocal reg) e
return (CmmReg (CmmLocal reg))
emitSaveThreadState :: FCode ()
emitSaveThreadState = do
profile <- getProfile
code <- saveThreadState profile
emit code
saveThreadState :: MonadUnique m => Profile -> m CmmAGraph
saveThreadState profile = do
let platform = profilePlatform profile
tso <- newTemp (gcWord platform)
close_nursery <- closeNursery profile tso
pure $ catAGraphs
[
mkAssign (CmmLocal tso) currentTSOExpr
,
mkStore (cmmOffset platform
(CmmLoad (cmmOffset platform
(CmmReg (CmmLocal tso))
(tso_stackobj profile))
(bWord platform))
(stack_SP profile))
spExpr
, close_nursery
,
if profileIsProfiling profile
then mkStore (cmmOffset platform (CmmReg (CmmLocal tso)) (tso_CCCS profile)) cccsExpr
else mkNop
]
emitSaveRegs :: FCode ()
emitSaveRegs = do
platform <- getPlatform
let regs = realArgRegsCover platform
save = catAGraphs (map (callerSaveGlobalReg platform) regs)
emit save
emitRestoreRegs :: FCode ()
emitRestoreRegs = do
platform <- getPlatform
let regs = realArgRegsCover platform
restore = catAGraphs (map (callerRestoreGlobalReg platform) regs)
emit restore
emitCloseNursery :: FCode ()
emitCloseNursery = do
profile <- getProfile
let platform = profilePlatform profile
tso <- newTemp (bWord platform)
code <- closeNursery profile tso
emit $ mkAssign (CmmLocal tso) currentTSOExpr <*> code
closeNursery :: MonadUnique m => Profile -> LocalReg -> m CmmAGraph
closeNursery profile tso = do
let tsoreg = CmmLocal tso
platform = profilePlatform profile
cnreg <- CmmLocal <$> newTemp (bWord platform)
pure $ catAGraphs [
mkAssign cnreg currentNurseryExpr,
mkStore (nursery_bdescr_free platform cnreg) (cmmOffsetW platform hpExpr 1),
let alloc =
CmmMachOp (mo_wordSub platform)
[ cmmOffsetW platform hpExpr 1
, CmmLoad (nursery_bdescr_start platform cnreg) (bWord platform)
]
alloc_limit = cmmOffset platform (CmmReg tsoreg) (tso_alloc_limit profile)
in
mkStore alloc_limit (CmmMachOp (MO_Sub W64)
[ CmmLoad alloc_limit b64
, CmmMachOp (mo_WordTo64 platform) [alloc] ])
]
emitLoadThreadState :: FCode ()
emitLoadThreadState = do
profile <- getProfile
code <- loadThreadState profile
emit code
loadThreadState :: MonadUnique m => Profile -> m CmmAGraph
loadThreadState profile = do
let platform = profilePlatform profile
tso <- newTemp (gcWord platform)
stack <- newTemp (gcWord platform)
open_nursery <- openNursery profile tso
pure $ catAGraphs [
mkAssign (CmmLocal tso) currentTSOExpr,
mkAssign (CmmLocal stack) (CmmLoad (cmmOffset platform (CmmReg (CmmLocal tso)) (tso_stackobj profile)) (bWord platform)),
mkAssign spReg (CmmLoad (cmmOffset platform (CmmReg (CmmLocal stack)) (stack_SP profile)) (bWord platform)),
mkAssign spLimReg (cmmOffsetW platform (cmmOffset platform (CmmReg (CmmLocal stack)) (stack_STACK profile))
(pc_RESERVED_STACK_WORDS (platformConstants platform))),
mkAssign hpAllocReg (zeroExpr platform),
open_nursery,
if profileIsProfiling profile
then storeCurCCS
(CmmLoad (cmmOffset platform (CmmReg (CmmLocal tso))
(tso_CCCS profile)) (ccsType platform))
else mkNop
]
emitOpenNursery :: FCode ()
emitOpenNursery = do
profile <- getProfile
let platform = profilePlatform profile
tso <- newTemp (bWord platform)
code <- openNursery profile tso
emit $ mkAssign (CmmLocal tso) currentTSOExpr <*> code
openNursery :: MonadUnique m => Profile -> LocalReg -> m CmmAGraph
openNursery profile tso = do
let tsoreg = CmmLocal tso
platform = profilePlatform profile
cnreg <- CmmLocal <$> newTemp (bWord platform)
bdfreereg <- CmmLocal <$> newTemp (bWord platform)
bdstartreg <- CmmLocal <$> newTemp (bWord platform)
pure $ catAGraphs [
mkAssign cnreg currentNurseryExpr,
mkAssign bdfreereg (CmmLoad (nursery_bdescr_free platform cnreg) (bWord platform)),
mkAssign hpReg (cmmOffsetW platform (CmmReg bdfreereg) (1)),
mkAssign bdstartreg (CmmLoad (nursery_bdescr_start platform cnreg) (bWord platform)),
mkAssign hpLimReg
(cmmOffsetExpr platform
(CmmReg bdstartreg)
(cmmOffset platform
(CmmMachOp (mo_wordMul platform) [
CmmMachOp (MO_SS_Conv W32 (wordWidth platform))
[CmmLoad (nursery_bdescr_blocks platform cnreg) b32],
mkIntExpr platform (pc_BLOCK_SIZE (platformConstants platform))
])
(1)
)
),
let alloc =
CmmMachOp (mo_wordSub platform) [CmmReg bdfreereg, CmmReg bdstartreg]
alloc_limit = cmmOffset platform (CmmReg tsoreg) (tso_alloc_limit profile)
in
mkStore alloc_limit (CmmMachOp (MO_Add W64)
[ CmmLoad alloc_limit b64
, CmmMachOp (mo_WordTo64 platform) [alloc] ])
]
nursery_bdescr_free, nursery_bdescr_start, nursery_bdescr_blocks
:: Platform -> CmmReg -> CmmExpr
nursery_bdescr_free platform cn =
cmmOffset platform (CmmReg cn) (pc_OFFSET_bdescr_free (platformConstants platform))
nursery_bdescr_start platform cn =
cmmOffset platform (CmmReg cn) (pc_OFFSET_bdescr_start (platformConstants platform))
nursery_bdescr_blocks platform cn =
cmmOffset platform (CmmReg cn) (pc_OFFSET_bdescr_blocks (platformConstants platform))
tso_stackobj, tso_CCCS, tso_alloc_limit, stack_STACK, stack_SP :: Profile -> ByteOff
tso_stackobj profile = closureField profile (pc_OFFSET_StgTSO_stackobj (profileConstants profile))
tso_alloc_limit profile = closureField profile (pc_OFFSET_StgTSO_alloc_limit (profileConstants profile))
tso_CCCS profile = closureField profile (pc_OFFSET_StgTSO_cccs (profileConstants profile))
stack_STACK profile = closureField profile (pc_OFFSET_StgStack_stack (profileConstants profile))
stack_SP profile = closureField profile (pc_OFFSET_StgStack_sp (profileConstants profile))
closureField :: Profile -> ByteOff -> ByteOff
closureField profile off = off + fixedHdrSize profile
getFCallArgs ::
[StgArg]
-> Type
-> FCode [(CmmExpr, ForeignHint)]
getFCallArgs args typ
= do { mb_cmms <- mapM get (zipEqual "getFCallArgs" args (collectStgFArgTypes typ))
; return (catMaybes mb_cmms) }
where
get (arg,typ)
| null arg_reps
= return Nothing
| otherwise
= do { cmm <- getArgAmode (NonVoid arg)
; profile <- getProfile
; return (Just (add_shim profile typ cmm, hint)) }
where
arg_ty = stgArgType arg
arg_reps = typePrimRep arg_ty
hint = typeForeignHint arg_ty
data StgFArgType
= StgPlainType
| StgArrayType
| StgSmallArrayType
| StgByteArrayType
add_shim :: Profile -> StgFArgType -> CmmExpr -> CmmExpr
add_shim profile ty expr = case ty of
StgPlainType -> expr
StgArrayType -> cmmOffsetB platform expr (arrPtrsHdrSize profile)
StgSmallArrayType -> cmmOffsetB platform expr (smallArrPtrsHdrSize profile)
StgByteArrayType -> cmmOffsetB platform expr (arrWordsHdrSize profile)
where
platform = profilePlatform profile
collectStgFArgTypes :: Type -> [StgFArgType]
collectStgFArgTypes = go []
where
go bs (ForAllTy _ res) = go bs res
go bs (AppTy{}) = reverse bs
go bs (TyConApp{}) = reverse bs
go bs (LitTy{}) = reverse bs
go bs (TyVarTy{}) = reverse bs
go _ (CastTy{}) = panic "myCollectTypeArgs: CastTy"
go _ (CoercionTy{}) = panic "myCollectTypeArgs: CoercionTy"
go bs (FunTy {ft_arg = arg, ft_res=res}) =
go (typeToStgFArgType arg:bs) res
typeToStgFArgType :: Type -> StgFArgType
typeToStgFArgType typ
| tycon == arrayPrimTyCon = StgArrayType
| tycon == mutableArrayPrimTyCon = StgArrayType
| tycon == arrayArrayPrimTyCon = StgArrayType
| tycon == mutableArrayArrayPrimTyCon = StgArrayType
| tycon == smallArrayPrimTyCon = StgSmallArrayType
| tycon == smallMutableArrayPrimTyCon = StgSmallArrayType
| tycon == byteArrayPrimTyCon = StgByteArrayType
| tycon == mutableByteArrayPrimTyCon = StgByteArrayType
| otherwise = StgPlainType
where
tycon = tyConAppTyCon (unwrapType typ)