diff --git a/src/Web/ActivityPub.hs b/src/Web/ActivityPub.hs
index edde43b..24d4d9f 100644
--- a/src/Web/ActivityPub.hs
+++ b/src/Web/ActivityPub.hs
@@ -54,6 +54,7 @@ import Data.Aeson.Types (Parser)
 import Data.Bifunctor (bimap, first)
 import Data.Bitraversable (bitraverse)
 import Data.ByteString (ByteString)
+import Data.Foldable (for_)
 import Data.List.NonEmpty (NonEmpty)
 import Data.PEM
 import Data.Semigroup (Endo)
@@ -429,7 +430,13 @@ fetchKey manager sigAlgo muActor uKey = runExceptT $ do
                         then case muActor of
                             Nothing -> throwE "Key is shared but actor header not specified!"
                             Just u  -> return u
-                        else return $ publicKeyOwner pkey
+                        else do
+                            let owner = publicKeyOwner pkey
+                            for_ muActor $ \ u ->
+                                if owner == u
+                                    then return ()
+                                    else throwE "Key's owner doesn't match actor header"
+                            return owner
                 actor <- fetch uActor
                 let PublicKeySet k1 mk2 = actorPublicKeys actor
                     match (Left uri) = uri == uKey
@@ -441,12 +448,10 @@ fetchKey manager sigAlgo muActor uKey = runExceptT $ do
                 if actorId actor == uKey { furiFragment = "" }
                     then return ()
                     else throwE "Actor ID doesn't match the keyid URI we fetched"
-                case muActor of
-                    Nothing -> return ()
-                    Just u ->
-                        if actorId actor == u
-                            then return ()
-                            else throwE "Key's owner doesn't match actor header"
+                for_ muActor $ \ u ->
+                    if actorId actor == u
+                        then return ()
+                        else throwE "Key's owner doesn't match actor header"
                 let PublicKeySet k1 mk2 = actorPublicKeys actor
                     match (Left _)   = Nothing
                     match (Right pk) =