Introduce a separate Address table

cardano-db-sync/src/Cardano/DbSync/Era/Byron/Genesis.hs

@@ -184,21 +184,39 @@ insertTxOuts blkId (address, value) = do
         , DB.txValidContract = True
         , DB.txScriptSize = 0
         }
+  addrId <- insertAddress address
   void . DB.insertTxOut $
     DB.TxOut
       { DB.txOutTxId = txId
       , DB.txOutIndex = 0
-      , DB.txOutAddress = Text.decodeUtf8 $ Byron.addrToBase58 address
-      , DB.txOutAddressRaw = Binary.serialize' address
-      , DB.txOutAddressHasScript = False
-      , DB.txOutPaymentCred = Nothing
+      , DB.txOutAddressId = addrId
       , DB.txOutStakeAddressId = Nothing
       , DB.txOutValue = DB.DbLovelace (Byron.unsafeGetLovelace value)
       , DB.txOutDataHash = Nothing
       , DB.txOutInlineDatumId = Nothing
       , DB.txOutReferenceScriptId = Nothing
       }
 
+insertAddress ::
+  (MonadBaseControl IO m, MonadIO m) =>
+  Byron.Address ->
+  ReaderT SqlBackend m DB.AddressId
+insertAddress address = do
+  mAddrId <- DB.queryAddress addrRaw
+  case mAddrId of
+    Nothing ->
+      DB.insertAddress
+        DB.Address
+          { DB.addressAddress = Text.decodeUtf8 $ Byron.addrToBase58 address
+          , DB.addressAddressRaw = addrRaw
+          , DB.addressHasScript = False
+          , DB.addressPaymentCred = Nothing
+          }
+    Just addrId -> pure addrId
+  where
+    addrRaw = Binary.serialize' address
+
+
 -- -----------------------------------------------------------------------------
 
 configGenesisHash :: Byron.Config -> ByteString

cardano-db-sync/src/Cardano/DbSync/Era/Byron/Insert.hs

@@ -234,21 +234,38 @@ insertTxOut ::
   Word32 ->
   Byron.TxOut ->
   ReaderT SqlBackend m ()
-insertTxOut _tracer txId index txout =
+insertTxOut _tracer txId index txout = do
+  addrId <- insertAddress
   void . DB.insertTxOut $
     DB.TxOut
       { DB.txOutTxId = txId
       , DB.txOutIndex = fromIntegral index
-      , DB.txOutAddress = Text.decodeUtf8 $ Byron.addrToBase58 (Byron.txOutAddress txout)
-      , DB.txOutAddressRaw = Binary.serialize' (Byron.txOutAddress txout)
-      , DB.txOutAddressHasScript = False
-      , DB.txOutPaymentCred = Nothing -- Byron does not have a payment credential.
+      , DB.txOutAddressId = addrId
       , DB.txOutStakeAddressId = Nothing -- Byron does not have a stake address.
       , DB.txOutValue = DbLovelace (Byron.unsafeGetLovelace $ Byron.txOutValue txout)
       , DB.txOutDataHash = Nothing
       , DB.txOutInlineDatumId = Nothing
       , DB.txOutReferenceScriptId = Nothing
       }
+  where
+    addrRaw :: ByteString
+    addrRaw = Binary.serialize' (Byron.txOutAddress txout)
+
+    insertAddress ::
+      (MonadBaseControl IO m, MonadIO m) =>
+      ReaderT SqlBackend m DB.AddressId
+    insertAddress = do
+      mAddrId <- DB.queryAddress addrRaw
+      case mAddrId of
+        Nothing ->
+          DB.insertAddress
+            DB.Address
+              { DB.addressAddress = Text.decodeUtf8 $ Byron.addrToBase58 (Byron.txOutAddress txout)
+              , DB.addressAddressRaw = addrRaw
+              , DB.addressHasScript = False
+              , DB.addressPaymentCred = Nothing
+              }
+        Just addrId -> pure addrId
 
 insertTxIn ::
   (MonadBaseControl IO m, MonadIO m) =>

cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Genesis.hs

@@ -41,7 +41,7 @@ import Data.Time.Clock (UTCTime (..))
 import qualified Data.Time.Clock as Time
 import Database.Persist.Sql (SqlBackend)
 import Lens.Micro
-import Ouroboros.Consensus.Cardano.Block (StandardShelley)
+import Ouroboros.Consensus.Cardano.Block (StandardCrypto, StandardShelley)
 import Ouroboros.Consensus.Shelley.Node (
   ShelleyGenesis (..),
   ShelleyGenesisStaking (..),
@@ -234,14 +234,12 @@ insertTxOuts trce blkId (ShelleyTx.TxIn txInId _, txOut) = do
         , DB.txScriptSize = 0
         }
   _ <- insertStakeAddressRefIfMissing trce uninitiatedCache txId (txOut ^. Core.addrTxOutL)
+  addrId <- insertAddress addr
   void . DB.insertTxOut $
     DB.TxOut
       { DB.txOutTxId = txId
       , DB.txOutIndex = 0
-      , DB.txOutAddress = Generic.renderAddress addr
-      , DB.txOutAddressRaw = Ledger.serialiseAddr addr
-      , DB.txOutAddressHasScript = hasScript
-      , DB.txOutPaymentCred = Generic.maybePaymentCred addr
+      , DB.txOutAddressId = addrId
       , DB.txOutStakeAddressId = Nothing -- No stake addresses in Shelley Genesis
       , DB.txOutValue = Generic.coinToDbLovelace (txOut ^. Core.valueTxOutL)
       , DB.txOutDataHash = Nothing -- No output datum in Shelley Genesis
@@ -251,7 +249,25 @@ insertTxOuts trce blkId (ShelleyTx.TxIn txInId _, txOut) = do
   where
     addr = txOut ^. Core.addrTxOutL
 
-    hasScript = maybe False Generic.hasCredScript (Generic.getPaymentCred addr)
+insertAddress ::
+  (MonadBaseControl IO m, MonadIO m) =>
+  Ledger.Addr StandardCrypto ->
+  ReaderT SqlBackend m DB.AddressId
+insertAddress addr = do
+  mAddrId <- DB.queryAddress addrRaw
+  case mAddrId of
+    Nothing ->
+      DB.insertAddress
+        DB.Address
+          { DB.addressAddress = Generic.renderAddress addr
+          , DB.addressAddressRaw = addrRaw
+          , DB.addressHasScript = maybe False Generic.hasCredScript (Generic.getPaymentCred addr)
+          , DB.addressPaymentCred = Generic.maybePaymentCred addr
+          }
+    Just addrId -> pure addrId
+  where
+    addrRaw = Ledger.serialiseAddr addr
+
 
 -- Insert pools and delegations coming from Genesis.
 insertStaking ::

cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Insert.hs

@@ -304,6 +304,7 @@ prepareTxOut ::
   Generic.TxOut ->
   ExceptT SyncNodeError (ReaderT SqlBackend m) (ExtendedTxOut, [MissingMaTxOut])
 prepareTxOut tracer cache iopts (txId, txHash) (Generic.TxOut index addr addrRaw value maMap mScript dt) = do
+  addrId <- lift insertAddress
   mSaId <- lift $ insertStakeAddressRefIfMissing tracer cache txId addr
   mDatumId <-
     whenFalseEmpty (ioPlutusExtra iopts) Nothing $
@@ -317,23 +318,39 @@ prepareTxOut tracer cache iopts (txId, txHash) (Generic.TxOut index addr addrRaw
         DB.TxOut
           { DB.txOutTxId = txId
           , DB.txOutIndex = index
-          , DB.txOutAddress = Generic.renderAddress addr
-          , DB.txOutAddressRaw = addrRaw
-          , DB.txOutAddressHasScript = hasScript
-          , DB.txOutPaymentCred = Generic.maybePaymentCred addr
+          , DB.txOutAddressId = addrId
           , DB.txOutStakeAddressId = mSaId
           , DB.txOutValue = Generic.coinToDbLovelace value
           , DB.txOutDataHash = Generic.dataHashToBytes <$> Generic.getTxOutDatumHash dt
           , DB.txOutInlineDatumId = mDatumId
           , DB.txOutReferenceScriptId = mScriptId
           }
-  let !eutxo = ExtendedTxOut txHash txOut
+  let !eutxo = ExtendedTxOut txHash txOut (Generic.maybePaymentCred addr)
   !maTxOuts <- whenFalseMempty (ioMultiAssets iopts) $ prepareMaTxOuts tracer cache maMap
   pure (eutxo, maTxOuts)
   where
     hasScript :: Bool
     hasScript = maybe False Generic.hasCredScript (Generic.getPaymentCred addr)
 
+    paymentCreds :: Maybe ByteString
+    paymentCreds = Generic.maybePaymentCred addr
+
+    insertAddress ::
+      (MonadBaseControl IO m, MonadIO m) =>
+      ReaderT SqlBackend m DB.AddressId
+    insertAddress = do
+      mAddrId <- DB.queryAddress addrRaw
+      case mAddrId of
+        Nothing ->
+          DB.insertAddress
+            DB.Address
+              { DB.addressAddress = Generic.renderAddress addr
+              , DB.addressAddressRaw = addrRaw
+              , DB.addressHasScript = hasScript
+              , DB.addressPaymentCred = paymentCreds
+              }
+        Just addrId -> pure addrId
+
 insertCollateralTxOut ::
   (MonadBaseControl IO m, MonadIO m) =>
   Trace IO Text ->

cardano-db-sync/src/Cardano/DbSync/Era/Shelley/Insert/Grouped.hs

@@ -54,6 +54,7 @@ data MissingMaTxOut = MissingMaTxOut
 data ExtendedTxOut = ExtendedTxOut
   { etoTxHash :: !ByteString
   , etoTxOut :: !DB.TxOut
+  , etoPaymentCred :: Maybe ByteString
   }
 
 instance Monoid BlockGroupedData where
@@ -137,7 +138,7 @@ resolveScriptHash groupedOutputs txIn =
       Left err ->
         case resolveInMemory txIn groupedOutputs of
           Nothing -> pure $ Left err
-          Just eutxo -> pure $ Right $ DB.txOutPaymentCred $ etoTxOut eutxo
+          Just eutxo -> pure $ Right $ etoPaymentCred eutxo
 
 resolveInMemory :: Generic.TxIn -> [ExtendedTxOut] -> Maybe ExtendedTxOut
 resolveInMemory txIn =

cardano-db-tool/src/Cardano/DbTool/UtxoSet.hs

@@ -2,6 +2,7 @@
 
 module Cardano.DbTool.UtxoSet (
   utxoSetAtSlot,
+  utxoSetSum,
 ) where
 
 import Cardano.Chain.Common (decodeAddressBase58, isRedeemAddress)
@@ -58,12 +59,12 @@ utxoSetAtSlot slotNo = do
 
 -- -----------------------------------------------------------------------------
 
-aggregateUtxos :: [(TxOut, a)] -> [(Text, Word64)]
+aggregateUtxos :: [(TxOut, Text, a)] -> [(Text, Word64)]
 aggregateUtxos xs =
   List.sortOn (Text.length . fst)
     . Map.toList
     . Map.fromListWith (+)
-    $ map (\(x, _) -> (txOutAddress x, unDbLovelace (txOutValue x))) xs
+    $ map (\(x, address, _) -> (address, unDbLovelace (txOutValue x))) xs
 
 isRedeemTextAddress :: Text -> Bool
 isRedeemTextAddress addr =
@@ -81,7 +82,7 @@ partitionUtxos =
     accept (addr, _) =
       Text.length addr <= 180 && not (isRedeemTextAddress addr)
 
-queryAtSlot :: Word64 -> IO (Ada, [(TxOut, ByteString)], Ada, Either LookupFail UTCTime)
+queryAtSlot :: Word64 -> IO (Ada, [(TxOut, Text, ByteString)], Ada, Either LookupFail UTCTime)
 queryAtSlot slotNo =
   -- Run the following queries in a single transaction.
   runDbNoLoggingEnv $ do
@@ -111,9 +112,9 @@ showUtxo (addr, value) =
     , "    }"
     ]
 
-utxoSetSum :: [(TxOut, a)] -> Ada
+utxoSetSum :: [(TxOut, b, a)] -> Ada
 utxoSetSum xs =
-  word64ToAda . sum $ map (unDbLovelace . txOutValue . fst) xs
+  word64ToAda . sum $ map (\(txOut, _, _) -> unDbLovelace $ txOutValue txOut) xs
 
 writeUtxos :: FilePath -> [(Text, Word64)] -> IO ()
 writeUtxos fname xs = do

cardano-db-tool/src/Cardano/DbTool/Validate/TotalSupply.hs

@@ -6,6 +6,7 @@ module Cardano.DbTool.Validate.TotalSupply (
 
 import Cardano.Db
 import Cardano.DbTool.Validate.Util
+import Cardano.DbTool.UtxoSet (utxoSetSum)
 import Data.Word (Word64)
 import System.Random (randomRIO)
 

cardano-db-tool/src/Cardano/DbTool/Validate/Util.hs

@@ -6,10 +6,8 @@ module Cardano.DbTool.Validate.Util (
   greenText,
   redText,
   putStrF,
-  utxoSetSum,
 ) where
 
-import Cardano.Db
 import System.Console.ANSI (setSGRCode)
 import System.Console.ANSI.Types (
   Color (..),
@@ -39,7 +37,3 @@ redText s = codeRed ++ s ++ codeReset
 
 putStrF :: String -> IO ()
 putStrF s = putStr s >> hFlush stdout
-
-utxoSetSum :: [(TxOut, a)] -> Ada
-utxoSetSum xs =
-  word64ToAda . sum $ map (unDbLovelace . txOutValue . fst) xs

cardano-db/src/Cardano/Db/Insert.hs

@@ -44,6 +44,7 @@ module Cardano.Db.Insert (
   insertTxOut,
   insertCollateralTxOut,
   insertManyTxOut,
+  insertAddress,
   insertWithdrawal,
   insertRedeemer,
   insertCostModel,
@@ -235,6 +236,9 @@ insertCollateralTxOut = insertUnchecked "CollateralTxOut"
 insertManyTxOut :: (MonadBaseControl IO m, MonadIO m) => [TxOut] -> ReaderT SqlBackend m [TxOutId]
 insertManyTxOut = insertMany' "TxOut"
 
+insertAddress :: (MonadBaseControl IO m, MonadIO m) => Address -> ReaderT SqlBackend m AddressId
+insertAddress = insertUnchecked "insertAddress"
+
 insertWithdrawal :: (MonadBaseControl IO m, MonadIO m) => Withdrawal -> ReaderT SqlBackend m WithdrawalId
 insertWithdrawal = insertUnchecked "Withdrawal"