Prepare 13.6.0.5

CHANGELOG.md

@@ -1,5 +1,13 @@
 # Revision history for cardano-db-sync
 
+## 13.6.0.5
+- Fix offchain data so it supports files up to 3MB [#1928]
+- Upgrade to PostgreSQL 17
+- Stop creating views and deleting them on every restart [#1950]
+- Maintain less snapshots
+- No longer crash if stage 1 or 2 migration have been altered
+- Improvements for Address variant
+
 ## 13.6.0.4
 - Fixes an issue from ledger where DRep delegations could be removed under some conditions
 - Fix an issue with migration from 13.5.x when consumed_tx_out config is enabled

cardano-chain-gen/cardano-chain-gen.cabal

@@ -1,7 +1,7 @@
 cabal-version:          3.6
 
 name:                   cardano-chain-gen
-version:                13.6.0.4
+version:                13.6.0.5
 synopsis:               A fake chain generator for testing cardano DB sync.
 description:            A fake chain generator for testing cardano DB sync.
 homepage:               https://github.com/IntersectMBO/cardano-db-sync

cardano-db-sync/cardano-db-sync.cabal

@@ -1,7 +1,7 @@
 cabal-version:          3.6
 
 name:                   cardano-db-sync
-version:                13.6.0.4
+version:                13.6.0.5
 synopsis:               The Cardano DB Sync node
 description:            A Cardano node that follows the Cardano chain and inserts data from the
                         chain into a PostgresQL database.

cardano-db-sync/src/Cardano/DbSync.hs

@@ -88,10 +88,7 @@ runDbSync metricsSetters knownMigrations iomgr trce params syncNodeConfigFromFil
   whenJust mErrors $ \(unknown, stage4orNewStage3) ->
     if stage4orNewStage3
       then logWarning trce $ Db.renderMigrationValidateError unknown
-      else do
-        let msg = Db.renderMigrationValidateError unknown
-        logError trce msg
-        throwIO unknown
+      else logError trce $ Db.renderMigrationValidateError unknown
 
   logInfo trce "Schema migration files validated"
 

cardano-db-sync/src/Cardano/DbSync/Ledger/State.hs

@@ -333,7 +333,7 @@ storeSnapshotAndCleanupMaybe env oldState appResult blkNo isCons syncState =
     Just newEpoch
       | newEpochNo <- unEpochNo (Generic.neEpoch newEpoch)
       , newEpochNo > 0
-      , isCons || (newEpochNo `mod` 10 == 0) || newEpochNo >= 503 ->
+      , isCons || (newEpochNo `mod` 10 == 0) || newEpochNo >= 530 ->
           do
             -- TODO: Instead of newEpochNo - 1, is there any way to get the epochNo from 'lssOldState'?
             liftIO $ saveCleanupState env oldState (Just $ EpochNo $ newEpochNo - 1)
@@ -480,10 +480,10 @@ cleanupLedgerStateFiles env slotNo = do
   let (epochBoundary, valid, invalid) = foldr groupFiles ([], [], []) files
   -- Remove invalid (ie SlotNo >= current) ledger state files (occurs on rollback).
   deleteAndLogFiles env "invalid" invalid
-  -- Remove all but 6 most recent state files.
-  deleteAndLogStateFile env "old" (List.drop 3 valid)
-  -- Remove all but 6 most recent epoch boundary state files.
-  deleteAndLogStateFile env "old epoch boundary" (List.drop 6 epochBoundary)
+  -- Remove all but 2 most recent state files.
+  deleteAndLogStateFile env "old" (List.drop 2 valid)
+  -- Remove all but 3 most recent epoch boundary state files.
+  deleteAndLogStateFile env "old epoch boundary" (List.drop 3 epochBoundary)
   where
     groupFiles ::
       LedgerStateFile ->

cardano-db-sync/src/Cardano/DbSync/OffChain/Http.hs

@@ -105,7 +105,7 @@ httpGetOffChainVoteDataSingle ::
 httpGetOffChainVoteDataSingle vurl metaHash anchorType = do
   manager <- liftIO $ Http.newManager tlsManagerSettings
   request <- parseOffChainUrl url
-  let req = httpGetBytes manager request 300000 300000 url
+  let req = httpGetBytes manager request 3000000 3000000 url
   httpRes <- handleExceptT (convertHttpException url) req
   (respBS, respLBS, mContentType) <- hoistEither httpRes
   (ocvd, decodedValue, metadataHash, mWarning) <- parseAndValidateVoteData respBS respLBS metaHash anchorType (Just $ OffChainVoteUrl vurl)

cardano-db-tool/cardano-db-tool.cabal

@@ -1,7 +1,7 @@
 cabal-version:          3.6
 
 name:                   cardano-db-tool
-version:                13.6.0.4
+version:                13.6.0.5
 synopsis:               Utilities to manage the cardano-db-sync databases.
 description:            Utilities and executable, used to manage and validate the
                         PostgreSQL db and the ledger database of the cardano-db-sync node

cardano-db/cardano-db.cabal

@@ -1,7 +1,7 @@
 cabal-version:          3.6
 
 name:                   cardano-db
-version:                13.6.0.4
+version:                13.6.0.5
 synopsis:               A base PostgreSQL component for the cardano-db-sync node.
 description:            Code for the Cardano DB Sync node that is shared between the
                         cardano-db-node and other components.

cardano-db/test/cardano-db-test.cabal

@@ -1,7 +1,7 @@
 cabal-version:          3.6
 
 name:                   cardano-db-test
-version:                13.6.0.4
+version:                13.6.0.5
 synopsis:               Tests for the base functionality of the cardano-db library
 description:            Code for the Cardano DB Sync node that is shared between the
                         cardano-db-node and other components.

cardano-smash-server/cardano-smash-server.cabal

@@ -1,7 +1,7 @@
 cabal-version:          3.6
 
 name:                   cardano-smash-server
-version:                13.6.0.4
+version:                13.6.0.5
 synopsis:               The Cardano smash server
 description:            Please see the README on GitHub at
                         <https://github.com/IntersectMBO/cardano-db-sync/cardano-smash-server/#readme>

docker-compose.example.yml

@@ -59,7 +59,7 @@ services:
         max-file: "10"
 
   cardano-db-sync:
-    image: ghcr.io/intersectmbo/cardano-db-sync:13.6.0.4
+    image: ghcr.io/intersectmbo/cardano-db-sync:13.6.0.5
     environment:
       - NETWORK=${NETWORK:-mainnet}
       - POSTGRES_HOST=postgres

docker-compose.yml

@@ -54,7 +54,7 @@ services:
         max-file: "10"
 
   cardano-db-sync:
-    image: ghcr.io/intersectmbo/cardano-db-sync:13.6.0.4
+    image: ghcr.io/intersectmbo/cardano-db-sync:13.6.0.5
     environment:
       - DB_SYNC_CONFIG=${DB_SYNC_CONFIG:-}
       - DISABLE_LEDGER=${DISABLE_LEDGER}

schema/migration-1-0002-20190912.sql

@@ -1,34 +1,19 @@
--- Drop all VIEWs linked to the 'cexplorer' table.
 
--- Many schema migration changes will fail if they operate on a table that is part of a VIEW.
--- So we drop all VIEWs here and recreate them later.
-
-CREATE FUNCTION drop_cexplorer_views () RETURNS void language plpgsql as $$
-
-DECLARE vname text;
-DECLARE next_version int;
 
+CREATE FUNCTION migrate() RETURNS void AS $$
+DECLARE
+  next_version int ;
 BEGIN
-  SELECT stage_one + 1 INTO next_version FROM "schema_version";
+  SELECT stage_one + 1 INTO next_version FROM "schema_version" ;
   IF next_version = 2 THEN
-    -- Only update the schema version if the versions are as expected.
-    UPDATE "schema_version" SET stage_one = next_version;
-    RAISE NOTICE 'DB has been migrated to stage_one version %', next_version;
-  END IF;
+    -- Empty migration file just to raise the version
 
-  -- Always run the following.
-  for vname in
-      select '"' || table_name || '"'
-	    from information_schema.views
-        where table_catalog = current_database ()
-          and table_schema = 'public'
-          and table_name !~* 'pg_stat_.*'
-    loop
-      execute format ('drop view if exists %s cascade ;', vname) ;
-      raise notice 'Dropping view : %', vname ;
-    end loop ;
-end $$ ;
+    UPDATE schema_version SET stage_one = next_version ;
+    RAISE NOTICE 'DB has been migrated to stage_one version %', next_version ;
+  END IF ;
+END ;
+$$ LANGUAGE plpgsql ;
 
-SELECT drop_cexplorer_views () ;
+SELECT migrate() ;
 
-DROP FUNCTION drop_cexplorer_views () ;
+DROP FUNCTION migrate() ;

schema/migration-3-0001-20190816.sql

@@ -1,28 +0,0 @@
--- Create all the views.
-
--- The views are always created because all views are deleted at the start ot the
--- migration process.
-
--- Conventions:
---  * Views use `_view` as a suffix to show they are views rather than table.
-
--- A utxo view which shows all unspent transaction outputs including the un-redeemed redeem
--- addresses.
-create or replace view  utxo_byron_view as select
-	tx_out.*
-  from tx_out left outer join tx_in
-	on tx_out.tx_id = tx_in.tx_out_id and tx_out.index = tx_in.tx_out_index
-  where tx_in.tx_in_id is null ;
-
-
--- A utxo view which shows all unspent transaction outputs *excluding* the un-redeemed redeem
--- addresses.
--- This should produce the same query results as the above `utxo_byron_view` for Shelley addresses
--- and non-redeem Byron addresses.
-create or replace view utxo_view as select
-	tx_out.*
-  from tx_out
-	left outer join tx_in on tx_out.tx_id = tx_in.tx_out_id and tx_out.index = tx_in.tx_out_index
-	left outer join tx on tx.id = tx_out.tx_id
-	left outer join block on tx.block_id = block.id
-  where tx_in.tx_in_id is null and block.epoch_no is not null ;