db: Add a field to the Tx table which holds the block index

cardano-db-sync/src/Cardano/DbSync/Genesis.hs

@@ -172,6 +172,7 @@ insertTxOuts blkId (address, value) = do
             DB.Tx
               { DB.txHash = unTxHash $ txHashOfAddress address
               , DB.txBlock = blkId
+              , DB.txBlockIndex = 0
               , DB.txOutSum = Ledger.unsafeGetLovelace value
               , DB.txFee = 0
               , DB.txSize = 0 -- Genesis distribution address to not have a size.

cardano-db-sync/src/Cardano/DbSync/Plugin/Default/Insert.hs

@@ -120,16 +120,16 @@ insertABlock tracer blk tip = do
                     , DB.blockTxCount = fromIntegral $ length (blockPayload blk)
                     }
 
-    mapMVExceptT (insertTx tracer blkId) $ blockPayload blk
+    mapMVExceptT (insertTx tracer blkId) $ zip (blockPayload blk) [ 0 .. ]
 
     liftIO $ do
       let followingClosely = withOrigin 0 unBlockNo (getTipBlockNo tip) - blockNumber blk < 20
-          (epoch, slotWithin) = slotNumber blk `divMod` slotsPerEpoch
-      when (followingClosely && slotWithin /= 0 && slotNumber blk > 0 && slotNumber blk `mod` 20 == 0) $ do
+          (epoch, slotWithinEpoch) = slotNumber blk `divMod` slotsPerEpoch
+      when (followingClosely && slotWithinEpoch /= 0 && slotNumber blk > 0 && slotNumber blk `mod` 20 == 0) $ do
         logInfo tracer $
           mconcat
             [ "insertABlock: continuing epoch ", textShow epoch
-            , " (slot ", textShow slotWithin, ")"
+            , " (slot ", textShow slotWithinEpoch, ")"
             ]
       logger tracer $ mconcat
         [ "insertABlock: slot ", textShow (slotNumber blk)
@@ -146,14 +146,15 @@ insertABlock tracer blk tip = do
 
 insertTx
     :: MonadIO m
-    => Trace IO Text -> DB.BlockId -> Ledger.TxAux
+    => Trace IO Text -> DB.BlockId -> (Ledger.TxAux, Word64)
     -> ExceptT DbSyncNodeError (ReaderT SqlBackend m) ()
-insertTx tracer blkId tx = do
+insertTx tracer blkId (tx, blockIndex) = do
     valFee <- firstExceptT annotateTx $ newExceptT (calculateTxFee $ Ledger.taTx tx)
     txId <- lift . DB.insertTx $
               DB.Tx
                 { DB.txHash = unTxHash $ Crypto.serializeCborHash (Ledger.taTx tx)
                 , DB.txBlock = blkId
+                , DB.txBlockIndex = blockIndex
                 , DB.txOutSum = vfValue valFee
                 , DB.txFee = vfFee valFee
                 -- Would be really nice to have a way to get the transaction size

cardano-db-sync/src/Cardano/DbSync/Plugin/Epoch.hs

@@ -36,7 +36,8 @@ import           Cardano.DbSync.Error
 import           Cardano.DbSync.Util
 
 import           Ouroboros.Consensus.Byron.Ledger (ByronBlock (..))
-import           Ouroboros.Network.Block (Point, Tip)
+import           Ouroboros.Network.Block (BlockNo (..), Point, Tip, getTipBlockNo)
+import           Ouroboros.Network.Point (withOrigin)
 
 import           System.IO.Unsafe (unsafePerformIO)
 
@@ -70,7 +71,7 @@ epochPluginOnStartup trce = do
     updateChainTipEpochVar trce
 
 epochPluginInsertBlock :: Trace IO Text -> ByronBlock -> Tip ByronBlock -> ReaderT SqlBackend (LoggingT IO) (Either DbSyncNodeError ())
-epochPluginInsertBlock trce rawBlk _tip =
+epochPluginInsertBlock trce rawBlk tip =
   case byronBlockRaw rawBlk of
     Ledger.ABOBBoundary _ ->
       -- For the OBFT era there are no boundary blocks so we ignore them even in
@@ -84,9 +85,14 @@ epochPluginInsertBlock trce rawBlk _tip =
       let epochNum = epochNumber blk slotsPerEpoch
           lastCachedEpoch = fromMaybe 0 mLatestCachedEpoch
 
+
+
       if  | epochNum == chainTipEpoch && lastCachedEpoch == chainTipEpoch ->
-              -- Following the chain quite closely.
-              updateEpochNum epochNum trce
+              if withOrigin 0 unBlockNo (getTipBlockNo tip) - blockNumber blk < 15
+                then -- Following the chain very closely.
+                     updateEpochNum epochNum trce
+                else pure $ Right ()
+
           | epochNum > 0 && mLatestCachedEpoch == Nothing ->
               updateEpochNum 0 trce
           | epochNum >= lastCachedEpoch + 2 ->

cardano-db/app/Cardano/Db/App/Validate/BlockTxs.hs

@@ -24,10 +24,11 @@ import qualified System.Random as Random
 
 validateEpochBlockTxs :: IO ()
 validateEpochBlockTxs = do
+  fullySynced <- runDbNoLogging queryIsFullySynced
   mLatestEpoch <- runDbNoLogging queryLatestCachedEpochNo
   case mLatestEpoch of
     Nothing -> putStrLn "Epoch table is empty"
-    Just latest -> validateLatestBlockTxs latest
+    Just latest -> validateLatestBlockTxs fullySynced latest
 
 -- -----------------------------------------------------------------------------
 
@@ -37,10 +38,15 @@ data ValidateError = ValidateError
   , veTxCountExpected :: !Word64
   }
 
-validateLatestBlockTxs :: Word64 -> IO ()
-validateLatestBlockTxs latestEpoch = do
-  validateBlockTxs latestEpoch
-  validateBlockTxs =<< Random.randomRIO (0, latestEpoch - 1)
+validateLatestBlockTxs :: Bool -> Word64 -> IO ()
+validateLatestBlockTxs fullySynced latestEpoch = do
+  if not fullySynced
+    then putStrLn "Not fully synced so not running BlockTx validation"
+    else do
+      -- This validation seems to be quite DB intensive, so only run it
+      -- when the DB is fully synced.
+      validateBlockTxs latestEpoch
+      validateBlockTxs =<< Random.randomRIO (0, latestEpoch - 1)
 
 validateBlockTxs :: Word64 -> IO ()
 validateBlockTxs epoch = do

cardano-db/src/Cardano/Db/Schema.hs

@@ -77,6 +77,7 @@ share
   Tx
     hash                ByteString          sqltype=hash32type
     block               BlockId             -- This type is the primary key for the 'block' table.
+    blockIndex          Word64              sqltype=uinteger  -- The index of this transaction within the block.
     outSum              Word64              sqltype=lovelace
     fee                 Word64              sqltype=lovelace
     size                Word64              sqltype=uinteger

cardano-db/test/Test/IO/Cardano/Db/Rollback.hs

@@ -96,7 +96,7 @@ createAndInsertBlocks blockCount =
         newMTxOutId <- if indx /= 0
                       then pure mTxOutId
                       else do
-                        txId <- insertTx $ Tx (mkTxHash blkId 0) blkId 0 0 12
+                        txId <- insertTx $ Tx (mkTxHash blkId 0) blkId 0 0 0 12
                         void $ insertTxOut (mkTxOut blkId txId)
                         pure $ Just txId
         case (indx, mTxOutId) of

cardano-db/test/Test/IO/Cardano/Db/TotalSupply.hs

@@ -36,7 +36,7 @@ initialSupplyTest =
 
     -- Spend from the Utxo set.
     bid1 <- insertBlock (mkBlock 1 slid)
-    tx1Id <- insertTx (Tx (mkTxHash bid1 1) bid1 500000000 100 123)
+    tx1Id <- insertTx (Tx (mkTxHash bid1 1) bid1 0 500000000 100 123)
     _ <- insertTxIn (TxIn tx1Id (head tx0Ids) 0)
     _ <- insertTxOut $ TxOut tx1Id 0 (mkAddressHash bid1 tx1Id) 500000000
     supply1 <- queryTotalSupply

cardano-db/test/Test/IO/Cardano/Db/Util.hs

@@ -72,7 +72,7 @@ mkTxs :: BlockId -> Word -> [Tx]
 mkTxs blkId count =
     take (fromIntegral count) $ map create [ 0 .. ]
   where
-    create w = Tx (mkTxHash blkId w) blkId 2 1 12
+    create w = Tx (mkTxHash blkId w) blkId 0 2 1 12
 
 testSlotLeader :: SlotLeader
 testSlotLeader =

schema/migration-2-0001-20190902.sql → schema/migration-2-0001-20200517.sql

@@ -6,13 +6,13 @@ DECLARE
 BEGIN
   SELECT stage_two + 1 INTO next_version FROM schema_version ;
   IF next_version = 1 THEN
-    EXECUTE 'CREATe TABLE "slot_leader"("id" SERIAL8  PRIMARY KEY UNIQUE,"hash" hash28type NOT NULL,"desciption" VARCHAR NOT NULL)' ;
+    EXECUTE 'CREATe TABLE "slot_leader"("id" SERIAL8  PRIMARY KEY UNIQUE,"hash" hash28type NOT NULL,"description" VARCHAR NOT NULL)' ;
     EXECUTE 'ALTER TABLE "slot_leader" ADD CONSTRAINT "unique_slot_leader" UNIQUE("hash")' ;
-    EXECUTE 'CREATe TABLE "block"("id" SERIAL8  PRIMARY KEY UNIQUE,"hash" hash32type NOT NULL,"slot_no" uinteger NULL,"block_no" uinteger,"previous" INT8 NULL,"merkel_root" hash32type NULL,"slot_leader" INT8 NOT NULL,"size" uinteger NOT NULL)' ;
+    EXECUTE 'CREATe TABLE "block"("id" SERIAL8  PRIMARY KEY UNIQUE,"hash" hash32type NOT NULL,"epoch_no" uinteger NULL,"slot_no" uinteger NULL,"block_no" uinteger NULL,"previous" INT8 NULL,"merkel_root" hash32type NULL,"slot_leader" INT8 NOT NULL,"size" uinteger NOT NULL,"time" timestamp NOT NULL,"tx_count" uinteger NOT NULL)' ;
     EXECUTE 'ALTER TABLE "block" ADD CONSTRAINT "unique_block" UNIQUE("hash")' ;
     EXECUTE 'ALTER TABLE "block" ADD CONSTRAINT "block_previous_fkey" FOREIGN KEY("previous") REFERENCES "block"("id")' ;
     EXECUTE 'ALTER TABLE "block" ADD CONSTRAINT "block_slot_leader_fkey" FOREIGN KEY("slot_leader") REFERENCES "slot_leader"("id")' ;
-    EXECUTE 'CREATe TABLE "tx"("id" SERIAL8  PRIMARY KEY UNIQUE,"hash" hash32type NOT NULL,"block" INT8 NOT NULL,"fee" lovelace NOT NULL)' ;
+    EXECUTE 'CREATe TABLE "tx"("id" SERIAL8  PRIMARY KEY UNIQUE,"hash" hash32type NOT NULL,"block" INT8 NOT NULL,"block_index" uinteger NOT NULL,"out_sum" lovelace NOT NULL,"fee" lovelace NOT NULL,"size" uinteger NOT NULL)' ;
     EXECUTE 'ALTER TABLE "tx" ADD CONSTRAINT "unique_tx" UNIQUE("hash")' ;
     EXECUTE 'ALTER TABLE "tx" ADD CONSTRAINT "tx_block_fkey" FOREIGN KEY("block") REFERENCES "block"("id")' ;
     EXECUTE 'CREATe TABLE "tx_out"("id" SERIAL8  PRIMARY KEY UNIQUE,"tx_id" INT8 NOT NULL,"index" txindex NOT NULL,"address" VARCHAR NOT NULL,"value" lovelace NOT NULL)' ;
@@ -22,8 +22,10 @@ BEGIN
     EXECUTE 'ALTER TABLE "tx_in" ADD CONSTRAINT "unique_txin" UNIQUE("tx_out_id","tx_out_index")' ;
     EXECUTE 'ALTER TABLE "tx_in" ADD CONSTRAINT "tx_in_tx_in_id_fkey" FOREIGN KEY("tx_in_id") REFERENCES "tx"("id")' ;
     EXECUTE 'ALTER TABLE "tx_in" ADD CONSTRAINT "tx_in_tx_out_id_fkey" FOREIGN KEY("tx_out_id") REFERENCES "tx"("id")' ;
-    EXECUTE 'CREATe TABLE "meta"("id" SERIAL8  PRIMARY KEY UNIQUE,"protocol_const" INT8 NOT NULL,"slot_duration" INT8 NOT NULL,"start_time" TIMESTAMP WITH TIME ZONE NOT NULL)' ;
+    EXECUTE 'CREATe TABLE "meta"("id" SERIAL8  PRIMARY KEY UNIQUE,"protocol_const" INT8 NOT NULL,"slot_duration" INT8 NOT NULL,"start_time" timestamp NOT NULL,"network_name" VARCHAR NULL)' ;
     EXECUTE 'ALTER TABLE "meta" ADD CONSTRAINT "unique_meta" UNIQUE("start_time")' ;
+    EXECUTE 'CREATe TABLE "epoch"("id" SERIAL8  PRIMARY KEY UNIQUE,"out_sum" outsum NOT NULL,"tx_count" uinteger NOT NULL,"blk_count" uinteger NOT NULL,"no" uinteger NOT NULL,"start_time" timestamp NOT NULL,"end_time" timestamp NOT NULL)' ;
+    EXECUTE 'ALTER TABLE "epoch" ADD CONSTRAINT "unique_epoch" UNIQUE("no")' ;
     -- Hand written SQL statements can be added here.
     UPDATE schema_version SET stage_two = 1 ;
     RAISE NOTICE 'DB has been migrated to stage_two version %', next_version ;