Merge branch 'main' into integrate-kurtosis

Author: MegaRedHand

.github/workflows/ci.yml

@@ -139,9 +139,9 @@ jobs:
     steps:
     - uses: actions/checkout@v4
     - name: Set up Docker Buildx
-      uses: docker/setup-buildx-action@v1
+      uses: docker/setup-buildx-action@v3
     - name: Build Docker image
-      uses: docker/build-push-action@v2
+      uses: docker/build-push-action@v5
       with:
         context: .
         file: ./Dockerfile

Dockerfile

@@ -33,19 +33,37 @@ RUN apt-get update && apt-get install -y cmake protobuf-compiler
 # Install protobuf for elixir
 RUN mix escript.install --force hex protobuf
 
+# Download openapi spec
+COPY Makefile .oapi_version /consensus/
+RUN make download-beacon-node-oapi
+
 # Install rust
 RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
 
 ENV PATH="${PATH}:/root/.cargo/bin:/root/.mix/escripts"
 
+# Precompile rust crates. Rustler stores targets under _build
+COPY ./native /consensus/native
+RUN cd /consensus/native/bls_nif && cargo build --release \
+    --target-dir="/consensus/_build/prod/lib/lambda_ethereum_consensus/native/bls_nif"
+RUN cd /consensus/native/kzg_nif && cargo build --release \
+    --target-dir="/consensus/_build/prod/lib/lambda_ethereum_consensus/native/kzg_nif"
+RUN cd /consensus/native/snappy_nif && cargo build --release \
+    --target-dir="/consensus/_build/prod/lib/lambda_ethereum_consensus/native/snappy_nif"
+RUN cd /consensus/native/ssz_nif && cargo build --release \
+    --target-dir="/consensus/_build/prod/lib/lambda_ethereum_consensus/native/ssz_nif"
+
+# Precompile elixir dependencies
+COPY mix.exs mix.lock .fork_version ./
+COPY ./config/config.exs /consensus/config/config.exs
+RUN mix deps.get
+RUN mix deps.compile
+
 COPY . .
 COPY --from=libp2p_builder /libp2p_port/libp2p_port /consensus/priv/native/libp2p_port
 
 RUN protoc --elixir_out=./lib proto/libp2p.proto
 
-RUN make download-beacon-node-oapi
-
-RUN mix deps.get
 RUN mix compile
 
 ENTRYPOINT [ "iex", "-S", "mix", "run", "--"]

lib/bls.ex

@@ -5,48 +5,50 @@ defmodule Bls do
   use Rustler, otp_app: :lambda_ethereum_consensus, crate: "bls_nif"
 
   @type privkey() :: <<_::256>>
+  @type pubkey :: <<_::384>>
+  @type signature :: <<_::768>>
 
-  @spec sign(privkey(), binary()) :: {:ok, Types.bls_signature()} | {:error, any()}
+  @spec sign(privkey(), binary()) :: {:ok, signature()} | {:error, any()}
   def sign(_private_key, _message) do
     :erlang.nif_error(:nif_not_loaded)
   end
 
-  @spec aggregate([Types.bls_signature()]) :: {:ok, Types.bls_signature()} | {:error, any()}
+  @spec aggregate([signature()]) :: {:ok, signature()} | {:error, any()}
   def aggregate(_signatures) do
     :erlang.nif_error(:nif_not_loaded)
   end
 
-  @spec verify(Types.bls_pubkey(), binary(), Types.bls_signature()) ::
+  @spec verify(pubkey(), binary(), signature()) ::
           {:ok, boolean} | {:error, binary()}
   def verify(_public_key, _message, _signature) do
     :erlang.nif_error(:nif_not_loaded)
   end
 
-  @spec fast_aggregate_verify([Types.bls_pubkey()], binary(), Types.bls_signature()) ::
+  @spec fast_aggregate_verify([pubkey()], binary(), signature()) ::
           {:ok, boolean} | {:error, binary()}
   def fast_aggregate_verify(_public_keys, _message, _signature) do
     :erlang.nif_error(:nif_not_loaded)
   end
 
-  @spec eth_fast_aggregate_verify([Types.bls_pubkey()], binary(), Types.bls_signature()) ::
+  @spec eth_fast_aggregate_verify([pubkey()], binary(), signature()) ::
           {:ok, boolean} | {:error, binary()}
   def eth_fast_aggregate_verify(_public_keys, _message, _signature) do
     :erlang.nif_error(:nif_not_loaded)
   end
 
-  @spec aggregate_verify([Types.bls_pubkey()], [binary()], Types.bls_signature()) ::
+  @spec aggregate_verify([pubkey()], [binary()], signature()) ::
           {:ok, boolean} | {:error, binary()}
   def aggregate_verify(_public_keys, _messages, _signature) do
     :erlang.nif_error(:nif_not_loaded)
   end
 
-  @spec eth_aggregate_pubkeys([Types.bls_pubkey()]) ::
-          {:ok, Types.bls_pubkey()} | {:error, any()}
+  @spec eth_aggregate_pubkeys([pubkey()]) ::
+          {:ok, pubkey()} | {:error, any()}
   def eth_aggregate_pubkeys(_public_keys) do
     :erlang.nif_error(:nif_not_loaded)
   end
 
-  @spec key_validate(Types.bls_pubkey()) ::
+  @spec key_validate(pubkey()) ::
           {:ok, boolean} | {:error, binary()}
   def key_validate(_public_key) do
     :erlang.nif_error(:nif_not_loaded)
@@ -56,7 +58,7 @@ defmodule Bls do
   @doc """
   Same as ``Bls.verify``, but treats errors as invalid signatures.
   """
-  @spec valid?(Types.bls_pubkey(), binary(), Types.bls_signature()) :: boolean()
+  @spec valid?(pubkey(), binary(), signature()) :: boolean()
   def valid?(public_key, message, signature) do
     case Bls.verify(public_key, message, signature) do
       {:ok, bool} -> bool
@@ -68,7 +70,7 @@ defmodule Bls do
   @doc """
   Same as ``Bls.fast_aggregate_verify``, but treats errors as invalid signatures.
   """
-  @spec fast_aggregate_valid?([Types.bls_pubkey()], binary(), Types.bls_signature()) :: boolean()
+  @spec fast_aggregate_valid?([pubkey()], binary(), signature()) :: boolean()
   def fast_aggregate_valid?(public_keys, message, signature) do
     case Bls.fast_aggregate_verify(public_keys, message, signature) do
       {:ok, bool} -> bool

lib/kzg.ex

@@ -4,44 +4,47 @@ defmodule Kzg do
   """
   use Rustler, otp_app: :lambda_ethereum_consensus, crate: "kzg_nif"
 
-  @spec blob_to_kzg_commitment(Types.blob()) :: {:ok, Types.kzg_commitment()} | {:error, binary()}
+  @type commitment :: <<_::384>>
+  @type proof :: <<_::768>>
+
+  @spec blob_to_kzg_commitment(Types.blob()) :: {:ok, commitment()} | {:error, binary()}
   def blob_to_kzg_commitment(_blob) do
     :erlang.nif_error(:nif_not_loaded)
   end
 
   @spec compute_kzg_proof(Types.blob(), Types.bytes32()) ::
-          {:ok, {Types.kzg_proof(), Types.bytes32()}} | {:error, binary()}
+          {:ok, {proof(), Types.bytes32()}} | {:error, binary()}
   def compute_kzg_proof(_blob, _z) do
     :erlang.nif_error(:nif_not_loaded)
   end
 
   @spec verify_kzg_proof(
-          Types.kzg_commitment(),
+          commitment(),
           Types.bytes32(),
           Types.bytes32(),
-          Types.kzg_proof()
+          proof()
         ) ::
           {:ok, boolean} | {:error, binary()}
   def verify_kzg_proof(_kzg_commitment, _z, _y, _kzg_proof) do
     :erlang.nif_error(:nif_not_loaded)
   end
 
-  @spec compute_blob_kzg_proof(Types.blob(), Types.kzg_commitment()) ::
-          {:ok, Types.kzg_proof()} | {:error, binary()}
+  @spec compute_blob_kzg_proof(Types.blob(), commitment()) ::
+          {:ok, proof()} | {:error, binary()}
   def compute_blob_kzg_proof(_blob, _kzg_commitment) do
     :erlang.nif_error(:nif_not_loaded)
   end
 
-  @spec verify_blob_kzg_proof(Types.blob(), Types.kzg_commitment(), Types.kzg_proof()) ::
+  @spec verify_blob_kzg_proof(Types.blob(), commitment(), proof()) ::
           {:ok, boolean} | {:error, binary()}
   def verify_blob_kzg_proof(_blob, _kzg_commitment, _kzg_proof) do
     :erlang.nif_error(:nif_not_loaded)
   end
 
   @spec verify_blob_kzg_proof_batch(
           list(Types.blob()),
-          list(Types.kzg_commitment()),
-          list(Types.kzg_proof())
+          list(commitment()),
+          list(proof())
         ) ::
           {:ok, boolean} | {:error, binary()}
   def verify_blob_kzg_proof_batch(_blobs, _kzg_commitments, _kzg_proofs) do
@@ -54,8 +57,8 @@ defmodule Kzg do
 
   @spec blob_kzg_proof_batch_valid?(
           list(Types.blob()),
-          list(Types.kzg_commitment()),
-          list(Types.kzg_proof())
+          list(commitment()),
+          list(proof())
         ) :: boolean()
   def blob_kzg_proof_batch_valid?(blobs, kzg_commitments, kzg_proofs) do
     case verify_blob_kzg_proof_batch(blobs, kzg_commitments, kzg_proofs) do

lib/lambda_ethereum_consensus/beacon/beacon_node.ex

@@ -50,10 +50,10 @@ defmodule LambdaEthereumConsensus.Beacon.BeaconNode do
         LambdaEthereumConsensus.P2P.IncomingRequests,
         LambdaEthereumConsensus.Beacon.PendingBlocks,
         LambdaEthereumConsensus.Beacon.SyncBlocks,
-        LambdaEthereumConsensus.P2P.GossipSub,
         LambdaEthereumConsensus.P2P.Gossip.Attestation,
         LambdaEthereumConsensus.P2P.Gossip.BeaconBlock,
-        LambdaEthereumConsensus.P2P.Gossip.BlobSideCar
+        LambdaEthereumConsensus.P2P.Gossip.BlobSideCar,
+        LambdaEthereumConsensus.P2P.Gossip.OperationsCollector
       ] ++ validator_children
 
     Supervisor.init(children, strategy: :one_for_all)

lib/lambda_ethereum_consensus/beacon/sync_blocks.ex

@@ -75,14 +75,19 @@ defmodule LambdaEthereumConsensus.Beacon.SyncBlocks do
 
     if Enum.empty?(chunks) do
       Logger.info("[Optimistic Sync] Sync completed")
-      Gossip.BeaconBlock.start()
-      Gossip.BlobSideCar.start()
+      start_subscriptions()
     else
       Process.sleep(1000)
       perform_sync(remaining_chunks)
     end
   end
 
+  defp start_subscriptions() do
+    Gossip.BeaconBlock.start()
+    Gossip.BlobSideCar.start()
+    Gossip.OperationsCollector.start()
+  end
+
   @spec fetch_blocks_by_slot(Types.slot(), non_neg_integer()) ::
           {:ok, [SignedBeaconBlock.t()]} | {:error, String.t()}
   def fetch_blocks_by_slot(from, count) do