Add Nix Flake (#936)

* flakes

* add nomad entrypoint

* add mantis-entrypoint to hydra jobs

* remove rocksdb lock before start

* entrypoint needs procps

* another attempt to fix the broken shutdown issue

* another attempt to make sure entrypoint is killable

* shorten the time between config updates when mantis is unstable

* Update mantis sha256

* Update mantis sha256 again

* Remove old mantis packages

Co-authored-by: Jonathan Ringer <[email protected]>
Co-authored-by: Jonathan Ringer <[email protected]>
Co-authored-by: Petra Bierleutgeb <[email protected]>

Author: 3 people

flake.lock

@@ -0,0 +1,47 @@
+{
+  description = "Mantis flake";
+
+  inputs.flake-utils.url = "github:numtide/flake-utils";
+  inputs.nixpkgs.url = "github:nixos/nixpkgs?rev=a98302aa9b9628915878a6ea9776c40a0bb02950";
+  inputs.sbt-derivation.url = "github:zaninime/sbt-derivation";
+
+  outputs = { self, nixpkgs, flake-utils, sbt-derivation }: #, libsonic, libsonic-jnr }:
+    let
+      overlay = import ./nix/overlay.nix self.rev;
+      pkgsForSystem = system: (import nixpkgs) { inherit system; overlays = [
+        #libsonic.overlay
+        #libsonic-jnr.overlay
+        sbt-derivation.overlay
+        overlay
+      ];};
+
+      mkHydraUtils = mkPkgs: let
+        # nothing in lib should really depend on the system
+        libPkgs = mkPkgs "x86_64-linux";
+        # [attrset] -> attrset
+        recursiveMerge = libPkgs.lib.foldr libPkgs.lib.recursiveUpdate {};
+        mkHydraJobsForSystem = attrs: system:
+            recursiveMerge (map (n: { "${n}"."${system}" = (mkPkgs system)."${n}"; }) attrs);
+      in {
+        collectHydraSets = jobSets: { hydraJobs = recursiveMerge jobSets; };
+        mkHydraSet = attrs: systems: recursiveMerge (map (mkHydraJobsForSystem attrs) systems);
+      };
+
+      hydraUtils = mkHydraUtils pkgsForSystem;
+      inherit (hydraUtils) collectHydraSets mkHydraSet;
+
+    in flake-utils.lib.eachDefaultSystem (system: rec {
+      pkgs = pkgsForSystem system;
+      legacyPackages = pkgs;
+
+      defaultPackage = pkgs.mantis;
+      devShell = pkgs.mkShell {
+        nativeBuildInputs = with pkgs; [ solc sbt ];
+      };
+      apps.mantis = flake-utils.lib.mkApp { drv = pkgs.mantis; };
+      defaultApp = apps.mantis;
+    }) // ( collectHydraSets [
+      (mkHydraSet [ "mantis" ] [ "x86_64-linux" "x86_64-darwin" ])
+      (mkHydraSet [ "mantis-entrypoint" ] [ "x86_64-linux" ])
+    ]);
+}

flake.nix

@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+
+set -exuo pipefail
+
+cleanup() {
+    # kill all processes whose parent is this process
+    pkill -P $$
+}
+
+for sig in INT QUIT HUP TERM; do
+  trap "
+    cleanup
+    trap - $sig EXIT
+    kill -s $sig "'"$$"' "$sig"
+done
+trap cleanup EXIT
+
+
+mkdir -p /tmp
+mkdir -p "$NOMAD_TASK_DIR/mantis"
+cd "$NOMAD_TASK_DIR"
+name="java"
+
+if [ -n "${DAG_NAME:-}" ]; then
+  if [ -f "ethash/$DAG_NAME" ]; then
+    echo "found existing DAG"
+    sha256sum "ethash/$DAG_NAME"
+  else
+    mkdir -p ethash
+    aws \
+      --endpoint-url "$MONITORING_ADDR" \
+      s3 cp \
+      "s3://$DAG_BUCKET/$DAG_NAME" \
+      "ethash/$DAG_NAME" \
+      || echo "Unable to download DAG, skipping."
+  fi
+fi
+
+if [ -d "$STORAGE_DIR" ]; then
+  echo "$STORAGE_DIR found, not restoring from backup..."
+else
+  echo "$STORAGE_DIR not found, restoring backup..."
+  restic restore latest \
+    --tag "$NAMESPACE" \
+    --target / \
+    || echo "couldn't restore backup, continue startup procedure..."
+      mkdir -p "$NOMAD_TASK_DIR/mantis"
+      rm -rf "$NOMAD_TASK_DIR/mantis/{keystore,node.key}"
+      rm -rf "$NOMAD_TASK_DIR/mantis/logs"
+fi
+
+until [ "$(grep -c enode mantis.conf)" -ge "$REQUIRED_PEER_COUNT" ]; do
+  sleep 1
+done
+
+ulimit -c unlimited
+cp mantis.conf running.conf
+
+(
+while true; do
+  set +x
+  while diff -u running.conf mantis.conf > /dev/stderr; do
+    sleep 900
+  done
+  set -x
+
+  if ! diff -u running.conf mantis.conf > /dev/stderr; then
+    echo "Found updated config file, restarting Mantis"
+    pkill "$name" || true
+  fi
+done
+) &
+
+starts=0
+while true; do
+  starts="$((starts+1))"
+  echo "Start Number $starts" > /dev/stderr
+  cp mantis.conf running.conf
+  cat running.conf > /dev/stderr
+  rm -f "$NOMAD_TASK_DIR/mantis/rocksdb/LOCK"
+  mantis "-Duser.home=$NOMAD_TASK_DIR" "$@" || true
+  sleep 10
+done

nix/entrypoint.sh

@@ -0,0 +1,93 @@
+{ src
+, lib
+, coreutils
+, stdenv
+, nix-gitignore
+, solc
+, makeWrapper
+, runtimeShell
+, jre
+, sbt
+, gawk
+, gnused
+, protobuf
+, substituteAll
+, writeShellScriptBin
+}:
+
+let
+  version = let
+    buildSbt = builtins.readFile ../build.sbt;
+    captures = builtins.match ''.*version := "([^"]+)".*'' buildSbt;
+  in builtins.elemAt captures 0;
+
+  PATH = lib.makeBinPath [ jre solc coreutils gawk gnused ];
+  LD_LIBRARY_PATH = ''''; #lib.makeLibraryPath [ libsonic ];
+
+  # filter out mentions of protobridge, which is unable to execute
+  protoc-wrapper = writeShellScriptBin "protoc" ''
+    set -e
+
+    for f in "$@"; do
+      echo ''${f##*=}
+    done | grep protocbridge | xargs sed -i "1s|.*|#!${runtimeShell}|"
+
+    exec ${protobuf}/bin/protoc "$@"
+  '';
+
+
+in sbt.mkDerivation rec {
+  pname = "mantis";
+  inherit src version;
+
+  nativeBuildInputs = [ solc protobuf makeWrapper ];
+
+  preConfigure = ''
+    HOME=$TMPDIR
+    PROTOC_CACHE=.nix/protoc-cache
+  '';
+
+  # used by sbt-derivation to modify vendor derivation
+  overrideDepsAttrs = oldAttrs: {
+    inherit preConfigure;
+  };
+  PROTOCBRIDGE_SHELL = runtimeShell;
+
+  patches = [
+    (substituteAll {
+      src = ./protoc.patch;
+      protobuf = protoc-wrapper;
+    })
+  ];
+
+  #patchPhase = lib.optionalString (libsonic != null) ''
+  #  rm -rf src/main/resources
+  #  cp -r ${libsonic}/lib src/main/resources
+  #'';
+
+  # This sha represents the change dependencies of mantis.
+  # Update this sha whenever you change the dependencies
+  depsSha256 = "0n7vv4k73cxjwg40qggr7gnkkg7vn8a179sf0wxnz3absj1700jj";
+
+  # this is the command used to to create the fixed-output-derivation
+  depsWarmupCommand = "PROTOC_CACHE=.nix/protoc-cache; HOME=$TMPDIR; PATH=${PATH}:$PATH; sbt clean; sbt compile --debug";
+
+  installPhase = ''
+    sbt stage
+    mkdir -p $out/
+    cp -r target/universal/stage/* $out/
+    mkdir -p $out/share/mantis
+    mv $out/{LICENSE,RELEASE,mantis_config.txt} $_
+
+    # wrap executable so that java is available at runtime
+    for p in $(find $out/bin/* -executable); do
+    wrapProgram "$p" \
+      --prefix PATH : ${PATH} \
+      ${lib.optionalString (!stdenv.isDarwin)
+        "--prefix LD_LIBRARY_PATH : ${LD_LIBRARY_PATH}"
+      }
+    done
+  '';
+
+
+}

nix/mantis.nix

@@ -0,0 +1,35 @@
+rev: final: prev: {
+  jre = prev.jdk8.jre;
+
+  mantis = final.callPackage ./mantis.nix {
+    src = builtins.fetchGit {
+      url = "https://github.com/input-output-hk/mantis";
+      ref = "flake";
+      rev = rev;
+      submodules = true;
+    };
+  };
+
+  writeBashChecked = final.writers.makeScriptWriter {
+    interpreter = "${final.bashInteractive}/bin/bash";
+    check = final.writers.writeBash "shellcheck-check" ''
+      ${final.shellcheck}/bin/shellcheck -x "$1"
+    '';
+  };
+
+  writeBashBinChecked = name: final.writeBashChecked "/bin/${name}";
+
+  mantis-entrypoint-script = final.writeBashBinChecked "mantis-entrypoint" ''
+    export PATH=${
+      final.lib.makeBinPath
+      (with final; [ coreutils restic gnugrep awscli diffutils mantis procps ])
+    }
+
+    ${builtins.readFile ./entrypoint.sh}
+  '';
+
+  mantis-entrypoint = final.symlinkJoin {
+    name = "mantis";
+    paths = with final; [ mantis mantis-entrypoint-script ];
+  };
+}