Merge pull request #20 from stackhpc/prod2312-state

Add state volume

Author: sjpb

environments/lab/terraform/main.tf

@@ -81,12 +81,29 @@ resource "openstack_networking_port_v2" "control_control" {
 resource "openstack_compute_instance_v2" "control" {
 
   name = "${var.cluster_name}-control"
-  image_name = var.control_image
+  image_id = data.openstack_images_image_v2.control.id
   flavor_name = var.control_flavor
   key_pair = var.key_pair
   config_drive = true
   availability_zone = var.cluster_availability_zone
 
+  # root device:
+  block_device {
+      uuid = data.openstack_images_image_v2.control.id
+      source_type  = "image"
+      destination_type = "local"
+      boot_index = 0
+      delete_on_termination = true
+  }
+
+  # state volume:
+  block_device {
+      destination_type = "volume"
+      source_type  = "volume"
+      boot_index = -1
+      uuid = data.openstack_blockstorage_volume_v3.state.id
+  }
+
   network {
     port = openstack_networking_port_v2.control_cluster.id
   }
@@ -100,6 +117,14 @@ resource "openstack_compute_instance_v2" "control" {
     access_network = true
   }
 
+  user_data = <<-EOF
+    #cloud-config
+    bootcmd:
+      - BLKDEV=$(readlink -f $(ls /dev/disk/by-id/*${substr(data.openstack_blockstorage_volume_v3.state.id, 0, 20)}* | head -n1 )); blkid -o value -s TYPE $BLKDEV ||  mke2fs -t ext4 -L state $BLKDEV
+      
+    mounts:
+      - [LABEL=state, /var/lib/state]
+  EOF
 }
 
 # --- slurm logins ---

environments/lab/terraform/prereqs/prereqs.tf

@@ -9,6 +9,10 @@ terraform {
   }
 }
 
+locals {
+  cluster_name = regex("cluster_name  = \"([a-z]+)\"", file("${path.module}/../terraform.tfvars"))[0]
+}
+
 resource "openstack_networking_network_v2" "storage" {
   name           = "lab-storage"
   admin_state_up = "true"
@@ -32,3 +36,10 @@ resource "openstack_networking_subnet_v2" "compute" {
   cidr       = "192.168.101.0/24"
   no_gateway = true
 }
+
+resource "openstack_blockstorage_volume_v3" "state" {
+  # read cluster_name from main tfvars file:
+  name = "${local.cluster_name}-state"
+  description = "State for control node"
+  size = 10 # GB, doesn't matter for lab
+}

environments/nrel/inventory/group_vars/all/defaults.yml

@@ -0,0 +1 @@
+appliances_state_dir: /var/lib/state # see volume mount in environments/*/terraform/main.tf:openstack_compute_instance_v2.control:userdata

environments/nrel/inventory/group_vars/openhpc/overrides.yml

@@ -111,7 +111,6 @@ openhpc_generic_packages:
 
 # Additional parameters to set in slurm.conf - use yaml format
 openhpc_slurmd_spool_dir: /var/spool/slurm/slurmd
-openhpc_state_save_location: /var/spool/slurm/slurmctld # TODO: move to persistent storage
 openhpc_config_extra:
   LaunchParameters: use_interactive_step
   FirstJobId: '50000000'

environments/prod/terraform/datasources.tf

@@ -25,3 +25,11 @@ data "openstack_networking_subnet_v2" "cluster" {
 data "openstack_networking_subnet_v2" "control" {
   name = var.control_subnet
 }
+
+data "openstack_images_image_v2" "control" {
+  name = var.control_image
+}
+
+data "openstack_blockstorage_volume_v3" "state" {
+    name = "${var.cluster_name}-state"
+}

environments/prod/terraform/main.tf

@@ -81,12 +81,29 @@ resource "openstack_networking_port_v2" "control_control" {
 resource "openstack_compute_instance_v2" "control" {
 
   name = "${var.cluster_name}-control"
-  image_name = var.control_image
+  image_id = data.openstack_images_image_v2.control.id
   flavor_name = var.control_flavor
   key_pair = var.key_pair
   config_drive = true
   availability_zone = var.cluster_availability_zone
 
+  # root device:
+  block_device {
+      uuid = data.openstack_images_image_v2.control.id
+      source_type  = "image"
+      destination_type = "local"
+      boot_index = 0
+      delete_on_termination = true
+  }
+
+  # state volume:
+  block_device {
+      destination_type = "volume"
+      source_type  = "volume"
+      boot_index = -1
+      uuid = data.openstack_blockstorage_volume_v3.state.id
+  }
+
   network {
     port = openstack_networking_port_v2.control_cluster.id
   }
@@ -100,6 +117,14 @@ resource "openstack_compute_instance_v2" "control" {
     access_network = true
   }
 
+  user_data = <<-EOF
+    #cloud-config
+    bootcmd:
+      - BLKDEV=$(readlink -f $(ls /dev/disk/by-id/*${substr(data.openstack_blockstorage_volume_v3.state.id, 0, 20)}* | head -n1 )); blkid -o value -s TYPE $BLKDEV ||  mke2fs -t ext4 -L state $BLKDEV
+      
+    mounts:
+      - [LABEL=state, /var/lib/state]
+  EOF
 }
 
 # --- slurm logins ---

environments/vtest/terraform/main.tf

@@ -81,12 +81,29 @@ resource "openstack_networking_port_v2" "control_control" {
 resource "openstack_compute_instance_v2" "control" {
 
   name = "${var.cluster_name}-vtcontrol"
-  image_name = var.control_image
+  image_id = data.openstack_images_image_v2.control.id
   flavor_name = var.control_flavor
   key_pair = var.key_pair
   config_drive = true
   availability_zone = var.cluster_availability_zone
 
+  # root device:
+  block_device {
+      uuid = data.openstack_images_image_v2.control.id
+      source_type  = "image"
+      destination_type = "local"
+      boot_index = 0
+      delete_on_termination = true
+  }
+
+  # state volume:
+  block_device {
+      destination_type = "volume"
+      source_type  = "volume"
+      boot_index = -1
+      uuid = data.openstack_blockstorage_volume_v3.state.id
+  }
+
   network {
     port = openstack_networking_port_v2.control_cluster.id
   }
@@ -100,6 +117,14 @@ resource "openstack_compute_instance_v2" "control" {
     access_network = true
   }
 
+  user_data = <<-EOF
+    #cloud-config
+    bootcmd:
+      - BLKDEV=$(readlink -f $(ls /dev/disk/by-id/*${substr(data.openstack_blockstorage_volume_v3.state.id, 0, 20)}* | head -n1 )); blkid -o value -s TYPE $BLKDEV ||  mke2fs -t ext4 -L state $BLKDEV
+      
+    mounts:
+      - [LABEL=state, /var/lib/state]
+  EOF
 }
 
 # --- slurm logins ---
@@ -179,12 +204,14 @@ resource "openstack_networking_port_v2" "login_control" {
   }
 }
 
+# flavor_name = each.value
 resource "openstack_compute_instance_v2" "logins" {
 
   for_each = var.login_names
 
   name = "${var.cluster_name}-${each.key}"
   image_name = var.login_image
+
   flavor_name = var.control_flavor
   key_pair = var.key_pair
   config_drive = true
@@ -204,7 +231,6 @@ resource "openstack_compute_instance_v2" "logins" {
   }
 
 }
-#  flavor_name = each.value
 
 # --- slurm compute ---
 
@@ -296,22 +322,18 @@ resource "openstack_compute_instance_v2" "computes" {
   key_pair = var.key_pair
   config_drive = true
   availability_zone = var.cluster_availability_zone
-  #availability_zone_hints = "{{ var.cluster_availability_zone }}:vs-0519-u03a"
 
   network {
     port = openstack_networking_port_v2.compute_control[each.key].id
-    #name = "control"
   }
 
   network {
     port = openstack_networking_port_v2.compute_cluster[each.key].id
-    #name = "compute"
     access_network = true
   }
 
   network {
     port = openstack_networking_port_v2.compute_storage[each.key].id
-    #name = "storage"
   }
 
 }