Merge pull request #63 from commjoen/hardening

Fix identation and add documentation, metrics server for eks

Author: commjoen

aws/README.md

@@ -57,6 +57,13 @@ Are you done playing? Please run `terraform destroy` twice to clean up.
 ### Test it
 When you have completed the installation steps, you can do `kubectl port-forward service/wrongsecrets-balancer 3000:3000` and then go to [http://localhost:3000](http://localhost:3000).
 
+Want to know how well your cluster is holding up? Check with 
+
+```sh
+    kubectl top nodes
+    kubectl top pods
+```
+
 ### Clean it up
 
 When you're done:

aws/build-an-deploy-aws.sh

@@ -103,10 +103,12 @@ aws secretsmanager put-secret-value --secret-id wrongsecret-2 --secret-string "$
 echo "Generate Parameter store challenge secret"
 aws ssm put-parameter --name wrongsecretvalue --overwrite --type SecureString --value "$(openssl rand -base64 24)" --region $AWS_REGION --output json --no-cli-pager
 
+echo "Installing metrics api-server"
+kubectl apply -f https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml
 
 wait
 
 DEFAULT_PASSWORD=thankyou
 #TODO: REWRITE ABOVE, REWRITE THE HARDCODED DEPLOYMENT VALS INTO VALUES AND OVERRIDE THEM HERE!
 echo "default password is ${DEFAULT_PASSWORD}"
-helm upgrade --install mj ../helm/wrongsecrets-ctf-party --set="imagePullPolicy=Always" --set="balancer.env.K8S_ENV=aws" --set"balancer.env.IRSA_ROLE=arn:aws:iam::${ACCOUNT_ID}:role/wrongsecrets-secret-manager" --set="balancer.env.REACT_APP_ACCESS_PASSWORD=${DEFAULT_PASSWORD}" --set="balancer.cookie.cookieParserSecret=thisisanewrandomvaluesowecanworkatit" --set="balancer.repository=jeroenwillemsen/wrongsecrets-balancer" --set="balancer.tag=0.89aws" --set="balancer.replicas=4" --set="wrongsecretsCleanup.repository=jeroenwillemsen/wrongsecrets-ctf-cleaner" --set="wrongsecretsCleanup.tag=0.2"
+helm upgrade --install mj ../helm/wrongsecrets-ctf-party --set="imagePullPolicy=Always" --set="balancer.env.K8S_ENV=aws" --set"balancer.env.IRSA_ROLE=arn:aws:iam::${ACCOUNT_ID}:role/wrongsecrets-secret-manager" --set="balancer.env.REACT_APP_ACCESS_PASSWORD=${DEFAULT_PASSWORD}" --set="balancer.cookie.cookieParserSecret=thisisanewrandomvaluesowecanworkatit" --set="balancer.repository=jeroenwillemsen/wrongsecrets-balancer" --set="balancer.tag=0.92aws" --set="balancer.replicas=4" --set="wrongsecretsCleanup.repository=jeroenwillemsen/wrongsecrets-ctf-cleaner" --set="wrongsecretsCleanup.tag=0.2"

helm/wrongsecrets-ctf-party/values.yaml

@@ -35,7 +35,7 @@ balancer:
     # -- Set this to a fixed random alpa-numeric string (recommended length 24 chars). If not set this get randomly generated with every helm upgrade, each rotation invalidates all active cookies / sessions requirering users to login again.
     cookieParserSecret: null
   repository: jeroenwillemsen/wrongsecrets-balancer
-  tag: 0.89aws
+  tag: 0.92aws
   # -- Number of replicas of the wrongsecrets-balancer deployment
   replicas: 1
   service:
@@ -198,7 +198,7 @@ virtualdesktop:
   maxInstances: 500
   # -- Juice Shop Image to use
   image: jeroenwillemsen/wrongsecrets-desktop
-  tag: latest
+  tag: test2
   repository: commjoenie/wrongSecrets
   resources:
     request:

readme.md

@@ -9,7 +9,6 @@ Note that we:
 - A working admin interface which can restart both or delete both (by deleting the full namespace)
 - Do not support any progress watchdog as you will have access to it, we therefore disabled it.
 
-
 ## Special thanks
 Special thanks to Madhu Akula, Ben de Haan, and Mike Woudenberg for making this port a reality!
 
@@ -20,8 +19,11 @@ This environment uses a webtop and an instance of wrongsecrets per user. This me
 - 3.5 GB RAM (min 2.5GB, limit = 3.5GB)
 - 8GB HD (min 3 GB, limit = 8GB)
 
-A 6 contestant game can be played on a local minikube with updated cpu & memory settings.
-A 100 contestant game can be played on the AWS setup, which will require at least 200 CPUs, 3500 GB Ram, and 800 GB of storage available in the cluster. 
+### Running this on minikube
+A 3-6 contestant game can be played on a local minikube with updated cpu & memory settings (e.g. 6 CPUs, 9 GB ram).
+
+### Running this on AWS EKS with larger groups
+A 100 contestant game can be played on the AWS setup, which will require around 200 (100-250) CPUs, 300 (250-350) GB Ram, and 800 GB of storage available in the cluster. Note that we have configured everything based on autoscaling in AWS. This means that you can often start with a cluster about 20% of the size of the "limit" numbers and then see how things evolve. If you see heavy under-utilization as players are not very actively engaged: you can often scale down the amount of nodes required.
 
 ## Status
 
@@ -49,6 +51,13 @@ eval $(minikube docker-env)
 ./build-an-deploy.sh
 kubectl port-forward service/wrongsecrets-balancer 3000:3000
 
+```
+Want to know whether your system is holding up? use 
+
+```shell
+minikube addons enable metrics-server
+kubectl top nodes
+kubectl top pods
 ```
 
 ### Action with AWS EKSS:

wrongsecrets-balancer/src/app.js

@@ -50,7 +50,7 @@ if (get('metrics.enabled')) {
 const teamRoutes = require('./teams/teams');
 const adminRoutes = require('./admin/admin');
 const proxyRoutes = require('./proxy/proxy');
-const scoreBoard = require('./score-board/score-board');
+//const scoreBoard = require('./score-board/score-board');
 
 app.get('/balancer/dynamics', (req, res) => {
   const accessPassword = process.env['REACT_APP_ACCESS_PASSWORD'];
@@ -108,7 +108,7 @@ app.get('/balancer/admin', (req, res) => {
   res.sendFile(indexFile);
 });
 app.use('/balancer/admin', adminRoutes);
-app.use('/balancer/score-board', scoreBoard);
+//app.use('/balancer/score-board', scoreBoard);
 
 app.use(proxyRoutes);
 

wrongsecrets-balancer/src/kubernetes.js

@@ -419,6 +419,10 @@ const createAWSDeploymentForTeam = async ({ team, passcodeHash }) => {
                   name: 'K8S_ENV',
                   value: 'aws',
                 },
+                {
+                  name: 'APP_VERSION',
+                  value: `${wrongSecretsContainterTag}-ctf`,
+                },
                 {
                   name: 'CTF_SERVER_ADDRESS',
                   value: `${heroku_wrongsecret_ctf_url}`,
@@ -1092,18 +1096,18 @@ const createDesktopDeploymentForTeam = async ({ team, passcodeHash }) => {
             {
               name: 'virtualdesktop',
               //TODO REPLACE HARDCODED BELOW WITH PROPPER GETS: image: `${get('wrongsecrets.image')}:${get('wrongsecrets.tag')}`,
-              image: 'jeroenwillemsen/wrongsecrets-desktop:1.5.4RC8',
+              image: 'jeroenwillemsen/wrongsecrets-desktop:latest',
               imagePullPolicy: get('virtualdesktop.imagePullPolicy'),
               resources: {
                 requests: {
                   memory: '2G',
                   cpu: '800m',
-                  'ephemeral-storage': '2Gi',
+                  'ephemeral-storage': '4Gi',
                 },
                 limits: {
                   memory: '3G',
                   cpu: '2000m',
-                  'ephemeral-storage': '4Gi',
+                  'ephemeral-storage': '8Gi',
                 },
               },
               // resources: get('virtualdesktop.resources'),
@@ -1118,6 +1122,28 @@ const createDesktopDeploymentForTeam = async ({ team, passcodeHash }) => {
                   containerPort: 3000,
                 },
               ],
+              volumeMounts: [
+                // {
+                //   mountPath: '/config',
+                //   name: 'ephemeral',
+                // },
+                // {
+                //   mountPath: '/defaults',
+                //   name: 'ephemeral-2',
+                // },
+                // {
+                //   mountPath: '/etc',
+                //   name: 'ephemeral-3',
+                // },
+                // {
+                //   mountPath: '/app',
+                //   name: 'ephemeral-4',
+                // },
+                // {
+                //   mountPath: '/run',
+                //   name: 'ephemeral-5',
+                // },
+              ],
               readinessProbe: {
                 httpGet: {
                   path: '/',
@@ -1137,6 +1163,29 @@ const createDesktopDeploymentForTeam = async ({ team, passcodeHash }) => {
               },
             },
           ],
+          volumes: [
+            // {
+            //   name: 'ephemeral',
+            //   emptyDir: {},
+            //   sizeLimit: '4Gi',
+            // },
+            // {
+            //   name: 'ephemeral-2',
+            //   emptyDir: {},
+            // },
+            // {
+            //   name: 'ephemeral-3',
+            //   emptyDir: {},
+            // },
+            // {
+            //   name: 'ephemeral-4',
+            //   emptyDir: {},
+            // },
+            // {
+            //   name: 'ephemeral-5',
+            //   emptyDir: {},
+            // },
+          ],
           tolerations: get('virtualdesktop.tolerations'),
           affinity: get('virtualdesktop.affinity'),
           runtimeClassName: get('virtualdesktop.runtimeClassName')

wrongsecrets-balancer/src/teams/teams.js

@@ -498,7 +498,9 @@ async function resetPasscode(req, res) {
       return res.status(404).send({ message: 'No instance to reset the passcode for.' });
     }
     logger.error(
-      `Encountered unknown error while resetting passcode hash for deployment: ${JSON.stringify(error)}`
+      `Encountered unknown error while resetting passcode hash for deployment: ${JSON.stringify(
+        error
+      )}`
     );
     // logger.error(error.message);
     return res.status(500).send({ message: 'Unknown error while resetting passcode.' });