Add basic matrix to test PHPLIB on evergreen

Author: alcaeus

.evergreen/auth_aws/README.md

@@ -0,0 +1,4 @@
+# Configuration Scripts for End-to-end Testing
+
+These scripts were taken from [mongo-enterprise-modules](https://github.com/10gen/mongo-enterprise-modules/tree/master/jstests/external_auth_aws) 
+and intended to simplify creating users, attaching roles to existing EC2 instances, launching an Amazon ECS container instance, etc.

.evergreen/auth_aws/aws_e2e_assume_role.js

@@ -0,0 +1,49 @@
+/**
+ * Verify the AWS IAM Auth works with temporary credentials from sts:AssumeRole
+ */
+
+load("lib/aws_e2e_lib.js");
+
+(function() {
+"use strict";
+
+const ASSUMED_ROLE = "arn:aws:sts::557821124784:assumed-role/authtest_user_assume_role/*";
+
+function getAssumeCredentials() {
+    const config = readSetupJson();
+
+    const env = {
+        AWS_ACCESS_KEY_ID: config["iam_auth_assume_aws_account"],
+        AWS_SECRET_ACCESS_KEY: config["iam_auth_assume_aws_secret_access_key"],
+    };
+
+    const role_name = config["iam_auth_assume_role_name"];
+
+    const python_command = getPython3Binary() +
+        ` -u lib/aws_assume_role.py --role_name=${role_name} > creds.json`;
+
+    const ret = runShellCmdWithEnv(python_command, env);
+    assert.eq(ret, 0, "Failed to assume role on the current machine");
+
+    const result = cat("creds.json");
+    try {
+        return JSON.parse(result);
+    } catch (e) {
+        jsTestLog("Failed to parse: " + result);
+        throw e;
+    }
+}
+
+const credentials = getAssumeCredentials();
+const admin = Mongo().getDB("admin");
+const external = admin.getMongo().getDB("$external");
+
+assert(admin.auth("bob", "pwd123"));
+assert.commandWorked(external.runCommand({createUser: ASSUMED_ROLE, roles:[{role: 'read', db: "aws"}]}));
+assert(external.auth({
+    user: credentials["AccessKeyId"],
+    pwd: credentials["SecretAccessKey"],
+    awsIamSessionToken: credentials["SessionToken"],
+    mechanism: 'MONGODB-AWS'
+}));
+}());

.evergreen/auth_aws/aws_e2e_ec2.js

@@ -0,0 +1,58 @@
+/**
+ * Verify the AWS IAM EC2 hosted auth works
+ */
+load("lib/aws_e2e_lib.js");
+
+(function() {
+"use strict";
+
+// This varies based on hosting EC2 as the account id and role name can vary
+const AWS_ACCOUNT_ARN = "arn:aws:sts::557821124784:assumed-role/authtest_instance_profile_role/*";
+
+function assignInstanceProfile() {
+    const config = readSetupJson();
+
+    const env = {
+        AWS_ACCESS_KEY_ID: config["iam_auth_ec2_instance_account"],
+        AWS_SECRET_ACCESS_KEY: config["iam_auth_ec2_instance_secret_access_key"],
+    };
+
+    const instanceProfileName = config["iam_auth_ec2_instance_profile"];
+    const python_command = getPython3Binary() +
+        ` -u lib/aws_assign_instance_profile.py --instance_profile_arn=${instanceProfileName}`;
+
+    const ret = runShellCmdWithEnv(python_command, env);
+    if (ret == 2) {
+        print("WARNING: Request limit exceeded for AWS API");
+        return false;
+    }
+
+    assert.eq(ret, 0, "Failed to assign an instance profile to the current machine");
+    return true;
+}
+
+if (!assignInstanceProfile()) {
+    return;
+}
+
+const admin = Mongo().getDB("admin");
+const external = admin.getMongo().getDB("$external");
+
+assert(admin.auth("bob", "pwd123"));
+assert.commandWorked(external.runCommand({createUser: AWS_ACCOUNT_ARN, roles:[{role: 'read', db: "aws"}]}));
+
+// Try the command line
+const smoke = runMongoProgram("mongo",
+                              "--host",
+                              "localhost",
+                              '--authenticationMechanism',
+                              'MONGODB-AWS',
+                              '--authenticationDatabase',
+                              '$external',
+                              "--eval",
+                              "1");
+assert.eq(smoke, 0, "Could not auth with smoke user");
+
+// Try the auth function
+assert(external.auth({mechanism: 'MONGODB-AWS'}));
+}());

.evergreen/auth_aws/aws_e2e_ecs.js

@@ -0,0 +1,44 @@
+/**
+ * Validate that MONGODB-AWS auth works from ECS temporary credentials.
+ */
+load("lib/aws_e2e_lib.js");
+
+(function() {
+   'use strict';
+
+   assert.eq(typeof mongo_binaries != 'undefined', true, "mongo_binaries must be set");
+   assert.eq(typeof project_dir != 'undefined', true, "project_dir must be set");
+
+   const config = readSetupJson();
+
+   const base_command = getPython3Binary() + " -u  lib/container_tester.py";
+   const run_prune_command = base_command + ' -v remote_gc_services ' +
+       ' --cluster ' + config['iam_auth_ecs_cluster'];
+
+   const run_test_command = base_command + ' -d -v run_e2e_test' +
+       ' --cluster ' + config['iam_auth_ecs_cluster'] + ' --task_definition ' +
+       config['iam_auth_ecs_task_definition'] + ' --subnets ' +
+       config['iam_auth_ecs_subnet_a'] + ' --subnets ' +
+       config['iam_auth_ecs_subnet_b'] + ' --security_group ' +
+       config['iam_auth_ecs_security_group'] +
+       ` --files ${mongo_binaries}/mongod:/root/mongod ${mongo_binaries}/mongo:/root/mongo ` +
+       " lib/ecs_hosted_test.js:/root/ecs_hosted_test.js " +
+       `${project_dir}:/root` +
+       " --script lib/ecs_hosted_test.sh";
+
+   // Pass in the AWS credentials as environment variables
+   // AWS_SHARED_CREDENTIALS_FILE does not work in evergreen for an unknown
+   // reason
+   const env = {
+      AWS_ACCESS_KEY_ID: config['iam_auth_ecs_account'],
+      AWS_SECRET_ACCESS_KEY: config['iam_auth_ecs_secret_access_key'],
+   };
+
+   // Prune other containers
+   let ret = runWithEnv(['/bin/sh', '-c', run_prune_command], env);
+   assert.eq(ret, 0, 'Prune Container failed');
+
+   // Run the test in a container
+   ret = runWithEnv(['/bin/sh', '-c', run_test_command], env);
+   assert.eq(ret, 0, 'Container Test failed');
+}());

.evergreen/auth_aws/aws_e2e_regular_aws.js

@@ -0,0 +1,23 @@
+/**
+ * Validate that the server supports real credentials from AWS and can talk to a real AWS STS
+ * service
+ */
+load("lib/aws_e2e_lib.js");
+
+(function() {
+"use strict";
+
+const admin = Mongo().getDB("admin");
+const external = admin.getMongo().getDB("$external");
+assert(admin.auth("bob", "pwd123"));
+
+const config = readSetupJson();
+assert.commandWorked(
+    external.runCommand({createUser: config["iam_auth_ecs_account_arn"], roles:[{role: 'read', db: "aws"}]}));
+
+assert(external.auth({
+    user: config["iam_auth_ecs_account"],
+    pwd: config["iam_auth_ecs_secret_access_key"],
+    mechanism: 'MONGODB-AWS'
+}));
+}());

.evergreen/auth_aws/lib/aws_assign_instance_profile.py

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+"""
+Script for assign an instance policy to the current machine.
+"""
+
+import argparse
+import urllib.request
+import logging
+import sys
+import time
+
+import boto3
+import botocore
+
+LOGGER = logging.getLogger(__name__)
+
+def _get_local_instance_id():
+    return urllib.request.urlopen('http://169.254.169.254/latest/meta-data/instance-id').read().decode()
+
+def _has_instance_profile():
+    base_url = "http://169.254.169.254/latest/meta-data/iam/security-credentials/"
+    try:
+        print("Reading: " + base_url)
+        iam_role = urllib.request.urlopen(base_url).read().decode()
+    except urllib.error.HTTPError as e:
+        print(e)
+        if e.code == 404:
+            return False
+        raise e
+
+    try:
+        url = base_url + iam_role
+        print("Reading: " + url)
+        req = urllib.request.urlopen(url)
+    except urllib.error.HTTPError as e:
+        print(e)
+        if e.code == 404:
+            return False
+        raise e
+
+    return True
+
+def _wait_instance_profile():
+    retry = 60
+    while not _has_instance_profile() and retry:
+        time.sleep(5)
+        retry -= 1
+
+    if retry == 0:
+        raise ValueError("Timeout on waiting for instance profile")
+
+def _assign_instance_policy(iam_instance_arn):
+
+    if _has_instance_profile():
+        print("IMPORTANT: Found machine already has instance profile, skipping the assignment")
+        return
+
+    instance_id = _get_local_instance_id()
+
+    ec2_client = boto3.client("ec2", 'us-east-1')
+
+    #https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ec2.html#EC2.Client.associate_iam_instance_profile
+    try:
+        response = ec2_client.associate_iam_instance_profile(
+            IamInstanceProfile={
+                'Arn' : iam_instance_arn,
+            },
+            InstanceId = instance_id)
+
+        print(response)
+
+        # Wait for the instance profile to be assigned by polling the local instance metadata service
+        _wait_instance_profile()
+
+    except botocore.exceptions.ClientError as ce:
+        if ce.response["Error"]["Code"] == "RequestLimitExceeded":
+            print("WARNING: RequestLimitExceeded, exiting with error code 2")
+            sys.exit(2)
+        raise
+
+def main() -> None:
+    """Execute Main entry point."""
+
+    parser = argparse.ArgumentParser(description='IAM Assign Instance frontend.')
+
+    parser.add_argument('-v', "--verbose", action='store_true', help="Enable verbose logging")
+    parser.add_argument('-d', "--debug", action='store_true', help="Enable debug logging")
+
+    parser.add_argument('--instance_profile_arn', type=str, help="Name of instance profile")
+
+    args = parser.parse_args()
+
+    if args.debug:
+        logging.basicConfig(level=logging.DEBUG)
+    elif args.verbose:
+        logging.basicConfig(level=logging.INFO)
+
+    _assign_instance_policy(args.instance_profile_arn)
+
+
+if __name__ == "__main__":
+    main()

.evergreen/auth_aws/lib/aws_assume_role.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+"""
+Script for assuming an aws role.
+"""
+
+import argparse
+import uuid
+import logging
+
+import boto3
+
+LOGGER = logging.getLogger(__name__)
+
+STS_DEFAULT_ROLE_NAME = "arn:aws:iam::579766882180:role/mark.benvenuto"
+
+def _assume_role(role_name):
+    sts_client = boto3.client("sts")
+
+    response = sts_client.assume_role(RoleArn=role_name, RoleSessionName=str(uuid.uuid4()), DurationSeconds=900)
+
+    creds = response["Credentials"]
+
+
+    print(f"""{{
+  "AccessKeyId" : "{creds["AccessKeyId"]}",
+  "SecretAccessKey" : "{creds["SecretAccessKey"]}",
+  "SessionToken" : "{creds["SessionToken"]}",
+  "Expiration" : "{str(creds["Expiration"])}"
+}}""")
+
+
+def main() -> None:
+    """Execute Main entry point."""
+
+    parser = argparse.ArgumentParser(description='Assume Role frontend.')
+
+    parser.add_argument('-v', "--verbose", action='store_true', help="Enable verbose logging")
+    parser.add_argument('-d', "--debug", action='store_true', help="Enable debug logging")
+
+    parser.add_argument('--role_name', type=str, default=STS_DEFAULT_ROLE_NAME, help="Role to assume")
+
+    args = parser.parse_args()
+
+    if args.debug:
+        logging.basicConfig(level=logging.DEBUG)
+    elif args.verbose:
+        logging.basicConfig(level=logging.INFO)
+
+    _assume_role(args.role_name)
+
+
+if __name__ == "__main__":
+    main()

.evergreen/auth_aws/lib/aws_e2e_lib.js

@@ -0,0 +1,39 @@
+
+function readSetupJson() {
+    let result;
+    try {
+        result = cat("aws_e2e_setup.json");
+    } catch (e) {
+        jsTestLog(
+            "Failed to parse read aws_e2e_setup.json. See evergreen.yml for how to generate this file which contains evergreen secrets.");
+        throw e;
+    }
+
+    try {
+        return JSON.parse(result);
+    } catch (e) {
+        jsTestLog("Failed to parse: aws_e2e_setup.json");
+        throw e;
+    }
+}
+
+function runWithEnv(args, env) {
+    const pid = _startMongoProgram({args: args, env: env});
+    return waitProgram(pid);
+}
+
+function runShellCmdWithEnv(argStr, env) {
+    if (_isWindows()) {
+        return runWithEnv(['cmd.exe', '/c', argStr], env);
+    } else {
+        return runWithEnv(['/bin/sh', '-c', argStr], env);
+    }
+}
+
+function getPython3Binary() {
+    if (_isWindows()) {
+        return "python.exe";
+    }
+
+    return "python3";
+}