Skip to content

Commit afba0a7

Browse files
bryantbiggsbryantbiggs
authored
feat: add S3 import functionality which is supported for MySQL instances (#197)
1 parent 9d7c8f2 commit afba0a7

File tree

11 files changed

+425
-31
lines changed

11 files changed

+425
-31
lines changed

‎.github/workflows/pre-commit.yml

Lines changed: 56 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -7,55 +7,95 @@ on:
77
- master
88

99
jobs:
10-
getBaseVersion:
11-
name: Get min/max versions
12-
runs-on: ubuntu-latest
10+
# Min Terraform version(s)
11+
getDirectories:
12+
name: Get root directories
13+
runs-on: ubuntu-latest
14+
steps:
15+
- name: Checkout
16+
uses: actions/checkout@v2
17+
- name: Install Python
18+
uses: actions/setup-python@v2
19+
- name: Build matrix
20+
id: matrix
21+
run: |
22+
DIRS=$(python -c "import json; import glob; print(json.dumps([x.replace('/versions.tf', '') for x in glob.glob('./**/versions.tf', recursive=True)]))")
23+
echo "::set-output name=directories::$DIRS"
24+
outputs:
25+
directories: ${{ steps.matrix.outputs.directories }}
1326

27+
preCommitMinVersions:
28+
name: Min TF validate
29+
needs: getDirectories
30+
runs-on: ubuntu-latest
31+
strategy:
32+
matrix:
33+
directory: ${{ fromJson(needs.getDirectories.outputs.directories) }}
1434
steps:
1535
- name: Checkout
1636
uses: actions/checkout@v2
37+
- name: Install Python
38+
uses: actions/setup-python@v2
39+
- name: Terraform min/max versions
40+
id: minMax
41+
uses: clowdhaus/[email protected]
42+
with:
43+
directory: ${{ matrix.directory }}
44+
- name: Install Terraform v${{ steps.minMax.outputs.minVersion }}
45+
uses: hashicorp/setup-terraform@v1
46+
with:
47+
terraform_version: ${{ steps.minMax.outputs.minVersion }}
48+
- name: Install pre-commit dependencies
49+
run: pip install pre-commit
50+
- name: Execute pre-commit
51+
# Run only validate pre-commit check on min version supported
52+
if: ${{ matrix.directory != '.' }}
53+
run:
54+
pre-commit run terraform_validate --color=always --show-diff-on-failure --files ${{ matrix.directory }}/*
55+
- name: Execute pre-commit
56+
# Run only validate pre-commit check on min version supported
57+
if: ${{ matrix.directory == '.' }}
58+
run:
59+
pre-commit run terraform_validate --color=always --show-diff-on-failure --files $(ls *.tf)
60+
1761

62+
# Max Terraform version
63+
getBaseVersion:
64+
name: Module max TF version
65+
runs-on: ubuntu-latest
66+
steps:
67+
- name: Checkout
68+
uses: actions/checkout@v2
1869
- name: Terraform min/max versions
1970
id: minMax
2071
uses: clowdhaus/[email protected]
2172
outputs:
2273
minVersion: ${{ steps.minMax.outputs.minVersion }}
2374
maxVersion: ${{ steps.minMax.outputs.maxVersion }}
2475

25-
preCommit:
26-
name: Pre-commit check
76+
preCommitMaxVersion:
77+
name: Max TF pre-commit
2778
runs-on: ubuntu-latest
2879
needs: getBaseVersion
2980
strategy:
3081
fail-fast: false
3182
matrix:
3283
version:
33-
- ${{ needs.getBaseVersion.outputs.minVersion }}
3484
- ${{ needs.getBaseVersion.outputs.maxVersion }}
35-
3685
steps:
3786
- name: Checkout
3887
uses: actions/checkout@v2
39-
4088
- name: Install Python
4189
uses: actions/setup-python@v2
42-
4390
- name: Install Terraform v${{ matrix.version }}
4491
uses: hashicorp/setup-terraform@v1
4592
with:
4693
terraform_version: ${{ matrix.version }}
47-
4894
- name: Install pre-commit dependencies
4995
run: |
5096
pip install pre-commit
5197
curl -L "$(curl -s https://api.github.com/repos/terraform-docs/terraform-docs/releases/latest | grep -o -E "https://.+?-linux-amd64" | head -n1)" > terraform-docs && chmod +x terraform-docs && sudo mv terraform-docs /usr/bin/
5298
curl -L "$(curl -s https://api.github.com/repos/terraform-linters/tflint/releases/latest | grep -o -E "https://.+?_linux_amd64.zip")" > tflint.zip && unzip tflint.zip && rm tflint.zip && sudo mv tflint /usr/bin/
53-
54-
- name: Execute pre-commit
55-
# Run only validate pre-commit check on min version supported
56-
if: ${{ matrix.version == needs.getBaseVersion.outputs.minVersion }}
57-
run: pre-commit run --color=always --show-diff-on-failure --all-files terraform_validate
58-
5999
- name: Execute pre-commit
60100
# Run all pre-commit checks on max version supported
61101
if: ${{ matrix.version == needs.getBaseVersion.outputs.maxVersion }}

‎.pre-commit-config.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
repos:
22
- repo: git://github.com/antonbabenko/pre-commit-terraform
3-
rev: v1.46.0
3+
rev: v1.47.0
44
hooks:
55
- id: terraform_fmt
66
- id: terraform_validate

‎README.md

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -107,18 +107,18 @@ No Modules.
107107

108108
| Name |
109109
|------|
110-
| [aws_appautoscaling_policy](https://registry.terraform.io/providers/hashicorp/aws/3.8/docs/resources/appautoscaling_policy) |
111-
| [aws_appautoscaling_target](https://registry.terraform.io/providers/hashicorp/aws/3.8/docs/resources/appautoscaling_target) |
112-
| [aws_db_subnet_group](https://registry.terraform.io/providers/hashicorp/aws/3.8/docs/resources/db_subnet_group) |
113-
| [aws_iam_policy_document](https://registry.terraform.io/providers/hashicorp/aws/3.8/docs/data-sources/iam_policy_document) |
114-
| [aws_iam_role_policy_attachment](https://registry.terraform.io/providers/hashicorp/aws/3.8/docs/resources/iam_role_policy_attachment) |
115-
| [aws_iam_role](https://registry.terraform.io/providers/hashicorp/aws/3.8/docs/resources/iam_role) |
116-
| [aws_rds_cluster_instance](https://registry.terraform.io/providers/hashicorp/aws/3.8/docs/resources/rds_cluster_instance) |
117-
| [aws_rds_cluster](https://registry.terraform.io/providers/hashicorp/aws/3.8/docs/resources/rds_cluster) |
118-
| [aws_security_group_rule](https://registry.terraform.io/providers/hashicorp/aws/3.8/docs/resources/security_group_rule) |
119-
| [aws_security_group](https://registry.terraform.io/providers/hashicorp/aws/3.8/docs/resources/security_group) |
120-
| [random_id](https://registry.terraform.io/providers/hashicorp/random/2.2/docs/resources/id) |
121-
| [random_password](https://registry.terraform.io/providers/hashicorp/random/2.2/docs/resources/password) |
110+
| [aws_appautoscaling_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/appautoscaling_policy) |
111+
| [aws_appautoscaling_target](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/appautoscaling_target) |
112+
| [aws_db_subnet_group](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/db_subnet_group) |
113+
| [aws_iam_policy_document](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) |
114+
| [aws_iam_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) |
115+
| [aws_iam_role_policy_attachment](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) |
116+
| [aws_rds_cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/rds_cluster) |
117+
| [aws_rds_cluster_instance](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/rds_cluster_instance) |
118+
| [aws_security_group](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) |
119+
| [aws_security_group_rule](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) |
120+
| [random_id](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) |
121+
| [random_password](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/password) |
122122

123123
## Inputs
124124

@@ -178,10 +178,11 @@ No Modules.
178178
| replica\_scale\_min | Minimum number of replicas to allow scaling for | `number` | `2` | no |
179179
| replica\_scale\_out\_cooldown | Cooldown in seconds before allowing further scaling operations after a scale out | `number` | `300` | no |
180180
| replication\_source\_identifier | ARN of a source DB cluster or DB instance if this DB cluster is to be created as a Read Replica. | `string` | `""` | no |
181+
| s3\_import | Restore from a Percona Xtrabackup in S3 (only MySQL is supported) | `map(string)` | `null` | no |
181182
| scaling\_configuration | Map of nested attributes with scaling properties. Only valid when engine\_mode is set to `serverless` | `map(string)` | `{}` | no |
182183
| security\_group\_description | The description of the security group. If value is set to empty string it will contain cluster name in the description. | `string` | `"Managed by Terraform"` | no |
183184
| skip\_final\_snapshot | Should a final snapshot be created on cluster destroy | `bool` | `false` | no |
184-
| snapshot\_identifier | DB snapshot to create this database from | `string` | `""` | no |
185+
| snapshot\_identifier | DB snapshot to create this database from | `string` | `null` | no |
185186
| source\_region | The source region for an encrypted replica DB cluster. | `string` | `""` | no |
186187
| storage\_encrypted | Specifies whether the underlying storage layer should be encrypted | `bool` | `true` | no |
187188
| subnets | List of subnet IDs to use | `list(string)` | `[]` | no |

‎examples/s3_import/README.md

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
# MySQL S3 Import Example
2+
3+
Configuration in this directory creates set of RDS resources including DB instance, DB subnet group and DB parameter group where the database itself is imported from a MySQL Percona Xtrabackup stored in S3.
4+
5+
## Usage
6+
7+
To run this example you need to execute:
8+
9+
```bash
10+
$ terraform init
11+
$ terraform plan
12+
$ terraform apply
13+
```
14+
15+
## Testing
16+
17+
In order to test this example, you will need a database backup in order to upload to S3 and import into the module. A backup has been provided under `backup/`, but in the case that a new backup needs to be created, the steps outlined below should suffice for creating a backup that can be used for the sake of testing and verifying module functionality/changes.
18+
19+
1. Create database container
20+
21+
```bash
22+
$ docker run -d --name percona-server-mysql-5.7.12 -e MYSQL_ROOT_PASSWORD=root percona/percona-server:5.7.12
23+
$ docker exec -it percona-server-mysql-5.7.12 bash
24+
$ mysql -u root -p # password is also root
25+
```
26+
27+
2. Once logged into container and database, create database and user used by RDS
28+
29+
```sql
30+
CREATE DATABASE s3Import;
31+
CREATE USER 's3_import_user'@'localhost' IDENTIFIED BY 'YourPwdShouldBeLongAndSecure!';
32+
GRANT ALL PRIVILEGES ON * . * TO 's3_import_user'@'localhost';
33+
FLUSH PRIVILEGES;
34+
```
35+
36+
3. Use Percona Xtrabackup container to dump database and upload to S3
37+
38+
```bash
39+
$ mkdir -p /tmp/backup
40+
$ docker run --name percona-xtrabackup-2.4 --mount type=bind,src=/tmp/backup,dst=/backup --volumes-from percona-server-mysql-5.7.12 percona/percona-xtrabackup:2.4 xtrabackup --backup --data-dir=/var/lib/mysql --target-dir=/backup --user=root --password=root
41+
$ mv /tmp/backup ./backup
42+
```
43+
44+
Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources.
45+
46+
<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
47+
## Requirements
48+
49+
| Name | Version |
50+
|------|---------|
51+
| terraform | >= 0.12.26 |
52+
| aws | >= 3.8 |
53+
| random | >= 2.2 |
54+
55+
## Providers
56+
57+
| Name | Version |
58+
|------|---------|
59+
| aws | >= 3.8 |
60+
| random | >= 2.2 |
61+
62+
## Modules
63+
64+
| Name | Source | Version |
65+
|------|--------|---------|
66+
| aurora | ../../ | |
67+
| import_s3_bucket | terraform-aws-modules/s3-bucket/aws | ~> 1.20 |
68+
| vpc | terraform-aws-modules/vpc/aws | ~> 2.77 |
69+
70+
## Resources
71+
72+
| Name |
73+
|------|
74+
| [aws_db_parameter_group](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/db_parameter_group) |
75+
| [aws_iam_policy_document](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) |
76+
| [aws_iam_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) |
77+
| [aws_iam_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) |
78+
| [aws_rds_cluster_parameter_group](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/rds_cluster_parameter_group) |
79+
| [random_password](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/password) |
80+
| [random_pet](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/pet) |
81+
82+
## Inputs
83+
84+
No input.
85+
86+
## Outputs
87+
88+
| Name | Description |
89+
|------|-------------|
90+
| this\_rds\_cluster\_database\_name | Name for an automatically created database on cluster creation |
91+
| this\_rds\_cluster\_endpoint | The cluster endpoint |
92+
| this\_rds\_cluster\_id | The ID of the cluster |
93+
| this\_rds\_cluster\_instance\_endpoints | A list of all cluster instance endpoints |
94+
| this\_rds\_cluster\_instance\_ids | A list of all cluster instance ids |
95+
| this\_rds\_cluster\_master\_password | The master password |
96+
| this\_rds\_cluster\_master\_username | The master username |
97+
| this\_rds\_cluster\_port | The port |
98+
| this\_rds\_cluster\_reader\_endpoint | The cluster reader endpoint |
99+
| this\_rds\_cluster\_resource\_id | The Resource ID of the cluster |
100+
| this\_security\_group\_id | The security group ID of the cluster |
101+
<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

‎examples/s3_import/backup.zip

3.2 MB
Binary file not shown.

0 commit comments

Comments
 (0)