Skip to content

Commit 3231a0a

Browse files
xs2suruchixs2suruchi
authored
Merge pull request #133 from oracle-devrel/develop
Automation Toolkit Release v2024.4.3
2 parents 15c4406 + 12b3f5a commit 3231a0a

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

42 files changed

+203
-137
lines changed

‎Dockerfile

Lines changed: 46 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,22 @@ FROM oraclelinux:9-slim
22
LABEL maintainer="Team at Oracle"
33
LABEL description="OCI format to generate CD3 image"
44

5-
ARG USERNAME=cd3user
5+
6+
########### Input Parameters for image creation ############
7+
# UID of user on underlying OS. eg 503 for Mac
68
ARG USER_UID=1001
9+
# Whether to download Jenkins as part of image creation
10+
ARG USE_DEVOPS=YES
11+
#############################################################
12+
13+
14+
ARG USERNAME=cd3user
715
ARG USER_GID=$USER_UID
16+
# Whether to download Provider as part of image creation
17+
ARG DOWNLOAD_PROVIDER=YES
18+
# TF Provider version
19+
ARG TF_OCI_PROVIDER=6.15.0
20+
ARG TF_NULL_PROVIDER=3.2.1
821

922
RUN microdnf install -y sudo && \
1023
groupadd --gid $USER_GID $USERNAME && \
@@ -15,6 +28,8 @@ RUN microdnf install -y sudo && \
1528
chown -R $USERNAME:$USERNAME /cd3user/tenancies/ && \
1629
microdnf install -y vim && \
1730
microdnf install -y dnf && \
31+
microdnf install -y wget && \
32+
microdnf install -y unzip && \
1833
microdnf install -y graphviz && \
1934
echo 'alias vi="vim"' >> /etc/bashrc
2035

@@ -29,12 +44,26 @@ RUN sudo dnf install -y oraclelinux-release-el9 && \
2944
sudo chown -R $USERNAME:$USERNAME /cd3user/ && \
3045
sudo sed -i -e 's/\r$//' /cd3user/oci_tools/cd3_automation_toolkit/shell_script.sh && \
3146
bash /cd3user/oci_tools/cd3_automation_toolkit/shell_script.sh && \
32-
sudo chown -R cd3user:cd3user /cd3user/ && \
3347
sudo dnf clean all && \
3448
sudo rm -rf /var/cache/dnf && \
35-
sudo chmod -R 740 /cd3user/
49+
sudo chmod -R 740 /cd3user/ && \
50+
sudo chown -R cd3user:cd3user /cd3user/
3651

3752

53+
RUN if [ "$DOWNLOAD_PROVIDER" == "YES" ]; then \
54+
# oci provider
55+
sudo wget https://releases.hashicorp.com/terraform-provider-oci/${TF_OCI_PROVIDER}/terraform-provider-oci_${TF_OCI_PROVIDER}_linux_amd64.zip && \
56+
sudo mkdir -p /cd3user/.terraform.d/plugins/registry.terraform.io/oracle/oci/${TF_OCI_PROVIDER}/linux_amd64 && \
57+
sudo unzip terraform-provider-oci_${TF_OCI_PROVIDER}_linux_amd64.zip -d /cd3user/.terraform.d/plugins/registry.terraform.io/oracle/oci/${TF_OCI_PROVIDER}/linux_amd64 && \
58+
# null provider
59+
sudo wget https://releases.hashicorp.com/terraform-provider-null/${TF_NULL_PROVIDER}/terraform-provider-null_${TF_NULL_PROVIDER}_linux_amd64.zip && \
60+
sudo mkdir -p /cd3user/.terraform.d/plugins/registry.terraform.io/hashicorp/null/${TF_NULL_PROVIDER}/linux_amd64 && \
61+
sudo unzip terraform-provider-null_${TF_NULL_PROVIDER}_linux_amd64.zip -d /cd3user/.terraform.d/plugins/registry.terraform.io/hashicorp/null/${TF_NULL_PROVIDER}/linux_amd64 && \
62+
sudo cp -r /cd3user/.terraform.d/plugins/registry.terraform.io /cd3user/.terraform.d/plugins/registry.opentofu.org && \
63+
sudo chown -R cd3user:cd3user /cd3user/ && \
64+
sudo rm -rf terraform-provider-null_${TF_NULL_PROVIDER}_linux_amd64.zip terraform-provider-oci_${TF_OCI_PROVIDER}_linux_amd64.zip ;\
65+
fi
66+
3867
##################################### START INSTALLING JENKINS ###################################
3968
ARG JENKINS_VERSION=2.444
4069
ARG JENKINS_SHA=ab093a455fc35951c9b46361002e17cc3ed7c59b0943bbee3a57a363f3370d2e
@@ -45,15 +74,7 @@ ARG JENKINS_HOME=/cd3user/tenancies/jenkins_home
4574
ARG JENKINS_INSTALL=/usr/share/jenkins
4675
ARG REF=/usr/share/jenkins/ref
4776

48-
RUN sudo microdnf install -y java-21-openjdk && \
49-
sudo microdnf install -y java-21-openjdk-devel && \
50-
sudo microdnf install git-2.39.3 -y && \
51-
sudo mkdir -p ${REF}/init.groovy.d && \
52-
sudo chown -R cd3user:cd3user ${JENKINS_INSTALL} && \
53-
sudo curl -fsSL http://updates.jenkins-ci.org/download/war/${JENKINS_VERSION}/jenkins.war -o ${JENKINS_INSTALL}/jenkins.war && \
54-
echo "${JENKINS_SHA} ${JENKINS_INSTALL}/jenkins.war" | sha256sum -c - && \
55-
sudo curl -fsSL ${PLUGIN_CLI_URL} -o ${JENKINS_INSTALL}/jenkins-plugin-manager.jar
56-
77+
ENV USE_DEVOPS ${USE_DEVOPS}
5778
ENV JAVA_HOME /usr/lib/jvm/java-21-openjdk
5879
ENV JENKINS_HOME ${JENKINS_HOME}
5980
ENV JENKINS_INSTALL ${JENKINS_INSTALL}
@@ -64,11 +85,21 @@ ENV JENKINS_INCREMENTALS_REPO_MIRROR=https://repo.jenkins-ci.org/incrementals
6485
ENV JAVA_OPTS="-Djenkins.install.runSetupWizard=false"
6586
ENV COPY_REFERENCE_FILE_LOG ${JENKINS_HOME}/copy_reference_file.log
6687
ENV CASC_JENKINS_CONFIG ${JENKINS_HOME}/jcasc.yaml
67-
6888
COPY --chown=cd3user:cd3user jenkins_install ${JENKINS_INSTALL}/
6989
COPY --chown=cd3user:cd3user jenkins_install/init/*.groovy ${REF}/init.groovy.d/
7090
COPY --chown=cd3user:cd3user jenkins_install/plugins.txt ${REF}/plugins.txt
7191

72-
RUN sudo java -jar ${JENKINS_INSTALL}/jenkins-plugin-manager.jar --war ${JENKINS_INSTALL}/jenkins.war --verbose -f ${REF}/plugins.txt && \
92+
93+
RUN if [ "$USE_DEVOPS" == "YES" ]; then \
94+
sudo microdnf install -y java-21-openjdk && \
95+
sudo microdnf install -y java-21-openjdk-devel && \
96+
sudo microdnf install git-2.39.3 -y && \
97+
sudo mkdir -p ${REF}/init.groovy.d && \
98+
sudo chown -R cd3user:cd3user ${JENKINS_INSTALL} && \
99+
sudo curl -fsSL http://updates.jenkins-ci.org/download/war/${JENKINS_VERSION}/jenkins.war -o ${JENKINS_INSTALL}/jenkins.war && \
100+
echo "${JENKINS_SHA} ${JENKINS_INSTALL}/jenkins.war" | sha256sum -c - && \
101+
sudo curl -fsSL ${PLUGIN_CLI_URL} -o ${JENKINS_INSTALL}/jenkins-plugin-manager.jar && \
102+
sudo java -jar ${JENKINS_INSTALL}/jenkins-plugin-manager.jar --war ${JENKINS_INSTALL}/jenkins.war --verbose -f ${REF}/plugins.txt && \
73103
sudo chown -R cd3user:cd3user ${JENKINS_INSTALL} && \
74-
sudo chmod +x ${JENKINS_INSTALL}/jenkins.sh
104+
sudo chmod +x ${JENKINS_INSTALL}/jenkins.sh ; \
105+
fi

‎OCIWorkVMStack/modules/network/locals.tf

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,5 @@ locals {
44
create_inet_gw = (var.vcn_strategy == "Create New VCN" && var.subnet_type == "Public") ? 1 : 0
55
create_nat_gw = (var.vcn_strategy == "Create New VCN" && var.subnet_type == "Private") ? 1 : 0
66
create_nsg_rule = (var.vcn_strategy == "Create New VCN" && length(var.source_cidr) != 0) ? 1 : 0
7+
route_rule_drg = var.drg_attachment == true ? ( length(var.source_cidr) > 0 ? var.source_cidr : [] ) : []
78
}

‎OCIWorkVMStack/modules/network/network.tf

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,12 +49,23 @@ resource "oci_core_route_table" "rt" {
4949
vcn_id = local.vcn_id
5050
display_name = "${var.subnet_name}-rt"
5151

52+
# Route rules to NGW or IGW
5253
route_rules {
5354
destination = "0.0.0.0/0"
5455
destination_type = "CIDR_BLOCK"
5556
network_entity_id = local.create_inet_gw == 1 ? oci_core_internet_gateway.internet_gw[0].id : oci_core_nat_gateway.nat_gw[0].id
5657
}
5758

59+
# Route rules to DRG
60+
dynamic route_rules {
61+
for_each = local.route_rule_drg
62+
content {
63+
destination = route_rules.value
64+
destination_type = "CIDR_BLOCK"
65+
network_entity_id = var.existing_drg_id
66+
}
67+
}
68+
5869
}
5970
resource "oci_core_security_list" "security_list" {
6071
count = local.create_vcn

‎OCIWorkVMStack/schema.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -406,7 +406,7 @@ variables:
406406
- drg_attachment
407407
- true
408408
type: string
409-
pattern: '^ocid1\.([a-z0-9_-]{1,32})\.([a-z0-9_-]{1,15})\.([a-z0-9]{0,24})\.([a-z0-9]{60})$'
409+
pattern: '^ocid1\.([a-z0-9_-]{1,32})\.([a-z0-9_-]{1,15})\.([a-z0-9_-]{0,24})\.([a-z0-9]{60})$'
410410
required: true
411411
title: Enter Existing DRG OCID
412412
description: Enter existing DRG OCID

‎README.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77

88
<br>
99

10-
[What's New](https://github.com/oracle-devrel/cd3-automation-toolkit/releases/tag/v2024.4.2) &nbsp;•&nbsp;[Excel Templates](https://oracle-devrel.github.io/cd3-automation-toolkit/latest/excel-templates/) &nbsp;•&nbsp;[CD3 Docs](https://oracle-devrel.github.io/cd3-automation-toolkit/)&nbsp;•&nbsp; [Watch & Learn](https://www.youtube.com/playlist?list=PLPIzp-E1msrbJ3WawXVhzimQnLw5iafcp) &nbsp;•&nbsp;[Blogs & Tutorials](https://oracle-devrel.github.io/cd3-automation-toolkit/latest/tutorials/) &nbsp;•&nbsp;[Livelabs](https://apexapps.oracle.com/pls/apex/f?p=133:180:112501098061930::::wid:3724) &nbsp;•&nbsp;[Slack Channel](https://oracle-devrel.github.io/cd3-automation-toolkit/latest/queries)
10+
[What's New](https://github.com/oracle-devrel/cd3-automation-toolkit/releases/tag/v2024.4.3) &nbsp;•&nbsp;[Excel Templates](https://oracle-devrel.github.io/cd3-automation-toolkit/latest/excel-templates/) &nbsp;•&nbsp;[CD3 Docs](https://oracle-devrel.github.io/cd3-automation-toolkit/)&nbsp;•&nbsp; [Watch & Learn](https://www.youtube.com/playlist?list=PLPIzp-E1msrbJ3WawXVhzimQnLw5iafcp) &nbsp;•&nbsp;[Blogs & Tutorials](https://oracle-devrel.github.io/cd3-automation-toolkit/latest/tutorials/) &nbsp;•&nbsp;[Livelabs](https://apexapps.oracle.com/pls/apex/f?p=133:180:112501098061930::::wid:3724) &nbsp;•&nbsp;[Slack Channel](https://oracle-devrel.github.io/cd3-automation-toolkit/latest/queries)
1111

1212
<br>
1313

@@ -81,7 +81,7 @@ Additionally, the toolkit also supports seamless resource management using OCI D
8181

8282
📝 Creating Terraform Code for each module/resource can be cumbersome and requires Terraform expertise.
8383

84-
🔁 Manually created infrastrucutre is hard to rebuild for different environments or regions.
84+
🔁 Manually created infrastructure is hard to rebuild for different environments or regions.
8585

8686
<br>
8787

‎cd3_automation_toolkit/Compute/create_terraform_instances.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -226,12 +226,12 @@ def create_terraform_instances(inputfile, outdir, service_dir, prefix, ct):
226226

227227
if columnname == "Source Details":
228228
if columnvalue.strip() != '' and columnvalue.strip().lower() != 'nan':
229-
if "ocid1.image.oc1" in columnvalue.strip():
229+
if "ocid1.image.oc" in columnvalue.strip():
230230
ocid = columnvalue.strip()
231231
type = "image"
232232
source_details.append(type)
233233
source_details.append(ocid)
234-
elif "ocid1.bootvolume.oc1" in columnvalue.strip():
234+
elif "ocid1.bootvolume.oc" in columnvalue.strip():
235235
ocid = columnvalue.strip()
236236
type = "bootVolume"
237237
source_details.append(type)

‎cd3_automation_toolkit/DeveloperServices/OKE/export_oke_nonGreenField.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -531,7 +531,7 @@ def export_oke(inputfile, outdir,service_dir, config, signer, ct, export_compart
531531

532532

533533
#Virtual NodePool
534-
if ("ocid1.virtualnodepool.oc1" in nodepool_info.id):
534+
if ("ocid1.virtualnodepool.oc" in nodepool_info.id):
535535
nodepool_display_name = nodepool_info.display_name
536536
np_tf_name = commonTools.check_tf_variable(nodepool_display_name)
537537
tf_resource = f'module.virtual-nodepools[\\"{cluster_tf_name}_{np_tf_name}\\"].oci_containerengine_virtual_node_pool.virtual_nodepool'
@@ -540,7 +540,7 @@ def export_oke(inputfile, outdir,service_dir, config, signer, ct, export_compart
540540
nodepool_type = "virtual"
541541

542542
# Managed NodePool
543-
if ("ocid1.nodepool.oc1" in nodepool_info.id):
543+
if ("ocid1.nodepool.oc" in nodepool_info.id):
544544
nodepool_display_name = nodepool_info.name
545545
np_tf_name = commonTools.check_tf_variable(nodepool_display_name)
546546
nodepool_type = "managed"

‎cd3_automation_toolkit/DeveloperServices/ResourceManager/create_resource_manager_stack.py

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -119,12 +119,14 @@ def create_resource_manager(outdir,var_file, outdir_struct,prefix,auth_mechanism
119119

120120
rm_dir = region_dir + '/RM/'
121121

122-
# 1. Copy all the TF files for specified regions to RM directory
122+
# 1. Copy all the TF files for specified regions to RM directory. Also copy modules directory
123123
try:
124124
shutil.copytree(region_dir, rm_dir, ignore=shutil.ignore_patterns('*.terraform.lock.hcl','*.terraform','provider.tf','*.zip*','*.safe*','*.log*','*cis_report','*.csv*','*cd3validator', 'variables_*.tf*'))
125+
shutil.copytree(outdir+"/modules", rm_dir)
125126
except FileExistsError as fe:
126127
shutil.rmtree(rm_dir)
127128
shutil.copytree(region_dir, rm_dir, ignore=shutil.ignore_patterns('*.terraform.lock.hcl','*.terraform','provider.tf','*.zip*','*.safe*','*.log*','*cis_report','*.csv*','*cd3validator', 'variables_*.tf*'))
129+
shutil.copytree(outdir+"/modules", rm_dir+"/modules")
128130

129131
#2. Change the provider.tf and variables_<region>.tf to include just the region variable in all stacks for specified regions
130132
tfStr[region]=''
@@ -286,6 +288,15 @@ def create_resource_manager(outdir,var_file, outdir_struct,prefix,auth_mechanism
286288
zip_name = rm_name + ".zip"
287289
# Fix for make_archive huge zip file issue - Ulag
288290
file_paths = []
291+
for file in os.listdir(rm_dir):
292+
if ".tf" in file and "variables" not in file and "provider" not in file and 'backend' not in file:
293+
with open(file, 'r') as tf_file:
294+
module_data = tf_file.read().rstrip()
295+
module_data = module_data.replace("\"../modules", "\"./modules")
296+
f = open(file, "w+")
297+
f.write(module_data)
298+
f.close()
299+
289300
for root, directories, files in os.walk(rm_dir):
290301
for filename in files:
291302
rel_dir = os.path.relpath(root, rm_dir)
@@ -368,7 +379,7 @@ def create_resource_manager(outdir,var_file, outdir_struct,prefix,auth_mechanism
368379
if os.path.exists(service_dir+"/"+ svc + ".tf"):
369380
with open(service_dir+"/"+ svc + ".tf", 'r') as tf_file:
370381
module_data = tf_file.read().rstrip()
371-
module_data = module_data.replace("\"../modules", "\"./modules")
382+
module_data = module_data.replace("\"../../modules", "\"./modules")
372383
if svc == 'rpc':
373384
f = open(service_dir+"/"+ svc + "-temp.tf", "w+")
374385
else:

‎cd3_automation_toolkit/Governance/Tagging/create_terraform_tags.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -231,7 +231,7 @@ def create_terraform_tags(inputfile, outdir, service_dir, prefix, ct):
231231
else:
232232
if str(df.loc[i, 'Validator']).strip() == '' or str(df.loc[i, 'Validator']).strip().lower() == 'nan':
233233
is_required_updated = 'true' #Uncomment this if needed
234-
default_value = '-'
234+
default_value = '[CANNOT_BE_EMPTY]'
235235
columnvalue = key_tf_name+"="+default_compartment+"="+default_value+"="+is_required_updated #Uncomment this if needed
236236
if columnvalue not in default_tags:
237237
default_tags.append(columnvalue)

‎cd3_automation_toolkit/Identity/Users/export_users_nonGreenField.py

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -112,9 +112,14 @@ def export_users(inputfile, outdir, service_dir, config, signer, ct,export_domai
112112
domain_name = domain_key.split("@")[1]
113113
domain_client = oci.identity_domains.IdentityDomainsClient(config=config, signer=signer,
114114
service_endpoint=idcs_endpoint)
115-
users = domain_client.list_users(limit=100000) # change this to pagination once api supports
115+
list_users_response = domain_client.list_users() # change this to pagination once api supports
116+
users = list_users_response.data.resources
117+
while list_users_response.has_next_page:
118+
list_users_response = domain_client.list_users(page=list_users_response.next_page)
119+
users.extend(list_users_response.data.resources)
120+
116121
index = 0
117-
for user in users.data.resources:
122+
for user in users:
118123
defined_tags_info = user.urn_ietf_params_scim_schemas_oracle_idcs_extension_oci_tags
119124
user_defined_tags = []
120125

‎cd3_automation_toolkit/Identity/export_identity_nonGreenField.py

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -359,17 +359,31 @@ def process_group(grp_info, members_list,membership_id_list, domain_name, is_dyn
359359
domain_name = domain_key.split("@")[1]
360360
domain_client = oci.identity_domains.IdentityDomainsClient(config=config, signer=signer,
361361
service_endpoint=idcs_endpoint)
362-
groups = domain_client.list_groups(attributes=['members'], attribute_sets=['all'])
363-
dyngroups = domain_client.list_dynamic_resource_groups(attributes=['matching_rule'], attribute_sets=['all'])
362+
list_groups_response = domain_client.list_groups(attributes=['members'], attribute_sets=['all'])
363+
groups = list_groups_response.data.resources
364+
while list_groups_response.has_next_page:
365+
list_groups_response = domain_client.list_groups(attributes=['members'], attribute_sets=['all'],page=list_groups_response.next_page)
366+
groups.extend(list_groups_response.data.resources)
364367

365-
for grp_info in groups.data.resources:
368+
for grp_info in groups:
366369
if grp_info.display_name in ["Domain_Administrators", "All Domain Users", "Administrators"]:
367370
continue
368371
total_g +=1
369372
members_list = [section.name for section in grp_info.members if section and section.name] if grp_info.members else []
370373
importCommands, values_for_column_groups = process_group(grp_info, members_list,[], domain_name, is_dynamic=False, importCommands=importCommands, values_for_column_groups=values_for_column_groups)
371374

372-
for dg in dyngroups.data.resources:
375+
dyngroups_response = domain_client.list_dynamic_resource_groups(attributes=['matching_rule'],
376+
attribute_sets=['all']
377+
)
378+
dyngroups = dyngroups_response.data.resources
379+
while dyngroups_response.has_next_page:
380+
dyngroups_response = domain_client.list_dynamic_resource_groups(attributes=['matching_rule'],
381+
attribute_sets=['all'],
382+
page=dyngroups_response.next_page
383+
)
384+
dyngroups.extend(dyngroups_response.data.resources)
385+
386+
for dg in dyngroups:
373387
total_g += 1
374388
importCommands, values_for_column_groups = process_group(dg, [],[], domain_name, is_dynamic=True, importCommands=importCommands, values_for_column_groups=values_for_column_groups)
375389
else:

0 commit comments

Comments
 (0)