Skip to content

Commit 727b6a6

Browse files
authored
Merge model kubernetes section into target output file (#1095)
* Resolved mustache problem with nested hashes * Updated copyright * Quote unresolved template fields so the result can be parsed * Read and re-write template output files if needed, find operator section to update * Removed extra argument * Merge model dictionaries and lists into target output * Move object list key lookup method to wko_schema_helper * Merge object lists based on object keys; code cleanup and comments * Added managed server object name key * Code cleanup and comments * Update POM versions to 2.1.0-SNAPSHOT * Added unit test for output_file_helper
1 parent 676a6aa commit 727b6a6

File tree

16 files changed

+361
-45
lines changed

16 files changed

+361
-45
lines changed

alias-test/pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
<parent>
1313
<artifactId>weblogic-deploy</artifactId>
1414
<groupId>com.oracle.weblogic.lifecycle</groupId>
15-
<version>2.0.1-SNAPSHOT</version>
15+
<version>2.1.0-SNAPSHOT</version>
1616
<relativePath>../pom.xml</relativePath>
1717
</parent>
1818

core/pom.xml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
<parent>
1212
<artifactId>weblogic-deploy</artifactId>
1313
<groupId>com.oracle.weblogic.lifecycle</groupId>
14-
<version>2.0.1-SNAPSHOT</version>
14+
<version>2.1.0-SNAPSHOT</version>
1515
<relativePath>../pom.xml</relativePath>
1616
</parent>
1717

core/src/main/python/wlsdeploy/tool/extract/domain_resource_extractor.py

Lines changed: 6 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -50,11 +50,11 @@
5050
DEFAULT_IMAGE_PULL_SECRETS = PASSWORD_TOKEN
5151
DEFAULT_SOURCE_TYPE = 'Image'
5252

53-
# deprecated - used for "named object list" format
54-
OBJECT_NAME_ATTRIBUTES = {
55-
'spec/adminServer/adminService/channels': 'channelName',
56-
'spec/clusters': 'clusterName'
57-
}
53+
# specific to Verrazzano
54+
COMPONENT = 'Component'
55+
TEMPLATE = 'template'
56+
VERRAZZANO_WEBLOGIC_WORKLOAD = 'VerrazzanoWebLogicWorkload'
57+
WORKLOAD = 'workload'
5858

5959
_secret_pattern = re.compile("@@SECRET:([\\w.-]+):[\\w.-]+@@")
6060

@@ -161,7 +161,7 @@ def _process_object_array(self, model_value, item_info, schema_path):
161161
self._process_object(object_map, item_info, next_target_dict, schema_path)
162162

163163
# see if the model name should become an attribute in the target dict
164-
mapped_name = get_mapped_key(schema_path)
164+
mapped_name = wko_schema_helper.get_object_list_key(schema_path)
165165
properties = wko_schema_helper.get_properties(item_info)
166166
if (mapped_name in properties.keys()) and (mapped_name not in next_target_dict.keys()):
167167
_add_to_top(next_target_dict, mapped_name, name)
@@ -332,22 +332,6 @@ def _add_secrets(folder, secrets, domain_uid):
332332
secrets.append(secret_name)
333333

334334

335-
# deprecated
336-
def get_mapped_key(schema_path):
337-
"""
338-
For the deprecated "named object list format", the name of each item in a
339-
multiple folder sometimes corresponds to one of its attributes, usually "name".
340-
If a different attribute name is used for the path, return that name.
341-
If the default 'name' is returned, caller should verify that it is an available attribute.
342-
:param schema_path: the slash-delimited path of the elements (no multi-element names)
343-
:return: the attribute key to be used
344-
"""
345-
mapped_key = dictionary_utils.get_element(OBJECT_NAME_ATTRIBUTES, schema_path)
346-
if mapped_key is not None:
347-
return mapped_key
348-
return 'name'
349-
350-
351335
def _add_to_top(dictionary, key, item):
352336
"""
353337
Add an item to the beginning of an ordered dictionary.

core/src/main/python/wlsdeploy/tool/extract/wko_schema_helper.py

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,13 @@
3232
'metadata/ownerReferences'
3333
]
3434

35+
# some object list members don't use 'name' as a key
36+
OBJECT_NAME_ATTRIBUTES = {
37+
'spec/adminServer/adminService/channels': 'channelName',
38+
'spec/clusters': 'clusterName',
39+
'spec/managedServers': 'serverName'
40+
}
41+
3542
_logger = platform_logger.PlatformLogger('wlsdeploy.deploy')
3643
_class_name = 'wko_schema_helper'
3744

@@ -160,6 +167,19 @@ def is_unsupported_folder(path):
160167
return path in UNSUPPORTED_FOLDERS
161168

162169

170+
def get_object_list_key(schema_path):
171+
"""
172+
Return the name of the attribute that acts as a key for objects in an object list.
173+
In most cases, this is 'name', but there are a few exceptions.
174+
:param schema_path: the path to be checked
175+
:return: the object key
176+
"""
177+
mapped_key = dictionary_utils.get_element(OBJECT_NAME_ATTRIBUTES, schema_path)
178+
if mapped_key is not None:
179+
return mapped_key
180+
return 'name'
181+
182+
163183
def append_path(path, element):
164184
if path:
165185
return path + "/" + element

core/src/main/python/wlsdeploy/tool/util/targets/additional_output_helper.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
from wlsdeploy.logging.platform_logger import PlatformLogger
1919
from wlsdeploy.tool.util import k8s_helper
2020
from wlsdeploy.tool.util.targets import file_template_helper
21+
from wlsdeploy.tool.util.targets import output_file_helper
2122
from wlsdeploy.util import dictionary_utils
2223
from wlsdeploy.util import path_utils
2324
from wlsdeploy.util import target_configuration_helper
@@ -70,22 +71,21 @@ def create_additional_output(model, model_context, aliases, credential_injector,
7071
template_hash = _build_template_hash(model, model_context, aliases, credential_injector)
7172
template_names = model_context.get_target_configuration().get_additional_output_types()
7273
for template_name in template_names:
73-
_create_file(template_name, template_hash, model_context, output_dir, exception_type)
74+
_create_file(template_name, template_hash, output_dir, exception_type)
75+
output_file_helper.update_from_model(output_dir, template_name, model)
7476

7577

76-
def _create_file(template_name, template_hash, model_context, output_dir, exception_type):
78+
def _create_file(template_name, template_hash, output_dir, exception_type):
7779
"""
7880
Read the template from the resource stream, perform any substitutions,
7981
and write it to a file with the same name in the output directory.
8082
:param template_name: the name of the template file, and the output file
8183
:param template_hash: a dictionary of substitution values
82-
:param model_context: used to determine location and content for the output
8384
:param output_dir: the directory to write the output file
8485
:param exception_type: the type of exception to throw if needed
8586
"""
8687
_method_name = '_create_file'
8788

88-
target_key = model_context.get_target()
8989
template_subdir = "targets/templates/" + template_name
9090
template_path = path_utils.find_config_path(template_subdir)
9191
output_file = File(output_dir, template_name)

core/src/main/python/wlsdeploy/tool/util/targets/file_template_helper.py

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
"""
2-
Copyright (c) 2020, 2021 Oracle Corporation and/or its affiliates.
2+
Copyright (c) 2020, 2022 Oracle Corporation and/or its affiliates.
33
Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
44
55
Methods for template substitution.
@@ -134,6 +134,10 @@ def _process_block(block_key, template_lines, template_hash, file_writer):
134134
nested_block_key = None
135135
nested_block_lines = []
136136

137+
nested_hash = dict(template_hash)
138+
if isinstance(list_element, dict):
139+
nested_hash.update(list_element)
140+
137141
for line in template_lines:
138142
block_start_key = _get_block_start_key(line)
139143

@@ -142,9 +146,6 @@ def _process_block(block_key, template_lines, template_hash, file_writer):
142146
if nested_block_key is not None:
143147
block_end_key = _get_block_end_key(line)
144148
if block_end_key == nested_block_key:
145-
nested_hash = dict(template_hash)
146-
if isinstance(list_element, dict):
147-
nested_hash.update(list_element)
148149
_process_block(nested_block_key, nested_block_lines, nested_hash, file_writer)
149150
nested_block_key = None
150151
else:
@@ -157,8 +158,7 @@ def _process_block(block_key, template_lines, template_hash, file_writer):
157158

158159
# otherwise, substitute and write the line
159160
else:
160-
if isinstance(list_element, dict):
161-
line = _substitute_line(line, list_element)
161+
line = _substitute_line(line, nested_hash)
162162
file_writer.write(line + "\n")
163163

164164

Lines changed: 178 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,178 @@
1+
"""
2+
Copyright (c) 2022 Oracle Corporation and/or its affiliates.
3+
Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
4+
5+
Methods to update an output file with information from the kubernetes section of the model.
6+
"""
7+
from java.io import File
8+
9+
from oracle.weblogic.deploy.yaml import YamlException
10+
11+
from oracle.weblogic.deploy.util import PyOrderedDict
12+
from wlsdeploy.aliases.model_constants import KUBERNETES
13+
from wlsdeploy.logging.platform_logger import PlatformLogger
14+
from wlsdeploy.tool.extract import wko_schema_helper
15+
from wlsdeploy.tool.extract.domain_resource_extractor import COMPONENT
16+
from wlsdeploy.tool.extract.domain_resource_extractor import DEFAULT_KIND
17+
from wlsdeploy.tool.extract.domain_resource_extractor import KIND
18+
from wlsdeploy.tool.extract.domain_resource_extractor import SPEC
19+
from wlsdeploy.tool.extract.domain_resource_extractor import TEMPLATE
20+
from wlsdeploy.tool.extract.domain_resource_extractor import VERRAZZANO_WEBLOGIC_WORKLOAD
21+
from wlsdeploy.tool.extract.domain_resource_extractor import WORKLOAD
22+
from wlsdeploy.util import dictionary_utils
23+
from wlsdeploy.yaml.yaml_translator import PythonToYaml
24+
from wlsdeploy.yaml.yaml_translator import YamlToPython
25+
26+
__class_name = 'output_file_helper'
27+
__logger = PlatformLogger('wlsdeploy.tool.util')
28+
29+
30+
def update_from_model(output_dir, output_file_name, model):
31+
"""
32+
Update the output content with information from the kubernetes section of the model.
33+
Output files are (currently) always Kubernetes resource files.
34+
:param output_dir: the directory of the output file to update
35+
:param output_file_name: the name of the output file
36+
:param model: the model to use for update
37+
"""
38+
_method_name = 'update_from_model'
39+
40+
# if model doesn't have kubernetes section, return
41+
kubernetes_content = model.get_model_kubernetes()
42+
if not kubernetes_content:
43+
return
44+
45+
# failures will be logged as severe, but not cause tool failure.
46+
# this will allow the unaltered output file to be examined for problems.
47+
48+
output_file = File(output_dir, output_file_name)
49+
__logger.info('WLSDPLY-01675', output_file, KUBERNETES, class_name=__class_name, method_name=_method_name)
50+
51+
try:
52+
reader = YamlToPython(output_file.getPath(), True)
53+
documents = reader.parse_documents()
54+
except YamlException, ex:
55+
__logger.severe('WLSDPLY-01673', output_file, str(ex), error=ex, class_name=__class_name,
56+
method_name=_method_name)
57+
return
58+
59+
_update_documents(documents, kubernetes_content, output_file.getPath())
60+
61+
try:
62+
writer = PythonToYaml(documents)
63+
writer.write_to_yaml_file(output_file.getPath())
64+
except YamlException, ex:
65+
__logger.severe('WLSDPLY-01674', output_file, str(ex), error=ex, class_name=__class_name,
66+
method_name=_method_name)
67+
return
68+
69+
70+
def _update_documents(documents, kubernetes_content, output_file_path):
71+
_method_name = '_update_documents'
72+
found = False
73+
74+
# update section(s) based on their kind, etc.
75+
for document in documents:
76+
if isinstance(document, dict):
77+
kind = dictionary_utils.get_element(document, KIND)
78+
79+
# is this a standard WKO document?
80+
if kind == DEFAULT_KIND:
81+
_update_dictionary(document, kubernetes_content, None, output_file_path)
82+
found = True
83+
84+
# is this a Verrazzano WebLogic workload document?
85+
elif kind == COMPONENT:
86+
spec = dictionary_utils.get_dictionary_element(document, SPEC)
87+
workload = dictionary_utils.get_dictionary_element(spec, WORKLOAD)
88+
component_kind = dictionary_utils.get_element(workload, KIND)
89+
if component_kind == VERRAZZANO_WEBLOGIC_WORKLOAD:
90+
component_spec = _get_or_create_dictionary(workload, SPEC)
91+
component_template = _get_or_create_dictionary(component_spec, TEMPLATE)
92+
_update_dictionary(component_template, kubernetes_content, None, output_file_path)
93+
found = True
94+
95+
if not found:
96+
__logger.warning('WLSDPLY-01676', output_file_path, class_name=__class_name, method_name=_method_name)
97+
98+
99+
def _update_dictionary(output_dictionary, model_dictionary, schema_path, output_file_path):
100+
"""
101+
Update output_dictionary with attributes from model_dictionary.
102+
:param output_dictionary: the dictionary to be updated
103+
:param model_dictionary: the dictionary to update from (type previously validated)
104+
:param schema_path: used for wko_schema_helper lookups and logging
105+
:param output_file_path: used for logging
106+
"""
107+
_method_name = '_update_dictionary'
108+
if not isinstance(output_dictionary, dict):
109+
__logger.warning('WLSDPLY-01677', schema_path, output_file_path, class_name=__class_name,
110+
method_name=_method_name)
111+
return
112+
113+
for key, value in model_dictionary.items():
114+
if key not in output_dictionary:
115+
output_dictionary[key] = value
116+
elif isinstance(value, dict):
117+
next_schema_path = wko_schema_helper.append_path(schema_path, key)
118+
_update_dictionary(output_dictionary[key], value, next_schema_path, output_file_path)
119+
elif isinstance(value, list):
120+
if not value:
121+
# if the model has an empty list, override output value
122+
output_dictionary[key] = value
123+
else:
124+
next_schema_path = wko_schema_helper.append_path(schema_path, key)
125+
_update_list(output_dictionary[key], value, next_schema_path, output_file_path)
126+
else:
127+
output_dictionary[key] = value
128+
pass
129+
130+
131+
def _update_list(output_list, model_list, schema_path, output_file_path):
132+
"""
133+
Update output_list from model_list, overriding or merging existing values
134+
:param output_list: the list to be updated
135+
:param model_list: the list to update from (type previously validated)
136+
:param schema_path: used for wko_schema_helper lookups and logging
137+
:param output_file_path: used for logging
138+
"""
139+
_method_name = '_update_list'
140+
if not isinstance(output_list, list):
141+
__logger.warning('WLSDPLY-01678', schema_path, output_file_path, class_name=__class_name,
142+
method_name=_method_name)
143+
return
144+
145+
for item in model_list:
146+
if isinstance(item, dict):
147+
match = _find_object_match(item, output_list, schema_path)
148+
if match:
149+
_update_dictionary(match, item, schema_path, output_file_path)
150+
else:
151+
output_list.append(item)
152+
elif item not in output_list:
153+
output_list.append(item)
154+
155+
156+
def _find_object_match(item, match_list, schema_path):
157+
"""
158+
Find an object in match_list that has a name matching the item.
159+
:param item: the item to be matched
160+
:param match_list: a list of items
161+
:param schema_path: used for wko_schema_helper key lookup
162+
:return: a matching dictionary object
163+
"""
164+
key = wko_schema_helper.get_object_list_key(schema_path)
165+
item_key = item[key]
166+
if item_key:
167+
for match_item in match_list:
168+
if isinstance(match_item, dict):
169+
if item_key == match_item[key]:
170+
return match_item
171+
return None
172+
173+
174+
def _get_or_create_dictionary(dictionary, key):
175+
if key not in dictionary:
176+
dictionary[key] = PyOrderedDict()
177+
178+
return dictionary[key]

core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -313,6 +313,12 @@ WLSDPLY-01669=To correct this, shorten the DOMAIN_UID or the secret key(s) in th
313313
WLSDPLY-01670=Update the corresponding secret references in the WDT model(s) to match these values before deployment.
314314
WLSDPLY-01671=Update {0} used to encrypt model and domain hashes
315315
WLSDPLY-01672=This secret is only required for model-in-image deployments
316+
WLSDPLY-01673=Unable to parse target output file "{0}" for update from model: {1}
317+
WLSDPLY-01674=Unable to write updated target output file "{0}": {1}
318+
WLSDPLY-01675=Updating target output file "{0}" from the {1} section of the model
319+
WLSDPLY-01676=Unable to find section for update in target output file "{0}"
320+
WLSDPLY-01677=Expected a map value for {0} in the target output file {1}, skipping update
321+
WLSDPLY-01678=Expected a list value for {0} in the target output file {1}, skipping update
316322

317323
# wlsdeploy/util/enum.py
318324
WLSDPLY-01700=The value {0} is not a valid value of the Enum type {1}

core/src/main/targetconfigs/templates/vz-application.yaml

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -51,9 +51,9 @@ spec:
5151
domainUID: {{{domainUid}}}
5252

5353
# WebLogic Image Tool provides domainHome, domainHomeSourceType, and imageName
54-
domainHome: {{{domainHome}}}
55-
domainHomeSourceType: {{{domainHomeSourceType}}}
56-
image: {{{imageName}}}
54+
domainHome: '{{{domainHome}}}'
55+
domainHomeSourceType: '{{{domainHomeSourceType}}}'
56+
image: '{{{imageName}}}'
5757

5858
imagePullSecrets:
5959
- name: {{{domainPrefix}}}-registry-credentials
@@ -67,7 +67,7 @@ spec:
6767
domainType: {{{domainType}}}
6868

6969
# WebLogic Image Tool provides modelHome
70-
modelHome: {{{modelHome}}}
70+
modelHome: '{{{modelHome}}}'
7171
{{#runtimeEncryptionSecret}}
7272

7373
# encryption for the WDT model and the SystemSerializedIni.data file.

0 commit comments

Comments
 (0)