diff --git a/CHANGELOG.next.md b/CHANGELOG.next.md index b46ee2c925..0b33f6f813 100644 --- a/CHANGELOG.next.md +++ b/CHANGELOG.next.md @@ -17,6 +17,10 @@ Thanks, you're awesome :-) --> #### Added #### Improvements +* Allow ECS from any directory #2019 +* Added ability to specify a prefix for es_templates #2019 +* Allow projects to create their own ACSIIDOC templates #2019 + * Define base encoding of `x509.serial_number`. #2383 * Restrict the encoding of `x509.serial_number` to base 16. #2398 diff --git a/scripts/generator.py b/scripts/generator.py index 0cb241e044..818da9ab3c 100644 --- a/scripts/generator.py +++ b/scripts/generator.py @@ -69,7 +69,11 @@ def main() -> None: ecs_generated_version += "+exp" print('Experimental ECS version ' + ecs_generated_version) - fields: dict[str, FieldEntry] = loader.load_schemas(ref=args.ref, included_files=args.include) + fields: dict[str, FieldEntry] = loader.load_schemas( + ref=args.ref, + included_files=args.include, + no_ecs=args.no_ecs + ) cleaner.clean(fields, strict=args.strict) finalizer.finalize(fields) fields, docs_only_fields = subset_filter.filter(fields, args.subset, out_dir) @@ -84,7 +88,8 @@ def main() -> None: exit() csv_generator.generate(flat, ecs_generated_version, out_dir) - es_template.generate(nested, ecs_generated_version, out_dir, args.mapping_settings, args.template_settings) + es_template.generate(nested, ecs_generated_version, out_dir, + args.mapping_settings, args.template_settings,ecs_component_name_prefix=args.component_name_prefix) es_template.generate_legacy(flat, ecs_generated_version, out_dir, args.mapping_settings, args.template_settings_legacy) beats.generate(nested, ecs_generated_version, out_dir) @@ -118,6 +123,10 @@ def argument_parser() -> argparse.Namespace: help='enforce strict checking at schema cleanup') parser.add_argument('--intermediate-only', action='store_true', help='generate intermediary files only') + parser.add_argument('--no-ecs', action='store_true', + help='do not include ECS schemas') + parser.add_argument('--component-name-prefix', action='store', default="ecs", + help='prefix to use for component names') parser.add_argument('--force-docs', action='store_true', help='generate ECS docs even if --subset, --include, or --exclude are set') parser.add_argument('--semconv-version', action='store', diff --git a/scripts/generators/asciidoc_fields.py b/scripts/generators/asciidoc_fields.py index a547d8e689..c1626b45cf 100644 --- a/scripts/generators/asciidoc_fields.py +++ b/scripts/generators/asciidoc_fields.py @@ -152,8 +152,14 @@ def save_asciidoc(f, text): # jinja2 setup +cur_dir = path.abspath(path.curdir) local_dir = path.dirname(path.abspath(__file__)) -TEMPLATE_DIR = path.join(local_dir, '../templates') +CUR_TEMPLATE_DIR = path.join(cur_dir, 'templates') +LOCAL_TEMPLATE_DIR = path.join(local_dir, '../templates') +if path.exists(CUR_TEMPLATE_DIR): + TEMPLATE_DIR = CUR_TEMPLATE_DIR +elif path.exists(LOCAL_TEMPLATE_DIR): + TEMPLATE_DIR = LOCAL_TEMPLATE_DIR template_loader = jinja2.FileSystemLoader(searchpath=TEMPLATE_DIR) template_env = jinja2.Environment(loader=template_loader, keep_trailing_newline=True) @@ -241,6 +247,7 @@ def page_field_values(nested, template_name='field_values_template.j2'): category_fields = ['event.kind', 'event.category', 'event.type', 'event.outcome'] nested_fields = [] for cat_field in category_fields: - nested_fields.append(nested['event']['fields'][cat_field]) + if nested.get("event", {}).get("fields", {}).get(cat_field) is not None: + nested_fields.append(nested['event']['fields'][cat_field]) return dict(fields=nested_fields) diff --git a/scripts/generators/beats.py b/scripts/generators/beats.py index fc9d46f972..c098c5c50d 100644 --- a/scripts/generators/beats.py +++ b/scripts/generators/beats.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -from os.path import join +from os.path import join, dirname from collections import OrderedDict from typing import ( Dict, @@ -29,6 +29,8 @@ FieldNestedEntry, ) +BEATS_DEFAULT_FIELDS = join(dirname(ecs_helpers.__file__), "beats_default_fields_allowlist.yml") + def generate( ecs_nested: Dict[str, FieldNestedEntry], @@ -36,8 +38,12 @@ def generate( out_dir: str ) -> None: # base first - ecs_nested = ecs_helpers.remove_top_level_reusable_false(ecs_nested) - beats_fields: List[OrderedDict] = fieldset_field_array(ecs_nested['base']['fields'], ecs_nested['base']['prefix']) + if 'base' in ecs_nested: + beats_fields: List[OrderedDict] = fieldset_field_array( + ecs_nested['base']['fields'], ecs_nested['base']['prefix']) + else: + beats_fields = [] + allowed_fieldset_keys: List[str] = ['name', 'title', 'group', 'description', 'footnote', 'type'] # other fieldsets @@ -56,7 +62,7 @@ def generate( beats_fields.append(beats_field) # Load temporary allowlist for default_fields workaround. - df_allowlist = ecs_helpers.yaml_load('scripts/generators/beats_default_fields_allowlist.yml') + df_allowlist = ecs_helpers.yaml_load(BEATS_DEFAULT_FIELDS) # Set default_field configuration. set_default_field(beats_fields, df_allowlist) diff --git a/scripts/generators/es_template.py b/scripts/generators/es_template.py index fa9fdda9c0..64ff6fa801 100644 --- a/scripts/generators/es_template.py +++ b/scripts/generators/es_template.py @@ -40,11 +40,12 @@ def generate( ecs_version: str, out_dir: str, mapping_settings_file: str, - template_settings_file: str + template_settings_file: str, + ecs_component_name_prefix: str = "ecs" ) -> None: """This generates all artifacts for the composable template approach""" all_component_templates(ecs_nested, ecs_version, out_dir) - component_names = component_name_convention(ecs_version, ecs_nested) + component_names = component_name_convention(ecs_version, ecs_nested, ecs_component_name_prefix) save_composable_template(ecs_version, component_names, out_dir, mapping_settings_file, template_settings_file) @@ -100,12 +101,13 @@ def save_component_template( def component_name_convention( ecs_version: str, - ecs_nested: Dict[str, FieldNestedEntry] + ecs_nested: Dict[str, FieldNestedEntry], + ecs_component_name_prefix: str="ecs" ) -> List[str]: version: str = ecs_version.replace('+', '-') names: List[str] = [] for (fieldset_name, fieldset) in ecs_helpers.remove_top_level_reusable_false(ecs_nested).items(): - names.append("ecs_{}_{}".format(version, fieldset_name.lower())) + names.append("{}_{}_{}".format(ecs_component_name_prefix, version, fieldset_name.lower())) return names diff --git a/scripts/schema/loader.py b/scripts/schema/loader.py index ef73805e5e..3e09c32cda 100644 --- a/scripts/schema/loader.py +++ b/scripts/schema/loader.py @@ -77,13 +77,18 @@ def load_schemas( ref: Optional[str] = None, - included_files: Optional[List[str]] = [] + included_files: Optional[List[str]] = [], + no_ecs: Optional[bool] = False ) -> Dict[str, FieldEntry]: """Loads ECS and custom schemas. They are returned deeply nested and merged.""" # ECS fields (from git ref or not) - schema_files_raw: Dict[str, FieldNestedEntry] = load_schemas_from_git( - ref) if ref else load_schema_files(ecs_helpers.ecs_files()) - fields: Dict[str, FieldEntry] = deep_nesting_representation(schema_files_raw) + if not no_ecs: + schema_files_raw: Dict[str, FieldNestedEntry] = load_schemas_from_git( + ref) if ref else load_schema_files(ecs_helpers.ecs_files()) + fields: Dict[str, FieldEntry] = deep_nesting_representation(schema_files_raw) + else: + print('Not loading ECS schemas') + fields = {} # Custom additional files if included_files and len(included_files) > 0: diff --git a/scripts/schema/subset_filter.py b/scripts/schema/subset_filter.py index 9f72205777..d006006061 100644 --- a/scripts/schema/subset_filter.py +++ b/scripts/schema/subset_filter.py @@ -179,8 +179,15 @@ def extract_matching_fields( subset_definitions: Dict[str, Any] ) -> Dict[str, FieldEntry]: """Removes fields that are not in the subset definition. Returns a copy without modifying the input fields dict.""" - retained_fields: Dict[str, FieldEntry] = {x: fields[x].copy() for x in subset_definitions} + retained_fields: Dict[str, FieldEntry] = {} + for x in subset_definitions: + if x not in fields: + print('{0} included in subset but has not been loaded'.format(x)) + else: + retained_fields[x] = fields[x].copy() for key, val in subset_definitions.items(): + if key not in fields: + continue retained_fields[key]['field_details'] = fields[key]['field_details'].copy() for option in val: if option != 'fields': diff --git a/scripts/tests/test_es_template.py b/scripts/tests/test_es_template.py index 24db749028..bbfc8a89a9 100644 --- a/scripts/tests/test_es_template.py +++ b/scripts/tests/test_es_template.py @@ -286,6 +286,18 @@ def test_component_composable_template_name(self): exp = ["ecs_{}_acme".format(version)] self.assertEqual(es_template.component_name_convention(version, test_map), exp) + def test_component_composable_template_name_with_custom_prefix(self): + version = "1.8" + prefix="custom" + test_map = { + "Acme": { + "name": "Acme", + } + } + + exp = ["{}_{}_acme".format(prefix,version)] + self.assertEqual(es_template.component_name_convention(version, test_map,prefix), exp) + def test_legacy_template_settings_override(self): ecs_version = 100 default = es_template.default_legacy_template_settings(ecs_version)