Create yaml for new layout of the communities' codex

.github/workflows/fetch_filter_resources.yaml

@@ -159,6 +159,7 @@ jobs:
           jq -s 'map(.[])' communities/all/resources/repositories*.list_tools.json > communities/all/resources/tools.json 
           rm communities/all/resources/repositories*.list_tools.json
           rm communities/all/resources/repositories*.list_tools.tsv
+          rm communities/all/resources/repositories*.list_tools.yml
       - name: Generate tools wordcloud and interactive table
         run: |
           bash sources/bin/format_tools.sh

sources/bin/extract_all_tools.sh

@@ -11,19 +11,22 @@ then
                         --all-tsv "communities/all/resources/test_tools.tsv" \
                         --all "communities/all/resources/test_tools.json" \
                         --all-workflows "communities/all/resources/test_workflows.json" \
+                        --all-yml 'communities/all/resources/test_tools.yml' \
                         --all-tutorials "communities/all/resources/test_tutorials.json" \
                         --planemo-repository-list "test.list" \
                         --test
         else
                 tsv_output="communities/all/resources/${1}_tools.tsv"
                 json_output="communities/all/resources/${1}_tools.json"
+                yml_output="communities/all/resources/${1}_tools.yml"
 
                 if [[ $1 =~ "01" ]]; then
                 python sources/bin/extract_galaxy_tools.py \
                         extract \
                         --api $GITHUB_API_KEY \
                         --all-tsv $tsv_output \
                         --all $json_output \
+                        --all-yml $yml_output \
                         --all-workflows "communities/all/resources/workflows.json" \
                         --all-tutorials "communities/all/resources/tutorials.json" \
                         --planemo-repository-list $1
@@ -33,6 +36,7 @@ then
                         --api $GITHUB_API_KEY \
                         --all-tsv $tsv_output \
                         --all $json_output \
+                        --all-yml $yml_output \
                         --all-workflows "communities/all/resources/workflows.json" \
                         --all-tutorials "communities/all/resources/tutorials.json" \
                         --planemo-repository-list $1 \
@@ -46,7 +50,11 @@ else
                 --api $GITHUB_API_KEY \
                 --all-tsv 'communities/all/resources/tools.tsv' \
                 --all 'communities/all/resources/tools.json' \
+                --all-yml 'communities/all/resources/tools.yml' \
                 --all-workflows "communities/all/resources/workflows.json" \
                 --all-tutorials "communities/all/resources/tutorials.json" 
+
+        # ln -s "./../../communities/all/resources/tools.yml" "./_data/tools.yml"
+
 fi
 

sources/bin/extract_all_tutorials.sh

@@ -18,7 +18,10 @@ else
         python sources/bin/extract_gtn_tutorials.py \
                 filter \
                 --all "communities/all/resources/tutorials.json" \
+                --yml "communities/all/resources/tutorials.yml" \
                 --filtered "communities/all/resources/tutorials.tsv"
+
+        # ln -s "./../../communities/all/resources/tutorials.yml" "./_data/tutorials.yml"
 
         python sources/bin/create_interactive_table.py \
                 --input "communities/all/resources/tutorials.tsv" \

sources/bin/extract_all_workflows.sh

@@ -4,10 +4,14 @@ if [ ! -z $1 ]
 then
         python sources/bin/extract_galaxy_workflows.py extract \
                 --all "communities/all/resources/test_workflows.json" \
+                --yml "communities/all/resources/test_workflows.yml" \
                 --tools "communities/all/resources/test_tools.json" \
                 --test
 else
         python sources/bin/extract_galaxy_workflows.py extract \
                 --all "communities/all/resources/workflows.json" \
+                --yml "communities/all/resources/workflows.yml" \
                 --tools "communities/all/resources/tools.json"
+
+        # ln -s "./../../communities/all/resources/workflows.yml" "./_data/workflows.yml"
 fi

sources/bin/extract_galaxy_tools.py

@@ -1035,6 +1035,27 @@ def export_missing_tools(missing_tools: dict, tool_dp: str) -> None:
         export_missing_tools_to_yaml(Path(top_d) / Path(server_fn), tools["top"])
 
 
+def export_tools_to_yml(tools: List[Dict], yml_output_path: str) -> None:
+    """
+    Export to YAML for rendering on the website
+    """
+    for tool in tools:
+        availability = {}
+        for field in tool:
+            field_value = tool[field]
+            availability_match_string = "[Nn]umber of tools"
+            if re.search(availability_match_string, field):
+                instance_match_string = "[Uu]se[Gg]alaxy\.[a-z]{2}"
+                if re.search(instance_match_string, field):
+                    match = re.search(instance_match_string, field)
+                    if match:
+                        field_name = match.group(0)
+                        if field_value != 0:
+                            availability[field_name] = field_value
+        tool["availability"] = availability
+    shared.export_to_yml(tools, yml_output_path)
+
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
         description="Extract Galaxy tools from GitHub repositories together with biotools and conda metadata"
@@ -1045,6 +1066,7 @@ def export_missing_tools(missing_tools: dict, tool_dp: str) -> None:
     extract.add_argument("--api", "-a", required=True, help="GitHub access token")
     extract.add_argument("--all", "-o", required=True, help="Filepath to JSON with all extracted tools")
     extract.add_argument("--all-tsv", "-j", required=True, help="Filepath to TSV with all extracted tools")
+    extract.add_argument("--all-yml", "-y", required=True, help="Filepath to yml with all extracted tools")
     extract.add_argument(
         "--all-workflows",
         "-aw",
@@ -1134,6 +1156,7 @@ def export_missing_tools(missing_tools: dict, tool_dp: str) -> None:
         "-s",
         help="Path to a TSV file with tool status - at least 3 columns: IDs of tool suites, Boolean with True to keep and False to exclude, Boolean with True if deprecated and False if not",
     )
+    curatetools.add_argument("--yml", "-y", required=True, help="Filepath to yml with community extracted tools")
 
     # Curate tools categories
     labpop = subparser.add_parser("popLabSection", help="Fill in Lab section tools")
@@ -1180,6 +1203,7 @@ def export_missing_tools(missing_tools: dict, tool_dp: str) -> None:
         tools = get_tools(repo_list, args.all_workflows, args.all_tutorials, edam_ontology)
         export_tools_to_json(tools, args.all)
         export_tools_to_tsv(tools, args.all_tsv, format_list_col=True)
+        export_tools_to_yml(tools, args.all_yml)
 
     elif args.command == "filter":
         with Path(args.all).open() as f:
@@ -1235,6 +1259,7 @@ def export_missing_tools(missing_tools: dict, tool_dp: str) -> None:
                 format_list_col=True,
                 to_keep_columns=["Suite ID", "bio.tool name", "EDAM operations", "EDAM topics"],
             )
+            export_tools_to_yml(curated_tools, args.yml)
         else:
             # if there are no ts filtered tools
             print("No tools left after curation")

sources/bin/extract_galaxy_workflows.py

@@ -466,6 +466,7 @@ def fill_lab_section(self, lab_fp: str) -> None:
     # Extract Workflows
     extract = subparser.add_parser("extract", help="Extract all workflows")
     extract.add_argument("--all", "-o", required=True, help="Filepath to JSON with all extracted workflows")
+    extract.add_argument("--yml", "-y", required=True, help="Filepath to YAML with all extracted workflows")
     extract.add_argument(
         "--tools",
         "-t",
@@ -536,6 +537,7 @@ def fill_lab_section(self, lab_fp: str) -> None:
         "-s",
         help="Path to a TSV file with workflow status",
     )
+    curatewf.add_argument("--yml", "-y", required=True, help="Filepath to yml with community extracted workflows")
 
     # Curate tools categories
     labpop = subparser.add_parser("popLabSection", help="Fill in Lab section workflows")
@@ -557,6 +559,7 @@ def fill_lab_section(self, lab_fp: str) -> None:
         wfs = Workflows(test=args.test)
         wfs.init_by_searching(args.tools)
         shared.export_to_json(wfs.export_workflows_to_dict(), args.all)
+        shared.export_to_yml(wfs.export_workflows_to_dict(), args.yml)
 
     elif args.command == "filter":
         wfs = Workflows()
@@ -600,6 +603,7 @@ def fill_lab_section(self, lab_fp: str) -> None:
         wfs.curate_workflows(status)
         shared.export_to_json(wfs.export_workflows_to_dict(), args.curated)
         wfs.export_workflows_to_tsv(args.tsv_curated)
+        shared.export_to_yml(wfs.export_workflows_to_dict(), args.yml)
 
     elif args.command == "popLabSection":
         wfs = Workflows()

sources/bin/extract_gtn_tutorials.py

@@ -420,6 +420,12 @@ def fill_lab_tutorial_section(
         required=True,
         help="Filepath to JSON with all extracted tutorials, generated by extracttutorials command",
     )
+    filtertuto.add_argument(
+        "--yml",
+        "-y",
+        required=True,
+        help="Filepath to yml file with all extracted tutorials, generated by extracttutorials command",
+    )
     filtertuto.add_argument(
         "--filtered",
         "-f",
@@ -459,6 +465,7 @@ def fill_lab_tutorial_section(
         # filter training lists
         filtered_tutorials = filter_tutorials(all_tutorials, tags)
         export_tutorials_to_tsv(filtered_tutorials, args.filtered)
+        shared.export_to_yml(filtered_tutorials, args.yml)
 
     elif args.command == "popLabSection":
         lab_section = shared.load_yaml(args.lab)

sources/bin/get_community_tools.sh

@@ -17,7 +17,11 @@ if [[ ! -z $1  && $1 == "test" ]]; then
                 --status "communities/microgalaxy/metadata/tool_status.tsv" \
                 --curated "communities/microgalaxy/resources/curated_tools.tsv" \
                 --wo-biotools "communities/microgalaxy/resources/curated_tools_wo_biotools.tsv" \
-                --w-biotools "communities/microgalaxy/resources/curated_tools_w_biotools.tsv"
+                --w-biotools "communities/microgalaxy/resources/curated_tools_w_biotools.tsv"\
+                --yml "communities/microgalaxy/resources/curated_tools.yml"
+
+        mkdir -p _data/communities/microgalaxy/
+        ln -sf ../../../communities/microgalaxy/resources/curated_tools.yml _data/communities/microgalaxy/curated_tools.yml
 
         python sources/bin/create_wordcloud.py \
                 --input  "communities/microgalaxy/resources/curated_tools.tsv" \
@@ -49,7 +53,11 @@ else
                                                 --status "communities/$community/metadata/tool_status.tsv" \
                                                 --curated "communities/$community/resources/curated_tools.tsv" \
                                                 --wo-biotools "communities/$community/resources/curated_tools_wo_biotools.tsv" \
-                                                --w-biotools "communities/$community/resources/curated_tools_w_biotools.tsv"
+                                                --w-biotools "communities/$community/resources/curated_tools_w_biotools.tsv"\
+                                                --yml "communities/$community/resources/curated_tools.yml"
+
+                                        mkdir -p _data/communities/$community/
+                                        ln -sf ../../../communities/$community/resources/curated_tools.yml _data/communities/$community/curated_tools.yml
 
                                         if [[ -e "communities/$community/resources/curated_tools.tsv" ]]; then
                                                 python sources/bin/create_wordcloud.py \

sources/bin/get_community_tutorials.sh

@@ -5,6 +5,7 @@ then
         python sources/bin/extract_gtn_tutorials.py \
                 filter \
                 --all "communities/all/resources/test_tutorials.json" \
+                --yml "communities/microgalaxy/resources/test_tutorials.yml" \
                 --filtered "communities/microgalaxy/resources/test_tutorials.tsv" \
                 --tags "communities/microgalaxy/metadata/tutorial_tags"
 
@@ -26,9 +27,13 @@ else
                                         python sources/bin/extract_gtn_tutorials.py \
                                                 filter \
                                                 --all "communities/all/resources/tutorials.json" \
+                                                --yml "communities/$community/resources/tutorials.yml" \
                                                 --filtered "communities/$community/resources/tutorials.tsv" \
                                                 --tags "communities/$community/metadata/tutorial_tags"
 
+                                        mkdir -p _data/communities/$community/
+                                        ln -sf ../../../communities/$community/resources/tutorials.yml _data/communities/$community/tutorials.yml
+
                                         python sources/bin/create_interactive_table.py \
                                                 --input "communities/$community/resources/tutorials.tsv" \
                                                 --template "sources/data/interactive_table_template.html" \

sources/bin/get_community_workflows.sh

@@ -14,7 +14,9 @@ then
                 curate \
                 --filtered "communities/microgalaxy/resources/test_workflows.json" \
                 --status "communities/microgalaxy/metadata/test_workflow_status.tsv" \
-                --curated "communities/microgalaxy/resources/test_curated_workflows.tsv"
+                --curated "communities/microgalaxy/resources/test_curated_workflows.tsv" \
+                --tsv-curated "communities/microgalaxy/resources/test_curated_workflows.tsv" \
+                --yml "communities/microgalaxy/resources/test_curated_workflows.yml"
 else
         for com_data_fp in communities/* ; do
                 if [[ -d "$com_data_fp" && ! -L "$com_data_fp" ]]; then
@@ -37,7 +39,11 @@ else
                                                 --filtered "communities/$community/resources/tag_filtered_workflows.json" \
                                                 --status "communities/$community/metadata/workflow_status.tsv" \
                                                 --curated "communities/$community/resources/curated_workflows.json" \
-                                                --tsv-curated "communities/$community/resources/curated_workflows.tsv"
+                                                --tsv-curated "communities/$community/resources/curated_workflows.tsv" \
+                                                --yml "communities/$community/resources/curated_workflows.yml"
+
+                                        mkdir -p _data/communities/$community/
+                                        ln -sf ../../../communities/$community/resources/curated_workflows.yml _data/communities/$community/curated_workflows.yml
                                 fi;
                         fi;
                 fi;

sources/bin/shared.py

@@ -87,6 +87,14 @@ def load_yaml(input_df: str) -> Dict:
     return content
 
 
+def export_to_yml(data: list, yml_output_path: str) -> None:
+    """
+    Export to YAML file
+    """
+    with Path(yml_output_path).open("w") as file:
+        yaml.dump(data, file, default_flow_style=False)
+
+
 def read_suite_per_tool_id(tool_fp: str) -> Dict:
     """
     Read the tool suite table and extract a dictionary per tool id