diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 6d49887fd..1891c520e 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -15,6 +15,4 @@ jobs: steps: - uses: actions/checkout@v3 - name: Build the Docker image - run: docker build . --file Dockerfile --tag linode/cli:$(date +%s) --build-arg="github_token=$GITHUB_TOKEN" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file + run: docker build . --file Dockerfile --tag linode/cli:$(date +%s) diff --git a/.github/workflows/e2e-suite-windows.yml b/.github/workflows/e2e-suite-windows.yml index 108c757d8..df7b1bbdf 100644 --- a/.github/workflows/e2e-suite-windows.yml +++ b/.github/workflows/e2e-suite-windows.yml @@ -69,8 +69,6 @@ jobs: - name: Install the CLI run: make install - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: make INTEGRATION_TEST_PATH="${{ inputs.test_path }}" testint env: @@ -102,4 +100,4 @@ jobs: status: 'completed', conclusion: process.env.conclusion }); - return result; + return result; \ No newline at end of file diff --git a/.github/workflows/e2e-suite.yml b/.github/workflows/e2e-suite.yml index b5e92fa60..3430652aa 100644 --- a/.github/workflows/e2e-suite.yml +++ b/.github/workflows/e2e-suite.yml @@ -101,8 +101,6 @@ jobs: - name: Install Package run: make install - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Set LINODE_CLI_TOKEN run: | diff --git a/.github/workflows/nightly-smoke-tests.yml b/.github/workflows/nightly-smoke-tests.yml index baa498028..f8808d1e5 100644 --- a/.github/workflows/nightly-smoke-tests.yml +++ b/.github/workflows/nightly-smoke-tests.yml @@ -34,8 +34,6 @@ jobs: - name: Install Linode CLI run: make install - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Run smoke tests id: smoke_tests diff --git a/.github/workflows/publish-oci.yml b/.github/workflows/publish-oci.yml index 1fe7f1583..a5227ef2d 100644 --- a/.github/workflows/publish-oci.yml +++ b/.github/workflows/publish-oci.yml @@ -57,4 +57,3 @@ jobs: tags: linode/cli:${{ steps.cli_version.outputs.result }},linode/cli:latest build-args: | linode_cli_version=${{ steps.cli_version.outputs.result }} - github_token=${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml index 1023ecd08..27f161194 100644 --- a/.github/workflows/publish-pypi.yml +++ b/.github/workflows/publish-pypi.yml @@ -30,7 +30,6 @@ jobs: - name: Build the package run: make build env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} LINODE_CLI_VERSION: ${{ github.event.release.tag_name }} - name: Publish the release artifacts to PyPI diff --git a/.github/workflows/remote-release-trigger.yml b/.github/workflows/remote-release-trigger.yml deleted file mode 100644 index 91b449518..000000000 --- a/.github/workflows/remote-release-trigger.yml +++ /dev/null @@ -1,74 +0,0 @@ -name: Remote Release Trigger -on: - repository_dispatch: - types: [ cli-release ] -jobs: - remote-release-trigger: - runs-on: ubuntu-22.04 - environment: CLI Automated Release - steps: - - name: Generate App Installation Token - id: generate_token - uses: tibdex/github-app-token@b62528385c34dbc9f38e5f4225ac829252d1ea92 # pin@v1 - with: - app_id: ${{ secrets.CLI_RELEASE_APP_ID }} - private_key: ${{ secrets.CLI_RELEASE_PRIVATE_KEY }} - - - name: Checkout - uses: actions/checkout@v3 - with: - # We want to checkout the main branch - ref: 'main' - fetch-depth: 0 - - - name: Get previous tag - id: previoustag - uses: WyriHaximus/github-action-get-previous-tag@385a2a0b6abf6c2efeb95adfac83d96d6f968e0c # pin@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Calculate the desired release version - id: calculate_version - uses: actions/github-script@v6 - env: - SPEC_VERSION: ${{ github.event.client_payload.spec_version }} - PREVIOUS_CLI_VERSION: ${{ steps.previoustag.outputs.tag }} - with: - result-encoding: string - version: ${{ steps.previoustag.outputs.tag }} - script: | - let spec_version_segments = process.env.SPEC_VERSION.replace("v", "").split("."); - let cli_version_segments = process.env.PREVIOUS_CLI_VERSION.replace("v", "").split("."); - - // Default to a patch version bump - let bump_idx = 2; - - // This is a minor version bump - if (spec_version_segments[2] == "0") { - bump_idx = 1; - - // The patch number should revert to 0 - cli_version_segments[2] = "0" - } - - // Bump the version - cli_version_segments[bump_idx] = (parseInt(cli_version_segments[bump_idx]) + 1).toString() - - return "v" + cli_version_segments.join(".") - - - name: Calculate the SHA of HEAD on the main branch - id: calculate_head_sha - run: echo "commit_sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" - - - uses: rickstaa/action-create-tag@84c90e6ba79b47b5147dcb11ff25d6a0e06238ba # pin@v1 - with: - tag: ${{ steps.calculate_version.outputs.result }} - commit_sha: ${{ steps.calculate_head_sha.outputs.commit_sha }} - - - name: Release - uses: softprops/action-gh-release@de2c0eb89ae2a093876385947365aca7b0e5f844 # pin@v1 - with: - target_commitish: 'main' - token: ${{ steps.generate_token.outputs.token }} - body: Built from Linode OpenAPI spec ${{ github.event.client_payload.spec_version }} - tag_name: ${{ steps.calculate_version.outputs.result }} diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index e4119a022..5e91239da 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -29,8 +29,6 @@ jobs: - name: Install Package run: make install - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Run the unit test suite run: make test @@ -59,8 +57,6 @@ jobs: shell: pwsh run: | make install - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Run the unit test suite run: make test \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 499343bc9..b51b03eda 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,6 @@ FROM python:3.11-slim AS builder ARG linode_cli_version -ARG github_token WORKDIR /src @@ -12,7 +11,7 @@ COPY . . RUN make requirements -RUN LINODE_CLI_VERSION=$linode_cli_version GITHUB_TOKEN=$github_token make build +RUN LINODE_CLI_VERSION=$linode_cli_version make build FROM python:3.11-slim diff --git a/Makefile b/Makefile index a8884fdf2..c0f719429 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,10 @@ # # Makefile for more convenient building of the Linode CLI and its baked content # + +SPEC := https://raw.githubusercontent.com/akamai/akamai-apis/main/apis/linode-api/v4/openapi.json + +# Test-related arguments MODULE := TEST_CASE_COMMAND := TEST_ARGS := @@ -9,12 +13,6 @@ ifdef TEST_CASE TEST_CASE_COMMAND = -k $(TEST_CASE) endif - -SPEC_VERSION ?= latest -ifndef SPEC -override SPEC = $(shell ./resolve_spec_url ${SPEC_VERSION}) -endif - # Version-related variables VERSION_FILE := ./linodecli/version.py VERSION_MODULE_DOCSTRING ?= \"\"\"\nThe version of the Linode CLI.\n\"\"\"\n\n diff --git a/linodecli/__init__.py b/linodecli/__init__.py index 776b6589f..70035eeb8 100755 --- a/linodecli/__init__.py +++ b/linodecli/__init__.py @@ -15,12 +15,7 @@ from linodecli import plugins from linodecli.exit_codes import ExitCodes -from .arg_helpers import ( - bake_command, - register_args, - register_plugin, - remove_plugin, -) +from .arg_helpers import register_args, register_plugin, remove_plugin from .cli import CLI from .completion import get_completions from .configuration import ENV_TOKEN_NAME @@ -103,7 +98,7 @@ def main(): # pylint: disable=too-many-branches,too-many-statements if parsed.action is None: print("No spec provided, cannot bake") sys.exit(ExitCodes.ARGUMENT_ERROR) - bake_command(cli, parsed.action) + cli.bake(parsed.action) sys.exit(ExitCodes.SUCCESS) elif cli.ops is None: # if not spec was found and we weren't baking, we're doomed diff --git a/linodecli/arg_helpers.py b/linodecli/arg_helpers.py index 36b0d2c59..e01d2e36c 100644 --- a/linodecli/arg_helpers.py +++ b/linodecli/arg_helpers.py @@ -3,15 +3,9 @@ Argument parser for the linode CLI """ -import os -import sys from importlib import import_module -import requests -import yaml - from linodecli import plugins -from linodecli.exit_codes import ExitCodes from linodecli.helpers import ( register_args_shared, register_debug_arg, @@ -166,24 +160,3 @@ def remove_plugin(plugin_name, config): config.write_config() return f"Plugin {plugin_name} removed", 0 - - -def bake_command(cli, spec_loc): - """ - Handle a bake command from args - """ - try: - if os.path.exists(os.path.expanduser(spec_loc)): - with open(os.path.expanduser(spec_loc), encoding="utf-8") as f: - spec = yaml.safe_load(f.read()) - else: # try to GET it - resp = requests.get(spec_loc, timeout=120) - if resp.status_code == 200: - spec = yaml.safe_load(resp.content) - else: - raise RuntimeError(f"Request failed to {spec_loc}") - except Exception as e: - print(f"Could not load spec: {e}") - sys.exit(ExitCodes.REQUEST_FAILED) - - cli.bake(spec) diff --git a/linodecli/baked/parsing.py b/linodecli/baked/parsing.py index 5125e45dc..dd41af5ef 100644 --- a/linodecli/baked/parsing.py +++ b/linodecli/baked/parsing.py @@ -9,7 +9,7 @@ # Sentence delimiter, split on a period followed by any type of # whitespace (space, new line, tab, etc.) -REGEX_SENTENCE_DELIMITER = re.compile(r"\.(?:\s|$)") +REGEX_SENTENCE_DELIMITER = re.compile(r"\W(?:\s|$)") # Matches on pattern __prefix__ at the beginning of a description # or after a comma diff --git a/linodecli/cli.py b/linodecli/cli.py index a9f61a86c..72f30af62 100644 --- a/linodecli/cli.py +++ b/linodecli/cli.py @@ -2,11 +2,17 @@ Responsible for managing spec and routing commands to operations. """ +import contextlib +import json import os import pickle import sys +from json import JSONDecodeError from sys import version_info +from typing import IO, Any, ContextManager, Dict +import requests +import yaml from openapi3 import OpenAPI from linodecli.api_request import do_request, get_all_pages @@ -40,11 +46,19 @@ def __init__(self, version, base_url, skip_config=False): self.config = CLIConfig(self.base_url, skip_config=skip_config) self.load_baked() - def bake(self, spec): + def bake(self, spec_location: str): """ - Generates ops and bakes them to a pickle + Generates ops and bakes them to a pickle. + + :param spec_location: The URL or file path of the OpenAPI spec to parse. """ - spec = OpenAPI(spec) + + try: + spec = self._load_openapi_spec(spec_location) + except Exception as e: + print(f"Failed to load spec: {e}") + sys.exit(ExitCodes.REQUEST_FAILED) + self.spec = spec self.ops = {} ext = { @@ -205,3 +219,85 @@ def user_agent(self) -> str: f"linode-api-docs/{self.spec_version} " f"python/{version_info[0]}.{version_info[1]}.{version_info[2]}" ) + + @staticmethod + def _load_openapi_spec(spec_location: str) -> OpenAPI: + """ + Attempts to load the raw OpenAPI spec (YAML or JSON) at the given location. + + :param spec_location: The location of the OpenAPI spec. + This can be a local path or a URL. + + :returns: A tuple containing the loaded OpenAPI object and the parsed spec in + dict format. + """ + + with CLI._get_spec_file_reader(spec_location) as f: + parsed = CLI._parse_spec_file(f) + + return OpenAPI(parsed) + + @staticmethod + @contextlib.contextmanager + def _get_spec_file_reader( + spec_location: str, + ) -> ContextManager[IO]: + """ + Returns a reader for an OpenAPI spec file from the given location. + + :param spec_location: The location of the OpenAPI spec. + This can be a local path or a URL. + + :returns: A context manager yielding the spec file's reader. + """ + + # Case for local file + local_path = os.path.expanduser(spec_location) + if os.path.exists(local_path): + f = open(local_path, "r", encoding="utf-8") + + try: + yield f + finally: + f.close() + + return + + # Case for remote file + resp = requests.get(spec_location, stream=True, timeout=120) + if resp.status_code != 200: + raise RuntimeError(f"Failed to GET {spec_location}") + + # We need to access the underlying urllib + # response here so we can return a reader + # usable in yaml.safe_load(...) and json.load(...) + resp.raw.decode_content = True + + try: + yield resp.raw + finally: + resp.close() + + @staticmethod + def _parse_spec_file(reader: IO) -> Dict[str, Any]: + """ + Parses the given file reader into a dict and returns a dict. + + :param reader: A reader for a YAML or JSON file. + + :returns: The parsed file. + """ + + errors = [] + + try: + return yaml.safe_load(reader) + except yaml.YAMLError as err: + errors.append(str(err)) + + try: + return json.load(reader) + except JSONDecodeError as err: + errors.append(str(err)) + + raise ValueError(f"Failed to parse spec file: {'; '.join(errors)}") diff --git a/resolve_spec_url b/resolve_spec_url deleted file mode 100755 index 641beedcd..000000000 --- a/resolve_spec_url +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python3 -# Usage: -# ./resolve_latest_spec -# Prints the URL of the latest Linode OpenAPI spec on GitHub -import os -import sys - -import requests - -LINODE_DOCS_REPO = "linode/linode-api-docs" - - -def get_latest_tag(): - headers = {} - - token = os.getenv("GITHUB_TOKEN") - if token is not None: - headers["Authorization"] = f"Bearer {token}" - - data = requests.get( - f"https://api.github.com/repos/{LINODE_DOCS_REPO}/releases/latest", - headers=headers, - ) - - if data.status_code != 200: - raise RuntimeError("Got error from GitHub API: {}".format(data.json())) - - return data.json()["tag_name"] - - -if __name__ == "__main__": - if len(sys.argv) != 2: - print(f"Invalid number of arguments: {len(sys.argv)}", file=sys.stderr) - exit(1) - - desired_version = sys.argv[1] - - if desired_version.lower() == "latest": - desired_version = get_latest_tag() - - print( - f"https://raw.githubusercontent.com/{LINODE_DOCS_REPO}/{desired_version}/openapi.yaml" - ) diff --git a/tests/fixtures/cli_test_load.json b/tests/fixtures/cli_test_load.json new file mode 100644 index 000000000..ba0a55a87 --- /dev/null +++ b/tests/fixtures/cli_test_load.json @@ -0,0 +1,95 @@ +{ + "openapi": "3.0.1", + "info": { + "title": "API Specification", + "version": "1.0.0" + }, + "servers": [ + { + "url": "http://localhost/v4" + } + ], + "paths": { + "/foo/bar": { + "get": { + "summary": "get info", + "operationId": "fooBarGet", + "description": "This is description", + "responses": { + "200": { + "description": "Successful response", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "data": { + "type": "array", + "items": { + "$ref": "#/components/schemas/OpenAPIResponseAttr" + } + }, + "page": { + "$ref": "#/components/schemas/PaginationEnvelope/properties/page" + }, + "pages": { + "$ref": "#/components/schemas/PaginationEnvelope/properties/pages" + }, + "results": { + "$ref": "#/components/schemas/PaginationEnvelope/properties/results" + } + } + } + } + } + } + } + } + } + }, + "components": { + "schemas": { + "OpenAPIResponseAttr": { + "type": "object", + "properties": { + "filterable_result": { + "x-linode-filterable": true, + "type": "string", + "description": "Filterable result value" + }, + "filterable_list_result": { + "x-linode-filterable": true, + "type": "array", + "items": { + "type": "string" + }, + "description": "Filterable result value" + } + } + }, + "PaginationEnvelope": { + "type": "object", + "properties": { + "pages": { + "type": "integer", + "readOnly": true, + "description": "The total number of pages.", + "example": 1 + }, + "page": { + "type": "integer", + "readOnly": true, + "description": "The current page.", + "example": 1 + }, + "results": { + "type": "integer", + "readOnly": true, + "description": "The total number of results.", + "example": 1 + } + } + } + } + } +} diff --git a/tests/fixtures/cli_test_load.yaml b/tests/fixtures/cli_test_load.yaml new file mode 100644 index 000000000..f7dd7704e --- /dev/null +++ b/tests/fixtures/cli_test_load.yaml @@ -0,0 +1,64 @@ +openapi: 3.0.1 +info: + title: API Specification + version: 1.0.0 +servers: + - url: http://localhost/v4 +paths: + /foo/bar: + get: + summary: get info + operationId: fooBarGet + description: This is description + responses: + '200': + description: Successful response + content: + application/json: + schema: + type: object + properties: + data: + type: array + items: + $ref: '#/components/schemas/OpenAPIResponseAttr' + page: + $ref: '#/components/schemas/PaginationEnvelope/properties/page' + pages: + $ref: '#/components/schemas/PaginationEnvelope/properties/pages' + results: + $ref: '#/components/schemas/PaginationEnvelope/properties/results' + +components: + schemas: + OpenAPIResponseAttr: + type: object + properties: + filterable_result: + x-linode-filterable: true + type: string + description: Filterable result value + filterable_list_result: + x-linode-filterable: true + type: array + items: + type: string + description: Filterable result value + PaginationEnvelope: + type: object + properties: + pages: + type: integer + readOnly: true + description: The total number of pages. + example: 1 + page: + type: integer + readOnly: true + description: The current page. + example: 1 + results: + type: integer + readOnly: true + description: The total number of results. + example: 1 \ No newline at end of file diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index fef49ab27..04fc5e79c 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -1,5 +1,7 @@ import configparser -from typing import List +import contextlib +import os +from typing import ContextManager, List, TextIO import pytest from openapi3 import OpenAPI @@ -23,6 +25,25 @@ LOADED_FILES = {} +FIXTURES_PATH = "tests/fixtures" + + +@contextlib.contextmanager +def open_fixture(filename: str) -> ContextManager[TextIO]: + """ + Gets the reader for a given fixture. + + :returns: A context manager yielding the fixture's reader. + """ + + f = open(os.path.join(FIXTURES_PATH, filename), "r") + + try: + yield f + finally: + f.close() + + def _get_parsed_yaml(filename): """ Returns a python dict that is a parsed yaml file from the tests/fixtures @@ -33,8 +54,9 @@ def _get_parsed_yaml(filename): :type filename: str """ if filename not in LOADED_FILES: - with open("tests/fixtures/" + filename) as f: + with open_fixture(filename) as f: raw = f.read() + parsed = safe_load(raw) LOADED_FILES[filename] = parsed diff --git a/tests/unit/test_arg_helpers.py b/tests/unit/test_arg_helpers.py index e5ad7b3a8..3743061c2 100644 --- a/tests/unit/test_arg_helpers.py +++ b/tests/unit/test_arg_helpers.py @@ -1,5 +1,4 @@ #!/usr/local/bin/python3 -import pytest from linodecli import arg_helpers @@ -150,10 +149,3 @@ def test_remove_plugin_not_available(self, mocked_config): msg, code = arg_helpers.remove_plugin("testing.plugin", mocked_config) assert "not a registered plugin" in msg assert code == 14 - - def test_bake_command_bad_website(self, capsys, mock_cli): - with pytest.raises(SystemExit) as ex: - arg_helpers.bake_command(mock_cli, "https://website.com") - captured = capsys.readouterr() - assert ex.value.code == 2 - assert "Request failed to https://website.com" in captured.out diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index b7f41a79c..e9c2189dc 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -2,12 +2,16 @@ import copy import math +import os import re import pytest import requests +import requests_mock from pytest import MonkeyPatch +from tests.unit.conftest import FIXTURES_PATH, open_fixture + if True: from linodecli import CLI from linodecli.api_request import get_all_pages @@ -78,6 +82,46 @@ def test_user_agent(self, mock_cli: CLI): r"linode-cli/[0-9]+\.[0-9]+\.[0-9]+ linode-api-docs/[0-9]+\.[0-9]+\.[0-9]+ python/[0-9]+\.[0-9]+\.[0-9]+" ).match(mock_cli.user_agent) + def test_load_openapi_spec_json(self): + url_base = "https://localhost/" + path = "cli_test_load.json" + url = f"{url_base}{path}" + + with open_fixture(path) as f: + content = f.read() + + with requests_mock.Mocker() as m: + m.get(url, text=content) + + parsed_json_local = CLI._load_openapi_spec( + str(os.path.join(FIXTURES_PATH, path)) + ) + + parsed_json_http = CLI._load_openapi_spec(url) + + assert m.call_count == 1 + assert parsed_json_http.raw_element == parsed_json_local.raw_element + + def test_load_openapi_spec_yaml(self): + url_base = "https://localhost/" + path = "cli_test_load.yaml" + url = f"{url_base}{path}" + + with open_fixture(path) as f: + content = f.read() + + with requests_mock.Mocker() as m: + m.get(url, text=content) + + parsed_json_local = CLI._load_openapi_spec( + str(os.path.join(FIXTURES_PATH, path)) + ) + + parsed_json_http = CLI._load_openapi_spec(url) + + assert m.call_count == 1 + assert parsed_json_http.raw_element == parsed_json_local.raw_element + def test_get_all_pages( mock_cli: CLI, list_operation: OpenAPIOperation, monkeypatch: MonkeyPatch