diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e2dd13..b7c3ccb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,10 @@ # Changelog -## v3.8.0 (2025-06-04) +## v3.8.0 (2025-06-13) ### Updates +- Added `--csv-download` argument for stackql magic commands - Refactor - Enhanced test coverage diff --git a/README.rst b/README.rst index 3c5c05c..1c5cef9 100644 --- a/README.rst +++ b/README.rst @@ -22,7 +22,7 @@ PyStackQL - Python Wrapper for StackQL StackQL is an open source developer tool which allows you to query and interact with cloud and SaaS provider APIs using SQL grammar. StackQL can be used for cloud inventory analysis, cloud cost optimization, cloud security and compliance, provisioning/IaC, assurance, XOps, and more. -PyStackQL is a Python wrapper for StackQL which allows you to use StackQL within Python applications and to use the power of Python to extend StackQL. +`PyStackQL `_ is a Python wrapper for StackQL which allows you to use StackQL within Python applications and to use the power of Python to extend StackQL. PyStackQL can be used with ``pandas``, ``matplotlib``, ``plotly``, ``jupyter`` and other Python libraries to create powerful data analysis and visualization applications. For detailed documentation, including the API reference, see `Read the Docs `_. @@ -52,19 +52,17 @@ The following example demonstrates how to run a query and return the results as :: from pystackql import StackQL - import pandas as pd region = "ap-southeast-2" - stackql = StackQL() + stackql = StackQL(output='pandas') query = """ - SELECT instanceType, COUNT(*) as num_instances + SELECT instance_type, COUNT(*) as num_instances FROM aws.ec2.instances WHERE region = '%s' - GROUP BY instanceType + GROUP BY instance_type """ % (region) - res = stackql.execute(query) - df = pd.read_json(res) + df = stackql.execute(query) print(df) Using PyStackQL with Jupyter Notebook @@ -76,7 +74,7 @@ To use the integrated Jupyter magic commands provided by PyStackQL: .. code-block:: python - %load_ext pystackql + %load_ext pystackql.magic 2. **Execute a Query Using Line Magic**: @@ -111,12 +109,11 @@ Supported Python Versions PyStackQL has been tested on: -- Python 3.7 -- Python 3.8 - Python 3.9 - Python 3.10 - Python 3.11 -- Python 3.12 (MacOS and Linux only) +- Python 3.12 +- Python 3.13 Licensing ~~~~~~~~~ diff --git a/docs/source/conf.py b/docs/source/conf.py index c4775d7..7a60863 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -20,13 +20,13 @@ # -- Project information ----------------------------------------------------- project = 'pystackql' -copyright = '2021-2024, StackQL Studios' +copyright = '2021-2025, StackQL Studios' author = 'StackQL Studios' # The short X.Y version version = '' # The full version, including alpha/beta/rc tags -release = 'v3.6.6' +release = 'v3.8.0' # -- General configuration --------------------------------------------------- diff --git a/docs/source/examples.rst b/docs/source/examples.rst index b7c5a4a..14e8e8c 100644 --- a/docs/source/examples.rst +++ b/docs/source/examples.rst @@ -49,10 +49,10 @@ StackQL can be used to collect, analyze, summarize, and report on cloud resource regions = ["ap-southeast-2", "us-east-1"] queries = [ f""" - SELECT '{region}' as region, instanceType, COUNT(*) as num_instances + SELECT '{region}' as region, instance_type, COUNT(*) as num_instances FROM aws.ec2.instances WHERE region = '{region}' - GROUP BY instanceType + GROUP BY instance_type """ for region in regions ] diff --git a/docs/source/intro.rst b/docs/source/intro.rst index 6cf8227..0833406 100644 --- a/docs/source/intro.rst +++ b/docs/source/intro.rst @@ -71,19 +71,17 @@ The following example demonstrates how to run a query and return the results as .. code-block:: python from pystackql import StackQL - import pandas as pd region = "ap-southeast-2" - stackql = StackQL() + stackql = StackQL(output='pandas') query = """ - SELECT instanceType, COUNT(*) as num_instances + SELECT instance_type, COUNT(*) as num_instances FROM aws.ec2.instances WHERE region = '%s' - GROUP BY instanceType + GROUP BY instance_type """ % (region) - res = stackql.execute(query) - df = pd.read_json(res) + df = stackql.execute(query) print(df) Using ``UNION`` and ``JOIN`` operators @@ -96,26 +94,25 @@ StackQL is a fully functional SQL programming environment, enabling the full set ... regions = ["ap-southeast-2", "us-east-1"] query = """ - SELECT '%s' as region, instanceType, COUNT(*) as num_instances + SELECT '%s' as region, instance_type, COUNT(*) as num_instances FROM aws.ec2.instances WHERE region = '%s' - GROUP BY instanceType + GROUP BY instance_type UNION - SELECT '%s' as region, instanceType, COUNT(*) as num_instances + SELECT '%s' as region, instance_type, COUNT(*) as num_instances FROM aws.ec2.instances WHERE region = '%s' - GROUP BY instanceType + GROUP BY instance_type """ % (regions[0], regions[0], regions[1], regions[1]) - res = stackql.execute(query) - df = pd.read_json(res) + df = stackql.execute(query) print(df) The preceding example will print a ``pandas.DataFrame`` which would look like this: .. code-block:: sh - instanceType num_instances region + instance_type num_instances region 0 t2.medium 2 ap-southeast-2 1 t2.micro 7 ap-southeast-2 2 t2.small 4 ap-southeast-2 @@ -133,17 +130,15 @@ In addition to ``UNION`` DML operators, you can also run a batch (list) of queri queries = [ f""" - SELECT '{region}' as region, instanceType, COUNT(*) as num_instances + SELECT '{region}' as region, instance_type, COUNT(*) as num_instances FROM aws.ec2.instances WHERE region = '{region}' - GROUP BY instanceType + GROUP BY instance_type """ for region in regions ] - res = stackql.executeQueriesAsync(queries) - df = pd.read_json(json.dumps(res)) - + df = stackql.executeQueriesAsync(queries) print(df) @@ -156,10 +151,9 @@ Here is an example of using the ``json_extract`` function to extract a field fro .. code-block:: python from pystackql import StackQL - import pandas as pd subscriptionId = "273769f6-545f-45b2-8ab8-2f14ec5768dc" resourceGroupName = "stackql-ops-cicd-dev-01" - stackql = StackQL() + stackql = StackQL() # output format defaults to 'dict' query = """ SELECT name, @@ -172,8 +166,7 @@ Here is an example of using the ``json_extract`` function to extract a field fro """ % (resourceGroupName, subscriptionId) res = stackql.execute(query) - df = pd.read_json(res) - print(df) + print(res) Using the Jupyter Magic Extension ================================= @@ -184,7 +177,7 @@ To get started with the magic extension, first load it into your Jupyter environ .. code-block:: ipython - %load_ext pystackql + %load_ext pystackql.magic After loading the magic extension, you can use the `%%stackql` magic to execute StackQL commands in a dedicated Jupyter cell. The output will be displayed directly below the cell, just like any other Jupyter command output. @@ -196,3 +189,9 @@ Example: SHOW SERVICES in aws This Jupyter magic extension provides a seamless integration of `pystackql` into your Jupyter workflows, allowing you to explore cloud and SaaS provider data interactively within your notebooks. + +To use the magic extension to run queries against a StackQL server, you can use the following command: + +.. code-block:: ipython + + %load_ext pystackql.magics diff --git a/docs/source/magic_ext.rst b/docs/source/magic_ext.rst index 5eeed67..898a8c2 100644 --- a/docs/source/magic_ext.rst +++ b/docs/source/magic_ext.rst @@ -38,9 +38,9 @@ The extension provides both line and cell magic functionalities: .. code-block:: python %%stackql - SELECT instanceType, COUNT(*) as num_instances + SELECT instance_type, COUNT(*) as num_instances FROM aws.ec2.instances - WHERE region = '$region' GROUP BY instanceType + WHERE region = '$region' GROUP BY instance_type Options ------- @@ -48,8 +48,13 @@ Options When using `StackqlMagic` as cell magic, you can pass in the following options: - ``--no-display`` : Suppresses the display of the results. Even when this option is enabled, the results are still saved in the `stackql_df` Pandas DataFrame. +- ``--csv-download`` : Adds a download button below the query results that allows you to download the data as a CSV file. -Example: +Examples +-------- + +Basic Query +~~~~~~~~~~ .. code-block:: python @@ -57,6 +62,15 @@ Example: zone = 'australia-southeast1-a' region = 'australia-southeast1' + %%stackql + SELECT SPLIT_PART(machineType, '/', -1) as machine_type, count(*) as num_instances + FROM google.compute.instances + WHERE project = '$project' AND zone = '$zone' + GROUP BY machine_type + +Suppressing Display +~~~~~~~~~~~~~~~~~~ + .. code-block:: python %%stackql --no-display @@ -67,6 +81,38 @@ Example: This will run the query but won't display the results in the notebook. Instead, you can later access the results via the `stackql_df` variable. +Downloading Results as CSV +~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: python + + %%stackql --csv-download + SELECT + 'Python' as language, + 'Development' as mode, + 'PyStackQL' as package + +This will display the query results in the notebook and add a download button below the results. Clicking the button will download the data as a CSV file named ``stackql_results.csv``. + +Combining Options +~~~~~~~~~~~~~~~ + +You can also combine options. For example, if you want to suppress the display but still want a download button: + +.. code-block:: python + + # First run the query with no display + %%stackql --no-display + SELECT instance_type, COUNT(*) as num_instances + FROM aws.ec2.instances + WHERE region = '$region' GROUP BY instance_type + + # Then manually display with the download button + from IPython.display import display + display(stackql_df) + from pystackql import StackqlMagic + StackqlMagic(get_ipython())._display_with_csv_download(stackql_df) + .. note:: The results of the queries are always saved in a Pandas DataFrame named `stackql_df` in the notebook's current namespace. This allows you to further process or visualize the data as needed. @@ -75,8 +121,8 @@ An example of visualizing the results using Pandas is shown below: .. code-block:: python - stackql_df.plot(kind='pie', y='num_instances', labels=_['machine_type'], title='Instances by Type', autopct='%1.1f%%') + stackql_df.plot(kind='pie', y='num_instances', labels=stackql_df['machine_type'], title='Instances by Type', autopct='%1.1f%%') -------- -This documentation provides a basic overview and usage guide for the `StackqlMagic` extension. For advanced usage or any additional features provided by the extension, refer to the source code or any other accompanying documentation. +This documentation provides a basic overview and usage guide for the `StackqlMagic` extension. For advanced usage or any additional features provided by the extension, refer to the source code or any other accompanying documentation. \ No newline at end of file diff --git a/notebooks/demo.ipynb b/notebooks/demo.ipynb index c980af3..7009386 100644 --- a/notebooks/demo.ipynb +++ b/notebooks/demo.ipynb @@ -1,150 +1,148 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# PyStackQL Development Demo\n", - "\n", - "This notebook demonstrates how to use the development version of PyStackQL directly from the source code. Any changes you make to the PyStackQL code will be immediately reflected here." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# First, let's check what version of PyStackQL we're using\n", - "import pystackql\n", - "print(f\"PyStackQL Version: {pystackql.__version__}\")\n", - "\n", - "# Check the location of the package to confirm we're using the development version\n", - "print(f\"Package Location: {pystackql.__file__}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Load the magic extension\n", - "%load_ext pystackql.magic" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Basic Query Test\n", - "\n", - "Let's run a simple query to test that everything is working:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%stackql SELECT 42 as answer" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## CSV Download Test\n", - "\n", - "Let's test the CSV download functionality:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%%stackql --csv-download\n", - "SELECT \n", - " 'Python' as language,\n", - " 'Development' as mode,\n", - " 'PyStackQL' as package" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Test Cloud Provider Functionality\n", - "\n", - "If you have credentials configured, you can test actual cloud provider queries:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Uncomment and run the appropriate provider query based on your available credentials\n", - "\n", - "# AWS Example\n", - "# %stackql DESCRIBE aws.ec2.instances\n", - "\n", - "# GitHub Example\n", - "# %stackql registry pull github\n", - "# %stackql SELECT login FROM github.users.followers WHERE username = 'stackql'" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Development Tips\n", - "\n", - "1. After modifying PyStackQL code, you don't need to reinstall the package - changes are reflected immediately\n", - "2. You can run tests from the terminal with `pytest tests/`\n", - "3. If you modify deep core functionality, you may need to restart the kernel\n", - "4. To debug issues, you can use Python's built-in debugging tools:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Example debugging a PyStackQL function\n", - "from pystackql.core import StackQL\n", - "\n", - "# Get instance properties\n", - "stackql = StackQL()\n", - "props = stackql.properties()\n", - "print(props)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python (PyStackQL Dev)", - "language": "python", - "name": "pystackql-dev" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.4" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} \ No newline at end of file +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# PyStackQL Development Demo\n", + "\n", + "This notebook demonstrates how to use the development version of PyStackQL directly from the source code. Any changes you make to the PyStackQL code will be immediately reflected here." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from pystackql import StackQL\n", + "stackql = StackQL()\n", + "print(stackql.version)\n", + "print(stackql.package_version)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Load the magic extension\n", + "%load_ext pystackql.magic" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Basic Query Test\n", + "\n", + "Let's run a simple query to test that everything is working:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%stackql SELECT 42 as answer" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## CSV Download Test\n", + "\n", + "Let's test the CSV download functionality:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%stackql --csv-download\n", + "SELECT \n", + " 'Python' as language,\n", + " 'Development' as mode,\n", + " 'PyStackQL' as package" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Test Cloud Provider Functionality\n", + "\n", + "If you have credentials configured, you can test actual cloud provider queries:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Uncomment and run the appropriate provider query based on your available credentials\n", + "\n", + "# AWS Example\n", + "# %stackql DESCRIBE aws.ec2.instances\n", + "\n", + "# GitHub Example\n", + "# %stackql registry pull github\n", + "# %stackql SELECT login FROM github.users.followers WHERE username = 'stackql'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Development Tips\n", + "\n", + "1. After modifying PyStackQL code, you don't need to reinstall the package - changes are reflected immediately\n", + "2. You can run tests from the terminal with `pytest tests/`\n", + "3. If you modify deep core functionality, you may need to restart the kernel\n", + "4. To debug issues, you can use Python's built-in debugging tools:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Example debugging a PyStackQL function\n", + "from pystackql.core import StackQL\n", + "\n", + "# Get instance properties\n", + "stackql = StackQL()\n", + "props = stackql.properties()\n", + "print(props)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python (PyStackQL Dev)", + "language": "python", + "name": "pystackql-dev" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.4" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/pystackql/core/stackql.py b/pystackql/core/stackql.py index db0d483..ab6c479 100644 --- a/pystackql/core/stackql.py +++ b/pystackql/core/stackql.py @@ -368,48 +368,6 @@ def execute(self, query, suppress_errors=True, custom_auth=None, env_vars=None): # Format the result return self.local_output_formatter.format_query_result(output, suppress_errors) - # async def executeQueriesAsync(self, queries): - # """Executes multiple StackQL queries asynchronously using the current StackQL instance. - - # This method utilizes an asyncio event loop to concurrently run a list of provided - # StackQL queries. Each query is executed independently, and the combined results of - # all the queries are returned as a list of JSON objects if 'dict' output mode is selected, - # or as a concatenated DataFrame if 'pandas' output mode is selected. - - # The order of the results in the returned list or DataFrame may not necessarily - # correspond to the order of the queries in the input list due to the asynchronous nature - # of execution. - - # :param queries: A list of StackQL query strings to be executed concurrently. - # :type queries: list[str], required - # :return: A list of results corresponding to each query. Each result is a JSON object or a DataFrame. - # :rtype: list[dict] or pd.DataFrame - # :raises ValueError: If method is used in `server_mode` on an unsupported OS (anything other than Linux). - # :raises ValueError: If an unsupported output mode is selected (anything other than 'dict' or 'pandas'). - - # Example: - # >>> from pystackql import StackQL - # >>> stackql = StackQL() - # >>> queries = [ - # >>> \"\"\"SELECT '%s' as region, instanceType, COUNT(*) as num_instances - # ... FROM aws.ec2.instances - # ... WHERE region = '%s' - # ... GROUP BY instanceType\"\"\" % (region, region) - # >>> for region in regions ] - # >>> result = stackql.executeQueriesAsync(queries) - - # Note: - # - When operating in `server_mode`, this method is not supported. - # """ - # if self.server_mode: - # raise ValueError( - # "The executeQueriesAsync method is not supported in server mode. " - # "Please use the standard execute method with individual queries instead, " - # "or switch to local mode if you need to run multiple queries concurrently." - # ) - - # return await self.async_executor.execute_queries(queries) - async def executeQueriesAsync(self, queries): """Executes multiple StackQL queries asynchronously using the current StackQL instance. @@ -433,10 +391,10 @@ async def executeQueriesAsync(self, queries): >>> from pystackql import StackQL >>> stackql = StackQL() >>> queries = [ - >>> \"\"\"SELECT '%s' as region, instanceType, COUNT(*) as num_instances + >>> \"\"\"SELECT '%s' as region, instance_type, COUNT(*) as num_instances ... FROM aws.ec2.instances ... WHERE region = '%s' - ... GROUP BY instanceType\"\"\" % (region, region) + ... GROUP BY instance_type\"\"\" % (region, region) >>> for region in regions ] >>> result = stackql.executeQueriesAsync(queries)