From 3502121affe8b783525fa64038353cd37105fc26 Mon Sep 17 00:00:00 2001 From: Jeffrey Aven Date: Tue, 19 Nov 2024 19:48:30 +1100 Subject: [PATCH 1/6] fixed json handling issue --- CHANGELOG.md | 6 +++++- pystackql/stackql.py | 44 ++++++++++++++++++++++++++-------------- setup.py | 2 +- tests/pystackql_tests.py | 14 +++++++++++++ 4 files changed, 49 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ee77f3c..2c45e53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,15 @@ # Changelog -## v3.7.1 (2024-11-19) +## v3.7.2 (2024-11-19) ### Updates - Added `http_debug` constructor argument to return HTTP log information +### Bug Fixes + +- Fixed issue passing JSON strings to queries, added test + ## v3.7.0 (2024-11-08) ### Updates diff --git a/pystackql/stackql.py b/pystackql/stackql.py index 4176b33..580dae5 100644 --- a/pystackql/stackql.py +++ b/pystackql/stackql.py @@ -11,7 +11,7 @@ import sys, subprocess, json, os, asyncio, functools from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor import pandas as pd -import tempfile +import tempfile, shlex from io import StringIO @@ -205,8 +205,9 @@ def _run_query(self, query, custom_auth=None, env_vars=None): :raises FileNotFoundError: If the StackQL binary isn't found. :raises Exception: For any other exceptions during the execution, providing a generic error message. """ + local_params = self.params.copy() - local_params.insert(1, f'"{query}"') + local_params.insert(1, shlex.quote(query)) script_path = None # Handle custom authentication if provided @@ -240,19 +241,32 @@ def _run_query(self, query, custom_auth=None, env_vars=None): full_command = " ".join([self.bin_path] + local_params) try: - with subprocess.Popen(full_command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as iqlPopen: - stdout, stderr = iqlPopen.communicate() - - if self.debug: - self._debug_log(f"query: {query}") - self._debug_log(f"stdout: {stdout}") - self._debug_log(f"stderr: {stderr}") - - # Process stdout and stderr - if stderr: - output["error"] = stderr.decode('utf-8') if isinstance(stderr, bytes) else str(stderr) - if stdout: - output["data"] = stdout.decode('utf-8') if isinstance(stdout, bytes) else str(stdout) + + full_command = full_command.replace("\n", " ") + + result = subprocess.run( + full_command, + shell=True, + text=True, + capture_output=True + ) + + stdout = result.stdout + stderr = result.stderr + returncode = result.returncode + + if self.debug: + self._debug_log(f"fullcommand: {full_command}") + self._debug_log(f"returncode: {returncode}") + self._debug_log(f"stdout: {stdout}") + self._debug_log(f"stderr: {stderr}") + + # Process stdout and stderr + if stderr: + output["error"] = stderr.decode('utf-8') if isinstance(stderr, bytes) else str(stderr) + if stdout: + output["data"] = stdout.decode('utf-8') if isinstance(stdout, bytes) else str(stdout) + except FileNotFoundError: output["exception"] = f"ERROR: {self.bin_path} not found" diff --git a/setup.py b/setup.py index 8ebc3b1..003cab8 100644 --- a/setup.py +++ b/setup.py @@ -10,7 +10,7 @@ setup( name='pystackql', - version='v3.7.1', + version='v3.7.2', description='A Python interface for StackQL', long_description=readme, author='Jeffrey Aven', diff --git a/tests/pystackql_tests.py b/tests/pystackql_tests.py index 32b94d3..aeaff14 100644 --- a/tests/pystackql_tests.py +++ b/tests/pystackql_tests.py @@ -263,6 +263,20 @@ def test_18_execute_custom_auth_env_vars(self): self.assertEqual(result, expected_result, f"Expected result: {expected_result}, got: {result}") print_test_result(f"Test 18 execute with custom auth and command-specific environment variables\nRESULT: {result}", result == expected_result) + @pystackql_test_setup() + def test_19_json_extract_function(self): + query = """ + SELECT + json_extract('{"Key":"StackName","Value":"aws-stack"}', '$.Key') as key, + json_extract('{"Key":"StackName","Value":"aws-stack"}', '$.Value') as value + """ + expected_result = [ + {'key': {'String': 'StackName', 'Valid': True}, 'value': {'String': 'aws-stack', 'Valid': True}} + ] + result = self.stackql.execute(query) + self.assertEqual(result, expected_result, f"Expected result: {expected_result}, got: {result}") + print_test_result(f"Test 19 complex object handling\nRESULT: {result}", result == expected_result) + @unittest.skipIf(platform.system() == "Windows", "Skipping async tests on Windows") class PyStackQLAsyncTests(PyStackQLTestsBase): From 2b221a18bea0a833fd3d9e7c90111ade54270cb1 Mon Sep 17 00:00:00 2001 From: Jeffrey Aven Date: Tue, 19 Nov 2024 20:04:08 +1100 Subject: [PATCH 2/6] fixed json handling issue --- pystackql/stackql.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/pystackql/stackql.py b/pystackql/stackql.py index 580dae5..1f85ce2 100644 --- a/pystackql/stackql.py +++ b/pystackql/stackql.py @@ -11,7 +11,7 @@ import sys, subprocess, json, os, asyncio, functools from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor import pandas as pd -import tempfile, shlex +import tempfile from io import StringIO @@ -207,7 +207,19 @@ def _run_query(self, query, custom_auth=None, env_vars=None): """ local_params = self.params.copy() - local_params.insert(1, shlex.quote(query)) + # local_params.insert(1, shlex.quote(query)) + script_path = None + + if sys.platform == "Windows": + # Escape double quotes and wrap in double quotes for Windows + escaped_query = query.replace('"', '\\"') # Escape double quotes properly + safe_query = f'"{escaped_query}"' + else: + # Use shlex.quote for Unix-like systems + import shlex + safe_query = shlex.quote(query) + + local_params.insert(1, safe_query) script_path = None # Handle custom authentication if provided From a761ca4d3b1ca6a2562d405c52db0af3efc8322f Mon Sep 17 00:00:00 2001 From: Jeffrey Aven Date: Tue, 19 Nov 2024 20:41:24 +1100 Subject: [PATCH 3/6] fixed json handling issue --- .github/workflows/test.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index b36ac10..09111f5 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -22,6 +22,8 @@ jobs: exclude: - os: macos-latest python-version: "3.8" + - os: macos-latest + python-version: "3.13" runs-on: ${{matrix.os}} name: 'Run Tests on ${{matrix.os}} with Python ${{matrix.python-version}}' From 9e7332f1ff720759cc4e8e528def9dbcc45a1f99 Mon Sep 17 00:00:00 2001 From: Jeffrey Aven Date: Tue, 19 Nov 2024 20:45:15 +1100 Subject: [PATCH 4/6] fixed json handling issue --- pystackql/stackql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pystackql/stackql.py b/pystackql/stackql.py index 1f85ce2..4ee8d5f 100644 --- a/pystackql/stackql.py +++ b/pystackql/stackql.py @@ -210,7 +210,7 @@ def _run_query(self, query, custom_auth=None, env_vars=None): # local_params.insert(1, shlex.quote(query)) script_path = None - if sys.platform == "Windows": + if sys.os.platform == "Windows": # Escape double quotes and wrap in double quotes for Windows escaped_query = query.replace('"', '\\"') # Escape double quotes properly safe_query = f'"{escaped_query}"' From bdd41dd56fa0d3a5a5a258d344398c01a269fb77 Mon Sep 17 00:00:00 2001 From: Jeffrey Aven Date: Tue, 19 Nov 2024 20:58:20 +1100 Subject: [PATCH 5/6] fixed json handling issue --- pystackql/stackql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pystackql/stackql.py b/pystackql/stackql.py index 4ee8d5f..3bb02b0 100644 --- a/pystackql/stackql.py +++ b/pystackql/stackql.py @@ -210,7 +210,7 @@ def _run_query(self, query, custom_auth=None, env_vars=None): # local_params.insert(1, shlex.quote(query)) script_path = None - if sys.os.platform == "Windows": + if self.platform.startswith("Windows"): # Escape double quotes and wrap in double quotes for Windows escaped_query = query.replace('"', '\\"') # Escape double quotes properly safe_query = f'"{escaped_query}"' From f134fddce2707ebd675c27b4678406e547f12cab Mon Sep 17 00:00:00 2001 From: Jeffrey Aven Date: Tue, 19 Nov 2024 21:03:49 +1100 Subject: [PATCH 6/6] fixed json handling issue --- pystackql/stackql.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pystackql/stackql.py b/pystackql/stackql.py index 3bb02b0..300be86 100644 --- a/pystackql/stackql.py +++ b/pystackql/stackql.py @@ -207,7 +207,6 @@ def _run_query(self, query, custom_auth=None, env_vars=None): """ local_params = self.params.copy() - # local_params.insert(1, shlex.quote(query)) script_path = None if self.platform.startswith("Windows"):