From d905882b6e78809f91012898caac8f33f709dc69 Mon Sep 17 00:00:00 2001
From: Vaclav Petras <wenzeslaus@gmail.com>
Date: Tue, 18 Apr 2023 17:39:11 -0400
Subject: [PATCH 01/29] grass.experimental: Add object to access modules as
functions
This adds a Tools class which allows to access GRASS tools (modules) to be accessed using methods. Once an instance is created, calling a tool is calling a function (method) similarly to grass.jupyter.Map. Unlike grass.script, this does not require generic function name and unlike grass.pygrass module shortcuts, this does not require special objects to mimic the module families.
Outputs are handled through a returned object which is result of automatic capture of outputs and can do conversions from known formats using properties.
Usage example is in the _test() function in the file.
The code is included under new grass.experimental package which allows merging the code even when further breaking changes are anticipated.
---
python/grass/experimental/__init__.py | 0
python/grass/experimental/tools.py | 104 ++++++++++++++++++++++++++
2 files changed, 104 insertions(+)
create mode 100644 python/grass/experimental/__init__.py
create mode 100644 python/grass/experimental/tools.py
diff --git a/python/grass/experimental/__init__.py b/python/grass/experimental/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/python/grass/experimental/tools.py b/python/grass/experimental/tools.py
new file mode 100644
index 00000000000..9ae922221bb
--- /dev/null
+++ b/python/grass/experimental/tools.py
@@ -0,0 +1,104 @@
+import sys
+import shutil
+
+import grass.script as gs
+
+
+class ExecutedTool:
+ def __init__(self, name, kwargs, stdout, stderr):
+ self._name = name
+ self._stdout = stdout
+
+ @property
+ def text(self):
+ return self._stdout
+
+ @property
+ def json(self):
+ import json
+
+ return json.loads(self._stdout)
+
+ @property
+ def keyval(self):
+ return gs.parse_key_val(self._stdout)
+
+
+class SubExecutor:
+ """use as tools().params(a="x", b="y").g_region()"""
+
+ # Can support other envs or all PIPE and encoding read command supports
+
+
+class Tools:
+ def __init__(self):
+ # TODO: fix region, so that external g.region call in the middle
+ # is not a problem
+ # i.e. region is independent/internal/fixed
+ pass
+
+ def run(self, name, /, **kwargs):
+ """Run modules from the GRASS display family (modules starting with "d.").
+
+ This function passes arguments directly to grass.script.run_command()
+ so the syntax is the same.
+
+ :param str module: name of GRASS module
+ :param `**kwargs`: named arguments passed to run_command()"""
+ # alternatively use dev null as default or provide it as convenient settings
+ kwargs["stdout"] = gs.PIPE
+ kwargs["stderr"] = gs.PIPE
+ process = gs.pipe_command(name, **kwargs)
+ stdout, stderr = process.communicate()
+ stderr = gs.utils.decode(stderr)
+ returncode = process.poll()
+ # TODO: instead of printing, do exception right away
+ if returncode:
+ # Print only when we are capturing it and there was some output.
+ # (User can request ignoring the subprocess stderr and then
+ # we get only None.)
+ if stderr:
+ sys.stderr.write(stderr)
+ gs.handle_errors(returncode, stdout, [name], kwargs)
+ return ExecutedTool(name=name, kwargs=kwargs, stdout=stdout, stderr=stderr)
+
+ def __getattr__(self, name):
+ """Parse attribute to GRASS display module. Attribute should be in
+ the form 'd_module_name'. For example, 'd.rast' is called with 'd_rast'.
+ """
+ # Reformat string
+ grass_module = name.replace("_", ".")
+ # Assert module exists
+ if not shutil.which(grass_module):
+ raise AttributeError(
+ _(
+ "Cannot find GRASS tool {}. "
+ "Is the session set up and the tool on path?"
+ ).format(grass_module)
+ )
+
+ def wrapper(**kwargs):
+ # Run module
+ return self.run(grass_module, **kwargs)
+
+ return wrapper
+
+
+def _test():
+ gs.setup.init("~/grassdata/nc_spm_08_grass7/user1")
+
+ tools = Tools()
+ tools.g_region(raster="elevation")
+ tools.r_slope_aspect(elevation="elevation", slope="slope", overwrite=True)
+ print(tools.r_univar(map="slope", flags="g").keyval)
+
+ print(tools.v_info(map="bridges", flags="c").text)
+ print(
+ tools.v_db_univar(map="bridges", column="YEAR_BUILT", format="json").json[
+ "statistics"
+ ]["mean"]
+ )
+
+
+if __name__ == "__main__":
+ _test()
From aaef183a2dc7226c032036bac9f92f136b3feb99 Mon Sep 17 00:00:00 2001
From: Vaclav Petras <wenzeslaus@gmail.com>
Date: Fri, 21 Apr 2023 17:40:29 -0400
Subject: [PATCH 02/29] Support verbosity, overwrite and region freezing
---
python/grass/experimental/tools.py | 97 ++++++++++++++++++++++++++++--
1 file changed, 92 insertions(+), 5 deletions(-)
diff --git a/python/grass/experimental/tools.py b/python/grass/experimental/tools.py
index 9ae922221bb..64b103b28fc 100644
--- a/python/grass/experimental/tools.py
+++ b/python/grass/experimental/tools.py
@@ -1,3 +1,4 @@
+import os
import sys
import shutil
@@ -8,10 +9,11 @@ class ExecutedTool:
def __init__(self, name, kwargs, stdout, stderr):
self._name = name
self._stdout = stdout
+ self._decoded_stdout = gs.decode(self._stdout)
@property
def text(self):
- return self._stdout
+ return self._decoded_stdout.strip()
@property
def json(self):
@@ -21,21 +23,80 @@ def json(self):
@property
def keyval(self):
+ # TODO: possibly use or add _text_to_key_value_dict
+ # which converts int and float automatically
return gs.parse_key_val(self._stdout)
+ @property
+ def comma_items(self):
+ return self.text_split(",")
+
+ @property
+ def space_items(self):
+ return self.text_split(None)
+
+ def text_split(self, separator=None):
+ # The use of strip is assuming that the output is one line which
+ # ends with a newline character which is for display only.
+ return self._decoded_stdout.strip("\n").split(separator)
+
class SubExecutor:
"""use as tools().params(a="x", b="y").g_region()"""
+ # a and b would be overwrite or stdin
+
# Can support other envs or all PIPE and encoding read command supports
class Tools:
- def __init__(self):
+ def __init__(
+ self,
+ *,
+ session=None,
+ env=None,
+ overwrite=True,
+ quiet=False,
+ verbose=False,
+ superquiet=False,
+ freeze_region=False,
+ ):
# TODO: fix region, so that external g.region call in the middle
# is not a problem
# i.e. region is independent/internal/fixed
- pass
+ if env:
+ self._env = env.copy()
+ elif session and hasattr(session, "env"):
+ self._env = session.env.copy()
+ else:
+ self._env = os.environ.copy()
+ self._region_is_frozen = False
+ if freeze_region:
+ self._freeze_region()
+ if overwrite:
+ self._overwrite()
+ # This hopefully sets the numbers directly. An alternative implementation would
+ # be to pass the parameter every time.
+ # Does not check for multiple set at the same time, but the most versbose wins
+ # for safety.
+ if superquiet:
+ self._env["GRASS_VERBOSE"] = "0"
+ if quiet:
+ self._env["GRASS_VERBOSE"] = "1"
+ if verbose:
+ self._env["GRASS_VERBOSE"] = "3"
+
+ # These could be public, not protected.
+ def _freeze_region(self):
+ self._env["GRASS_REGION"] = gs.region_env(env=self._env)
+ self._region_is_frozen = True
+
+ def _overwrite(self):
+ self._env["GRASS_OVERWRITE"] = "1"
+
+ @property
+ def env(self):
+ return self._env
def run(self, name, /, **kwargs):
"""Run modules from the GRASS display family (modules starting with "d.").
@@ -48,7 +109,7 @@ def run(self, name, /, **kwargs):
# alternatively use dev null as default or provide it as convenient settings
kwargs["stdout"] = gs.PIPE
kwargs["stderr"] = gs.PIPE
- process = gs.pipe_command(name, **kwargs)
+ process = gs.pipe_command(name, env=self._env, **kwargs)
stdout, stderr = process.communicate()
stderr = gs.utils.decode(stderr)
returncode = process.poll()
@@ -85,7 +146,7 @@ def wrapper(**kwargs):
def _test():
- gs.setup.init("~/grassdata/nc_spm_08_grass7/user1")
+ session = gs.setup.init("~/grassdata/nc_spm_08_grass7/user1")
tools = Tools()
tools.g_region(raster="elevation")
@@ -99,6 +160,32 @@ def _test():
]["mean"]
)
+ print(tools.g_mapset(flags="p").text)
+ print(tools.g_mapsets(flags="l").text_split())
+ print(tools.g_mapsets(flags="l").space_items)
+ print(tools.g_gisenv(get="GISDBASE,LOCATION_NAME,MAPSET", sep="comma").comma_items)
+
+ print(tools.g_region(flags="g").keyval)
+
+ env = os.environ.copy()
+ env["GRASS_REGION"] = gs.region_env(res=250)
+ coarse_computation = Tools(env=env)
+ current_region = coarse_computation.g_region(flags="g").keyval
+ print(
+ current_region["ewres"], current_region["nsres"]
+ ) # TODO: should keyval convert?
+ coarse_computation.r_slope_aspect(
+ elevation="elevation", slope="slope", flags="a", overwrite=True
+ )
+ print(coarse_computation.r_info(map="slope", flags="g").keyval)
+
+ independent_computation = Tools(session=session, freeze_region=True)
+ tools.g_region(res=500) # we would do this for another computation elsewhere
+ print(independent_computation.g_region(flags="g").keyval["ewres"])
+
+ tools_pro = Tools(session=session, freeze_region=True, superquiet=True)
+ tools_pro.r_slope_aspect(elevation="elevation", slope="slope")
+
if __name__ == "__main__":
_test()
From 54db575a33db12bc3d7e0665da3358c8e9776fba Mon Sep 17 00:00:00 2001
From: Vaclav Petras <wenzeslaus@gmail.com>
Date: Sat, 22 Apr 2023 14:31:14 -0400
Subject: [PATCH 03/29] Raise exception instead of calling handle_errors
---
python/grass/experimental/tools.py | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/python/grass/experimental/tools.py b/python/grass/experimental/tools.py
index 64b103b28fc..ebf5e16bf91 100644
--- a/python/grass/experimental/tools.py
+++ b/python/grass/experimental/tools.py
@@ -114,7 +114,16 @@ def run(self, name, /, **kwargs):
stderr = gs.utils.decode(stderr)
returncode = process.poll()
# TODO: instead of printing, do exception right away
+ # but right now, handle errors does not accept stderr
+ # or don't use handle errors and raise instead
if returncode:
+ raise gs.CalledModuleError(
+ name,
+ code=" ".join([f"{key}={value}" for key, value in kwargs.items()]),
+ returncode=returncode,
+ errors=stderr,
+ )
+
# Print only when we are capturing it and there was some output.
# (User can request ignoring the subprocess stderr and then
# we get only None.)
From 82f5894cf324700e9e7642f4e49cfaedd587d8e8 Mon Sep 17 00:00:00 2001
From: Vaclav Petras <wenzeslaus@gmail.com>
Date: Sat, 22 Apr 2023 14:33:29 -0400
Subject: [PATCH 04/29] Allow to specify stdin and use a new instance of Tools
itself to execute with that stdin
---
python/grass/experimental/tools.py | 66 ++++++++++++++++++++++++++----
1 file changed, 59 insertions(+), 7 deletions(-)
diff --git a/python/grass/experimental/tools.py b/python/grass/experimental/tools.py
index ebf5e16bf91..7bab6c2a4c5 100644
--- a/python/grass/experimental/tools.py
+++ b/python/grass/experimental/tools.py
@@ -9,7 +9,10 @@ class ExecutedTool:
def __init__(self, name, kwargs, stdout, stderr):
self._name = name
self._stdout = stdout
- self._decoded_stdout = gs.decode(self._stdout)
+ if self._stdout:
+ self._decoded_stdout = gs.decode(self._stdout)
+ else:
+ self._decoded_stdout = ""
@property
def text(self):
@@ -45,8 +48,14 @@ class SubExecutor:
"""use as tools().params(a="x", b="y").g_region()"""
# a and b would be overwrite or stdin
-
# Can support other envs or all PIPE and encoding read command supports
+ def __init__(self, *, tools, env, stdin=None):
+ self._tools = tools
+ self._env = env
+ self._stdin = stdin
+
+ def run(self, name, /, **kwargs):
+ pass
class Tools:
@@ -60,6 +69,7 @@ def __init__(
verbose=False,
superquiet=False,
freeze_region=False,
+ stdin=None,
):
# TODO: fix region, so that external g.region call in the middle
# is not a problem
@@ -85,6 +95,7 @@ def __init__(
self._env["GRASS_VERBOSE"] = "1"
if verbose:
self._env["GRASS_VERBOSE"] = "3"
+ self._set_stdin(stdin)
# These could be public, not protected.
def _freeze_region(self):
@@ -94,6 +105,10 @@ def _freeze_region(self):
def _overwrite(self):
self._env["GRASS_OVERWRITE"] = "1"
+ def _set_stdin(self, stdin, /):
+ print("_set_stdin", stdin)
+ self._stdin = stdin
+
@property
def env(self):
return self._env
@@ -107,10 +122,23 @@ def run(self, name, /, **kwargs):
:param str module: name of GRASS module
:param `**kwargs`: named arguments passed to run_command()"""
# alternatively use dev null as default or provide it as convenient settings
- kwargs["stdout"] = gs.PIPE
- kwargs["stderr"] = gs.PIPE
- process = gs.pipe_command(name, env=self._env, **kwargs)
- stdout, stderr = process.communicate()
+ stdout_pipe = gs.PIPE
+ stderr_pipe = gs.PIPE
+ if self._stdin:
+ stdin_pipe = gs.PIPE
+ stdin = gs.utils.encode(self._stdin)
+ else:
+ stdin_pipe = None
+ stdin = None
+ process = gs.start_command(
+ name,
+ env=self._env,
+ **kwargs,
+ stdin=stdin_pipe,
+ stdout=stdout_pipe,
+ stderr=stderr_pipe,
+ )
+ stdout, stderr = process.communicate(input=stdin)
stderr = gs.utils.decode(stderr)
returncode = process.poll()
# TODO: instead of printing, do exception right away
@@ -131,6 +159,11 @@ def run(self, name, /, **kwargs):
sys.stderr.write(stderr)
gs.handle_errors(returncode, stdout, [name], kwargs)
return ExecutedTool(name=name, kwargs=kwargs, stdout=stdout, stderr=stderr)
+ # executor = SubExecutor(tools=self, env=self._env, stdin=self._stdin)
+ # return executor.run(name, **kwargs)
+
+ def feed_input_to(self, stdin, /):
+ return Tools(env=self._env, stdin=stdin)
def __getattr__(self, name):
"""Parse attribute to GRASS display module. Attribute should be in
@@ -192,8 +225,27 @@ def _test():
tools.g_region(res=500) # we would do this for another computation elsewhere
print(independent_computation.g_region(flags="g").keyval["ewres"])
- tools_pro = Tools(session=session, freeze_region=True, superquiet=True)
+ tools_pro = Tools(
+ session=session, freeze_region=True, overwrite=True, superquiet=True
+ )
+ # gs.feed_command("v.in.ascii",
+ # input="-", output="point", separator=",",
+ # stdin="13.45,29.96,200", overwrite=True)
tools_pro.r_slope_aspect(elevation="elevation", slope="slope")
+ tools_pro.feed_input_to("13.45,29.96,200").v_in_ascii(
+ input="-", output="point", separator=","
+ )
+ print(tools_pro.v_info(map="point", flags="t").keyval["points"])
+
+ # try:
+ tools_pro.feed_input_to("13.45,29.96,200").v_in_ascii(
+ input="-",
+ output="point",
+ format="xstandard",
+ )
+ # except gs.CalledModuleError as error:
+ # print("Exception text:")
+ # print(error)
if __name__ == "__main__":
From 0f1e210a9148bb7ffe5f9411712ca20d71f64efc Mon Sep 17 00:00:00 2001
From: Vaclav Petras <wenzeslaus@gmail.com>
Date: Sat, 22 Apr 2023 17:56:57 -0400
Subject: [PATCH 05/29] Add ignore errors, r_mapcalc example, draft tests
---
.../experimental/tests/grass_tools_test.py | 26 +++++++++++++++++++
python/grass/experimental/tools.py | 14 +++++++++-
2 files changed, 39 insertions(+), 1 deletion(-)
create mode 100644 python/grass/experimental/tests/grass_tools_test.py
diff --git a/python/grass/experimental/tests/grass_tools_test.py b/python/grass/experimental/tests/grass_tools_test.py
new file mode 100644
index 00000000000..7a39c187e40
--- /dev/null
+++ b/python/grass/experimental/tests/grass_tools_test.py
@@ -0,0 +1,26 @@
+from grass.experimental.tools import Tools
+
+
+def test_key_value_parser(xy_dataset_session):
+ tools = Tools(session=xy_dataset_session)
+ assert tools.g_region(flags="g").keyval["nsres"] == 1
+
+
+# def test_json_parser(xy_dataset_session):
+# print(
+# tools.v_db_univar(map="bridges", column="YEAR_BUILT", format="json").json[
+# "statistics"
+# ]["mean"]
+# )
+
+# def test_direct_overwrite(xy_dataset_session):
+# tools = Tools(session=xy_dataset_session)
+# tools.r_slope_aspect(elevation="elevation", slope="slope")
+# tools.r_slope_aspect(elevation="elevation", slope="slope", overwrite=True)
+
+
+def test_stdin(xy_dataset_session):
+ tools = Tools(session=xy_dataset_session)
+ tools.feed_input_to("13.45,29.96,200").v_in_ascii(
+ input="-", output="point", separator=","
+ )
diff --git a/python/grass/experimental/tools.py b/python/grass/experimental/tools.py
index 7bab6c2a4c5..964194e8ac4 100644
--- a/python/grass/experimental/tools.py
+++ b/python/grass/experimental/tools.py
@@ -70,6 +70,7 @@ def __init__(
superquiet=False,
freeze_region=False,
stdin=None,
+ errors=None,
):
# TODO: fix region, so that external g.region call in the middle
# is not a problem
@@ -96,6 +97,7 @@ def __init__(
if verbose:
self._env["GRASS_VERBOSE"] = "3"
self._set_stdin(stdin)
+ self._errors = errors
# These could be public, not protected.
def _freeze_region(self):
@@ -144,7 +146,7 @@ def run(self, name, /, **kwargs):
# TODO: instead of printing, do exception right away
# but right now, handle errors does not accept stderr
# or don't use handle errors and raise instead
- if returncode:
+ if returncode and self._errors != "ignore":
raise gs.CalledModuleError(
name,
code=" ".join([f"{key}={value}" for key, value in kwargs.items()]),
@@ -165,6 +167,9 @@ def run(self, name, /, **kwargs):
def feed_input_to(self, stdin, /):
return Tools(env=self._env, stdin=stdin)
+ def ignore_errors_of(self):
+ return Tools(env=self._env, errors="ignore")
+
def __getattr__(self, name):
"""Parse attribute to GRASS display module. Attribute should be in
the form 'd_module_name'. For example, 'd.rast' is called with 'd_rast'.
@@ -237,6 +242,13 @@ def _test():
)
print(tools_pro.v_info(map="point", flags="t").keyval["points"])
+ print(tools_pro.ignore_errors_of().g_version(flags="rge").keyval)
+
+ elevation = "elevation"
+ exaggerated = "exaggerated"
+ tools_pro.r_mapcalc(expression=f"{exaggerated} = 5 * {elevation}")
+ tools_pro.feed_input_to(f"{exaggerated} = 5 * {elevation}").r_mapcalc(file="-")
+
# try:
tools_pro.feed_input_to("13.45,29.96,200").v_in_ascii(
input="-",
From f4e3fede7578c74d7710f1997995afa501d8ff7b Mon Sep 17 00:00:00 2001
From: Vaclav Petras <wenzeslaus@gmail.com>
Date: Mon, 24 Apr 2023 15:58:35 -0400
Subject: [PATCH 06/29] Add test for exceptions
---
python/grass/experimental/tests/grass_tools_test.py | 13 +++++++++++++
1 file changed, 13 insertions(+)
diff --git a/python/grass/experimental/tests/grass_tools_test.py b/python/grass/experimental/tests/grass_tools_test.py
index 7a39c187e40..3d471373176 100644
--- a/python/grass/experimental/tests/grass_tools_test.py
+++ b/python/grass/experimental/tests/grass_tools_test.py
@@ -1,3 +1,6 @@
+import pytest
+
+import grass.script as gs
from grass.experimental.tools import Tools
@@ -24,3 +27,13 @@ def test_stdin(xy_dataset_session):
tools.feed_input_to("13.45,29.96,200").v_in_ascii(
input="-", output="point", separator=","
)
+
+
+def test_raises(xy_dataset_session):
+ tools = Tools(session=xy_dataset_session)
+ with pytest.raises(gs.CalledModuleError, match="xstandard"):
+ tools.feed_input_to("13.45,29.96,200").v_in_ascii(
+ input="-",
+ output="point",
+ format="xstandard",
+ )
From 04087e827c51baffef88e20bb4d4751d29d9d0be Mon Sep 17 00:00:00 2001
From: Vaclav Petras <wenzeslaus@gmail.com>
Date: Thu, 4 May 2023 11:53:11 -0400
Subject: [PATCH 07/29] Add tests and Makefile
---
python/grass/experimental/Makefile | 20 +++
python/grass/experimental/tests/conftest.py | 11 ++
.../experimental/tests/grass_tools_test.py | 115 ++++++++++++++++--
3 files changed, 133 insertions(+), 13 deletions(-)
create mode 100644 python/grass/experimental/Makefile
create mode 100644 python/grass/experimental/tests/conftest.py
diff --git a/python/grass/experimental/Makefile b/python/grass/experimental/Makefile
new file mode 100644
index 00000000000..2ce55963c3c
--- /dev/null
+++ b/python/grass/experimental/Makefile
@@ -0,0 +1,20 @@
+MODULE_TOPDIR = ../../..
+
+include $(MODULE_TOPDIR)/include/Make/Other.make
+include $(MODULE_TOPDIR)/include/Make/Python.make
+
+DSTDIR = $(ETC)/python/grass/experimental
+
+MODULES = \
+ tools
+
+PYFILES := $(patsubst %,$(DSTDIR)/%.py,$(MODULES) __init__)
+PYCFILES := $(patsubst %,$(DSTDIR)/%.pyc,$(MODULES) __init__)
+
+default: $(PYFILES) $(PYCFILES)
+
+$(DSTDIR):
+ $(MKDIR) $@
+
+$(DSTDIR)/%: % | $(DSTDIR)
+ $(INSTALL_DATA) $< $@
diff --git a/python/grass/experimental/tests/conftest.py b/python/grass/experimental/tests/conftest.py
new file mode 100644
index 00000000000..9f8ad95b436
--- /dev/null
+++ b/python/grass/experimental/tests/conftest.py
@@ -0,0 +1,11 @@
+import pytest
+
+import grass.script as gs
+
+
+@pytest.fixture
+def xy_dataset_session(tmp_path):
+ """Creates a session with a mapset which has vector with a float column"""
+ gs.core._create_location_xy(tmp_path, "test") # pylint: disable=protected-access
+ with gs.setup.init(tmp_path / "test") as session:
+ yield session
diff --git a/python/grass/experimental/tests/grass_tools_test.py b/python/grass/experimental/tests/grass_tools_test.py
index 3d471373176..4088a3ce0a8 100644
--- a/python/grass/experimental/tests/grass_tools_test.py
+++ b/python/grass/experimental/tests/grass_tools_test.py
@@ -1,25 +1,113 @@
+import os
import pytest
import grass.script as gs
from grass.experimental.tools import Tools
-def test_key_value_parser(xy_dataset_session):
+def test_key_value_parser_number(xy_dataset_session):
+ """Check that numbers are parsed as numbers"""
tools = Tools(session=xy_dataset_session)
assert tools.g_region(flags="g").keyval["nsres"] == 1
-# def test_json_parser(xy_dataset_session):
-# print(
-# tools.v_db_univar(map="bridges", column="YEAR_BUILT", format="json").json[
-# "statistics"
-# ]["mean"]
-# )
+@pytest.mark.fails
+def test_key_value_parser_multiple_values(xy_dataset_session):
+ """Check that strings and floats are parsed"""
+ tools = Tools(session=xy_dataset_session)
+ name = "surface"
+ tools.r_surf_gauss(output=name) # needs seed
+ result = tools.r_info(map=name, flags="g").keyval
+ assert result["datatype"] == "DCELL"
+ assert result["nsres"] == 1
+ result = tools.r_univar(map=name, flags="g").keyval
+ assert result["mean"] == pytest.approx(-0.756762744552762)
+
+
+def test_json_parser(xy_dataset_session):
+ """Check that JSON is parsed"""
+ tools = Tools(session=xy_dataset_session)
+ assert (
+ tools.g_search_modules(keyword="random", flags="j").json[0]["name"]
+ == "r.random"
+ )
+
+
+def test_stdout_as_text(xy_dataset_session):
+ """Check that simple text is parsed and has no whitespace"""
+ tools = Tools(session=xy_dataset_session)
+ assert tools.g_mapset(flags="p").text == "PERMANENT"
-# def test_direct_overwrite(xy_dataset_session):
-# tools = Tools(session=xy_dataset_session)
-# tools.r_slope_aspect(elevation="elevation", slope="slope")
-# tools.r_slope_aspect(elevation="elevation", slope="slope", overwrite=True)
+
+def test_stdout_as_space_items(xy_dataset_session):
+ """Check that whitespace-separated items are parsed"""
+ tools = Tools(session=xy_dataset_session)
+ assert tools.g_mapset(flags="l").space_items == ["PERMANENT"]
+
+
+def test_stdout_split_whitespace(xy_dataset_session):
+ tools = Tools(session=xy_dataset_session)
+ assert tools.g_mapset(flags="l").text_split() == ["PERMANENT"]
+
+
+def test_stdout_split_space(xy_dataset_session):
+ tools = Tools(session=xy_dataset_session)
+ # Not a good example usage, but it tests the functionality.
+ assert tools.g_mapset(flags="l").text_split(" ") == ["PERMANENT", ""]
+
+
+def test_direct_overwrite(xy_dataset_session):
+ """Check overwrite as a parameter"""
+ tools = Tools(session=xy_dataset_session)
+ tools.r_random_surface(output="surface", seed=42)
+ tools.r_random_surface(output="surface", seed=42, overwrite=True)
+
+
+def test_object_overwrite(xy_dataset_session):
+ """Check overwrite as parameter of the tools object"""
+ tools = Tools(session=xy_dataset_session, overwrite=True)
+ tools.r_random_surface(output="surface", seed=42)
+ tools.r_random_surface(output="surface", seed=42)
+
+
+def test_no_overwrite(xy_dataset_session):
+ """Check that it fails without overwrite"""
+ tools = Tools(session=xy_dataset_session)
+ tools.r_random_surface(output="surface", seed=42)
+ with pytest.raises(gs.CalledModuleError, match="overwrite"):
+ tools.r_random_surface(output="surface", seed=42)
+
+
+def test_env_overwrite(xy_dataset_session):
+ """Check that overwrite from env parameter is used"""
+ # env = xy_dataset_session.env.copy() # ideally
+ env = os.environ.copy() # for now
+ env["GRASS_OVERWRITE"] = "1"
+ tools = Tools(session=xy_dataset_session, env=env)
+ tools.r_random_surface(output="surface", seed=42)
+ tools.r_random_surface(output="surface", seed=42)
+
+
+def test_global_overwrite_vs_env(xy_dataset_session):
+ """Check that global overwrite is not used when separate env is used"""
+ # env = xy_dataset_session.env.copy() # ideally
+ env = os.environ.copy() # for now
+ os.environ["GRASS_OVERWRITE"] = "1" # change to xy_dataset_session.env
+ tools = Tools(session=xy_dataset_session, env=env)
+ tools.r_random_surface(output="surface", seed=42)
+ with pytest.raises(gs.CalledModuleError, match="overwrite"):
+ tools.r_random_surface(output="surface", seed=42)
+ del os.environ["GRASS_OVERWRITE"] # check or ideally remove this
+
+
+def test_global_overwrite_vs_init(xy_dataset_session):
+ """Check that global overwrite is not used when separate env is used"""
+ tools = Tools(session=xy_dataset_session)
+ os.environ["GRASS_OVERWRITE"] = "1" # change to xy_dataset_session.env
+ tools.r_random_surface(output="surface", seed=42)
+ with pytest.raises(gs.CalledModuleError, match="overwrite"):
+ tools.r_random_surface(output="surface", seed=42)
+ del os.environ["GRASS_OVERWRITE"] # check or ideally remove this
def test_stdin(xy_dataset_session):
@@ -31,9 +119,10 @@ def test_stdin(xy_dataset_session):
def test_raises(xy_dataset_session):
tools = Tools(session=xy_dataset_session)
- with pytest.raises(gs.CalledModuleError, match="xstandard"):
+ wrong_name = "wrong_standard"
+ with pytest.raises(gs.CalledModuleError, match=wrong_name):
tools.feed_input_to("13.45,29.96,200").v_in_ascii(
input="-",
output="point",
- format="xstandard",
+ format=wrong_name,
)
From 6ab8e40d68e211d44666279c1a51aaa0cf792b6b Mon Sep 17 00:00:00 2001
From: Vaclav Petras <wenzeslaus@gmail.com>
Date: Thu, 4 May 2023 11:54:21 -0400
Subject: [PATCH 08/29] Convert values to ints and floats in keyval
---
python/grass/experimental/tools.py | 15 ++++++++++++---
1 file changed, 12 insertions(+), 3 deletions(-)
diff --git a/python/grass/experimental/tools.py b/python/grass/experimental/tools.py
index 964194e8ac4..9ffdea873fa 100644
--- a/python/grass/experimental/tools.py
+++ b/python/grass/experimental/tools.py
@@ -26,9 +26,18 @@ def json(self):
@property
def keyval(self):
- # TODO: possibly use or add _text_to_key_value_dict
- # which converts int and float automatically
- return gs.parse_key_val(self._stdout)
+ def conversion(value):
+ try:
+ return int(value)
+ except ValueError:
+ pass
+ try:
+ return float(value)
+ except ValueError:
+ pass
+ return value
+
+ return gs.parse_key_val(self._stdout, val_type=conversion)
@property
def comma_items(self):
From 744cfacce9c11ad830226c9741f5cb3516539eac Mon Sep 17 00:00:00 2001
From: Vaclav Petras <wenzeslaus@gmail.com>
Date: Thu, 4 May 2023 11:54:53 -0400
Subject: [PATCH 09/29] Do not overwrite by default to follow default behavior
in GRASS GIS
---
python/grass/experimental/tools.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/python/grass/experimental/tools.py b/python/grass/experimental/tools.py
index 9ffdea873fa..54287bd91cb 100644
--- a/python/grass/experimental/tools.py
+++ b/python/grass/experimental/tools.py
@@ -73,7 +73,7 @@ def __init__(
*,
session=None,
env=None,
- overwrite=True,
+ overwrite=False,
quiet=False,
verbose=False,
superquiet=False,
From 24c27e62a61461f357a04034bf1378ef88505a1d Mon Sep 17 00:00:00 2001
From: Vaclav Petras <wenzeslaus@gmail.com>
Date: Sun, 4 Jun 2023 00:55:34 +0200
Subject: [PATCH 10/29] Add doc, remove old code and todos
---
python/grass/experimental/tools.py | 79 +++++++++++++++++-------------
1 file changed, 45 insertions(+), 34 deletions(-)
diff --git a/python/grass/experimental/tools.py b/python/grass/experimental/tools.py
index 54287bd91cb..e7641792d2c 100644
--- a/python/grass/experimental/tools.py
+++ b/python/grass/experimental/tools.py
@@ -1,32 +1,59 @@
+#!/usr/bin/env python
+
+##############################################################################
+# AUTHOR(S): Vaclav Petras <wenzeslaus gmail com>
+#
+# PURPOSE: API to call GRASS tools (modules) as Python functions
+#
+# COPYRIGHT: (C) 2023 Vaclav Petras and the GRASS Development Team
+#
+# This program is free software under the GNU General Public
+# License (>=v2). Read the file COPYING that comes with GRASS
+# for details.
+##############################################################################
+
+"""API to call GRASS tools (modules) as Python functions"""
+
+import json
import os
-import sys
import shutil
import grass.script as gs
class ExecutedTool:
+ """Result returned after executing a tool"""
+
def __init__(self, name, kwargs, stdout, stderr):
self._name = name
+ self._kwargs = kwargs
self._stdout = stdout
+ self._stderr = stderr
if self._stdout:
self._decoded_stdout = gs.decode(self._stdout)
else:
self._decoded_stdout = ""
@property
- def text(self):
+ def text(self) -> str:
+ """Text output as decoded string"""
return self._decoded_stdout.strip()
@property
def json(self):
- import json
+ """Text output read as JSON
+ This returns the nested structure of dictionaries and lists or fails when
+ the output is not JSON.
+ """
return json.loads(self._stdout)
@property
def keyval(self):
+ """Text output read as key-value pairs separated by equal signs"""
+
def conversion(value):
+ """Convert text to int or float if possible, otherwise return it as is"""
try:
return int(value)
except ValueError:
@@ -41,33 +68,30 @@ def conversion(value):
@property
def comma_items(self):
+ """Text output read as comma-separated list"""
return self.text_split(",")
@property
def space_items(self):
+ """Text output read as whitespace-separated list"""
return self.text_split(None)
def text_split(self, separator=None):
+ """Parse text output read as list separated by separators
+
+ Any leading or trailing newlines are removed prior to parsing.
+ """
# The use of strip is assuming that the output is one line which
# ends with a newline character which is for display only.
return self._decoded_stdout.strip("\n").split(separator)
-class SubExecutor:
- """use as tools().params(a="x", b="y").g_region()"""
-
- # a and b would be overwrite or stdin
- # Can support other envs or all PIPE and encoding read command supports
- def __init__(self, *, tools, env, stdin=None):
- self._tools = tools
- self._env = env
- self._stdin = stdin
-
- def run(self, name, /, **kwargs):
- pass
+class Tools:
+ """Call GRASS tools as methods
+ GRASS tools (modules) can be executed as methods of this class.
+ """
-class Tools:
def __init__(
self,
*,
@@ -81,9 +105,6 @@ def __init__(
stdin=None,
errors=None,
):
- # TODO: fix region, so that external g.region call in the middle
- # is not a problem
- # i.e. region is independent/internal/fixed
if env:
self._env = env.copy()
elif session and hasattr(session, "env"):
@@ -122,6 +143,7 @@ def _set_stdin(self, stdin, /):
@property
def env(self):
+ """Internally used environment (reference to it, not a copy)"""
return self._env
def run(self, name, /, **kwargs):
@@ -152,9 +174,6 @@ def run(self, name, /, **kwargs):
stdout, stderr = process.communicate(input=stdin)
stderr = gs.utils.decode(stderr)
returncode = process.poll()
- # TODO: instead of printing, do exception right away
- # but right now, handle errors does not accept stderr
- # or don't use handle errors and raise instead
if returncode and self._errors != "ignore":
raise gs.CalledModuleError(
name,
@@ -162,21 +181,14 @@ def run(self, name, /, **kwargs):
returncode=returncode,
errors=stderr,
)
-
- # Print only when we are capturing it and there was some output.
- # (User can request ignoring the subprocess stderr and then
- # we get only None.)
- if stderr:
- sys.stderr.write(stderr)
- gs.handle_errors(returncode, stdout, [name], kwargs)
return ExecutedTool(name=name, kwargs=kwargs, stdout=stdout, stderr=stderr)
- # executor = SubExecutor(tools=self, env=self._env, stdin=self._stdin)
- # return executor.run(name, **kwargs)
def feed_input_to(self, stdin, /):
+ """Get a new object which will feed text input to a tool or tools"""
return Tools(env=self._env, stdin=stdin)
def ignore_errors_of(self):
+ """Get a new object which will ignore errors of the called tools"""
return Tools(env=self._env, errors="ignore")
def __getattr__(self, name):
@@ -202,6 +214,7 @@ def wrapper(**kwargs):
def _test():
+ """Ad-hoc tests and examples of the Tools class"""
session = gs.setup.init("~/grassdata/nc_spm_08_grass7/user1")
tools = Tools()
@@ -227,9 +240,7 @@ def _test():
env["GRASS_REGION"] = gs.region_env(res=250)
coarse_computation = Tools(env=env)
current_region = coarse_computation.g_region(flags="g").keyval
- print(
- current_region["ewres"], current_region["nsres"]
- ) # TODO: should keyval convert?
+ print(current_region["ewres"], current_region["nsres"])
coarse_computation.r_slope_aspect(
elevation="elevation", slope="slope", flags="a", overwrite=True
)
From ff187a6e3fe052b01a751ebf759778fe088f8df9 Mon Sep 17 00:00:00 2001
From: Vaclav Petras <wenzeslaus@gmail.com>
Date: Sun, 4 Jun 2023 01:21:47 +0200
Subject: [PATCH 11/29] Add to top Makefile
---
python/grass/Makefile | 1 +
1 file changed, 1 insertion(+)
diff --git a/python/grass/Makefile b/python/grass/Makefile
index 9e34f1281bf..cc54ca583b6 100644
--- a/python/grass/Makefile
+++ b/python/grass/Makefile
@@ -9,6 +9,7 @@ SUBDIRS = \
app \
benchmark \
exceptions \
+ experimental \
grassdb \
gunittest \
imaging \
From 22773c89b696c3f1a4d10cb74bf9b022b197d8d3 Mon Sep 17 00:00:00 2001
From: Vaclav Petras <wenzeslaus@gmail.com>
Date: Sun, 4 Jun 2023 01:25:19 +0200
Subject: [PATCH 12/29] Add docs for tests
---
python/grass/experimental/tests/grass_tools_test.py | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/python/grass/experimental/tests/grass_tools_test.py b/python/grass/experimental/tests/grass_tools_test.py
index 4088a3ce0a8..6c2a2950067 100644
--- a/python/grass/experimental/tests/grass_tools_test.py
+++ b/python/grass/experimental/tests/grass_tools_test.py
@@ -1,3 +1,5 @@
+"""Test grass.experimental.Tools class"""
+
import os
import pytest
@@ -46,11 +48,13 @@ def test_stdout_as_space_items(xy_dataset_session):
def test_stdout_split_whitespace(xy_dataset_session):
+ """Check that whitespace-based split function works"""
tools = Tools(session=xy_dataset_session)
assert tools.g_mapset(flags="l").text_split() == ["PERMANENT"]
def test_stdout_split_space(xy_dataset_session):
+ """Check that the split function works with space"""
tools = Tools(session=xy_dataset_session)
# Not a good example usage, but it tests the functionality.
assert tools.g_mapset(flags="l").text_split(" ") == ["PERMANENT", ""]
@@ -111,6 +115,7 @@ def test_global_overwrite_vs_init(xy_dataset_session):
def test_stdin(xy_dataset_session):
+ """Test that stdin is accepted"""
tools = Tools(session=xy_dataset_session)
tools.feed_input_to("13.45,29.96,200").v_in_ascii(
input="-", output="point", separator=","
@@ -118,6 +123,7 @@ def test_stdin(xy_dataset_session):
def test_raises(xy_dataset_session):
+ """Test that exception is raised for wrong parameter value"""
tools = Tools(session=xy_dataset_session)
wrong_name = "wrong_standard"
with pytest.raises(gs.CalledModuleError, match=wrong_name):
From 29110652bba1519ab43e95e7506db3ea398e5f2a Mon Sep 17 00:00:00 2001
From: Vaclav Petras <wenzeslaus@gmail.com>
Date: Sun, 4 Jun 2023 10:32:55 +0200
Subject: [PATCH 13/29] Allow test to fail because of the missing seed
parameter (so results are different now)
---
python/grass/experimental/tests/grass_tools_test.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/python/grass/experimental/tests/grass_tools_test.py b/python/grass/experimental/tests/grass_tools_test.py
index 6c2a2950067..c311c5f5855 100644
--- a/python/grass/experimental/tests/grass_tools_test.py
+++ b/python/grass/experimental/tests/grass_tools_test.py
@@ -13,7 +13,7 @@ def test_key_value_parser_number(xy_dataset_session):
assert tools.g_region(flags="g").keyval["nsres"] == 1
-@pytest.mark.fails
+@pytest.mark.xfail
def test_key_value_parser_multiple_values(xy_dataset_session):
"""Check that strings and floats are parsed"""
tools = Tools(session=xy_dataset_session)
From 437d46e24f4d2f8224401ce7bc44abb5e1a10e3e Mon Sep 17 00:00:00 2001
From: Vaclav Petras <wenzeslaus@gmail.com>
Date: Wed, 23 Apr 2025 16:40:10 -0400
Subject: [PATCH 14/29] Allow for optional output capture (error handling and
printing still needs to be improved there). Allow usage through attributes,
run with run_command syntax, and subprocess-like execution.
---
.../experimental/tests/grass_tools_test.py | 10 ++---
python/grass/experimental/tools.py | 38 +++++++++++++------
2 files changed, 32 insertions(+), 16 deletions(-)
diff --git a/python/grass/experimental/tests/grass_tools_test.py b/python/grass/experimental/tests/grass_tools_test.py
index c311c5f5855..498b2073332 100644
--- a/python/grass/experimental/tests/grass_tools_test.py
+++ b/python/grass/experimental/tests/grass_tools_test.py
@@ -3,8 +3,8 @@
import os
import pytest
-import grass.script as gs
from grass.experimental.tools import Tools
+from grass.exceptions import CalledModuleError
def test_key_value_parser_number(xy_dataset_session):
@@ -78,7 +78,7 @@ def test_no_overwrite(xy_dataset_session):
"""Check that it fails without overwrite"""
tools = Tools(session=xy_dataset_session)
tools.r_random_surface(output="surface", seed=42)
- with pytest.raises(gs.CalledModuleError, match="overwrite"):
+ with pytest.raises(CalledModuleError, match="overwrite"):
tools.r_random_surface(output="surface", seed=42)
@@ -99,7 +99,7 @@ def test_global_overwrite_vs_env(xy_dataset_session):
os.environ["GRASS_OVERWRITE"] = "1" # change to xy_dataset_session.env
tools = Tools(session=xy_dataset_session, env=env)
tools.r_random_surface(output="surface", seed=42)
- with pytest.raises(gs.CalledModuleError, match="overwrite"):
+ with pytest.raises(CalledModuleError, match="overwrite"):
tools.r_random_surface(output="surface", seed=42)
del os.environ["GRASS_OVERWRITE"] # check or ideally remove this
@@ -109,7 +109,7 @@ def test_global_overwrite_vs_init(xy_dataset_session):
tools = Tools(session=xy_dataset_session)
os.environ["GRASS_OVERWRITE"] = "1" # change to xy_dataset_session.env
tools.r_random_surface(output="surface", seed=42)
- with pytest.raises(gs.CalledModuleError, match="overwrite"):
+ with pytest.raises(CalledModuleError, match="overwrite"):
tools.r_random_surface(output="surface", seed=42)
del os.environ["GRASS_OVERWRITE"] # check or ideally remove this
@@ -126,7 +126,7 @@ def test_raises(xy_dataset_session):
"""Test that exception is raised for wrong parameter value"""
tools = Tools(session=xy_dataset_session)
wrong_name = "wrong_standard"
- with pytest.raises(gs.CalledModuleError, match=wrong_name):
+ with pytest.raises(CalledModuleError, match=wrong_name):
tools.feed_input_to("13.45,29.96,200").v_in_ascii(
input="-",
output="point",
diff --git a/python/grass/experimental/tools.py b/python/grass/experimental/tools.py
index e7641792d2c..898a92f0695 100644
--- a/python/grass/experimental/tools.py
+++ b/python/grass/experimental/tools.py
@@ -19,6 +19,7 @@
import shutil
import grass.script as gs
+from grass.exceptions import CalledModuleError
class ExecutedTool:
@@ -104,6 +105,7 @@ def __init__(
freeze_region=False,
stdin=None,
errors=None,
+ capture_output=True,
):
if env:
self._env = env.copy()
@@ -128,6 +130,7 @@ def __init__(
self._env["GRASS_VERBOSE"] = "3"
self._set_stdin(stdin)
self._errors = errors
+ self._capture_output = capture_output
# These could be public, not protected.
def _freeze_region(self):
@@ -154,34 +157,47 @@ def run(self, name, /, **kwargs):
:param str module: name of GRASS module
:param `**kwargs`: named arguments passed to run_command()"""
+ args, popen_options = gs.popen_args_command(name, **kwargs)
+ return self._execute_tool(args, **popen_options)
+
+ def _execute_tool(self, command, **popen_options):
# alternatively use dev null as default or provide it as convenient settings
- stdout_pipe = gs.PIPE
- stderr_pipe = gs.PIPE
+ if self._capture_output:
+ stdout_pipe = gs.PIPE
+ stderr_pipe = gs.PIPE
+ else:
+ stdout_pipe = None
+ stderr_pipe = None
if self._stdin:
stdin_pipe = gs.PIPE
stdin = gs.utils.encode(self._stdin)
else:
stdin_pipe = None
stdin = None
- process = gs.start_command(
- name,
- env=self._env,
- **kwargs,
+ # Allowing to overwrite env, but that's just to have maximum flexibility when
+ # the session is actually set up, but it may be confusing.
+ if "env" not in popen_options:
+ popen_options["env"] = self._env
+ process = gs.Popen(
+ command,
stdin=stdin_pipe,
stdout=stdout_pipe,
stderr=stderr_pipe,
+ **popen_options,
)
stdout, stderr = process.communicate(input=stdin)
- stderr = gs.utils.decode(stderr)
+ if stderr:
+ stderr = gs.utils.decode(stderr)
returncode = process.poll()
if returncode and self._errors != "ignore":
- raise gs.CalledModuleError(
- name,
- code=" ".join([f"{key}={value}" for key, value in kwargs.items()]),
+ raise CalledModuleError(
+ command[0],
+ code=" ".join(command),
returncode=returncode,
errors=stderr,
)
- return ExecutedTool(name=name, kwargs=kwargs, stdout=stdout, stderr=stderr)
+ # We don't have the keyword arguments to pass to the resulting object.
+ return ExecutedTool(name=command[0], kwargs=None, stdout=stdout, stderr=stderr)
def feed_input_to(self, stdin, /):
"""Get a new object which will feed text input to a tool or tools"""
From 61972d4ce5b79ffe748c7787ca8b839690652f18 Mon Sep 17 00:00:00 2001
From: Vaclav Petras <wenzeslaus@gmail.com>
Date: Fri, 25 Apr 2025 00:14:31 -0400
Subject: [PATCH 15/29] Access JSON as dict directly without an attribute using
getitem. Suggest a tool when there is a close match for the name.
---
.../experimental/tests/grass_tools_test.py | 81 +++++++++++++++++
python/grass/experimental/tools.py | 90 ++++++++++++++-----
2 files changed, 149 insertions(+), 22 deletions(-)
diff --git a/python/grass/experimental/tests/grass_tools_test.py b/python/grass/experimental/tests/grass_tools_test.py
index 498b2073332..2f3a8f202a0 100644
--- a/python/grass/experimental/tests/grass_tools_test.py
+++ b/python/grass/experimental/tests/grass_tools_test.py
@@ -1,8 +1,13 @@
"""Test grass.experimental.Tools class"""
import os
+import json
+
import pytest
+
+import grass.script as gs
+from grass.experimental.mapset import TemporaryMapsetSession
from grass.experimental.tools import Tools
from grass.exceptions import CalledModuleError
@@ -34,6 +39,32 @@ def test_json_parser(xy_dataset_session):
== "r.random"
)
+def test_json_direct_access(xy_dataset_session):
+ """Check that JSON is parsed"""
+ tools = Tools(session=xy_dataset_session)
+ assert (
+ tools.g_search_modules(keyword="random", flags="j")[0]["name"]
+ == "r.random"
+ )
+
+def test_json_direct_access_bad_key_type(xy_dataset_session):
+ """Check that JSON is parsed"""
+ tools = Tools(session=xy_dataset_session)
+ with pytest.raises(TypeError):
+ tools.g_search_modules(keyword="random", flags="j")["name"]
+
+def test_json_direct_access_bad_key_value(xy_dataset_session):
+ """Check that JSON is parsed"""
+ tools = Tools(session=xy_dataset_session)
+ high_number = 100_000_000
+ with pytest.raises(IndexError):
+ tools.g_search_modules(keyword="random", flags="j")[high_number]
+
+def test_json_direct_access_not_json(xy_dataset_session):
+ """Check that JSON is parsed"""
+ tools = Tools(session=xy_dataset_session)
+ with pytest.raises(json.JSONDecodeError):
+ tools.g_search_modules(keyword="random")[0]["name"]
def test_stdout_as_text(xy_dataset_session):
"""Check that simple text is parsed and has no whitespace"""
@@ -132,3 +163,53 @@ def test_raises(xy_dataset_session):
output="point",
format=wrong_name,
)
+
+def test_run_command(xy_dataset_session):
+ """Check run_command and its overwrite parameter"""
+ tools = Tools(session=xy_dataset_session)
+ tools.run_command("r.random.surface", output="surface", seed=42)
+ tools.run_command("r.random.surface", output="surface", seed=42, overwrite=True)
+
+
+def test_parse_command_key_value(xy_dataset_session):
+ tools = Tools(session=xy_dataset_session)
+ assert tools.parse_command("g.region", flags="g")["nsres"] == "1"
+
+
+def test_parse_command_json(xy_dataset_session):
+ tools = Tools(session=xy_dataset_session)
+ assert tools.parse_command("g.region", flags="g", format="json")["region"]["ns-res"] == 1
+
+
+def test_with_context_managers(tmpdir):
+ project = tmpdir / "project"
+ gs.create_project(project)
+ with gs.setup.init(project) as session:
+ tools = Tools(session=session)
+ tools.r_random_surface(output="surface", seed=42)
+ with TemporaryMapsetSession(env=tools.env) as mapset:
+ tools.r_random_surface(output="surface", seed=42, env=mapset.env)
+ with gs.MaskManager(env=mapset.env) as mask:
+ # TODO: Do actual test
+ tools.r_univar(map="surface", env=mask.env, format="json")[0]["mean"]
+
+def test_misspelling(xy_dataset_session):
+ tools = Tools(session=xy_dataset_session)
+ with pytest.raises(AttributeError, match="r.slope.aspect"):
+ tools.r_sloppy_respect()
+
+def test_multiple_suggestions(xy_dataset_session):
+ tools = Tools(session=xy_dataset_session)
+ with pytest.raises(AttributeError, match="v.db.univar|db.univar"):
+ tools.db_v_uni_var()
+
+
+def test_tool_group_vs_model_name(xy_dataset_session):
+ tools = Tools(session=xy_dataset_session)
+ with pytest.raises(AttributeError, match="r.sim.water"):
+ tools.rSIMWEwater()
+
+def test_wrong_attribute(xy_dataset_session):
+ tools = Tools(session=xy_dataset_session)
+ with pytest.raises(AttributeError, match="execute_big_command"):
+ tools.execute_big_command()
diff --git a/python/grass/experimental/tools.py b/python/grass/experimental/tools.py
index 898a92f0695..a31c8e1e2dd 100644
--- a/python/grass/experimental/tools.py
+++ b/python/grass/experimental/tools.py
@@ -5,7 +5,7 @@
#
# PURPOSE: API to call GRASS tools (modules) as Python functions
#
-# COPYRIGHT: (C) 2023 Vaclav Petras and the GRASS Development Team
+# COPYRIGHT: (C) 2023-2025 Vaclav Petras and the GRASS Development Team
#
# This program is free software under the GNU General Public
# License (>=v2). Read the file COPYING that comes with GRASS
@@ -86,6 +86,16 @@ def text_split(self, separator=None):
# ends with a newline character which is for display only.
return self._decoded_stdout.strip("\n").split(separator)
+ def __getitem__(self, name):
+ # TODO: cache parsed JSON
+ if self._stdout:
+ # We are testing just std out and letting rest to the parse and the user.
+ # This makes no assumption about how JSON is produced by the tool.
+ print(self.json, name)
+ return self.json[name]
+ msg = f"Output of the tool {self._name} is not JSON"
+ raise ValueError(msg)
+
class Tools:
"""Call GRASS tools as methods
@@ -158,9 +168,17 @@ def run(self, name, /, **kwargs):
:param str module: name of GRASS module
:param `**kwargs`: named arguments passed to run_command()"""
args, popen_options = gs.popen_args_command(name, **kwargs)
- return self._execute_tool(args, **popen_options)
+ # We approximate tool_kwargs as original kwargs.
+ return self._execute_tool(args, tool_kwargs=kwargs, **popen_options)
+
+ def run_command(self, name, /, **kwargs):
+ # Adjust error handling or provide custom implementation for full control?
+ return gs.run_command(name, **kwargs, env=self._env)
- def _execute_tool(self, command, **popen_options):
+ def parse_command(self, name, /, **kwargs):
+ return gs.parse_command(name, **kwargs, env=self._env)
+
+ def _execute_tool(self, command, tool_kwargs=None, **popen_options):
# alternatively use dev null as default or provide it as convenient settings
if self._capture_output:
stdout_pipe = gs.PIPE
@@ -196,8 +214,11 @@ def _execute_tool(self, command, **popen_options):
returncode=returncode,
errors=stderr,
)
+ # TODO: solve tool_kwargs is None
# We don't have the keyword arguments to pass to the resulting object.
- return ExecutedTool(name=command[0], kwargs=None, stdout=stdout, stderr=stderr)
+ return ExecutedTool(
+ name=command[0], kwargs=tool_kwargs, stdout=stdout, stderr=stderr
+ )
def feed_input_to(self, stdin, /):
"""Get a new object which will feed text input to a tool or tools"""
@@ -207,6 +228,37 @@ def ignore_errors_of(self):
"""Get a new object which will ignore errors of the called tools"""
return Tools(env=self._env, errors="ignore")
+ def levenshtein_distance(self, text1: str, text2: str) -> int:
+ if len(text1) < len(text2):
+ return self.levenshtein_distance(text2, text1)
+
+ if len(text2) == 0:
+ return len(text1)
+
+ previous_row = list(range(len(text2) + 1))
+ for i, char1 in enumerate(text1):
+ current_row = [i + 1]
+ for j, char2 in enumerate(text2):
+ insertions = previous_row[j + 1] + 1
+ deletions = current_row[j] + 1
+ substitutions = previous_row[j] + (char1 != char2)
+ current_row.append(min(insertions, deletions, substitutions))
+ previous_row = current_row
+
+ return previous_row[-1]
+
+ def suggest_tools(self, tool):
+ # TODO: cache commands also for dir
+ all_names = list(gs.get_commands()[0])
+ result = []
+ max_suggestions = 10
+ for name in all_names:
+ if self.levenshtein_distance(tool, name) < len(tool) / 2:
+ result.append(name)
+ if len(result) >= max_suggestions:
+ break
+ return result
+
def __getattr__(self, name):
"""Parse attribute to GRASS display module. Attribute should be in
the form 'd_module_name'. For example, 'd.rast' is called with 'd_rast'.
@@ -215,12 +267,19 @@ def __getattr__(self, name):
grass_module = name.replace("_", ".")
# Assert module exists
if not shutil.which(grass_module):
- raise AttributeError(
- _(
- "Cannot find GRASS tool {}. "
- "Is the session set up and the tool on path?"
- ).format(grass_module)
+ suggesions = self.suggest_tools(grass_module)
+ if suggesions:
+ msg = (
+ f"Tool {grass_module} not found. "
+ f"Did you mean: {', '.join(suggesions)}?"
+ )
+ raise AttributeError(msg)
+ msg = (
+ f"Tool or attribute {name} not found. "
+ "If you are executing a tool, is the session set up and the tool on path? "
+ "If you are looking for an attribute, is it in the documentation?"
)
+ raise AttributeError(msg)
def wrapper(**kwargs):
# Run module
@@ -269,9 +328,6 @@ def _test():
tools_pro = Tools(
session=session, freeze_region=True, overwrite=True, superquiet=True
)
- # gs.feed_command("v.in.ascii",
- # input="-", output="point", separator=",",
- # stdin="13.45,29.96,200", overwrite=True)
tools_pro.r_slope_aspect(elevation="elevation", slope="slope")
tools_pro.feed_input_to("13.45,29.96,200").v_in_ascii(
input="-", output="point", separator=","
@@ -285,16 +341,6 @@ def _test():
tools_pro.r_mapcalc(expression=f"{exaggerated} = 5 * {elevation}")
tools_pro.feed_input_to(f"{exaggerated} = 5 * {elevation}").r_mapcalc(file="-")
- # try:
- tools_pro.feed_input_to("13.45,29.96,200").v_in_ascii(
- input="-",
- output="point",
- format="xstandard",
- )
- # except gs.CalledModuleError as error:
- # print("Exception text:")
- # print(error)
-
if __name__ == "__main__":
_test()
From c86d8ffadfb9935543820ade251c13231a3cbe77 Mon Sep 17 00:00:00 2001
From: Vaclav Petras <wenzeslaus@gmail.com>
Date: Fri, 25 Apr 2025 15:10:06 -0400
Subject: [PATCH 16/29] Fix whitespace and regexp
---
.../experimental/tests/grass_tools_test.py | 29 ++++++++++++-------
1 file changed, 19 insertions(+), 10 deletions(-)
diff --git a/python/grass/experimental/tests/grass_tools_test.py b/python/grass/experimental/tests/grass_tools_test.py
index 2f3a8f202a0..182ab344196 100644
--- a/python/grass/experimental/tests/grass_tools_test.py
+++ b/python/grass/experimental/tests/grass_tools_test.py
@@ -39,13 +39,12 @@ def test_json_parser(xy_dataset_session):
== "r.random"
)
+
def test_json_direct_access(xy_dataset_session):
"""Check that JSON is parsed"""
tools = Tools(session=xy_dataset_session)
- assert (
- tools.g_search_modules(keyword="random", flags="j")[0]["name"]
- == "r.random"
- )
+ assert tools.g_search_modules(keyword="random", flags="j")[0]["name"] == "r.random"
+
def test_json_direct_access_bad_key_type(xy_dataset_session):
"""Check that JSON is parsed"""
@@ -53,6 +52,7 @@ def test_json_direct_access_bad_key_type(xy_dataset_session):
with pytest.raises(TypeError):
tools.g_search_modules(keyword="random", flags="j")["name"]
+
def test_json_direct_access_bad_key_value(xy_dataset_session):
"""Check that JSON is parsed"""
tools = Tools(session=xy_dataset_session)
@@ -60,12 +60,14 @@ def test_json_direct_access_bad_key_value(xy_dataset_session):
with pytest.raises(IndexError):
tools.g_search_modules(keyword="random", flags="j")[high_number]
+
def test_json_direct_access_not_json(xy_dataset_session):
"""Check that JSON is parsed"""
tools = Tools(session=xy_dataset_session)
with pytest.raises(json.JSONDecodeError):
tools.g_search_modules(keyword="random")[0]["name"]
+
def test_stdout_as_text(xy_dataset_session):
"""Check that simple text is parsed and has no whitespace"""
tools = Tools(session=xy_dataset_session)
@@ -164,6 +166,7 @@ def test_raises(xy_dataset_session):
format=wrong_name,
)
+
def test_run_command(xy_dataset_session):
"""Check run_command and its overwrite parameter"""
tools = Tools(session=xy_dataset_session)
@@ -178,7 +181,10 @@ def test_parse_command_key_value(xy_dataset_session):
def test_parse_command_json(xy_dataset_session):
tools = Tools(session=xy_dataset_session)
- assert tools.parse_command("g.region", flags="g", format="json")["region"]["ns-res"] == 1
+ assert (
+ tools.parse_command("g.region", flags="g", format="json")["region"]["ns-res"]
+ == 1
+ )
def test_with_context_managers(tmpdir):
@@ -193,22 +199,25 @@ def test_with_context_managers(tmpdir):
# TODO: Do actual test
tools.r_univar(map="surface", env=mask.env, format="json")[0]["mean"]
+
def test_misspelling(xy_dataset_session):
tools = Tools(session=xy_dataset_session)
- with pytest.raises(AttributeError, match="r.slope.aspect"):
+ with pytest.raises(AttributeError, match=r"r\.slope\.aspect"):
tools.r_sloppy_respect()
-
+
+
def test_multiple_suggestions(xy_dataset_session):
tools = Tools(session=xy_dataset_session)
- with pytest.raises(AttributeError, match="v.db.univar|db.univar"):
- tools.db_v_uni_var()
+ with pytest.raises(AttributeError, match=r"v\.db\.univar|db\.univar"):
+ tools.db_v_uni_var()
def test_tool_group_vs_model_name(xy_dataset_session):
tools = Tools(session=xy_dataset_session)
- with pytest.raises(AttributeError, match="r.sim.water"):
+ with pytest.raises(AttributeError, match=r"r\.sim\.water"):
tools.rSIMWEwater()
+
def test_wrong_attribute(xy_dataset_session):
tools = Tools(session=xy_dataset_session)
with pytest.raises(AttributeError, match="execute_big_command"):
From 3b995c984dc32bd5edc6a4113c76ecb2a4746789 Mon Sep 17 00:00:00 2001
From: Vaclav Petras <wenzeslaus@gmail.com>
Date: Fri, 25 Apr 2025 15:55:35 -0400
Subject: [PATCH 17/29] Represent not captured stdout as None, not empty
string.
---
.../experimental/tests/grass_tools_test.py | 7 ++++++
python/grass/experimental/tools.py | 23 +++++++++++--------
2 files changed, 20 insertions(+), 10 deletions(-)
diff --git a/python/grass/experimental/tests/grass_tools_test.py b/python/grass/experimental/tests/grass_tools_test.py
index 182ab344196..d02b5337fa3 100644
--- a/python/grass/experimental/tests/grass_tools_test.py
+++ b/python/grass/experimental/tests/grass_tools_test.py
@@ -93,6 +93,13 @@ def test_stdout_split_space(xy_dataset_session):
assert tools.g_mapset(flags="l").text_split(" ") == ["PERMANENT", ""]
+def test_stdout_without_capturing(xy_dataset_session):
+ """Check that text is not present when not capturing it"""
+ tools = Tools(session=xy_dataset_session, capture_output=False)
+ assert not tools.g_mapset(flags="p").text
+ assert tools.g_mapset(flags="p").text is None
+
+
def test_direct_overwrite(xy_dataset_session):
"""Check overwrite as a parameter"""
tools = Tools(session=xy_dataset_session)
diff --git a/python/grass/experimental/tools.py b/python/grass/experimental/tools.py
index a31c8e1e2dd..6c4d8fb4a0c 100644
--- a/python/grass/experimental/tools.py
+++ b/python/grass/experimental/tools.py
@@ -30,14 +30,17 @@ def __init__(self, name, kwargs, stdout, stderr):
self._kwargs = kwargs
self._stdout = stdout
self._stderr = stderr
- if self._stdout:
+ if self._stdout is not None:
self._decoded_stdout = gs.decode(self._stdout)
else:
- self._decoded_stdout = ""
+ self._decoded_stdout = None
+ self._cached_json = None
@property
def text(self) -> str:
"""Text output as decoded string"""
+ if self._decoded_stdout is None:
+ return None
return self._decoded_stdout.strip()
@property
@@ -47,7 +50,9 @@ def json(self):
This returns the nested structure of dictionaries and lists or fails when
the output is not JSON.
"""
- return json.loads(self._stdout)
+ if self._cached_json is None:
+ self._cached_json = json.loads(self._stdout)
+ return self._cached_json
@property
def keyval(self):
@@ -91,7 +96,6 @@ def __getitem__(self, name):
if self._stdout:
# We are testing just std out and letting rest to the parse and the user.
# This makes no assumption about how JSON is produced by the tool.
- print(self.json, name)
return self.json[name]
msg = f"Output of the tool {self._name} is not JSON"
raise ValueError(msg)
@@ -151,7 +155,6 @@ def _overwrite(self):
self._env["GRASS_OVERWRITE"] = "1"
def _set_stdin(self, stdin, /):
- print("_set_stdin", stdin)
self._stdin = stdin
@property
@@ -264,13 +267,13 @@ def __getattr__(self, name):
the form 'd_module_name'. For example, 'd.rast' is called with 'd_rast'.
"""
# Reformat string
- grass_module = name.replace("_", ".")
+ tool_name = name.replace("_", ".")
# Assert module exists
- if not shutil.which(grass_module):
- suggesions = self.suggest_tools(grass_module)
+ if not shutil.which(tool_name):
+ suggesions = self.suggest_tools(tool_name)
if suggesions:
msg = (
- f"Tool {grass_module} not found. "
+ f"Tool {tool_name} not found. "
f"Did you mean: {', '.join(suggesions)}?"
)
raise AttributeError(msg)
@@ -283,7 +286,7 @@ def __getattr__(self, name):
def wrapper(**kwargs):
# Run module
- return self.run(grass_module, **kwargs)
+ return self.run(tool_name, **kwargs)
return wrapper
From 4cc5a325c49abdcd7679982a0dd1f16bbb100901 Mon Sep 17 00:00:00 2001
From: Vaclav Petras <wenzeslaus@gmail.com>
Date: Tue, 29 Apr 2025 13:32:01 -0400
Subject: [PATCH 18/29] Add run subcommand to have a CLI use case for the
tools. It runs one tool in XY project, so useful only for things like
g.extension or m.proj, but, with a significant workaround for argparse
--help, it can do --help for a tool.
---
python/grass/app/cli.py | 98 +++++++++++++++++++++++++++++-
python/grass/experimental/tools.py | 3 +-
2 files changed, 99 insertions(+), 2 deletions(-)
diff --git a/python/grass/app/cli.py b/python/grass/app/cli.py
index c2bc8f40d0b..5f55f414caf 100644
--- a/python/grass/app/cli.py
+++ b/python/grass/app/cli.py
@@ -20,9 +20,48 @@
import tempfile
import os
import sys
+import subprocess
from pathlib import Path
+
import grass.script as gs
+from grass.app.data import lock_mapset, unlock_mapset, MapsetLockingException
+from grass.experimental.tools import Tools
+
+
+def subcommand_run_tool(args, tool_args: list, help: bool):
+ command = [args.tool_name, *tool_args]
+ with tempfile.TemporaryDirectory() as tmp_dir_name:
+ project_name = "project"
+ project_path = Path(tmp_dir_name) / project_name
+ gs.create_project(project_path)
+ with gs.setup.init(project_path) as session:
+ if help:
+ result = subprocess.run(command, env=session.env)
+ return result.returncode
+ tools = Tools(capture_output=False)
+ try:
+ tools.run_from_list(command)
+ except subprocess.CalledProcessError as error:
+ return error.returncode
+
+
+def subcommand_lock_mapset(args):
+ gs.setup.setup_runtime_env()
+ try:
+ lock_mapset(
+ args.mapset_path,
+ force_lock_removal=args.force_remove_lock,
+ timeout=args.timeout,
+ message_callback=print,
+ process_id=args.process_id,
+ )
+ except MapsetLockingException as e:
+ print(str(e), file=sys.stderr)
+
+
+def subcommand_unlock_mapset(args):
+ unlock_mapset(args.mapset_path)
def call_g_manual(**kwargs):
@@ -60,6 +99,41 @@ def main(args=None, program=None):
# Subcommand parsers
+ subparser = subparsers.add_parser("run", help="run a tool")
+ subparser.add_argument("tool_name", type=str)
+ subparser.set_defaults(func=subcommand_run_tool)
+
+ subparser = subparsers.add_parser("lock", help="lock a mapset")
+ subparser.add_argument("mapset_path", type=str)
+ subparser.add_argument(
+ "--process-id",
+ metavar="PID",
+ type=int,
+ default=1,
+ help=_(
+ "process ID of the process locking the mapset (a mapset can be "
+ "automatically unlocked if there is no process with this PID)"
+ ),
+ )
+ subparser.add_argument(
+ "--timeout",
+ metavar="TIMEOUT",
+ type=float,
+ default=30,
+ help=_("mapset locking timeout in seconds"),
+ )
+ subparser.add_argument(
+ "-f",
+ "--force-remove-lock",
+ action="store_true",
+ help=_("remove lock if present"),
+ )
+ subparser.set_defaults(func=subcommand_lock_mapset)
+
+ subparser = subparsers.add_parser("unlock", help="unlock a mapset")
+ subparser.add_argument("mapset_path", type=str)
+ subparser.set_defaults(func=subcommand_unlock_mapset)
+
subparser = subparsers.add_parser(
"help", help="show HTML documentation for a tool or topic"
)
@@ -72,5 +146,27 @@ def main(args=None, program=None):
subparser.add_argument("page", type=str)
subparser.set_defaults(func=subcommand_show_man)
- parsed_args = parser.parse_args(args)
+ # Parsing
+
+ if not args:
+ args = sys.argv[1:]
+ raw_args = args.copy()
+ add_back = None
+ if len(raw_args) > 2 and raw_args[0] == "run":
+ # Getting the --help of tools needs to work around the standard help mechanism
+ # of argparse.
+ # Maybe a better workaround is to use custom --help, action="help", print_help,
+ # and dedicated tool help function complimentary with g.manual subcommand
+ # interface.
+ if "--help" in raw_args[2:]:
+ raw_args.remove("--help")
+ add_back = "--help"
+ elif "--h" in raw_args[2:]:
+ raw_args.remove("--h")
+ add_back = "--h"
+ parsed_args, other_args = parser.parse_known_args(raw_args)
+ if parsed_args.subcommand == "run":
+ if add_back:
+ other_args.append(add_back)
+ return parsed_args.func(parsed_args, other_args, help=bool(add_back))
return parsed_args.func(parsed_args)
diff --git a/python/grass/experimental/tools.py b/python/grass/experimental/tools.py
index 6c4d8fb4a0c..a88a3fd93f2 100644
--- a/python/grass/experimental/tools.py
+++ b/python/grass/experimental/tools.py
@@ -181,7 +181,8 @@ def run_command(self, name, /, **kwargs):
def parse_command(self, name, /, **kwargs):
return gs.parse_command(name, **kwargs, env=self._env)
- def _execute_tool(self, command, tool_kwargs=None, **popen_options):
+ # Make this an overload of run.
+ def run_from_list(self, command, tool_kwargs=None, **popen_options):
# alternatively use dev null as default or provide it as convenient settings
if self._capture_output:
stdout_pipe = gs.PIPE
From 459b2ad1c4c388313b0a51fccdfad496143e764f Mon Sep 17 00:00:00 2001
From: Vaclav Petras <wenzeslaus@gmail.com>
Date: Wed, 30 Apr 2025 18:20:28 -0400
Subject: [PATCH 19/29] Update function name
---
python/grass/experimental/tools.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/python/grass/experimental/tools.py b/python/grass/experimental/tools.py
index a88a3fd93f2..56ec27725a7 100644
--- a/python/grass/experimental/tools.py
+++ b/python/grass/experimental/tools.py
@@ -172,7 +172,7 @@ def run(self, name, /, **kwargs):
:param `**kwargs`: named arguments passed to run_command()"""
args, popen_options = gs.popen_args_command(name, **kwargs)
# We approximate tool_kwargs as original kwargs.
- return self._execute_tool(args, tool_kwargs=kwargs, **popen_options)
+ return self.run_from_list(args, tool_kwargs=kwargs, **popen_options)
def run_command(self, name, /, **kwargs):
# Adjust error handling or provide custom implementation for full control?
From 513c9f8c92baff716db8c19c581bd5d80f624e2e Mon Sep 17 00:00:00 2001
From: Vaclav Petras <wenzeslaus@gmail.com>
Date: Mon, 2 Jun 2025 08:49:40 -0400
Subject: [PATCH 20/29] Add prototype code for numpy support
---
.../experimental/tests/grass_tools_test.py | 35 ++++++++++
python/grass/experimental/tools.py | 64 ++++++++++++++++++-
2 files changed, 98 insertions(+), 1 deletion(-)
diff --git a/python/grass/experimental/tests/grass_tools_test.py b/python/grass/experimental/tests/grass_tools_test.py
index d02b5337fa3..c92b8dd65e4 100644
--- a/python/grass/experimental/tests/grass_tools_test.py
+++ b/python/grass/experimental/tests/grass_tools_test.py
@@ -229,3 +229,38 @@ def test_wrong_attribute(xy_dataset_session):
tools = Tools(session=xy_dataset_session)
with pytest.raises(AttributeError, match="execute_big_command"):
tools.execute_big_command()
+
+import numpy as np
+
+def test_numpy_one_input(xy_dataset_session):
+ """Check that global overwrite is not used when separate env is used"""
+ tools = Tools(session=xy_dataset_session)
+ tools.r_slope_aspect(elevation=np.ones((1, 1)), slope="slope")
+ assert tools.r_info(map="slope", format="json")["datatype"] == "FCELL"
+
+# Other possible ways how to handle the syntax:
+
+# class ToNumpy:
+# pass
+
+# class AsInput:
+# pass
+
+# def test_numpy_one_input(xy_dataset_session):
+# """Check that global overwrite is not used when separate env is used"""
+# tools = Tools(session=xy_dataset_session)
+# tools.r_slope_aspect(elevation=np.ones((1, 1)), slope="slope", aspect="aspect", force_numpy_for_output=True)
+# tools.r_slope_aspect(elevation=np.ones((1, 1)), slope=np.nulls(0,0), aspect="aspect")
+# tools.r_slope_aspect(elevation=np.ones((1, 1)), slope=ToNumpy(), aspect="aspect")
+# tools.r_slope_aspect(elevation=np.ones((1, 1)), slope=np.ndarray, aspect="aspect")
+# tools.r_slope_aspect.ufunc(np.ones((1, 1)), slope=True, aspect=True, overwrite=True) # (np.array, np.array)
+# tools.r_slope_aspect(elevation=np.ones((1, 1)), slope=AsInput, aspect=AsInput) # {"slope": np.array(...), "aspect": np.array(...) }
+# assert tools.r_info(map="slope", format="json")["datatype"] == "FCELL"
+
+def test_numpy_one_input_one_output(xy_dataset_session):
+ """Check that global overwrite is not used when separate env is used"""
+ tools = Tools(session=xy_dataset_session)
+ tools.g_region(rows=2, cols=3)
+ slope = tools.r_slope_aspect(elevation=np.ones((2, 3)), slope=np.ndarray)
+ assert slope.shape == (2, 3)
+ assert slope[0] == 0
diff --git a/python/grass/experimental/tools.py b/python/grass/experimental/tools.py
index 56ec27725a7..ec315a734d5 100644
--- a/python/grass/experimental/tools.py
+++ b/python/grass/experimental/tools.py
@@ -162,6 +162,29 @@ def env(self):
"""Internally used environment (reference to it, not a copy)"""
return self._env
+ def _digest_data_parameters(self, parameters, command):
+ # Uses parameters, but modifies the command.
+ input_rasters = []
+ if "inputs" in parameters:
+ for item in parameters["inputs"]:
+ if item["value"].endswith(".grass_raster"):
+ input_rasters.append(Path(item["value"]))
+ for i, arg in enumerate(command):
+ if arg.startswith(f"{item['param']}="):
+ arg = arg.replace(item["value"], Path(item["value"]).stem)
+ command[i] = arg
+ output_rasters = []
+ if "outputs" in parameters:
+ for item in parameters["outputs"]:
+ if item["value"].endswith(".grass_raster"):
+ output_rasters.append(Path(item["value"]))
+ for i, arg in enumerate(command):
+ if arg.startswith(f"{item['param']}="):
+ arg = arg.replace(item["value"], Path(item["value"]).stem)
+ command[i] = arg
+ return input_rasters, output_rasters
+
+
def run(self, name, /, **kwargs):
"""Run modules from the GRASS display family (modules starting with "d.").
@@ -170,9 +193,48 @@ def run(self, name, /, **kwargs):
:param str module: name of GRASS module
:param `**kwargs`: named arguments passed to run_command()"""
+ original = {}
+ original_outputs = {}
+ import grass.script.array as garray
+ import numpy as np
+ for key, value in kwargs.items():
+ if isinstance(value, np.ndarray):
+ kwargs[key] = "tmp_serialized_array"
+ original[key] = value
+ elif value == np.ndarray:
+ kwargs[key] = "tmp_future_serialized_array"
+ original_outputs[key] = value
+
args, popen_options = gs.popen_args_command(name, **kwargs)
+
+ env = popen_options.get("env", self._env)
+
+ import subprocess
+ parameters = json.loads(
+ subprocess.check_output(
+ [*args, "--json"], text=True, env=env
+ )
+ )
+ if "inputs" in parameters:
+ for param in parameters["inputs"]:
+ if param["param"] not in original:
+ continue
+ map2d = garray.array()
+ print(param)
+ map2d[:] = original[param["param"]]
+ map2d.write("tmp_serialized_array", overwrite=True)
+
# We approximate tool_kwargs as original kwargs.
- return self.run_from_list(args, tool_kwargs=kwargs, **popen_options)
+ result = self.run_from_list(args, tool_kwargs=kwargs, **popen_options)
+
+ if "outputs" in parameters:
+ for param in parameters["outputs"]:
+ if param["param"] not in original_outputs:
+ continue
+ output_array = garray.array("tmp_future_serialized_array")
+ result = output_array
+
+ return result
def run_command(self, name, /, **kwargs):
# Adjust error handling or provide custom implementation for full control?
From 4a1e3745bc3dcccd5c19be5b6bdaaef97c59aff3 Mon Sep 17 00:00:00 2001
From: Vaclav Petras <wenzeslaus@gmail.com>
Date: Wed, 11 Jun 2025 08:35:38 -0400
Subject: [PATCH 21/29] Make the special features standalone objects used by
composition
---
python/grass/experimental/tests/conftest.py | 38 ++
.../experimental/tests/grass_tools_test.py | 230 +++++++++-
python/grass/experimental/tools.py | 425 +++++++++++++-----
python/grass/script/core.py | 9 +-
4 files changed, 562 insertions(+), 140 deletions(-)
diff --git a/python/grass/experimental/tests/conftest.py b/python/grass/experimental/tests/conftest.py
index b33ae757f8e..910036b3717 100644
--- a/python/grass/experimental/tests/conftest.py
+++ b/python/grass/experimental/tests/conftest.py
@@ -77,3 +77,41 @@ def xy_mapset_non_permament(xy_session): # pylint: disable=redefined-outer-name
"test1", create=True, env=xy_session.env
) as session:
yield session
+
+
+@pytest.fixture
+def rows_raster_file3x3(tmp_path):
+ project = tmp_path / "xy_test3x3"
+ gs.create_project(project)
+ with gs.setup.init(project, env=os.environ.copy()) as session:
+ gs.run_command("g.region", rows=3, cols=3, env=session.env)
+ gs.mapcalc("rows = row()", env=session.env)
+ output_file = tmp_path / "rows3x3.grass_raster"
+ gs.run_command(
+ "r.pack",
+ input="rows",
+ output=output_file,
+ flags="c",
+ superquiet=True,
+ env=session.env,
+ )
+ return output_file
+
+
+@pytest.fixture
+def rows_raster_file4x5(tmp_path):
+ project = tmp_path / "xy_test4x5"
+ gs.create_project(project)
+ with gs.setup.init(project, env=os.environ.copy()) as session:
+ gs.run_command("g.region", rows=4, cols=5, env=session.env)
+ gs.mapcalc("rows = row()", env=session.env)
+ output_file = tmp_path / "rows4x5.grass_raster"
+ gs.run_command(
+ "r.pack",
+ input="rows",
+ output=output_file,
+ flags="c",
+ superquiet=True,
+ env=session.env,
+ )
+ return output_file
diff --git a/python/grass/experimental/tests/grass_tools_test.py b/python/grass/experimental/tests/grass_tools_test.py
index c92b8dd65e4..1cba3e9e1c9 100644
--- a/python/grass/experimental/tests/grass_tools_test.py
+++ b/python/grass/experimental/tests/grass_tools_test.py
@@ -1,11 +1,11 @@
"""Test grass.experimental.Tools class"""
import os
-import json
+import io
+import numpy as np
import pytest
-
import grass.script as gs
from grass.experimental.mapset import TemporaryMapsetSession
from grass.experimental.tools import Tools
@@ -40,6 +40,24 @@ def test_json_parser(xy_dataset_session):
)
+def test_json_with_name_and_parameter_call(xy_dataset_session):
+ """Check that JSON is parsed with a name-and-parameters style call"""
+ tools = Tools(session=xy_dataset_session)
+ assert (
+ tools.run("g.search.modules", keyword="random", flags="j")[0]["name"]
+ == "r.random"
+ )
+
+
+def test_json_with_subprocess_run_like_call(xy_dataset_session):
+ """Check that JSON is parsed with a name-and-parameters style call"""
+ tools = Tools(session=xy_dataset_session)
+ assert (
+ tools.run_from_list(["g.search.modules", "keyword=random", "-j"])[0]["name"]
+ == "r.random"
+ )
+
+
def test_json_direct_access(xy_dataset_session):
"""Check that JSON is parsed"""
tools = Tools(session=xy_dataset_session)
@@ -62,9 +80,12 @@ def test_json_direct_access_bad_key_value(xy_dataset_session):
def test_json_direct_access_not_json(xy_dataset_session):
- """Check that JSON is parsed"""
+ """Check that JSON parsing creates an ValueError
+
+ Specifically, this tests the case when format="json" is not set.
+ """
tools = Tools(session=xy_dataset_session)
- with pytest.raises(json.JSONDecodeError):
+ with pytest.raises(ValueError, match=r"format.*json"):
tools.g_search_modules(keyword="random")[0]["name"]
@@ -204,7 +225,7 @@ def test_with_context_managers(tmpdir):
tools.r_random_surface(output="surface", seed=42, env=mapset.env)
with gs.MaskManager(env=mapset.env) as mask:
# TODO: Do actual test
- tools.r_univar(map="surface", env=mask.env, format="json")[0]["mean"]
+ tools.r_univar(map="surface", env=mask.env, format="json")["mean"]
def test_misspelling(xy_dataset_session):
@@ -230,7 +251,6 @@ def test_wrong_attribute(xy_dataset_session):
with pytest.raises(AttributeError, match="execute_big_command"):
tools.execute_big_command()
-import numpy as np
def test_numpy_one_input(xy_dataset_session):
"""Check that global overwrite is not used when separate env is used"""
@@ -238,29 +258,191 @@ def test_numpy_one_input(xy_dataset_session):
tools.r_slope_aspect(elevation=np.ones((1, 1)), slope="slope")
assert tools.r_info(map="slope", format="json")["datatype"] == "FCELL"
-# Other possible ways how to handle the syntax:
-
-# class ToNumpy:
-# pass
-# class AsInput:
-# pass
+# NumPy syntax for outputs
+# While inputs are straightforward, there is several possible ways how to handle
+# syntax for outputs.
+# Output is the type of function for creating NumPy arrays, return value is now the arrays:
+# tools.r_slope_aspect(elevation=np.ones((1, 1)), slope=np.ndarray, aspect=np.array)
+# Output is explicitly requested:
+# tools.r_slope_aspect(elevation=np.ones((1, 1)), slope="slope", aspect="aspect", force_numpy_for_output=True)
+# Output is explicitly requested at the object level:
+# Tools(force_numpy_for_output=True).r_slope_aspect(elevation=np.ones((1, 1)), slope="slope", aspect="aspect")
+# Output is always array or arrays when at least on input is an array:
+# tools.r_slope_aspect(elevation=np.ones((1, 1)), slope="slope", aspect="aspect")
+# An empty array is passed to signal the desired output:
+# tools.r_slope_aspect(elevation=np.ones((1, 1)), slope=np.nulls((0, 0)))
+# An array to be filled with data is passed, the return value is kept as is:
+# tools.r_slope_aspect(elevation=np.ones((1, 1)), slope=np.nulls((1, 1)))
+# NumPy universal function concept can be used explicitly to indicate,
+# possibly more easily allowing for nameless args as opposed to keyword arguments,
+# but outputs still need to be explicitly requested:
+# Returns by value (tuple: (np.array, np.array)):
+# tools.r_slope_aspect.ufunc(np.ones((1, 1)), slope=True, aspect=True)
+# Modifies its arguments in-place:
+# tools.r_slope_aspect.ufunc(np.ones((1, 1)), slope=True, aspect=True, out=(np.array((1, 1)), np.array((1, 1))))
+# Custom signaling classes or objects are passed (assuming empty classes AsNumpy and AsInput):
+# tools.r_slope_aspect(elevation=np.ones((1, 1)), slope=ToNumpy(), aspect=ToNumpy())
+# tools.r_slope_aspect(elevation=np.ones((1, 1)), slope=AsInput, aspect=AsInput)
+# NumPy functions usually return a tuple, for multiple outputs. Universal function does
+# unless the output is written to out parameter which is also provided as a tuple. We
+# have names, so generally, we can return a dictionary:
+# {"slope": np.array(...), "aspect": np.array(...) }.
-# def test_numpy_one_input(xy_dataset_session):
-# """Check that global overwrite is not used when separate env is used"""
-# tools = Tools(session=xy_dataset_session)
-# tools.r_slope_aspect(elevation=np.ones((1, 1)), slope="slope", aspect="aspect", force_numpy_for_output=True)
-# tools.r_slope_aspect(elevation=np.ones((1, 1)), slope=np.nulls(0,0), aspect="aspect")
-# tools.r_slope_aspect(elevation=np.ones((1, 1)), slope=ToNumpy(), aspect="aspect")
-# tools.r_slope_aspect(elevation=np.ones((1, 1)), slope=np.ndarray, aspect="aspect")
-# tools.r_slope_aspect.ufunc(np.ones((1, 1)), slope=True, aspect=True, overwrite=True) # (np.array, np.array)
-# tools.r_slope_aspect(elevation=np.ones((1, 1)), slope=AsInput, aspect=AsInput) # {"slope": np.array(...), "aspect": np.array(...) }
-# assert tools.r_info(map="slope", format="json")["datatype"] == "FCELL"
def test_numpy_one_input_one_output(xy_dataset_session):
- """Check that global overwrite is not used when separate env is used"""
+ """Check that a NumPy array works as input and for signaling output
+
+ It tests that the np.ndarray class is supported to signal output.
+ Return type is not strictly defined, so we are not testing for it explicitly
+ (only by actually using it as an NumPy array).
+ """
tools = Tools(session=xy_dataset_session)
tools.g_region(rows=2, cols=3)
slope = tools.r_slope_aspect(elevation=np.ones((2, 3)), slope=np.ndarray)
assert slope.shape == (2, 3)
- assert slope[0] == 0
+ assert np.all(slope == np.full((2, 3), 0))
+
+
+def test_numpy_with_name_and_parameter(xy_dataset_session):
+ """Check that a NumPy array works as input and for signaling output
+
+ It tests that the np.ndarray class is supported to signal output.
+ Return type is not strictly defined, so we are not testing for it explicitly
+ (only by actually using it as an NumPy array).
+ """
+ tools = Tools(session=xy_dataset_session)
+ tools.g_region(rows=2, cols=3)
+ slope = tools.run("r.slope.aspect", elevation=np.ones((2, 3)), slope=np.ndarray)
+ assert slope.shape == (2, 3)
+ assert np.all(slope == np.full((2, 3), 0))
+
+
+def test_numpy_one_input_multiple_outputs(xy_dataset_session):
+ """Check that a NumPy array function works for signaling multiple outputs
+
+ Besides multiple outputs it tests that np.array is supported to signal output.
+ """
+ tools = Tools(session=xy_dataset_session)
+ tools.g_region(rows=2, cols=3)
+ (slope, aspect) = tools.r_slope_aspect(
+ elevation=np.ones((2, 3)), slope=np.array, aspect=np.array
+ )
+ assert slope.shape == (2, 3)
+ assert np.all(slope == np.full((2, 3), 0))
+ assert aspect.shape == (2, 3)
+ assert np.all(aspect == np.full((2, 3), 0))
+
+
+def test_numpy_multiple_inputs_one_output(xy_dataset_session):
+ """Check that a NumPy array works for multiple inputs"""
+ tools = Tools(session=xy_dataset_session)
+ tools.g_region(rows=2, cols=3)
+ result = tools.r_mapcalc_simple(
+ expression="A + B", a=np.full((2, 3), 2), b=np.full((2, 3), 5), output=np.array
+ )
+ assert result.shape == (2, 3)
+ assert np.all(result == np.full((2, 3), 7))
+
+
+def test_numpy_grass_array_input_output(xy_dataset_session):
+ """Check that global overwrite is not used when separate env is used
+
+ When grass array output is requested, we explicitly test the return value type.
+ """
+ tools = Tools(session=xy_dataset_session)
+ rows = 2
+ cols = 3
+ tools.g_region(rows=rows, cols=cols)
+ tools.r_mapcalc_simple(expression="5", output="const_5")
+ const_5 = gs.array.array("const_5")
+ result = tools.r_mapcalc_simple(
+ expression="2 * A", a=const_5, output=gs.array.array
+ )
+ assert result.shape == (rows, cols)
+ assert np.all(result == np.full((rows, cols), 10))
+ assert isinstance(result, gs.array.array)
+
+
+def test_pack_input_output(xy_dataset_session, rows_raster_file3x3):
+ """Check that global overwrite is not used when separate env is used"""
+ tools = Tools(session=xy_dataset_session)
+ tools.g_region(rows=3, cols=3)
+ assert os.path.exists(rows_raster_file3x3)
+ tools.r_slope_aspect(elevation=rows_raster_file3x3, slope="file.grass_raster")
+ assert os.path.exists("file.grass_raster")
+
+
+def test_pack_input_output_with_name_and_parameter_call(
+ xy_dataset_session, rows_raster_file3x3
+):
+ """Check that global overwrite is not used when separate env is used"""
+ tools = Tools(session=xy_dataset_session)
+ tools.g_region(rows=3, cols=3)
+ assert os.path.exists(rows_raster_file3x3)
+ tools.run(
+ "r.slope.aspect", elevation=rows_raster_file3x3, slope="file.grass_raster"
+ )
+ assert os.path.exists("file.grass_raster")
+
+
+def test_pack_input_output_with_subprocess_run_like_call(
+ xy_dataset_session, rows_raster_file3x3
+):
+ tools = Tools(session=xy_dataset_session)
+ assert os.path.exists(rows_raster_file3x3)
+ tools.run_from_list(
+ [
+ "r.slope.aspect",
+ f"elevation={rows_raster_file3x3}",
+ "aspect=file.grass_raster",
+ ]
+ )
+ assert os.path.exists("file.grass_raster")
+
+
+def test_tool_groups_raster(xy_dataset_session):
+ """Check that global overwrite is not used when separate env is used"""
+ raster = Tools(session=xy_dataset_session, prefix="r")
+ raster.mapcalc(expression="streams = if(row() > 1, 1, null())")
+ raster.buffer(input="streams", output="buffer", distance=1)
+ assert raster.info(map="streams", format="json")["datatype"] == "CELL"
+
+
+def test_tool_groups_vector(xy_dataset_session):
+ """Check that global overwrite is not used when separate env is used"""
+ vector = Tools(prefix="v")
+ vector.edit(map="points", type="point", tool="create", env=xy_dataset_session.env)
+ # Here, the feed_input_to style does not make sense, but we are not using StringIO
+ # here to test the feed_input_to functionality and avoid dependence on the StringIO
+ # functionality.
+ # The ASCII format is for one point with no categories.
+ vector.feed_input_to("P 1 0\n 10 20").edit(
+ map="points",
+ type="point",
+ tool="add",
+ input="-",
+ flags="n",
+ env=xy_dataset_session.env,
+ )
+ vector.buffer(
+ input="points", output="buffer", distance=1, env=xy_dataset_session.env
+ )
+ assert (
+ vector.info(map="buffer", format="json", env=xy_dataset_session.env)["areas"]
+ == 1
+ )
+
+
+def test_stdin_as_stringio_object(xy_dataset_session):
+ """Check that global overwrite is not used when separate env is used"""
+ tools = Tools(session=xy_dataset_session)
+ tools.v_edit(map="points", type="point", tool="create")
+ tools.v_edit(
+ map="points",
+ type="point",
+ tool="add",
+ input=io.StringIO("P 1 0\n 10 20"),
+ flags="n",
+ )
+ assert tools.v_info(map="points", format="json")["points"] == 1
diff --git a/python/grass/experimental/tools.py b/python/grass/experimental/tools.py
index ec315a734d5..b6a4a2f5539 100644
--- a/python/grass/experimental/tools.py
+++ b/python/grass/experimental/tools.py
@@ -17,11 +17,211 @@
import json
import os
import shutil
+import subprocess
+from pathlib import Path
+from io import StringIO
+
+import numpy as np
import grass.script as gs
+import grass.script.array as garray
from grass.exceptions import CalledModuleError
+class PackImporterExporter:
+ def __init__(self, *, run_function, env=None):
+ self._run_function = run_function
+ self._env = env
+
+ @classmethod
+ def is_raster_pack_file(cls, value):
+ return value.endswith((".grass_raster", ".pack", ".rpack", ".grr"))
+
+ def modify_and_ingest_argument_list(self, args, parameters):
+ # Uses parameters, but modifies the command, generates list of rasters and vectors.
+ self.input_rasters = []
+ if "inputs" in parameters:
+ for item in parameters["inputs"]:
+ if self.is_raster_pack_file(item["value"]):
+ self.input_rasters.append(Path(item["value"]))
+ # No need to change that for the original kwargs.
+ # kwargs[item["param"]] = Path(item["value"]).stem
+ # Actual parameters to execute are now a list.
+ for i, arg in enumerate(args):
+ if arg.startswith(f"{item['param']}="):
+ arg = arg.replace(item["value"], Path(item["value"]).stem)
+ args[i] = arg
+ self.output_rasters = []
+ if "outputs" in parameters:
+ for item in parameters["outputs"]:
+ if self.is_raster_pack_file(item["value"]):
+ self.output_rasters.append(Path(item["value"]))
+ # kwargs[item["param"]] = Path(item["value"]).stem
+ for i, arg in enumerate(args):
+ if arg.startswith(f"{item['param']}="):
+ arg = arg.replace(item["value"], Path(item["value"]).stem)
+ args[i] = arg
+
+ def import_rasters(self):
+ for raster_file in self.input_rasters:
+ # Currently we override the projection check.
+ self._run_function(
+ "r.unpack",
+ input=raster_file,
+ output=raster_file.stem,
+ overwrite=True,
+ superquiet=True,
+ # flags="o",
+ env=self._env,
+ )
+
+ def export_rasters(self):
+ # Pack the output raster
+ for raster in self.output_rasters:
+ # Overwriting a file is a warning, so to avoid it, we delete the file first.
+ Path(raster).unlink(missing_ok=True)
+
+ self._run_function(
+ "r.pack",
+ input=raster.stem,
+ output=raster,
+ flags="c",
+ overwrite=True,
+ superquiet=True,
+ )
+
+ def import_data(self):
+ self.import_rasters()
+
+ def export_data(self):
+ self.export_rasters()
+
+
+class ObjectParameterHandler:
+ def __init__(self):
+ self._numpy_inputs = {}
+ self._numpy_outputs = {}
+ self._numpy_inputs_ordered = []
+ self.stdin = None
+
+ def process_parameters(self, kwargs):
+ for key, value in kwargs.items():
+ if isinstance(value, np.ndarray):
+ kwargs[key] = gs.append_uuid("tmp_serialized_input_array")
+ self._numpy_inputs[key] = value
+ self._numpy_inputs_ordered.append(value)
+ elif value in (np.ndarray, np.array, garray.array):
+ # We test for class or the function.
+ kwargs[key] = gs.append_uuid("tmp_serialized_output_array")
+ self._numpy_outputs[key] = value
+ elif isinstance(value, StringIO):
+ kwargs[key] = "-"
+ self.stdin = value.getvalue()
+
+ def translate_objects_to_data(self, kwargs, parameters, env):
+ if "inputs" in parameters:
+ for param in parameters["inputs"]:
+ if param["param"] in self._numpy_inputs:
+ map2d = garray.array(env=env)
+ map2d[:] = self._numpy_inputs[param["param"]]
+ map2d.write(kwargs[param["param"]])
+
+ def input_rows_columns(self):
+ if not len(self._numpy_inputs_ordered):
+ return None
+ return self._numpy_inputs_ordered[0].shape
+
+ def translate_data_to_objects(self, kwargs, parameters, env):
+ output_arrays = []
+ if "outputs" in parameters:
+ for param in parameters["outputs"]:
+ if param["param"] not in self._numpy_outputs:
+ continue
+ output_array = garray.array(kwargs[param["param"]], env=env)
+ output_arrays.append(output_array)
+ if len(output_arrays) == 1:
+ self.result = output_arrays[0]
+ return True
+ if len(output_arrays) > 1:
+ self.result = tuple(output_arrays)
+ return True
+ self.result = None
+ return False
+
+
+class ToolFunctionNameHelper:
+ def __init__(self, *, run_function, env, prefix=None):
+ self._run_function = run_function
+ self._env = env
+ self._prefix = prefix
+
+ # def __getattr__(self, name):
+ # self.get_function(name, exception_type=AttributeError)
+
+ def get_function(self, name, exception_type):
+ """Parse attribute to GRASS display module. Attribute should be in
+ the form 'd_module_name'. For example, 'd.rast' is called with 'd_rast'.
+ """
+ if self._prefix:
+ name = f"{self._prefix}.{name}"
+ # Reformat string
+ tool_name = name.replace("_", ".")
+ # Assert module exists
+ if not shutil.which(tool_name, path=self._env["PATH"]):
+ suggestions = self.suggest_tools(tool_name)
+ if suggestions:
+ msg = (
+ f"Tool {tool_name} not found. "
+ f"Did you mean: {', '.join(suggestions)}?"
+ )
+ raise AttributeError(msg)
+ msg = (
+ f"Tool or attribute {name} not found. "
+ "If you are executing a tool, is the session set up and the tool on path? "
+ "If you are looking for an attribute, is it in the documentation?"
+ )
+ raise AttributeError(msg)
+
+ def wrapper(**kwargs):
+ # Run module
+ return self._run_function(tool_name, **kwargs)
+
+ return wrapper
+
+ @staticmethod
+ def levenshtein_distance(text1: str, text2: str) -> int:
+ if len(text1) < len(text2):
+ return ToolFunctionNameHelper.levenshtein_distance(text2, text1)
+
+ if len(text2) == 0:
+ return len(text1)
+
+ previous_row = list(range(len(text2) + 1))
+ for i, char1 in enumerate(text1):
+ current_row = [i + 1]
+ for j, char2 in enumerate(text2):
+ insertions = previous_row[j + 1] + 1
+ deletions = current_row[j] + 1
+ substitutions = previous_row[j] + (char1 != char2)
+ current_row.append(min(insertions, deletions, substitutions))
+ previous_row = current_row
+
+ return previous_row[-1]
+
+ @staticmethod
+ def suggest_tools(tool):
+ # TODO: cache commands also for dir
+ all_names = list(gs.get_commands()[0])
+ result = []
+ max_suggestions = 10
+ for name in all_names:
+ if ToolFunctionNameHelper.levenshtein_distance(tool, name) < len(tool) / 2:
+ result.append(name)
+ if len(result) >= max_suggestions:
+ break
+ return result
+
+
class ExecutedTool:
"""Result returned after executing a tool"""
@@ -92,12 +292,20 @@ def text_split(self, separator=None):
return self._decoded_stdout.strip("\n").split(separator)
def __getitem__(self, name):
- # TODO: cache parsed JSON
if self._stdout:
# We are testing just std out and letting rest to the parse and the user.
# This makes no assumption about how JSON is produced by the tool.
- return self.json[name]
- msg = f"Output of the tool {self._name} is not JSON"
+ try:
+ return self.json[name]
+ except json.JSONDecodeError as error:
+ if self._kwargs.get("format") == "json":
+ raise
+ msg = (
+ f"Output of {self._name} cannot be parsed as JSON. "
+ 'Did you use format="json"?'
+ )
+ raise ValueError(msg) from error
+ msg = f"No text output for {self._name} to be parsed as JSON"
raise ValueError(msg)
@@ -120,6 +328,7 @@ def __init__(
stdin=None,
errors=None,
capture_output=True,
+ prefix=None,
):
if env:
self._env = env.copy()
@@ -134,7 +343,7 @@ def __init__(
self._overwrite()
# This hopefully sets the numbers directly. An alternative implementation would
# be to pass the parameter every time.
- # Does not check for multiple set at the same time, but the most versbose wins
+ # Does not check for multiple set at the same time, but the most verbose wins
# for safety.
if superquiet:
self._env["GRASS_VERBOSE"] = "0"
@@ -145,6 +354,8 @@ def __init__(
self._set_stdin(stdin)
self._errors = errors
self._capture_output = capture_output
+ self._prefix = prefix
+ self._name_helper = None
# These could be public, not protected.
def _freeze_region(self):
@@ -162,28 +373,12 @@ def env(self):
"""Internally used environment (reference to it, not a copy)"""
return self._env
- def _digest_data_parameters(self, parameters, command):
- # Uses parameters, but modifies the command.
- input_rasters = []
- if "inputs" in parameters:
- for item in parameters["inputs"]:
- if item["value"].endswith(".grass_raster"):
- input_rasters.append(Path(item["value"]))
- for i, arg in enumerate(command):
- if arg.startswith(f"{item['param']}="):
- arg = arg.replace(item["value"], Path(item["value"]).stem)
- command[i] = arg
- output_rasters = []
- if "outputs" in parameters:
- for item in parameters["outputs"]:
- if item["value"].endswith(".grass_raster"):
- output_rasters.append(Path(item["value"]))
- for i, arg in enumerate(command):
- if arg.startswith(f"{item['param']}="):
- arg = arg.replace(item["value"], Path(item["value"]).stem)
- command[i] = arg
- return input_rasters, output_rasters
+ def _process_parameters(self, command, popen_options):
+ env = popen_options.get("env", self._env)
+ return subprocess.run(
+ [*command, "--json"], text=True, capture_output=True, env=env
+ )
def run(self, name, /, **kwargs):
"""Run modules from the GRASS display family (modules starting with "d.").
@@ -193,58 +388,99 @@ def run(self, name, /, **kwargs):
:param str module: name of GRASS module
:param `**kwargs`: named arguments passed to run_command()"""
- original = {}
- original_outputs = {}
- import grass.script.array as garray
- import numpy as np
- for key, value in kwargs.items():
- if isinstance(value, np.ndarray):
- kwargs[key] = "tmp_serialized_array"
- original[key] = value
- elif value == np.ndarray:
- kwargs[key] = "tmp_future_serialized_array"
- original_outputs[key] = value
- args, popen_options = gs.popen_args_command(name, **kwargs)
+ object_parameter_handler = ObjectParameterHandler()
+ object_parameter_handler.process_parameters(kwargs)
- env = popen_options.get("env", self._env)
+ args, popen_options = gs.popen_args_command(name, **kwargs)
- import subprocess
- parameters = json.loads(
- subprocess.check_output(
- [*args, "--json"], text=True, env=env
+ interface_result = self._process_parameters(args, popen_options)
+ if interface_result.returncode != 0:
+ # This is only for the error states.
+ return gs.handle_errors(
+ interface_result.returncode,
+ result=None,
+ args=[name],
+ kwargs=kwargs,
+ stderr=interface_result.stderr,
+ handler="raise",
)
+ parameters = json.loads(interface_result.stdout)
+ object_parameter_handler.translate_objects_to_data(
+ kwargs, parameters, env=self._env
)
- if "inputs" in parameters:
- for param in parameters["inputs"]:
- if param["param"] not in original:
- continue
- map2d = garray.array()
- print(param)
- map2d[:] = original[param["param"]]
- map2d.write("tmp_serialized_array", overwrite=True)
# We approximate tool_kwargs as original kwargs.
- result = self.run_from_list(args, tool_kwargs=kwargs, **popen_options)
+ result = self.run_from_list(
+ args,
+ tool_kwargs=kwargs,
+ processed_parameters=parameters,
+ stdin=object_parameter_handler.stdin,
+ **popen_options,
+ )
+ use_objects = object_parameter_handler.translate_data_to_objects(
+ kwargs, parameters, env=self._env
+ )
+ if use_objects:
+ result = object_parameter_handler.result
+ return result
- if "outputs" in parameters:
- for param in parameters["outputs"]:
- if param["param"] not in original_outputs:
- continue
- output_array = garray.array("tmp_future_serialized_array")
- result = output_array
+ def run_from_list(
+ self,
+ command,
+ tool_kwargs=None,
+ stdin=None,
+ processed_parameters=None,
+ **popen_options,
+ ):
+ if not processed_parameters:
+ interface_result = self._process_parameters(command, popen_options)
+ if interface_result.returncode != 0:
+ # This is only for the error states.
+ return gs.handle_errors(
+ interface_result.returncode,
+ result=None,
+ args=[command],
+ kwargs=tool_kwargs,
+ stderr=interface_result.stderr,
+ handler="raise",
+ )
+ processed_parameters = json.loads(interface_result.stdout)
+ pack_importer_exporter = PackImporterExporter(run_function=self.no_nonsense_run)
+ pack_importer_exporter.modify_and_ingest_argument_list(
+ command, processed_parameters
+ )
+ pack_importer_exporter.import_data()
+
+ # We approximate tool_kwargs as original kwargs.
+ result = self.no_nonsense_run_from_list(
+ command,
+ tool_kwargs=tool_kwargs,
+ stdin=stdin,
+ **popen_options,
+ )
+ pack_importer_exporter.export_data()
return result
def run_command(self, name, /, **kwargs):
- # Adjust error handling or provide custom implementation for full control?
+ # TODO: Provide custom implementation for full control
return gs.run_command(name, **kwargs, env=self._env)
def parse_command(self, name, /, **kwargs):
+ # TODO: Provide custom implementation for full control
return gs.parse_command(name, **kwargs, env=self._env)
+ def no_nonsense_run(self, name, /, *, tool_kwargs=None, stdin=None, **kwargs):
+ args, popen_options = gs.popen_args_command(name, **kwargs)
+ return self.no_nonsense_run_from_list(
+ args, tool_kwargs=tool_kwargs, stdin=stdin, **popen_options
+ )
+
# Make this an overload of run.
- def run_from_list(self, command, tool_kwargs=None, **popen_options):
+ def no_nonsense_run_from_list(
+ self, command, tool_kwargs=None, stdin=None, **popen_options
+ ):
# alternatively use dev null as default or provide it as convenient settings
if self._capture_output:
stdout_pipe = gs.PIPE
@@ -255,6 +491,9 @@ def run_from_list(self, command, tool_kwargs=None, **popen_options):
if self._stdin:
stdin_pipe = gs.PIPE
stdin = gs.utils.encode(self._stdin)
+ elif stdin:
+ stdin_pipe = gs.PIPE
+ stdin = gs.utils.encode(stdin)
else:
stdin_pipe = None
stdin = None
@@ -288,70 +527,30 @@ def run_from_list(self, command, tool_kwargs=None, **popen_options):
def feed_input_to(self, stdin, /):
"""Get a new object which will feed text input to a tool or tools"""
- return Tools(env=self._env, stdin=stdin)
+ return Tools(
+ env=self._env,
+ stdin=stdin,
+ freeze_region=self._region_is_frozen,
+ errors=self._errors,
+ capture_output=self._capture_output,
+ prefix=self._prefix,
+ )
def ignore_errors_of(self):
"""Get a new object which will ignore errors of the called tools"""
return Tools(env=self._env, errors="ignore")
- def levenshtein_distance(self, text1: str, text2: str) -> int:
- if len(text1) < len(text2):
- return self.levenshtein_distance(text2, text1)
-
- if len(text2) == 0:
- return len(text1)
-
- previous_row = list(range(len(text2) + 1))
- for i, char1 in enumerate(text1):
- current_row = [i + 1]
- for j, char2 in enumerate(text2):
- insertions = previous_row[j + 1] + 1
- deletions = current_row[j] + 1
- substitutions = previous_row[j] + (char1 != char2)
- current_row.append(min(insertions, deletions, substitutions))
- previous_row = current_row
-
- return previous_row[-1]
-
- def suggest_tools(self, tool):
- # TODO: cache commands also for dir
- all_names = list(gs.get_commands()[0])
- result = []
- max_suggestions = 10
- for name in all_names:
- if self.levenshtein_distance(tool, name) < len(tool) / 2:
- result.append(name)
- if len(result) >= max_suggestions:
- break
- return result
-
def __getattr__(self, name):
"""Parse attribute to GRASS display module. Attribute should be in
the form 'd_module_name'. For example, 'd.rast' is called with 'd_rast'.
"""
- # Reformat string
- tool_name = name.replace("_", ".")
- # Assert module exists
- if not shutil.which(tool_name):
- suggesions = self.suggest_tools(tool_name)
- if suggesions:
- msg = (
- f"Tool {tool_name} not found. "
- f"Did you mean: {', '.join(suggesions)}?"
- )
- raise AttributeError(msg)
- msg = (
- f"Tool or attribute {name} not found. "
- "If you are executing a tool, is the session set up and the tool on path? "
- "If you are looking for an attribute, is it in the documentation?"
+ if not self._name_helper:
+ self._name_helper = ToolFunctionNameHelper(
+ run_function=self.run,
+ env=self.env,
+ prefix=self._prefix,
)
- raise AttributeError(msg)
-
- def wrapper(**kwargs):
- # Run module
- return self.run(tool_name, **kwargs)
-
- return wrapper
+ return self._name_helper.get_function(name, exception_type=AttributeError)
def _test():
diff --git a/python/grass/script/core.py b/python/grass/script/core.py
index 8d8065ac0d8..838e80a39a4 100644
--- a/python/grass/script/core.py
+++ b/python/grass/script/core.py
@@ -307,7 +307,7 @@ def make_command(
return args
-def handle_errors(returncode, result, args, kwargs):
+def handle_errors(returncode, result, args, kwargs, handler=None, stderr=None):
"""Error handler for :func:`run_command()` and similar functions
The functions which are using this function to handle errors,
@@ -352,7 +352,8 @@ def get_module_and_code(args, kwargs):
code = " ".join(args)
return module, code
- handler = kwargs.get("errors", "raise")
+ if handler is None:
+ handler = kwargs.get("errors", "raise")
if handler.lower() == "status":
return returncode
if returncode == 0:
@@ -370,7 +371,9 @@ def get_module_and_code(args, kwargs):
sys.exit(returncode)
else:
module, code = get_module_and_code(args, kwargs)
- raise CalledModuleError(module=module, code=code, returncode=returncode)
+ raise CalledModuleError(
+ module=module, code=code, returncode=returncode, errors=stderr
+ )
def popen_args_command(
From 41ad7ae6dfb0149a51c3898ea94e1bf58291edcb Mon Sep 17 00:00:00 2001
From: Vaclav Petras <wenzeslaus@gmail.com>
Date: Wed, 11 Jun 2025 13:12:46 -0400
Subject: [PATCH 22/29] Remove r.pack IO
---
python/grass/experimental/tests/conftest.py | 38 ---------
.../experimental/tests/grass_tools_test.py | 37 ---------
python/grass/experimental/tools.py | 80 +------------------
3 files changed, 1 insertion(+), 154 deletions(-)
diff --git a/python/grass/experimental/tests/conftest.py b/python/grass/experimental/tests/conftest.py
index 910036b3717..b33ae757f8e 100644
--- a/python/grass/experimental/tests/conftest.py
+++ b/python/grass/experimental/tests/conftest.py
@@ -77,41 +77,3 @@ def xy_mapset_non_permament(xy_session): # pylint: disable=redefined-outer-name
"test1", create=True, env=xy_session.env
) as session:
yield session
-
-
-@pytest.fixture
-def rows_raster_file3x3(tmp_path):
- project = tmp_path / "xy_test3x3"
- gs.create_project(project)
- with gs.setup.init(project, env=os.environ.copy()) as session:
- gs.run_command("g.region", rows=3, cols=3, env=session.env)
- gs.mapcalc("rows = row()", env=session.env)
- output_file = tmp_path / "rows3x3.grass_raster"
- gs.run_command(
- "r.pack",
- input="rows",
- output=output_file,
- flags="c",
- superquiet=True,
- env=session.env,
- )
- return output_file
-
-
-@pytest.fixture
-def rows_raster_file4x5(tmp_path):
- project = tmp_path / "xy_test4x5"
- gs.create_project(project)
- with gs.setup.init(project, env=os.environ.copy()) as session:
- gs.run_command("g.region", rows=4, cols=5, env=session.env)
- gs.mapcalc("rows = row()", env=session.env)
- output_file = tmp_path / "rows4x5.grass_raster"
- gs.run_command(
- "r.pack",
- input="rows",
- output=output_file,
- flags="c",
- superquiet=True,
- env=session.env,
- )
- return output_file
diff --git a/python/grass/experimental/tests/grass_tools_test.py b/python/grass/experimental/tests/grass_tools_test.py
index 1cba3e9e1c9..367f160ca26 100644
--- a/python/grass/experimental/tests/grass_tools_test.py
+++ b/python/grass/experimental/tests/grass_tools_test.py
@@ -364,43 +364,6 @@ def test_numpy_grass_array_input_output(xy_dataset_session):
assert isinstance(result, gs.array.array)
-def test_pack_input_output(xy_dataset_session, rows_raster_file3x3):
- """Check that global overwrite is not used when separate env is used"""
- tools = Tools(session=xy_dataset_session)
- tools.g_region(rows=3, cols=3)
- assert os.path.exists(rows_raster_file3x3)
- tools.r_slope_aspect(elevation=rows_raster_file3x3, slope="file.grass_raster")
- assert os.path.exists("file.grass_raster")
-
-
-def test_pack_input_output_with_name_and_parameter_call(
- xy_dataset_session, rows_raster_file3x3
-):
- """Check that global overwrite is not used when separate env is used"""
- tools = Tools(session=xy_dataset_session)
- tools.g_region(rows=3, cols=3)
- assert os.path.exists(rows_raster_file3x3)
- tools.run(
- "r.slope.aspect", elevation=rows_raster_file3x3, slope="file.grass_raster"
- )
- assert os.path.exists("file.grass_raster")
-
-
-def test_pack_input_output_with_subprocess_run_like_call(
- xy_dataset_session, rows_raster_file3x3
-):
- tools = Tools(session=xy_dataset_session)
- assert os.path.exists(rows_raster_file3x3)
- tools.run_from_list(
- [
- "r.slope.aspect",
- f"elevation={rows_raster_file3x3}",
- "aspect=file.grass_raster",
- ]
- )
- assert os.path.exists("file.grass_raster")
-
-
def test_tool_groups_raster(xy_dataset_session):
"""Check that global overwrite is not used when separate env is used"""
raster = Tools(session=xy_dataset_session, prefix="r")
diff --git a/python/grass/experimental/tools.py b/python/grass/experimental/tools.py
index b6a4a2f5539..a59abb00a03 100644
--- a/python/grass/experimental/tools.py
+++ b/python/grass/experimental/tools.py
@@ -18,7 +18,6 @@
import os
import shutil
import subprocess
-from pathlib import Path
from io import StringIO
import numpy as np
@@ -28,75 +27,6 @@
from grass.exceptions import CalledModuleError
-class PackImporterExporter:
- def __init__(self, *, run_function, env=None):
- self._run_function = run_function
- self._env = env
-
- @classmethod
- def is_raster_pack_file(cls, value):
- return value.endswith((".grass_raster", ".pack", ".rpack", ".grr"))
-
- def modify_and_ingest_argument_list(self, args, parameters):
- # Uses parameters, but modifies the command, generates list of rasters and vectors.
- self.input_rasters = []
- if "inputs" in parameters:
- for item in parameters["inputs"]:
- if self.is_raster_pack_file(item["value"]):
- self.input_rasters.append(Path(item["value"]))
- # No need to change that for the original kwargs.
- # kwargs[item["param"]] = Path(item["value"]).stem
- # Actual parameters to execute are now a list.
- for i, arg in enumerate(args):
- if arg.startswith(f"{item['param']}="):
- arg = arg.replace(item["value"], Path(item["value"]).stem)
- args[i] = arg
- self.output_rasters = []
- if "outputs" in parameters:
- for item in parameters["outputs"]:
- if self.is_raster_pack_file(item["value"]):
- self.output_rasters.append(Path(item["value"]))
- # kwargs[item["param"]] = Path(item["value"]).stem
- for i, arg in enumerate(args):
- if arg.startswith(f"{item['param']}="):
- arg = arg.replace(item["value"], Path(item["value"]).stem)
- args[i] = arg
-
- def import_rasters(self):
- for raster_file in self.input_rasters:
- # Currently we override the projection check.
- self._run_function(
- "r.unpack",
- input=raster_file,
- output=raster_file.stem,
- overwrite=True,
- superquiet=True,
- # flags="o",
- env=self._env,
- )
-
- def export_rasters(self):
- # Pack the output raster
- for raster in self.output_rasters:
- # Overwriting a file is a warning, so to avoid it, we delete the file first.
- Path(raster).unlink(missing_ok=True)
-
- self._run_function(
- "r.pack",
- input=raster.stem,
- output=raster,
- flags="c",
- overwrite=True,
- superquiet=True,
- )
-
- def import_data(self):
- self.import_rasters()
-
- def export_data(self):
- self.export_rasters()
-
-
class ObjectParameterHandler:
def __init__(self):
self._numpy_inputs = {}
@@ -447,21 +377,13 @@ def run_from_list(
)
processed_parameters = json.loads(interface_result.stdout)
- pack_importer_exporter = PackImporterExporter(run_function=self.no_nonsense_run)
- pack_importer_exporter.modify_and_ingest_argument_list(
- command, processed_parameters
- )
- pack_importer_exporter.import_data()
-
# We approximate tool_kwargs as original kwargs.
- result = self.no_nonsense_run_from_list(
+ return self.no_nonsense_run_from_list(
command,
tool_kwargs=tool_kwargs,
stdin=stdin,
**popen_options,
)
- pack_importer_exporter.export_data()
- return result
def run_command(self, name, /, **kwargs):
# TODO: Provide custom implementation for full control
From e3797ed1e8373026d70876ecbe0429f27676a108 Mon Sep 17 00:00:00 2001
From: Vaclav Petras <wenzeslaus@gmail.com>
Date: Sun, 27 Jul 2025 08:30:47 -0400
Subject: [PATCH 23/29] Integrate numpy code into grass.tool and remove the
experimental tools. Make tests separate.
---
.../experimental/tests/grass_tools_test.py | 411 --------------
python/grass/experimental/tools.py | 533 ------------------
python/grass/tools/session_tools.py | 17 +-
python/grass/tools/support.py | 37 +-
.../grass_tools_session_tools_numpy_test.py | 116 ++++
5 files changed, 167 insertions(+), 947 deletions(-)
delete mode 100644 python/grass/experimental/tests/grass_tools_test.py
delete mode 100644 python/grass/experimental/tools.py
create mode 100644 python/grass/tools/tests/grass_tools_session_tools_numpy_test.py
diff --git a/python/grass/experimental/tests/grass_tools_test.py b/python/grass/experimental/tests/grass_tools_test.py
deleted file mode 100644
index 367f160ca26..00000000000
--- a/python/grass/experimental/tests/grass_tools_test.py
+++ /dev/null
@@ -1,411 +0,0 @@
-"""Test grass.experimental.Tools class"""
-
-import os
-import io
-
-import numpy as np
-import pytest
-
-import grass.script as gs
-from grass.experimental.mapset import TemporaryMapsetSession
-from grass.experimental.tools import Tools
-from grass.exceptions import CalledModuleError
-
-
-def test_key_value_parser_number(xy_dataset_session):
- """Check that numbers are parsed as numbers"""
- tools = Tools(session=xy_dataset_session)
- assert tools.g_region(flags="g").keyval["nsres"] == 1
-
-
-@pytest.mark.xfail
-def test_key_value_parser_multiple_values(xy_dataset_session):
- """Check that strings and floats are parsed"""
- tools = Tools(session=xy_dataset_session)
- name = "surface"
- tools.r_surf_gauss(output=name) # needs seed
- result = tools.r_info(map=name, flags="g").keyval
- assert result["datatype"] == "DCELL"
- assert result["nsres"] == 1
- result = tools.r_univar(map=name, flags="g").keyval
- assert result["mean"] == pytest.approx(-0.756762744552762)
-
-
-def test_json_parser(xy_dataset_session):
- """Check that JSON is parsed"""
- tools = Tools(session=xy_dataset_session)
- assert (
- tools.g_search_modules(keyword="random", flags="j").json[0]["name"]
- == "r.random"
- )
-
-
-def test_json_with_name_and_parameter_call(xy_dataset_session):
- """Check that JSON is parsed with a name-and-parameters style call"""
- tools = Tools(session=xy_dataset_session)
- assert (
- tools.run("g.search.modules", keyword="random", flags="j")[0]["name"]
- == "r.random"
- )
-
-
-def test_json_with_subprocess_run_like_call(xy_dataset_session):
- """Check that JSON is parsed with a name-and-parameters style call"""
- tools = Tools(session=xy_dataset_session)
- assert (
- tools.run_from_list(["g.search.modules", "keyword=random", "-j"])[0]["name"]
- == "r.random"
- )
-
-
-def test_json_direct_access(xy_dataset_session):
- """Check that JSON is parsed"""
- tools = Tools(session=xy_dataset_session)
- assert tools.g_search_modules(keyword="random", flags="j")[0]["name"] == "r.random"
-
-
-def test_json_direct_access_bad_key_type(xy_dataset_session):
- """Check that JSON is parsed"""
- tools = Tools(session=xy_dataset_session)
- with pytest.raises(TypeError):
- tools.g_search_modules(keyword="random", flags="j")["name"]
-
-
-def test_json_direct_access_bad_key_value(xy_dataset_session):
- """Check that JSON is parsed"""
- tools = Tools(session=xy_dataset_session)
- high_number = 100_000_000
- with pytest.raises(IndexError):
- tools.g_search_modules(keyword="random", flags="j")[high_number]
-
-
-def test_json_direct_access_not_json(xy_dataset_session):
- """Check that JSON parsing creates an ValueError
-
- Specifically, this tests the case when format="json" is not set.
- """
- tools = Tools(session=xy_dataset_session)
- with pytest.raises(ValueError, match=r"format.*json"):
- tools.g_search_modules(keyword="random")[0]["name"]
-
-
-def test_stdout_as_text(xy_dataset_session):
- """Check that simple text is parsed and has no whitespace"""
- tools = Tools(session=xy_dataset_session)
- assert tools.g_mapset(flags="p").text == "PERMANENT"
-
-
-def test_stdout_as_space_items(xy_dataset_session):
- """Check that whitespace-separated items are parsed"""
- tools = Tools(session=xy_dataset_session)
- assert tools.g_mapset(flags="l").space_items == ["PERMANENT"]
-
-
-def test_stdout_split_whitespace(xy_dataset_session):
- """Check that whitespace-based split function works"""
- tools = Tools(session=xy_dataset_session)
- assert tools.g_mapset(flags="l").text_split() == ["PERMANENT"]
-
-
-def test_stdout_split_space(xy_dataset_session):
- """Check that the split function works with space"""
- tools = Tools(session=xy_dataset_session)
- # Not a good example usage, but it tests the functionality.
- assert tools.g_mapset(flags="l").text_split(" ") == ["PERMANENT", ""]
-
-
-def test_stdout_without_capturing(xy_dataset_session):
- """Check that text is not present when not capturing it"""
- tools = Tools(session=xy_dataset_session, capture_output=False)
- assert not tools.g_mapset(flags="p").text
- assert tools.g_mapset(flags="p").text is None
-
-
-def test_direct_overwrite(xy_dataset_session):
- """Check overwrite as a parameter"""
- tools = Tools(session=xy_dataset_session)
- tools.r_random_surface(output="surface", seed=42)
- tools.r_random_surface(output="surface", seed=42, overwrite=True)
-
-
-def test_object_overwrite(xy_dataset_session):
- """Check overwrite as parameter of the tools object"""
- tools = Tools(session=xy_dataset_session, overwrite=True)
- tools.r_random_surface(output="surface", seed=42)
- tools.r_random_surface(output="surface", seed=42)
-
-
-def test_no_overwrite(xy_dataset_session):
- """Check that it fails without overwrite"""
- tools = Tools(session=xy_dataset_session)
- tools.r_random_surface(output="surface", seed=42)
- with pytest.raises(CalledModuleError, match="overwrite"):
- tools.r_random_surface(output="surface", seed=42)
-
-
-def test_env_overwrite(xy_dataset_session):
- """Check that overwrite from env parameter is used"""
- # env = xy_dataset_session.env.copy() # ideally
- env = os.environ.copy() # for now
- env["GRASS_OVERWRITE"] = "1"
- tools = Tools(session=xy_dataset_session, env=env)
- tools.r_random_surface(output="surface", seed=42)
- tools.r_random_surface(output="surface", seed=42)
-
-
-def test_global_overwrite_vs_env(xy_dataset_session):
- """Check that global overwrite is not used when separate env is used"""
- # env = xy_dataset_session.env.copy() # ideally
- env = os.environ.copy() # for now
- os.environ["GRASS_OVERWRITE"] = "1" # change to xy_dataset_session.env
- tools = Tools(session=xy_dataset_session, env=env)
- tools.r_random_surface(output="surface", seed=42)
- with pytest.raises(CalledModuleError, match="overwrite"):
- tools.r_random_surface(output="surface", seed=42)
- del os.environ["GRASS_OVERWRITE"] # check or ideally remove this
-
-
-def test_global_overwrite_vs_init(xy_dataset_session):
- """Check that global overwrite is not used when separate env is used"""
- tools = Tools(session=xy_dataset_session)
- os.environ["GRASS_OVERWRITE"] = "1" # change to xy_dataset_session.env
- tools.r_random_surface(output="surface", seed=42)
- with pytest.raises(CalledModuleError, match="overwrite"):
- tools.r_random_surface(output="surface", seed=42)
- del os.environ["GRASS_OVERWRITE"] # check or ideally remove this
-
-
-def test_stdin(xy_dataset_session):
- """Test that stdin is accepted"""
- tools = Tools(session=xy_dataset_session)
- tools.feed_input_to("13.45,29.96,200").v_in_ascii(
- input="-", output="point", separator=","
- )
-
-
-def test_raises(xy_dataset_session):
- """Test that exception is raised for wrong parameter value"""
- tools = Tools(session=xy_dataset_session)
- wrong_name = "wrong_standard"
- with pytest.raises(CalledModuleError, match=wrong_name):
- tools.feed_input_to("13.45,29.96,200").v_in_ascii(
- input="-",
- output="point",
- format=wrong_name,
- )
-
-
-def test_run_command(xy_dataset_session):
- """Check run_command and its overwrite parameter"""
- tools = Tools(session=xy_dataset_session)
- tools.run_command("r.random.surface", output="surface", seed=42)
- tools.run_command("r.random.surface", output="surface", seed=42, overwrite=True)
-
-
-def test_parse_command_key_value(xy_dataset_session):
- tools = Tools(session=xy_dataset_session)
- assert tools.parse_command("g.region", flags="g")["nsres"] == "1"
-
-
-def test_parse_command_json(xy_dataset_session):
- tools = Tools(session=xy_dataset_session)
- assert (
- tools.parse_command("g.region", flags="g", format="json")["region"]["ns-res"]
- == 1
- )
-
-
-def test_with_context_managers(tmpdir):
- project = tmpdir / "project"
- gs.create_project(project)
- with gs.setup.init(project) as session:
- tools = Tools(session=session)
- tools.r_random_surface(output="surface", seed=42)
- with TemporaryMapsetSession(env=tools.env) as mapset:
- tools.r_random_surface(output="surface", seed=42, env=mapset.env)
- with gs.MaskManager(env=mapset.env) as mask:
- # TODO: Do actual test
- tools.r_univar(map="surface", env=mask.env, format="json")["mean"]
-
-
-def test_misspelling(xy_dataset_session):
- tools = Tools(session=xy_dataset_session)
- with pytest.raises(AttributeError, match=r"r\.slope\.aspect"):
- tools.r_sloppy_respect()
-
-
-def test_multiple_suggestions(xy_dataset_session):
- tools = Tools(session=xy_dataset_session)
- with pytest.raises(AttributeError, match=r"v\.db\.univar|db\.univar"):
- tools.db_v_uni_var()
-
-
-def test_tool_group_vs_model_name(xy_dataset_session):
- tools = Tools(session=xy_dataset_session)
- with pytest.raises(AttributeError, match=r"r\.sim\.water"):
- tools.rSIMWEwater()
-
-
-def test_wrong_attribute(xy_dataset_session):
- tools = Tools(session=xy_dataset_session)
- with pytest.raises(AttributeError, match="execute_big_command"):
- tools.execute_big_command()
-
-
-def test_numpy_one_input(xy_dataset_session):
- """Check that global overwrite is not used when separate env is used"""
- tools = Tools(session=xy_dataset_session)
- tools.r_slope_aspect(elevation=np.ones((1, 1)), slope="slope")
- assert tools.r_info(map="slope", format="json")["datatype"] == "FCELL"
-
-
-# NumPy syntax for outputs
-# While inputs are straightforward, there is several possible ways how to handle
-# syntax for outputs.
-# Output is the type of function for creating NumPy arrays, return value is now the arrays:
-# tools.r_slope_aspect(elevation=np.ones((1, 1)), slope=np.ndarray, aspect=np.array)
-# Output is explicitly requested:
-# tools.r_slope_aspect(elevation=np.ones((1, 1)), slope="slope", aspect="aspect", force_numpy_for_output=True)
-# Output is explicitly requested at the object level:
-# Tools(force_numpy_for_output=True).r_slope_aspect(elevation=np.ones((1, 1)), slope="slope", aspect="aspect")
-# Output is always array or arrays when at least on input is an array:
-# tools.r_slope_aspect(elevation=np.ones((1, 1)), slope="slope", aspect="aspect")
-# An empty array is passed to signal the desired output:
-# tools.r_slope_aspect(elevation=np.ones((1, 1)), slope=np.nulls((0, 0)))
-# An array to be filled with data is passed, the return value is kept as is:
-# tools.r_slope_aspect(elevation=np.ones((1, 1)), slope=np.nulls((1, 1)))
-# NumPy universal function concept can be used explicitly to indicate,
-# possibly more easily allowing for nameless args as opposed to keyword arguments,
-# but outputs still need to be explicitly requested:
-# Returns by value (tuple: (np.array, np.array)):
-# tools.r_slope_aspect.ufunc(np.ones((1, 1)), slope=True, aspect=True)
-# Modifies its arguments in-place:
-# tools.r_slope_aspect.ufunc(np.ones((1, 1)), slope=True, aspect=True, out=(np.array((1, 1)), np.array((1, 1))))
-# Custom signaling classes or objects are passed (assuming empty classes AsNumpy and AsInput):
-# tools.r_slope_aspect(elevation=np.ones((1, 1)), slope=ToNumpy(), aspect=ToNumpy())
-# tools.r_slope_aspect(elevation=np.ones((1, 1)), slope=AsInput, aspect=AsInput)
-# NumPy functions usually return a tuple, for multiple outputs. Universal function does
-# unless the output is written to out parameter which is also provided as a tuple. We
-# have names, so generally, we can return a dictionary:
-# {"slope": np.array(...), "aspect": np.array(...) }.
-
-
-def test_numpy_one_input_one_output(xy_dataset_session):
- """Check that a NumPy array works as input and for signaling output
-
- It tests that the np.ndarray class is supported to signal output.
- Return type is not strictly defined, so we are not testing for it explicitly
- (only by actually using it as an NumPy array).
- """
- tools = Tools(session=xy_dataset_session)
- tools.g_region(rows=2, cols=3)
- slope = tools.r_slope_aspect(elevation=np.ones((2, 3)), slope=np.ndarray)
- assert slope.shape == (2, 3)
- assert np.all(slope == np.full((2, 3), 0))
-
-
-def test_numpy_with_name_and_parameter(xy_dataset_session):
- """Check that a NumPy array works as input and for signaling output
-
- It tests that the np.ndarray class is supported to signal output.
- Return type is not strictly defined, so we are not testing for it explicitly
- (only by actually using it as an NumPy array).
- """
- tools = Tools(session=xy_dataset_session)
- tools.g_region(rows=2, cols=3)
- slope = tools.run("r.slope.aspect", elevation=np.ones((2, 3)), slope=np.ndarray)
- assert slope.shape == (2, 3)
- assert np.all(slope == np.full((2, 3), 0))
-
-
-def test_numpy_one_input_multiple_outputs(xy_dataset_session):
- """Check that a NumPy array function works for signaling multiple outputs
-
- Besides multiple outputs it tests that np.array is supported to signal output.
- """
- tools = Tools(session=xy_dataset_session)
- tools.g_region(rows=2, cols=3)
- (slope, aspect) = tools.r_slope_aspect(
- elevation=np.ones((2, 3)), slope=np.array, aspect=np.array
- )
- assert slope.shape == (2, 3)
- assert np.all(slope == np.full((2, 3), 0))
- assert aspect.shape == (2, 3)
- assert np.all(aspect == np.full((2, 3), 0))
-
-
-def test_numpy_multiple_inputs_one_output(xy_dataset_session):
- """Check that a NumPy array works for multiple inputs"""
- tools = Tools(session=xy_dataset_session)
- tools.g_region(rows=2, cols=3)
- result = tools.r_mapcalc_simple(
- expression="A + B", a=np.full((2, 3), 2), b=np.full((2, 3), 5), output=np.array
- )
- assert result.shape == (2, 3)
- assert np.all(result == np.full((2, 3), 7))
-
-
-def test_numpy_grass_array_input_output(xy_dataset_session):
- """Check that global overwrite is not used when separate env is used
-
- When grass array output is requested, we explicitly test the return value type.
- """
- tools = Tools(session=xy_dataset_session)
- rows = 2
- cols = 3
- tools.g_region(rows=rows, cols=cols)
- tools.r_mapcalc_simple(expression="5", output="const_5")
- const_5 = gs.array.array("const_5")
- result = tools.r_mapcalc_simple(
- expression="2 * A", a=const_5, output=gs.array.array
- )
- assert result.shape == (rows, cols)
- assert np.all(result == np.full((rows, cols), 10))
- assert isinstance(result, gs.array.array)
-
-
-def test_tool_groups_raster(xy_dataset_session):
- """Check that global overwrite is not used when separate env is used"""
- raster = Tools(session=xy_dataset_session, prefix="r")
- raster.mapcalc(expression="streams = if(row() > 1, 1, null())")
- raster.buffer(input="streams", output="buffer", distance=1)
- assert raster.info(map="streams", format="json")["datatype"] == "CELL"
-
-
-def test_tool_groups_vector(xy_dataset_session):
- """Check that global overwrite is not used when separate env is used"""
- vector = Tools(prefix="v")
- vector.edit(map="points", type="point", tool="create", env=xy_dataset_session.env)
- # Here, the feed_input_to style does not make sense, but we are not using StringIO
- # here to test the feed_input_to functionality and avoid dependence on the StringIO
- # functionality.
- # The ASCII format is for one point with no categories.
- vector.feed_input_to("P 1 0\n 10 20").edit(
- map="points",
- type="point",
- tool="add",
- input="-",
- flags="n",
- env=xy_dataset_session.env,
- )
- vector.buffer(
- input="points", output="buffer", distance=1, env=xy_dataset_session.env
- )
- assert (
- vector.info(map="buffer", format="json", env=xy_dataset_session.env)["areas"]
- == 1
- )
-
-
-def test_stdin_as_stringio_object(xy_dataset_session):
- """Check that global overwrite is not used when separate env is used"""
- tools = Tools(session=xy_dataset_session)
- tools.v_edit(map="points", type="point", tool="create")
- tools.v_edit(
- map="points",
- type="point",
- tool="add",
- input=io.StringIO("P 1 0\n 10 20"),
- flags="n",
- )
- assert tools.v_info(map="points", format="json")["points"] == 1
diff --git a/python/grass/experimental/tools.py b/python/grass/experimental/tools.py
deleted file mode 100644
index a59abb00a03..00000000000
--- a/python/grass/experimental/tools.py
+++ /dev/null
@@ -1,533 +0,0 @@
-#!/usr/bin/env python
-
-##############################################################################
-# AUTHOR(S): Vaclav Petras <wenzeslaus gmail com>
-#
-# PURPOSE: API to call GRASS tools (modules) as Python functions
-#
-# COPYRIGHT: (C) 2023-2025 Vaclav Petras and the GRASS Development Team
-#
-# This program is free software under the GNU General Public
-# License (>=v2). Read the file COPYING that comes with GRASS
-# for details.
-##############################################################################
-
-"""API to call GRASS tools (modules) as Python functions"""
-
-import json
-import os
-import shutil
-import subprocess
-from io import StringIO
-
-import numpy as np
-
-import grass.script as gs
-import grass.script.array as garray
-from grass.exceptions import CalledModuleError
-
-
-class ObjectParameterHandler:
- def __init__(self):
- self._numpy_inputs = {}
- self._numpy_outputs = {}
- self._numpy_inputs_ordered = []
- self.stdin = None
-
- def process_parameters(self, kwargs):
- for key, value in kwargs.items():
- if isinstance(value, np.ndarray):
- kwargs[key] = gs.append_uuid("tmp_serialized_input_array")
- self._numpy_inputs[key] = value
- self._numpy_inputs_ordered.append(value)
- elif value in (np.ndarray, np.array, garray.array):
- # We test for class or the function.
- kwargs[key] = gs.append_uuid("tmp_serialized_output_array")
- self._numpy_outputs[key] = value
- elif isinstance(value, StringIO):
- kwargs[key] = "-"
- self.stdin = value.getvalue()
-
- def translate_objects_to_data(self, kwargs, parameters, env):
- if "inputs" in parameters:
- for param in parameters["inputs"]:
- if param["param"] in self._numpy_inputs:
- map2d = garray.array(env=env)
- map2d[:] = self._numpy_inputs[param["param"]]
- map2d.write(kwargs[param["param"]])
-
- def input_rows_columns(self):
- if not len(self._numpy_inputs_ordered):
- return None
- return self._numpy_inputs_ordered[0].shape
-
- def translate_data_to_objects(self, kwargs, parameters, env):
- output_arrays = []
- if "outputs" in parameters:
- for param in parameters["outputs"]:
- if param["param"] not in self._numpy_outputs:
- continue
- output_array = garray.array(kwargs[param["param"]], env=env)
- output_arrays.append(output_array)
- if len(output_arrays) == 1:
- self.result = output_arrays[0]
- return True
- if len(output_arrays) > 1:
- self.result = tuple(output_arrays)
- return True
- self.result = None
- return False
-
-
-class ToolFunctionNameHelper:
- def __init__(self, *, run_function, env, prefix=None):
- self._run_function = run_function
- self._env = env
- self._prefix = prefix
-
- # def __getattr__(self, name):
- # self.get_function(name, exception_type=AttributeError)
-
- def get_function(self, name, exception_type):
- """Parse attribute to GRASS display module. Attribute should be in
- the form 'd_module_name'. For example, 'd.rast' is called with 'd_rast'.
- """
- if self._prefix:
- name = f"{self._prefix}.{name}"
- # Reformat string
- tool_name = name.replace("_", ".")
- # Assert module exists
- if not shutil.which(tool_name, path=self._env["PATH"]):
- suggestions = self.suggest_tools(tool_name)
- if suggestions:
- msg = (
- f"Tool {tool_name} not found. "
- f"Did you mean: {', '.join(suggestions)}?"
- )
- raise AttributeError(msg)
- msg = (
- f"Tool or attribute {name} not found. "
- "If you are executing a tool, is the session set up and the tool on path? "
- "If you are looking for an attribute, is it in the documentation?"
- )
- raise AttributeError(msg)
-
- def wrapper(**kwargs):
- # Run module
- return self._run_function(tool_name, **kwargs)
-
- return wrapper
-
- @staticmethod
- def levenshtein_distance(text1: str, text2: str) -> int:
- if len(text1) < len(text2):
- return ToolFunctionNameHelper.levenshtein_distance(text2, text1)
-
- if len(text2) == 0:
- return len(text1)
-
- previous_row = list(range(len(text2) + 1))
- for i, char1 in enumerate(text1):
- current_row = [i + 1]
- for j, char2 in enumerate(text2):
- insertions = previous_row[j + 1] + 1
- deletions = current_row[j] + 1
- substitutions = previous_row[j] + (char1 != char2)
- current_row.append(min(insertions, deletions, substitutions))
- previous_row = current_row
-
- return previous_row[-1]
-
- @staticmethod
- def suggest_tools(tool):
- # TODO: cache commands also for dir
- all_names = list(gs.get_commands()[0])
- result = []
- max_suggestions = 10
- for name in all_names:
- if ToolFunctionNameHelper.levenshtein_distance(tool, name) < len(tool) / 2:
- result.append(name)
- if len(result) >= max_suggestions:
- break
- return result
-
-
-class ExecutedTool:
- """Result returned after executing a tool"""
-
- def __init__(self, name, kwargs, stdout, stderr):
- self._name = name
- self._kwargs = kwargs
- self._stdout = stdout
- self._stderr = stderr
- if self._stdout is not None:
- self._decoded_stdout = gs.decode(self._stdout)
- else:
- self._decoded_stdout = None
- self._cached_json = None
-
- @property
- def text(self) -> str:
- """Text output as decoded string"""
- if self._decoded_stdout is None:
- return None
- return self._decoded_stdout.strip()
-
- @property
- def json(self):
- """Text output read as JSON
-
- This returns the nested structure of dictionaries and lists or fails when
- the output is not JSON.
- """
- if self._cached_json is None:
- self._cached_json = json.loads(self._stdout)
- return self._cached_json
-
- @property
- def keyval(self):
- """Text output read as key-value pairs separated by equal signs"""
-
- def conversion(value):
- """Convert text to int or float if possible, otherwise return it as is"""
- try:
- return int(value)
- except ValueError:
- pass
- try:
- return float(value)
- except ValueError:
- pass
- return value
-
- return gs.parse_key_val(self._stdout, val_type=conversion)
-
- @property
- def comma_items(self):
- """Text output read as comma-separated list"""
- return self.text_split(",")
-
- @property
- def space_items(self):
- """Text output read as whitespace-separated list"""
- return self.text_split(None)
-
- def text_split(self, separator=None):
- """Parse text output read as list separated by separators
-
- Any leading or trailing newlines are removed prior to parsing.
- """
- # The use of strip is assuming that the output is one line which
- # ends with a newline character which is for display only.
- return self._decoded_stdout.strip("\n").split(separator)
-
- def __getitem__(self, name):
- if self._stdout:
- # We are testing just std out and letting rest to the parse and the user.
- # This makes no assumption about how JSON is produced by the tool.
- try:
- return self.json[name]
- except json.JSONDecodeError as error:
- if self._kwargs.get("format") == "json":
- raise
- msg = (
- f"Output of {self._name} cannot be parsed as JSON. "
- 'Did you use format="json"?'
- )
- raise ValueError(msg) from error
- msg = f"No text output for {self._name} to be parsed as JSON"
- raise ValueError(msg)
-
-
-class Tools:
- """Call GRASS tools as methods
-
- GRASS tools (modules) can be executed as methods of this class.
- """
-
- def __init__(
- self,
- *,
- session=None,
- env=None,
- overwrite=False,
- quiet=False,
- verbose=False,
- superquiet=False,
- freeze_region=False,
- stdin=None,
- errors=None,
- capture_output=True,
- prefix=None,
- ):
- if env:
- self._env = env.copy()
- elif session and hasattr(session, "env"):
- self._env = session.env.copy()
- else:
- self._env = os.environ.copy()
- self._region_is_frozen = False
- if freeze_region:
- self._freeze_region()
- if overwrite:
- self._overwrite()
- # This hopefully sets the numbers directly. An alternative implementation would
- # be to pass the parameter every time.
- # Does not check for multiple set at the same time, but the most verbose wins
- # for safety.
- if superquiet:
- self._env["GRASS_VERBOSE"] = "0"
- if quiet:
- self._env["GRASS_VERBOSE"] = "1"
- if verbose:
- self._env["GRASS_VERBOSE"] = "3"
- self._set_stdin(stdin)
- self._errors = errors
- self._capture_output = capture_output
- self._prefix = prefix
- self._name_helper = None
-
- # These could be public, not protected.
- def _freeze_region(self):
- self._env["GRASS_REGION"] = gs.region_env(env=self._env)
- self._region_is_frozen = True
-
- def _overwrite(self):
- self._env["GRASS_OVERWRITE"] = "1"
-
- def _set_stdin(self, stdin, /):
- self._stdin = stdin
-
- @property
- def env(self):
- """Internally used environment (reference to it, not a copy)"""
- return self._env
-
- def _process_parameters(self, command, popen_options):
- env = popen_options.get("env", self._env)
-
- return subprocess.run(
- [*command, "--json"], text=True, capture_output=True, env=env
- )
-
- def run(self, name, /, **kwargs):
- """Run modules from the GRASS display family (modules starting with "d.").
-
- This function passes arguments directly to grass.script.run_command()
- so the syntax is the same.
-
- :param str module: name of GRASS module
- :param `**kwargs`: named arguments passed to run_command()"""
-
- object_parameter_handler = ObjectParameterHandler()
- object_parameter_handler.process_parameters(kwargs)
-
- args, popen_options = gs.popen_args_command(name, **kwargs)
-
- interface_result = self._process_parameters(args, popen_options)
- if interface_result.returncode != 0:
- # This is only for the error states.
- return gs.handle_errors(
- interface_result.returncode,
- result=None,
- args=[name],
- kwargs=kwargs,
- stderr=interface_result.stderr,
- handler="raise",
- )
- parameters = json.loads(interface_result.stdout)
- object_parameter_handler.translate_objects_to_data(
- kwargs, parameters, env=self._env
- )
-
- # We approximate tool_kwargs as original kwargs.
- result = self.run_from_list(
- args,
- tool_kwargs=kwargs,
- processed_parameters=parameters,
- stdin=object_parameter_handler.stdin,
- **popen_options,
- )
- use_objects = object_parameter_handler.translate_data_to_objects(
- kwargs, parameters, env=self._env
- )
- if use_objects:
- result = object_parameter_handler.result
- return result
-
- def run_from_list(
- self,
- command,
- tool_kwargs=None,
- stdin=None,
- processed_parameters=None,
- **popen_options,
- ):
- if not processed_parameters:
- interface_result = self._process_parameters(command, popen_options)
- if interface_result.returncode != 0:
- # This is only for the error states.
- return gs.handle_errors(
- interface_result.returncode,
- result=None,
- args=[command],
- kwargs=tool_kwargs,
- stderr=interface_result.stderr,
- handler="raise",
- )
- processed_parameters = json.loads(interface_result.stdout)
-
- # We approximate tool_kwargs as original kwargs.
- return self.no_nonsense_run_from_list(
- command,
- tool_kwargs=tool_kwargs,
- stdin=stdin,
- **popen_options,
- )
-
- def run_command(self, name, /, **kwargs):
- # TODO: Provide custom implementation for full control
- return gs.run_command(name, **kwargs, env=self._env)
-
- def parse_command(self, name, /, **kwargs):
- # TODO: Provide custom implementation for full control
- return gs.parse_command(name, **kwargs, env=self._env)
-
- def no_nonsense_run(self, name, /, *, tool_kwargs=None, stdin=None, **kwargs):
- args, popen_options = gs.popen_args_command(name, **kwargs)
- return self.no_nonsense_run_from_list(
- args, tool_kwargs=tool_kwargs, stdin=stdin, **popen_options
- )
-
- # Make this an overload of run.
- def no_nonsense_run_from_list(
- self, command, tool_kwargs=None, stdin=None, **popen_options
- ):
- # alternatively use dev null as default or provide it as convenient settings
- if self._capture_output:
- stdout_pipe = gs.PIPE
- stderr_pipe = gs.PIPE
- else:
- stdout_pipe = None
- stderr_pipe = None
- if self._stdin:
- stdin_pipe = gs.PIPE
- stdin = gs.utils.encode(self._stdin)
- elif stdin:
- stdin_pipe = gs.PIPE
- stdin = gs.utils.encode(stdin)
- else:
- stdin_pipe = None
- stdin = None
- # Allowing to overwrite env, but that's just to have maximum flexibility when
- # the session is actually set up, but it may be confusing.
- if "env" not in popen_options:
- popen_options["env"] = self._env
- process = gs.Popen(
- command,
- stdin=stdin_pipe,
- stdout=stdout_pipe,
- stderr=stderr_pipe,
- **popen_options,
- )
- stdout, stderr = process.communicate(input=stdin)
- if stderr:
- stderr = gs.utils.decode(stderr)
- returncode = process.poll()
- if returncode and self._errors != "ignore":
- raise CalledModuleError(
- command[0],
- code=" ".join(command),
- returncode=returncode,
- errors=stderr,
- )
- # TODO: solve tool_kwargs is None
- # We don't have the keyword arguments to pass to the resulting object.
- return ExecutedTool(
- name=command[0], kwargs=tool_kwargs, stdout=stdout, stderr=stderr
- )
-
- def feed_input_to(self, stdin, /):
- """Get a new object which will feed text input to a tool or tools"""
- return Tools(
- env=self._env,
- stdin=stdin,
- freeze_region=self._region_is_frozen,
- errors=self._errors,
- capture_output=self._capture_output,
- prefix=self._prefix,
- )
-
- def ignore_errors_of(self):
- """Get a new object which will ignore errors of the called tools"""
- return Tools(env=self._env, errors="ignore")
-
- def __getattr__(self, name):
- """Parse attribute to GRASS display module. Attribute should be in
- the form 'd_module_name'. For example, 'd.rast' is called with 'd_rast'.
- """
- if not self._name_helper:
- self._name_helper = ToolFunctionNameHelper(
- run_function=self.run,
- env=self.env,
- prefix=self._prefix,
- )
- return self._name_helper.get_function(name, exception_type=AttributeError)
-
-
-def _test():
- """Ad-hoc tests and examples of the Tools class"""
- session = gs.setup.init("~/grassdata/nc_spm_08_grass7/user1")
-
- tools = Tools()
- tools.g_region(raster="elevation")
- tools.r_slope_aspect(elevation="elevation", slope="slope", overwrite=True)
- print(tools.r_univar(map="slope", flags="g").keyval)
-
- print(tools.v_info(map="bridges", flags="c").text)
- print(
- tools.v_db_univar(map="bridges", column="YEAR_BUILT", format="json").json[
- "statistics"
- ]["mean"]
- )
-
- print(tools.g_mapset(flags="p").text)
- print(tools.g_mapsets(flags="l").text_split())
- print(tools.g_mapsets(flags="l").space_items)
- print(tools.g_gisenv(get="GISDBASE,LOCATION_NAME,MAPSET", sep="comma").comma_items)
-
- print(tools.g_region(flags="g").keyval)
-
- env = os.environ.copy()
- env["GRASS_REGION"] = gs.region_env(res=250)
- coarse_computation = Tools(env=env)
- current_region = coarse_computation.g_region(flags="g").keyval
- print(current_region["ewres"], current_region["nsres"])
- coarse_computation.r_slope_aspect(
- elevation="elevation", slope="slope", flags="a", overwrite=True
- )
- print(coarse_computation.r_info(map="slope", flags="g").keyval)
-
- independent_computation = Tools(session=session, freeze_region=True)
- tools.g_region(res=500) # we would do this for another computation elsewhere
- print(independent_computation.g_region(flags="g").keyval["ewres"])
-
- tools_pro = Tools(
- session=session, freeze_region=True, overwrite=True, superquiet=True
- )
- tools_pro.r_slope_aspect(elevation="elevation", slope="slope")
- tools_pro.feed_input_to("13.45,29.96,200").v_in_ascii(
- input="-", output="point", separator=","
- )
- print(tools_pro.v_info(map="point", flags="t").keyval["points"])
-
- print(tools_pro.ignore_errors_of().g_version(flags="rge").keyval)
-
- elevation = "elevation"
- exaggerated = "exaggerated"
- tools_pro.r_mapcalc(expression=f"{exaggerated} = 5 * {elevation}")
- tools_pro.feed_input_to(f"{exaggerated} = 5 * {elevation}").r_mapcalc(file="-")
-
-
-if __name__ == "__main__":
- _test()
diff --git a/python/grass/tools/session_tools.py b/python/grass/tools/session_tools.py
index 410fc8256ea..6a3e30e1657 100644
--- a/python/grass/tools/session_tools.py
+++ b/python/grass/tools/session_tools.py
@@ -167,13 +167,28 @@ def run(self, name: str, /, **kwargs):
# Get a fixed env parameter at at the beginning of each execution,
# but repeat it every time in case the referenced environment is modified.
args, popen_options = gs.popen_args_command(name, **kwargs)
+
+ # Compute the environment for subprocesses and store it for later use.
+ if "env" not in popen_options:
+ popen_options["env"] = self._modified_env_if_needed()
+
+ object_parameter_handler.translate_objects_to_data(
+ kwargs, env=popen_options["env"]
+ )
+
# We approximate original kwargs with the possibly-modified kwargs.
- return self.run_cmd(
+ result = self.run_cmd(
args,
tool_kwargs=kwargs,
input=object_parameter_handler.stdin,
**popen_options,
)
+ use_objects = object_parameter_handler.translate_data_to_objects(
+ kwargs, env=popen_options["env"]
+ )
+ if use_objects:
+ result = object_parameter_handler.result
+ return result
def run_cmd(
self,
diff --git a/python/grass/tools/support.py b/python/grass/tools/support.py
index 00844871de8..9416df1a8e8 100644
--- a/python/grass/tools/support.py
+++ b/python/grass/tools/support.py
@@ -25,22 +25,55 @@
import shutil
from io import StringIO
+import numpy as np
+
import grass.script as gs
+import grass.script.array as garray
class ParameterConverter:
def __init__(self):
self._numpy_inputs = {}
- self._numpy_outputs = {}
+ self._numpy_outputs = []
self._numpy_inputs_ordered = []
self.stdin = None
+ self.result = None
def process_parameters(self, kwargs):
for key, value in kwargs.items():
- if isinstance(value, StringIO):
+ if isinstance(value, np.ndarray):
+ name = gs.append_uuid("tmp_serialized_input_array")
+ kwargs[key] = name
+ self._numpy_inputs[key] = (name, value)
+ elif value in (np.ndarray, np.array, garray.array):
+ # We test for class or the function.
+ name = gs.append_uuid("tmp_serialized_output_array")
+ kwargs[key] = name
+ self._numpy_outputs.append((name, value))
+ elif isinstance(value, StringIO):
kwargs[key] = "-"
self.stdin = value.getvalue()
+ def translate_objects_to_data(self, kwargs, env):
+ for name, value in self._numpy_inputs.values():
+ map2d = garray.array(env=env)
+ map2d[:] = value
+ map2d.write(name)
+
+ def translate_data_to_objects(self, kwargs, env):
+ output_arrays = []
+ for name, value in self._numpy_outputs:
+ output_array = garray.array(name, env=env)
+ output_arrays.append(output_array)
+ if len(output_arrays) == 1:
+ self.result = output_arrays[0]
+ return True
+ if len(output_arrays) > 1:
+ self.result = tuple(output_arrays)
+ return True
+ self.result = None
+ return False
+
class ToolFunctionResolver:
def __init__(self, *, run_function, env):
diff --git a/python/grass/tools/tests/grass_tools_session_tools_numpy_test.py b/python/grass/tools/tests/grass_tools_session_tools_numpy_test.py
new file mode 100644
index 00000000000..b3877b108b5
--- /dev/null
+++ b/python/grass/tools/tests/grass_tools_session_tools_numpy_test.py
@@ -0,0 +1,116 @@
+import numpy as np
+
+import grass.script as gs
+from grass.tools import Tools
+
+
+def test_numpy_one_input(xy_dataset_session):
+ """Check that global overwrite is not used when separate env is used"""
+ tools = Tools(session=xy_dataset_session)
+ tools.r_slope_aspect(elevation=np.ones((1, 1)), slope="slope")
+ assert tools.r_info(map="slope", format="json")["datatype"] == "FCELL"
+
+
+# NumPy syntax for outputs
+# While inputs are straightforward, there is several possible ways how to handle
+# syntax for outputs.
+# Output is the type of function for creating NumPy arrays, return value is now the arrays:
+# tools.r_slope_aspect(elevation=np.ones((1, 1)), slope=np.ndarray, aspect=np.array)
+# Output is explicitly requested:
+# tools.r_slope_aspect(elevation=np.ones((1, 1)), slope="slope", aspect="aspect", force_numpy_for_output=True)
+# Output is explicitly requested at the object level:
+# Tools(force_numpy_for_output=True).r_slope_aspect(elevation=np.ones((1, 1)), slope="slope", aspect="aspect")
+# Output is always array or arrays when at least on input is an array:
+# tools.r_slope_aspect(elevation=np.ones((1, 1)), slope="slope", aspect="aspect")
+# An empty array is passed to signal the desired output:
+# tools.r_slope_aspect(elevation=np.ones((1, 1)), slope=np.nulls((0, 0)))
+# An array to be filled with data is passed, the return value is kept as is:
+# tools.r_slope_aspect(elevation=np.ones((1, 1)), slope=np.nulls((1, 1)))
+# NumPy universal function concept can be used explicitly to indicate,
+# possibly more easily allowing for nameless args as opposed to keyword arguments,
+# but outputs still need to be explicitly requested:
+# Returns by value (tuple: (np.array, np.array)):
+# tools.r_slope_aspect.ufunc(np.ones((1, 1)), slope=True, aspect=True)
+# Modifies its arguments in-place:
+# tools.r_slope_aspect.ufunc(np.ones((1, 1)), slope=True, aspect=True, out=(np.array((1, 1)), np.array((1, 1))))
+# Custom signaling classes or objects are passed (assuming empty classes AsNumpy and AsInput):
+# tools.r_slope_aspect(elevation=np.ones((1, 1)), slope=ToNumpy(), aspect=ToNumpy())
+# tools.r_slope_aspect(elevation=np.ones((1, 1)), slope=AsInput, aspect=AsInput)
+# NumPy functions usually return a tuple, for multiple outputs. Universal function does
+# unless the output is written to out parameter which is also provided as a tuple. We
+# have names, so generally, we can return a dictionary:
+# {"slope": np.array(...), "aspect": np.array(...) }.
+
+
+def test_numpy_one_input_one_output(xy_dataset_session):
+ """Check that a NumPy array works as input and for signaling output
+
+ It tests that the np.ndarray class is supported to signal output.
+ Return type is not strictly defined, so we are not testing for it explicitly
+ (only by actually using it as an NumPy array).
+ """
+ tools = Tools(session=xy_dataset_session)
+ tools.g_region(rows=2, cols=3)
+ slope = tools.r_slope_aspect(elevation=np.ones((2, 3)), slope=np.ndarray)
+ assert slope.shape == (2, 3)
+ assert np.all(slope == np.full((2, 3), 0))
+
+
+def test_numpy_with_name_and_parameter(xy_dataset_session):
+ """Check that a NumPy array works as input and for signaling output
+
+ It tests that the np.ndarray class is supported to signal output.
+ Return type is not strictly defined, so we are not testing for it explicitly
+ (only by actually using it as an NumPy array).
+ """
+ tools = Tools(session=xy_dataset_session)
+ tools.g_region(rows=2, cols=3)
+ slope = tools.run("r.slope.aspect", elevation=np.ones((2, 3)), slope=np.ndarray)
+ assert slope.shape == (2, 3)
+ assert np.all(slope == np.full((2, 3), 0))
+
+
+def test_numpy_one_input_multiple_outputs(xy_dataset_session):
+ """Check that a NumPy array function works for signaling multiple outputs
+
+ Besides multiple outputs it tests that np.array is supported to signal output.
+ """
+ tools = Tools(session=xy_dataset_session)
+ tools.g_region(rows=2, cols=3)
+ (slope, aspect) = tools.r_slope_aspect(
+ elevation=np.ones((2, 3)), slope=np.array, aspect=np.array
+ )
+ assert slope.shape == (2, 3)
+ assert np.all(slope == np.full((2, 3), 0))
+ assert aspect.shape == (2, 3)
+ assert np.all(aspect == np.full((2, 3), 0))
+
+
+def test_numpy_multiple_inputs_one_output(xy_dataset_session):
+ """Check that a NumPy array works for multiple inputs"""
+ tools = Tools(session=xy_dataset_session)
+ tools.g_region(rows=2, cols=3)
+ result = tools.r_mapcalc_simple(
+ expression="A + B", a=np.full((2, 3), 2), b=np.full((2, 3), 5), output=np.array
+ )
+ assert result.shape == (2, 3)
+ assert np.all(result == np.full((2, 3), 7))
+
+
+def test_numpy_grass_array_input_output(xy_dataset_session):
+ """Check that global overwrite is not used when separate env is used
+
+ When grass array output is requested, we explicitly test the return value type.
+ """
+ tools = Tools(session=xy_dataset_session)
+ rows = 2
+ cols = 3
+ tools.g_region(rows=rows, cols=cols)
+ tools.r_mapcalc_simple(expression="5", output="const_5")
+ const_5 = gs.array.array("const_5", env=xy_dataset_session.env)
+ result = tools.r_mapcalc_simple(
+ expression="2 * A", a=const_5, output=gs.array.array
+ )
+ assert result.shape == (rows, cols)
+ assert np.all(result == np.full((rows, cols), 10))
+ assert isinstance(result, gs.array.array)
From 52a14291b87139b763235a1241d4867a7c1853ce Mon Sep 17 00:00:00 2001
From: Vaclav Petras <wenzeslaus@gmail.com>
Date: Sun, 27 Jul 2025 08:40:10 -0400
Subject: [PATCH 24/29] Remove tools from experimental Makefile
---
python/grass/experimental/Makefile | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/python/grass/experimental/Makefile b/python/grass/experimental/Makefile
index e3c0c6d3c84..e28f932a0a8 100644
--- a/python/grass/experimental/Makefile
+++ b/python/grass/experimental/Makefile
@@ -7,8 +7,7 @@ DSTDIR = $(ETC)/python/grass/experimental
MODULES = \
create \
- mapset \
- tools
+ mapset
PYFILES := $(patsubst %,$(DSTDIR)/%.py,$(MODULES) __init__)
PYCFILES := $(patsubst %,$(DSTDIR)/%.pyc,$(MODULES) __init__)
From 3a4629e24b385176bb738c4dda17c41fc580541b Mon Sep 17 00:00:00 2001
From: Vaclav Petras <wenzeslaus@gmail.com>
Date: Sun, 27 Jul 2025 10:43:31 -0400
Subject: [PATCH 25/29] Remove unused fixture and move comment with discussion
to PR comment
---
python/grass/experimental/tests/conftest.py | 8 -----
.../grass_tools_session_tools_numpy_test.py | 31 -------------------
2 files changed, 39 deletions(-)
diff --git a/python/grass/experimental/tests/conftest.py b/python/grass/experimental/tests/conftest.py
index d9bb0d5403d..5bd44da1de0 100644
--- a/python/grass/experimental/tests/conftest.py
+++ b/python/grass/experimental/tests/conftest.py
@@ -33,14 +33,6 @@ def xy_session_for_module(tmp_path_factory):
yield session
-@pytest.fixture
-def xy_dataset_session(tmp_path):
- """Creates a session with a mapset which has vector with a float column"""
- gs.core._create_location_xy(tmp_path, "test") # pylint: disable=protected-access
- with gs.setup.init(tmp_path / "test") as session:
- yield session
-
-
@pytest.fixture
def unique_id():
"""A unique alphanumeric identifier"""
diff --git a/python/grass/tools/tests/grass_tools_session_tools_numpy_test.py b/python/grass/tools/tests/grass_tools_session_tools_numpy_test.py
index b3877b108b5..2fa639c4cca 100644
--- a/python/grass/tools/tests/grass_tools_session_tools_numpy_test.py
+++ b/python/grass/tools/tests/grass_tools_session_tools_numpy_test.py
@@ -11,37 +11,6 @@ def test_numpy_one_input(xy_dataset_session):
assert tools.r_info(map="slope", format="json")["datatype"] == "FCELL"
-# NumPy syntax for outputs
-# While inputs are straightforward, there is several possible ways how to handle
-# syntax for outputs.
-# Output is the type of function for creating NumPy arrays, return value is now the arrays:
-# tools.r_slope_aspect(elevation=np.ones((1, 1)), slope=np.ndarray, aspect=np.array)
-# Output is explicitly requested:
-# tools.r_slope_aspect(elevation=np.ones((1, 1)), slope="slope", aspect="aspect", force_numpy_for_output=True)
-# Output is explicitly requested at the object level:
-# Tools(force_numpy_for_output=True).r_slope_aspect(elevation=np.ones((1, 1)), slope="slope", aspect="aspect")
-# Output is always array or arrays when at least on input is an array:
-# tools.r_slope_aspect(elevation=np.ones((1, 1)), slope="slope", aspect="aspect")
-# An empty array is passed to signal the desired output:
-# tools.r_slope_aspect(elevation=np.ones((1, 1)), slope=np.nulls((0, 0)))
-# An array to be filled with data is passed, the return value is kept as is:
-# tools.r_slope_aspect(elevation=np.ones((1, 1)), slope=np.nulls((1, 1)))
-# NumPy universal function concept can be used explicitly to indicate,
-# possibly more easily allowing for nameless args as opposed to keyword arguments,
-# but outputs still need to be explicitly requested:
-# Returns by value (tuple: (np.array, np.array)):
-# tools.r_slope_aspect.ufunc(np.ones((1, 1)), slope=True, aspect=True)
-# Modifies its arguments in-place:
-# tools.r_slope_aspect.ufunc(np.ones((1, 1)), slope=True, aspect=True, out=(np.array((1, 1)), np.array((1, 1))))
-# Custom signaling classes or objects are passed (assuming empty classes AsNumpy and AsInput):
-# tools.r_slope_aspect(elevation=np.ones((1, 1)), slope=ToNumpy(), aspect=ToNumpy())
-# tools.r_slope_aspect(elevation=np.ones((1, 1)), slope=AsInput, aspect=AsInput)
-# NumPy functions usually return a tuple, for multiple outputs. Universal function does
-# unless the output is written to out parameter which is also provided as a tuple. We
-# have names, so generally, we can return a dictionary:
-# {"slope": np.array(...), "aspect": np.array(...) }.
-
-
def test_numpy_one_input_one_output(xy_dataset_session):
"""Check that a NumPy array works as input and for signaling output
From d3136d60ae6ef18741b06826510fbdb4127cfd22 Mon Sep 17 00:00:00 2001
From: Vaclav Petras <wenzeslaus@gmail.com>
Date: Tue, 29 Jul 2025 05:59:13 -0400
Subject: [PATCH 26/29] Add cleanup. Improve benchmark message.
---
python/grass/benchmark/runners.py | 2 +-
python/grass/tools/session_tools.py | 7 +++++++
python/grass/tools/support.py | 3 +++
3 files changed, 11 insertions(+), 1 deletion(-)
diff --git a/python/grass/benchmark/runners.py b/python/grass/benchmark/runners.py
index e733e9b64de..3dc8c8310f0 100644
--- a/python/grass/benchmark/runners.py
+++ b/python/grass/benchmark/runners.py
@@ -173,7 +173,7 @@ def benchmark_resolutions(module, resolutions, label, repeat=5, nprocs=None):
region = gs.region()
n_cells.append(region["cells"])
print("\u2500" * term_size.columns)
- print(f"Benchmark with {resolution} resolution...\n")
+ print(f"Benchmark with resolution {resolution}...\n")
time_sum = 0
measured_times = []
for _ in range(repeat):
diff --git a/python/grass/tools/session_tools.py b/python/grass/tools/session_tools.py
index 6a3e30e1657..03ddc7c28a2 100644
--- a/python/grass/tools/session_tools.py
+++ b/python/grass/tools/session_tools.py
@@ -188,6 +188,13 @@ def run(self, name: str, /, **kwargs):
)
if use_objects:
result = object_parameter_handler.result
+ if object_parameter_handler.temporary_rasters:
+ self.call(
+ "g.remove",
+ type="raster",
+ name=object_parameter_handler.temporary_rasters,
+ flags="f",
+ )
return result
def run_cmd(
diff --git a/python/grass/tools/support.py b/python/grass/tools/support.py
index 9416df1a8e8..004a9f3232f 100644
--- a/python/grass/tools/support.py
+++ b/python/grass/tools/support.py
@@ -38,6 +38,7 @@ def __init__(self):
self._numpy_inputs_ordered = []
self.stdin = None
self.result = None
+ self.temporary_rasters = []
def process_parameters(self, kwargs):
for key, value in kwargs.items():
@@ -59,12 +60,14 @@ def translate_objects_to_data(self, kwargs, env):
map2d = garray.array(env=env)
map2d[:] = value
map2d.write(name)
+ self.temporary_rasters.append(name)
def translate_data_to_objects(self, kwargs, env):
output_arrays = []
for name, value in self._numpy_outputs:
output_array = garray.array(name, env=env)
output_arrays.append(output_array)
+ self.temporary_rasters.append(name)
if len(output_arrays) == 1:
self.result = output_arrays[0]
return True
From 13cc850b0b61258405869dda5466832081e0a9aa Mon Sep 17 00:00:00 2001
From: Vaclav Petras <wenzeslaus@gmail.com>
Date: Tue, 29 Jul 2025 06:16:49 -0400
Subject: [PATCH 27/29] Add benchmark
---
.../benchmark/benchmark_grass_tools_numpy.py | 121 ++++++++++++++++++
1 file changed, 121 insertions(+)
create mode 100644 python/grass/tools/benchmark/benchmark_grass_tools_numpy.py
diff --git a/python/grass/tools/benchmark/benchmark_grass_tools_numpy.py b/python/grass/tools/benchmark/benchmark_grass_tools_numpy.py
new file mode 100644
index 00000000000..4077f849935
--- /dev/null
+++ b/python/grass/tools/benchmark/benchmark_grass_tools_numpy.py
@@ -0,0 +1,121 @@
+import time
+import numpy as np
+
+
+from grass.tools import Tools
+from grass.benchmark import (
+ num_cells_plot,
+ benchmark_resolutions,
+ load_results,
+ save_results,
+)
+
+
+class TimeMeasurer:
+ def __init__(self):
+ self._time = None
+ self._start = None
+
+ @property
+ def time(self):
+ return self._time
+
+ def start(self):
+ self._start = time.perf_counter()
+
+ def stop(self):
+ self._time = time.perf_counter() - self._start
+
+
+class PlainNumPyBenchmark(TimeMeasurer):
+ def run(self):
+ tools = Tools()
+ region = tools.g_region(flags="p", format="json")
+ a = np.full((region["rows"], region["cols"]), 1)
+ b = np.full((region["rows"], region["cols"]), 1)
+
+ self.start()
+ c = 2 * np.sqrt(a + b) * np.sqrt(a) + np.sqrt(b) + a / 2
+ self.stop()
+
+ print(c.sum())
+ print(c.size)
+
+ del a
+ del b
+ del c
+
+
+class PlainGRASSBenchmark(TimeMeasurer):
+ def run(self):
+ tools = Tools(overwrite=True)
+ tools.r_mapcalc(expression="a = 1")
+ tools.r_mapcalc(expression="b = 1")
+
+ self.start()
+ tools.r_mapcalc(expression="c = 2 * sqrt(a + b) * sqrt(a) * sqrt(b) + a / 2")
+ self.stop()
+
+ c_stats = tools.r_univar(map="c", format="json")
+ print(c_stats["sum"])
+ print(c_stats["cells"])
+
+
+class NumPyGRASSBenchmark(TimeMeasurer):
+ def run(self):
+ tools = Tools()
+ region = tools.g_region(flags="p", format="json")
+ a = np.full((region["rows"], region["cols"]), 1)
+ b = np.full((region["rows"], region["cols"]), 1)
+
+ self.start()
+ c = tools.r_mapcalc_simple(
+ expression="2* sqrt(A + B) * sqrt(A) * sqrt(B) + A / 2",
+ a=a,
+ b=b,
+ output=np.array,
+ )
+ self.stop()
+
+ c_stats = tools.r_univar(map=c, format="json")
+ print(c_stats["sum"])
+ print(c_stats["cells"])
+
+ del a
+ del b
+ del c
+
+
+def main():
+ resolutions = [5, 2, 1, 0.5]
+ repeat = 10
+ results = [
+ benchmark_resolutions(
+ module=PlainNumPyBenchmark(),
+ label="NumPy",
+ resolutions=resolutions,
+ repeat=repeat,
+ ),
+ benchmark_resolutions(
+ module=PlainGRASSBenchmark(),
+ label="GRASS",
+ resolutions=resolutions,
+ repeat=repeat,
+ ),
+ benchmark_resolutions(
+ module=NumPyGRASSBenchmark(),
+ label="NumPy GRASS",
+ resolutions=resolutions,
+ repeat=repeat,
+ ),
+ ]
+ print(results)
+ results = load_results(save_results(results))
+ print(results)
+ plot_file = "test_res_plot.png"
+ num_cells_plot(results.results, filename=plot_file)
+ print(plot_file)
+
+
+if __name__ == "__main__":
+ main()
From 697053f7933154c21519973f331d8f14f39b12f8 Mon Sep 17 00:00:00 2001
From: Vaclav Petras <wenzeslaus@gmail.com>
Date: Tue, 29 Jul 2025 07:01:53 -0400
Subject: [PATCH 28/29] Add NumPy arrays to docstring
---
python/grass/tools/session_tools.py | 13 +++++++++++++
1 file changed, 13 insertions(+)
diff --git a/python/grass/tools/session_tools.py b/python/grass/tools/session_tools.py
index 03ddc7c28a2..edd62e5cd88 100644
--- a/python/grass/tools/session_tools.py
+++ b/python/grass/tools/session_tools.py
@@ -52,6 +52,19 @@ class Tools:
the *ToolResult* object:
>>> tools.g_region(flags="p").text # doctest: +SKIP
+
+ Raster input and outputs can be NumPy arrays:
+
+ >>> import numpy as np
+ >>> tools.g_region(rows=2, cols=3)
+ ToolResult(...)
+ >>> slope = tools.r_slope_aspect(elevation=np.ones((2, 3)), slope=np.ndarray)
+
+ When multiple outputs are returned, they are returned as a tuple:
+
+ >>> (slope, aspect) = tools.r_slope_aspect(
+ ... elevation=np.ones((2, 3)), slope=np.array, aspect=np.array
+ ... )
"""
def __init__(
From 79cf950a0ff8b78e54326be9b4c109247f619ed6 Mon Sep 17 00:00:00 2001
From: Vaclav Petras <wenzeslaus@gmail.com>
Date: Tue, 29 Jul 2025 11:16:24 -0400
Subject: [PATCH 29/29] Document the support class
---
python/grass/tools/support.py | 23 +++++++++++++++++++++++
1 file changed, 23 insertions(+)
diff --git a/python/grass/tools/support.py b/python/grass/tools/support.py
index 004a9f3232f..496f09ddf0b 100644
--- a/python/grass/tools/support.py
+++ b/python/grass/tools/support.py
@@ -32,6 +32,8 @@
class ParameterConverter:
+ """Converts parameter values to strings and facilitates flow of the data."""
+
def __init__(self):
self._numpy_inputs = {}
self._numpy_outputs = []
@@ -41,6 +43,21 @@ def __init__(self):
self.temporary_rasters = []
def process_parameters(self, kwargs):
+ """Converts high level parameter values to strings.
+
+ Converts io.StringIO to dash and stores the string in the *stdin* attribute.
+ Replaces NumPy arrays by temporary raster names and stores the arrays.
+ Replaces NumPy array types by temporary raster names.
+
+ Temporary names are accessible in the *temporary_rasters* attribute and need
+ to be cleaned.
+ The functions *translate_objects_to_data* and *translate_data_to_objects*
+ need to be called before and after the computation to do the translations
+ from NumPy arrays to GRASS data and from GRASS data to NumPy arrays.
+
+ Simple type conversions from numbers and iterables to strings are expected to
+ be done by lower level code.
+ """
for key, value in kwargs.items():
if isinstance(value, np.ndarray):
name = gs.append_uuid("tmp_serialized_input_array")
@@ -56,6 +73,7 @@ def process_parameters(self, kwargs):
self.stdin = value.getvalue()
def translate_objects_to_data(self, kwargs, env):
+ """Convert NumPy arrays to GRASS data"""
for name, value in self._numpy_inputs.values():
map2d = garray.array(env=env)
map2d[:] = value
@@ -63,6 +81,11 @@ def translate_objects_to_data(self, kwargs, env):
self.temporary_rasters.append(name)
def translate_data_to_objects(self, kwargs, env):
+ """Convert GRASS data to NumPy arrays
+
+ Returns True if there is one or more output arrays, False otherwise.
+ The arrays are stored in the *result* attribute.
+ """
output_arrays = []
for name, value in self._numpy_outputs:
output_array = garray.array(name, env=env)