diff --git a/docs/source/api.rst b/docs/source/api.rst index 3958bc1..4d42213 100644 --- a/docs/source/api.rst +++ b/docs/source/api.rst @@ -143,18 +143,22 @@ Utilities --------- .. autosummary:: + library.ProjectLibrary utils.AttrDict utils.Enum utils.IsInstalled utils.get_cls proj.base.ParseFailed + config.get_conf +.. autoclass:: library.ProjectLibrary .. autoclass:: projspec.utils.AttrDict .. autoclass:: projspec.utils.Enum .. autofunction:: projspec.utils.get_cls .. autoclass:: projspec.utils.IsInstalled :members: .. autoclass:: projspec.proj.base.ParseFailed +.. autofunction:: config.get_conf .. raw:: html diff --git a/pyproject.toml b/pyproject.toml index 4613696..6fe2daf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,15 +26,18 @@ dependencies = [ "fsspec >=2025.5.1", "pyyaml", "toml", + "fsspec", "click", "jinja2" ] [project.optional-dependencies] test = ["pytest", "pytest-cov"] +qt = ["pyqt>5,<6", "pyqtwebengin>5,<6"] [project.scripts] projspec = "projspec.__main__:main" +projspec-qt = "projspec.qtapp.main:main" [tool.poetry.extras] po_test = ["pytest"] @@ -53,6 +56,11 @@ raw-options = {'version_scheme'='post-release'} [tool.hatch.build.hooks.vcs] version-file = "src/projspec/_version.py" +[tool.coverage.run] +omit = [ + "src/projspec/qtapp/*.py" +] + [tool.ruff] target-version = "py39" exclude = [".tox", "build", "docs/source/conf.py", "projspec/_version"] diff --git a/src/projspec/__init__.py b/src/projspec/__init__.py index 3c75dca..d9160db 100644 --- a/src/projspec/__init__.py +++ b/src/projspec/__init__.py @@ -2,6 +2,8 @@ from projspec.proj import Project, ProjectSpec import projspec.content import projspec.artifact +import projspec.config +import projspec.library from projspec.utils import get_cls __all__ = ["Project", "ProjectSpec", "get_cls"] diff --git a/src/projspec/config.py b/src/projspec/config.py new file mode 100644 index 0000000..bf976fe --- /dev/null +++ b/src/projspec/config.py @@ -0,0 +1,28 @@ +import json +import os + +from typing import Any + +conf: dict[str, dict[str, Any]] = {} +default_conf_dir = os.path.join(os.path.expanduser("~"), ".config/projspec") +conf_dir = os.environ.get("PROJSPEC_CONFIG_DIR", default_conf_dir) + +defaults = { + # location of persisted project objects + "library_path": f"{conf_dir}/library.json", +} + + +def load_conf(path: str | None = None): + fn = f"{path or default_conf_dir}/projspec.json" + if os.path.exists(fn): + with open(fn) as f: + conf.update(json.load(f)) + + +load_conf() + + +def get_conf(name: str): + """Fetch the value of the given conf parameter from the current config or defaults""" + return conf[name] if name in conf else defaults[name] diff --git a/src/projspec/library.py b/src/projspec/library.py new file mode 100644 index 0000000..358926b --- /dev/null +++ b/src/projspec/library.py @@ -0,0 +1,74 @@ +import json +import os + +import fsspec + +from projspec.config import get_conf +from projspec.proj import Project + + +class ProjectLibrary: + """Stores scanned project objects at a given path in JSON format + + An instance of this library ``library`` is created on import. + + In the future, alternative serialisations will be implemented. + """ + + # TODO: support for remote libraries + + def __init__(self, library_path: str | None = None, auto_save: bool = True): + self.path = library_path or get_conf("library_path") + self.load() + self.entries: dict[str, Project] = {} + self.auto_save = auto_save + + def load(self): + """Loads scanned project objects from JSON file""" + try: + with fsspec.open(self.path, "r") as f: + self.entries = { + k: Project.from_dict(v) for k, v in json.load(f).items() + } + except FileNotFoundError: + self.entries = {} + + def clear(self): + """Clears scanned project objects from JSON file and memory""" + if os.path.isfile(self.path): + os.unlink(self.path) + self.entries = {} + + def add_entry(self, path: str, entry: Project): + """Adds an entry to the scanned project object""" + self.entries[path] = entry + if self.auto_save: + self.save() + + def save(self): + """Serialise the state of the scanned project objects to file""" + # don't catch + data = {k: v.to_dict(compact=False) for k, v in self.entries.items()} + with fsspec.open(self.path, "w") as f: + json.dump(data, f) + + def filter(self, filters: list[tuple[str, str]]) -> dict[str, Project]: + return {k: v for k, v in self.entries.items() if _match(v, filters)} + + +# move to Project definition? +def _match(proj: Project, filters: list[tuple[str, str | tuple[str]]]) -> bool: + # TODO: this is all AND, but you can get OR by passing a tuple of values + for cat, value in filters: + # TODO: make categories an enum + if cat == "spec" and value not in proj: + return False + if cat == "artifact" and not proj.all_artifacts(value): + return False + if cat == "content" and not proj.all_contents(value): + return False + return True + + +library = ProjectLibrary() +library.load() diff --git a/src/projspec/proj/base.py b/src/projspec/proj/base.py index 27d2cf7..3081218 100644 --- a/src/projspec/proj/base.py +++ b/src/projspec/proj/base.py @@ -32,6 +32,12 @@ class ParseFailed(ValueError): class Project: + """Top level representation of a project directory + + This holds any parsed project metadata specs, top level contents and artifacts and + any project details from nested child directories. + """ + def __init__( self, path: str, @@ -209,7 +215,7 @@ def pyproject(self): pass return {} - def all_artifacts(self, names=None) -> list: + def all_artifacts(self, names: str | None = None) -> list: """A flat list of all the artifact objects nested in this project.""" arts = set(self.artifacts.values()) for spec in self.specs.values(): @@ -270,14 +276,17 @@ def __contains__(self, item) -> bool: return item in self.specs or any(item in _ for _ in self.children.values()) def to_dict(self, compact=True) -> dict: - dic = AttrDict( - specs=self.specs, - children=self.children, - url=self.url, - storage_options=self.storage_options, - artifacts=self.artifacts, - contents=self.contents, - ) + try: + dic = AttrDict( + specs=self.specs, + children=self.children, + url=self.url, + storage_options=self.storage_options, + artifacts=self.artifacts, + contents=self.contents, + ) + except AttributeError: + print(list(self.__dict__.keys())) if not compact: dic["klass"] = "project" return dic.to_dict(compact=compact) @@ -296,6 +305,8 @@ def from_dict(dic): proj = object.__new__(Project) proj.specs = from_dict(dic["specs"], proj) proj.children = from_dict(dic["children"], proj) + proj.contents = from_dict(dic["contents"], proj) + proj.artifacts = from_dict(dic["artifacts"], proj) proj.url = dic["url"] proj.storage_options = dic["storage_options"] proj.fs, _ = fsspec.url_to_fs(proj.url, **proj.storage_options) @@ -386,8 +397,7 @@ def to_dict(self, compact=True) -> dict: _contents=self.contents, _artifacts=self.artifacts, ) - if self.subpath: - dic["subpath"] = self.subpath + dic["subpath"] = self.subpath if not compact: dic["klass"] = ["projspec", self.snake_name()] return dic.to_dict(compact=compact) diff --git a/src/projspec/proj/conda_package.py b/src/projspec/proj/conda_package.py index 946b3da..bbddde3 100644 --- a/src/projspec/proj/conda_package.py +++ b/src/projspec/proj/conda_package.py @@ -38,7 +38,9 @@ def parse(self) -> None: pass if meta is None: raise ParseFailed - art = CondaPackage(proj=self.proj, cmd=["conda-build", self.proj.url]) + art = CondaPackage( + proj=self.proj, cmd=["conda-build", f"{self.proj.url}/*.conda"] + ) self._artifacts = AttrDict(conda_package=art) # TODO: read envs from "outputs" like for Rattler, below? # or use ``conda render`` to "lock" a templated recipe diff --git a/src/projspec/proj/documentation.py b/src/projspec/proj/documentation.py index 4361fd6..9273309 100644 --- a/src/projspec/proj/documentation.py +++ b/src/projspec/proj/documentation.py @@ -39,7 +39,7 @@ def parse(self) -> None: # point to a requirements.txt or environment.yaml for conda env. # Artifact of HTML output. Classically with `make html` (in docs/ unless otherwise - # specified), and the output goes into docs/build/html unless the conf.py file says different. - # RTD actually does + # specified), and the output goes into docs/build/html unless the config.py file + # says different. RTD actually does # > python -m sphinx -T -W --keep-going -b html -d _build/doctrees -D language=en . $READTHEDOCS_OUTPUT/html pass diff --git a/src/projspec/proj/pixi.py b/src/projspec/proj/pixi.py index 0bb23fa..2ccd1c7 100644 --- a/src/projspec/proj/pixi.py +++ b/src/projspec/proj/pixi.py @@ -136,7 +136,7 @@ def parse(self) -> None: arts["conda_package"] = CondaPackage( proj=self.proj, name=pkg["name"], - fn=f"{pkg['name']}-{pkg['version']}*.conda", + fn=f"{self.proj.url}/{pkg['name']}-{pkg['version']}*.conda", cmd=["pixi", "build"], ) diff --git a/src/projspec/proj/rust.py b/src/projspec/proj/rust.py index bbb4060..2726e1f 100644 --- a/src/projspec/proj/rust.py +++ b/src/projspec/proj/rust.py @@ -16,9 +16,8 @@ def parse(self): with self.proj.fs.open(f"{self.proj.url}/Cargo.toml", "rt") as f: meta = toml.load(f) self.contents["desciptive_metadata"] = DescriptiveMetadata( - proj=self.proj, meta=meta["package"], artifacts=set() + proj=self.proj, meta=meta.get("package"), artifacts=set() ) - 1 class RustPython(Rust, PythonLibrary): diff --git a/src/projspec/proj/uv.py b/src/projspec/proj/uv.py index e158ae1..d55dd98 100644 --- a/src/projspec/proj/uv.py +++ b/src/projspec/proj/uv.py @@ -124,7 +124,7 @@ def match(self): try: with self.proj.fs.open(f"{self.proj.url}/.venv/pyvenv.cfg", "rt") as f: txt = f.read() - return b"uv =" in txt + return "uv =" in txt except (OSError, FileNotFoundError): pass return False diff --git a/src/projspec/qtapp/__init__.py b/src/projspec/qtapp/__init__.py index e69de29..d1942bd 100644 --- a/src/projspec/qtapp/__init__.py +++ b/src/projspec/qtapp/__init__.py @@ -0,0 +1,4 @@ +"""Example projspec apps, based on qt + +See also ``ipywidget`` or ``panel`` implementations, when they are ready. +""" diff --git a/src/projspec/qtapp/main.py b/src/projspec/qtapp/main.py index d72215a..a7bfa38 100644 --- a/src/projspec/qtapp/main.py +++ b/src/projspec/qtapp/main.py @@ -1,32 +1,59 @@ +import os.path import sys from pathlib import Path + +import fsspec from PyQt5.QtWidgets import ( QApplication, + QDialog, + QPushButton, + QComboBox, QMainWindow, QTreeWidget, QTreeWidgetItem, - QVBoxLayout, QWidget, - QLabel, + QStyle, QHBoxLayout, - QDialog, + QVBoxLayout, + QDockWidget, + QLineEdit, ) from PyQt5.QtWebEngineWidgets import QWebEngineView -from PyQt5.QtCore import Qt, QUuid, QUrl -from PyQt5.QtGui import QIcon +from PyQt5.QtCore import Qt, pyqtSignal # just Signal in PySide + +from projspec.library import library +import projspec class FileBrowserWindow(QMainWindow): - def __init__(self, parent=None): + """A mini filesystem browser with project information + + The right-hand pane will populate with an HTML view of the selected item, + if that item is a directory and can be interpreted as any project type. + """ + + def __init__(self, path=None, parent=None): super().__init__(parent) + self.library = Library() + if path is None: + # implicitly local + path = os.path.expanduser("~") + self.fs, self.path = fsspec.url_to_fs(path) + self.addDockWidget(Qt.BottomDockWidgetArea, self.library) + self.setWindowTitle("Projspec Browser") self.setGeometry(100, 100, 950, 600) + left = QVBoxLayout() # Create tree widget + self.path_text = QLineEdit(path) + self.path_text.returnPressed.connect(self.path_set) self.tree = QTreeWidget(self) - self.tree.setHeaderLabels(["Name", "Type", "Size"]) - self.tree.setColumnWidth(0, 400) + self.tree.setHeaderLabels(["Name", "Size"]) + self.tree.setColumnWidth(0, 300) self.tree.setColumnWidth(1, 50) + left.addWidget(self.path_text) + left.addWidget(self.tree) # Connect signals self.tree.itemExpanded.connect(self.on_item_expanded) @@ -35,13 +62,14 @@ def __init__(self, parent=None): self.detail = QWebEngineView(self) # self.detail.load(QUrl("https://qt-project.org/")) self.detail.setFixedWidth(600) + self.library.project_selected.connect(self.detail.setHtml) # Create central widget and layout central_widget = QWidget(self) self.setCentralWidget(central_widget) layout = QHBoxLayout(central_widget) - layout.addWidget(self.tree) + layout.addLayout(left) layout.addWidget(self.detail) central_widget.setLayout(layout) @@ -51,16 +79,19 @@ def __init__(self, parent=None): # Populate with home directory self.populate_tree() + def path_set(self): + self.fs, _ = fsspec.url_to_fs(self.path_text.text()) + self.path = self.path_text.text() + self.populate_tree() + def populate_tree(self): """Populate the tree with the user's home directory""" - home_path = Path.home() + self.tree.clear() root_item = QTreeWidgetItem(self.tree) - root_item.setText(0, home_path.name or str(home_path)) - root_item.setText(1, "Folder") - root_item.setData(0, Qt.ItemDataRole.UserRole, str(home_path)) + root_item.setText(0, self.path) # Add a dummy child to make it expandable - self.add_children(root_item, home_path) + self.add_children(root_item, self.path) # Expand the root root_item.setExpanded(True) @@ -68,35 +99,35 @@ def populate_tree(self): def add_children(self, parent_item, path): """Add child items for a directory""" try: - path_obj = Path(path) - # Get all items in directory - items = sorted( - path_obj.iterdir(), key=lambda x: (not x.is_dir(), x.name.lower()) - ) + details = self.fs.ls(path, detail=True) + items = sorted(details, key=lambda x: (x["type"], x["name"].lower())) for item in items: # Skip hidden files (optional) - if item.name.startswith("."): + name = item["name"].rsplit("/", 1)[-1] + if name.startswith("."): continue child_item = QTreeWidgetItem(parent_item) - child_item.setText(0, item.name) - child_item.setData(0, Qt.ItemDataRole.UserRole, str(item)) + child_item.setText(0, name) + child_item.setData(0, Qt.ItemDataRole.UserRole, item) - if item.is_dir(): - child_item.setText(1, "Folder") - child_item.setText(2, "") + style = app.style() + if item["type"] == "directory": + # TODO: change icon if it is in the library # Add dummy child to make it expandable dummy = QTreeWidgetItem(child_item) dummy.setText(0, "Loading...") + if item["name"] in library.entries: + child_item.setIcon( + 0, style.standardIcon(QStyle.SP_FileDialogInfoView) + ) + else: + child_item.setIcon(0, style.standardIcon(QStyle.SP_DirIcon)) else: - child_item.setText(1, "File") - try: - size = item.stat().st_size - child_item.setText(2, self.format_size(size)) - except: - child_item.setText(2, "") + child_item.setText(1, format_size(item["size"])) + child_item.setIcon(0, style.standardIcon(QStyle.SP_FileIcon)) except PermissionError: error_item = QTreeWidgetItem(parent_item) @@ -113,37 +144,193 @@ def on_item_expanded(self, item): if item.childCount() == 1 and item.child(0).text(0) == "Loading...": # Remove dummy child item.removeChild(item.child(0)) - - # Get path from item data - path = item.data(0, Qt.ItemDataRole.UserRole) - - # Add real children + path = item.data(0, Qt.ItemDataRole.UserRole)["name"] if path: self.add_children(item, path) self.statusBar().showMessage(f"Loaded: {path}") - def on_item_changed(self, item): + def on_item_changed(self, item: QTreeWidgetItem): import projspec - if item.text(1) == "Folder": - proj = projspec.Project(item.data(0, Qt.ItemDataRole.UserRole), walk=False) + detail = item.data(0, Qt.ItemDataRole.UserRole) + if detail["type"] == "directory": + path = detail["name"] + proj = projspec.Project(path, walk=False, fs=self.fs) if proj.specs: - print(proj.text_summary()) - html = f"{proj._repr_html_()}" - self.detail.setHtml(html) + style = app.style() + item.setIcon(0, style.standardIcon(QStyle.SP_FileDialogInfoView)) + body = f"{proj._repr_html_()}" + library.add_entry(path, proj) else: - self.detail.setHtml("") + body = "" + self.library.refresh() # only on new item? + self.detail.setHtml(f"{body}") + + +def format_size(size: None | int) -> str: + """Format file size in human-readable format""" + if size is None: + return "" + for unit in ["B", "KB", "MB", "GB", "TB"]: + if size < 1024.0: + return f"{size:.1f} {unit}" + size /= 1024.0 + return f"{size:.1f} PB" + + +class Library(QDockWidget): + """Shows all scanned projects and allows filtering by various criteria""" + + project_selected = pyqtSignal(str) + + def __init__(self): + super().__init__() + self.setWindowTitle("Project Library") + self.widget = QWidget(self) + + # search control + swidget = QWidget(self.widget) + upper_layout = QHBoxLayout() + search = QPushButton("๐Ÿ”") + search.clicked.connect(self.on_search_clicked) + clear = QPushButton("๐Ÿงน") + upper_layout.addWidget(search) + upper_layout.addWidget(clear) + upper_layout.addStretch() + swidget.setLayout(upper_layout) + + # main list + self.list = QTreeWidget(self.widget) + self.list.setHeaderLabels(["Path", "Types"]) + self.list.itemClicked.connect(self.on_selection_changed) + self.list.setColumnWidth(0, 300) + + # main layout + layout = QVBoxLayout(self.widget) + layout.addWidget(self.list) + layout.addWidget(swidget) + self.setWidget(self.widget) + self.dia = SearchDialog(self) + self.dia.accepted.connect(self.refresh) + clear.clicked.connect(self.dia.clear) + + self.refresh() + + def on_search_clicked(self): + self.dia.exec_() + + def on_selection_changed(self, item: QTreeWidgetItem): + path = item.text(0) + proj = library.entries[path] + body = f"{proj._repr_html_()}" + self.project_selected.emit(body) + + def refresh(self): + # any refresh reopens the pane if it was closed + self.list.clear() + data = library.filter(self.dia.search_criteria) + for path in sorted(data): + self.list.addTopLevelItem( + QTreeWidgetItem([path, " ".join(library.entries[path].specs)]) + ) + self.show() + + +class SearchItem(QWidget): + """A single search criterion""" + + removed = pyqtSignal(QWidget) + + def __init__(self, parent=None): + super().__init__(parent) + layout = QHBoxLayout() + self.which = QComboBox(parent=self) + self.which.addItems(["..", "spec", "artifact", "content"]) + self.which.currentTextChanged.connect(self.on_which_changed) + layout.addWidget(self.which, 1) + + self.select = QComboBox(parent=self) + self.select.addItem("..") + layout.addWidget(self.select, 1) + + self.x = QPushButton("โŒ") + self.x.clicked.connect(self.on_x_clicked) + layout.addWidget(self.x) + self.setLayout(layout) + + @property + def criterion(self): + sel = self.select.currentText() + return (self.which.currentText(), sel) if sel != ".." else None + + def on_x_clicked(self, _): + self.removed.emit(self) + + def on_which_changed(self, text): + self.select.clear() + self.select.addItem("..") + if text == "spec": + self.select.addItems([str(_) for _ in projspec.proj.base.registry]) + elif text == "artifact": + self.select.addItems([str(_) for _ in projspec.artifact.base.registry]) + elif text == "content": + self.select.addItems([str(_) for _ in projspec.content.base.registry]) + + +class SearchDialog(QDialog): + """Set search criteria""" + + def __init__(self, parent=None): + super().__init__(parent) + self.criteria = [] + + right = QVBoxLayout() + ok = QPushButton("OK") + ok.clicked.connect(self.accept) + cancel = QPushButton("Cancel") + cancel.clicked.connect(self.reject) + right.addWidget(ok) + right.addWidget(cancel) + right.addStretch(0) + + mini_layout = QHBoxLayout() + add = QPushButton("+") + add.clicked.connect(self.on_add) + mini_layout.addWidget(add) + mini_layout.addStretch(0) + + self.layout = QVBoxLayout() + self.layout.addLayout(mini_layout) + self.layout.addStretch(0) + + all_layout = QHBoxLayout(self) + all_layout.addLayout(self.layout, 1) + all_layout.addLayout(right) + self.setLayout(all_layout) + + def on_add(self): + search = SearchItem(self) + search.removed.connect(self._on_search_removed) + self.layout.insertWidget(0, search) + self.criteria.append(search) + + @property + def search_criteria(self): + return [_.criterion for _ in self.criteria if _.criterion is not None] + + def clear(self): + for item in self.criteria: + self.layout.removeWidget(item) + self.criteria = [] + self.accepted.emit() - def format_size(self, size): - """Format file size in human-readable format""" - for unit in ["B", "KB", "MB", "GB", "TB"]: - if size < 1024.0: - return f"{size:.1f} {unit}" - size /= 1024.0 - return f"{size:.1f} PB" + def _on_search_removed(self, search_widget): + self.layout.removeWidget(search_widget) + self.criteria.remove(search_widget) def main(): + global app app = QApplication(sys.argv) window = FileBrowserWindow() window.show() diff --git a/tests/test_config.py b/tests/test_config.py new file mode 100644 index 0000000..b5b22a6 --- /dev/null +++ b/tests/test_config.py @@ -0,0 +1,23 @@ +import json + +import pytest + +from projspec.config import conf, get_conf, defaults, load_conf + + +@pytest.fixture +def blank_conf(monkeypatch): + old_conf = conf.copy() + conf.clear() + yield + conf.update(old_conf) + + +def test_get_conf(blank_conf, tmpdir): + assert get_conf("library_path") == defaults["library_path"] + fn = str(tmpdir.join("projspec.json")) + with open(fn, "wt") as f: + json.dump({"temp": True}, f) + + load_conf(str(tmpdir)) + assert get_conf("temp") is True diff --git a/tests/test_library.py b/tests/test_library.py new file mode 100644 index 0000000..276e66a --- /dev/null +++ b/tests/test_library.py @@ -0,0 +1,42 @@ +import os + +from projspec import Project +from projspec.library import ProjectLibrary + +here = os.path.abspath(os.path.dirname(__file__)) +root = os.path.dirname(here) + + +def test_library(tmp_path): + fn = str(tmp_path / "library") + library = ProjectLibrary(fn, auto_save=True) + proj = Project(root) + + assert not os.path.exists(fn) + assert not library.entries + + library.add_entry(root, proj) + + assert os.path.exists(fn) + assert library.entries + + library.clear() + + assert not os.path.exists(fn) + assert not library.entries + + +def test_filter(tmp_path): + fn = str(tmp_path / "library") + library = ProjectLibrary(None, auto_save=False) + proj = Project(root) + library.add_entry(root, proj) + + # empty filter + assert library.filter([]) + + # filter hit + assert library.filter([("spec", "python_library")]) + + # miss + assert not library.filter([("spec", "xx")])