Cross platform packaging

.github/workflows/package_windows.yml

@@ -0,0 +1,59 @@
+name: windows-build
+
+on:
+  push:
+    branches: [ main ]
+    tags: [ "v*" ]
+  pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build-windows:
+    runs-on: windows-latest
+
+    defaults:
+      run:
+        shell: powershell
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Conda
+        uses: conda-incubator/setup-miniconda@v3
+        with:
+          python-version: ${{ matrix.python-version }}
+          auto-update-conda: true
+          channels: conda-forge
+          activate-environment: "datashuttle-test"
+
+      - name: Install rclone
+        run: |
+          conda activate datashuttle-test
+          conda install -c conda-forge rclone
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install .[dev]
+          pip install pyinstaller
+
+      # Install Inno Setup silently
+      - name: Install Inno Setup
+        run: |
+          choco install innosetup --yes
+
+      - name: Verify Inno Setup installation
+        run: |
+          Get-Command "C:\Program Files (x86)\Inno Setup 6\ISCC.exe"
+
+      - name: Run Windows packaging script
+        run: |
+          python package/package_windows.py
+
+      - name: Upload installer artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: windows-installer
+          path: package\Output\datashuttle_0.0.0.exe

.gitignore

@@ -1,7 +1,13 @@
+# Packaging
+package/dist
+package/build
+package/_vendored
+package/Output
+package/inno_complie_script.iss
+
 # python-dotenv environment file for test account credentials.
 .env
 
-
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
@@ -35,7 +41,6 @@ MANIFEST
 #  Usually these files are written by a python script from a template
 #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 *.manifest
-*.spec
 
 # Installer logs
 pip-log.txt

datashuttle/tui/css/tui_menu.tcss

@@ -4,6 +4,7 @@
 #mainwindow_contents_container {
     align: center top;
     padding: 1 0 0 0;
+    overflow: hidden auto;
 }
 #mainwindow_contents_container > Button {
     width: 50%;

datashuttle/tui_launcher.py

@@ -42,4 +42,7 @@ def main() -> None:
 
 
 if __name__ == "__main__":
-    main()
+    try:
+        main()
+    except:
+        input("hello")

datashuttle/utils/rclone.py

@@ -27,12 +27,32 @@
 import platform
 import shlex
 import subprocess
+import sys
 import tempfile
 
 from datashuttle.configs import canonical_configs
 from datashuttle.utils import rclone_encryption, utils
 
 
+def get_command(command: str) -> str:
+    """ """
+    from pathlib import Path
+
+    if getattr(sys, "frozen", False):
+        # PyInstaller: binary extracted to _MEIPASS
+        if sys.platform == "win32":
+            format_command = (
+                f"{str(Path(sys._MEIPASS) / 'rclone.exe')} {command}"
+            )
+        else:
+            format_command = f"{sys._MEIPASS}/rclone {command}"
+    else:
+        # Normal Python execution: use PATH or fixed path
+        format_command = f"rclone {command}"  # or provide full path if needed
+
+    return format_command
+
+
 def call_rclone(command: str, pipe_std: bool = False) -> CompletedProcess:
     """Call rclone with the specified command.
 
@@ -49,13 +69,17 @@ def call_rclone(command: str, pipe_std: bool = False) -> CompletedProcess:
     subprocess.CompletedProcess with `stdout` and `stderr` attributes.
 
     """
-    command = "rclone " + command
+    format_command = get_command(command)
+
     if pipe_std:
         output = subprocess.run(
-            command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True
+            format_command,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            shell=True,
         )
     else:
-        output = subprocess.run(command, shell=True)
+        output = subprocess.run(format_command, shell=True)
 
     if output.returncode != 0:
         prompt_rclone_download_if_does_not_exist()
@@ -101,18 +125,18 @@ def call_rclone_through_script_for_central_connection(
     """
     system = platform.system()
 
-    command = "rclone " + command
-
     if system == "Windows":
         suffix = ".bat"
     else:
         suffix = ".sh"
         command = "#!/bin/bash\n" + command
 
+    format_command = get_command(command)
+
     with tempfile.NamedTemporaryFile(
         mode="w", suffix=suffix, delete=False
     ) as tmp_script:
-        tmp_script.write(command)
+        tmp_script.write(format_command)
         tmp_script_path = tmp_script.name
 
     try:
@@ -594,7 +618,7 @@ def check_rclone_with_default_call() -> bool:
     """
     try:
         output = subprocess.run(
-            "rclone -h",
+            get_command("-h"),
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
             shell=True,

package/README.md

@@ -0,0 +1,64 @@
+# Packaging
+
+Packaging datashuttle is slightly more complex than packaging a simple Python application,
+because we need to vendor the terminal to run datashuttle in, because it can render strangely
+depending on the platform (e.g. macOS terminal, older Windows terminals).
+
+Wezterm, a cross-platform terminal that renders datashuttle well. It is vendored as part of
+this distribution.
+
+Packaging is a two-step process. First, we package datashuttle into an executable.
+This executable contains all datashuttle dependencies and can be run from any terminal.
+
+Next, a simple script ('terminal launcher') that runs the datashuttle executable in the vendored
+Wezterm terminal is packaged using pyinstaller. This is the main entry point for the distribution.
+
+Finally, this executable is distributed via an installer (inno setup on Windows, TODO on macos (`.dmg`)
+This installs the executable in the proper place on the system, adds shortcuts etc.
+
+So the call chain is:
+1) The main executable is 'terminal launcher'
+2) This opens the vendored Wezterm, and uses it to start the second, datashuttle executable.
+
+## The `package` directory
+
+The path handling is a little fiddly here, and some scrips are shared  between platform.
+Therefore, all materials are currently stored in this folder and not refactored into their own folder.
+
+The key files are:
+
+- **datashuttle.spec**: The pyinstaller command file that packages datashuttle, through `datashuttle_launcher.py` script.
+    This outputs a datashuttle executable that contains all datashuttle dependencies, and can be run through a terminal.
+- **datashuttle_launcher.py**: A wrapper script that `datashuttle.spec` packages, that launches datashuttle.
+- **license.txt**: The datashuttle license
+- **package_<macos_or_windows>.py**: The entry point script that coordinates the packaging of datashuttle on macOS / Windows.
+- **terminal_launcher_<macos_or_windows>.py**: The entry point script for the final datashuttle distribution.
+    It opens Wezterm and runs the datashuttle executable in it.
+- **terminal_launcher_macos.spec**: The Pyinstaller command file that packages the `terminal_launcher` script into an executable.
+- **wezterm_config.lua**: The Wezterm config. Use this to change settings that affect how the vendored Wezterm appears.
+- **inno_compile_script.iss**:  The Inno script used to package the Windows distribution into an installer. The output
+    of this script is shipped.
+- **packaging_utils.py**: General utils used in the packaging scripts.
+
+## Windows
+
+When packaging on Windows, a number of intermediate folders will be created. Under normal circumstances
+they will be deleted, but knowing their contents can be useful for debugging:
+
+- **build**: Generated by pyinstaller, it contains intermediate files generating during the
+packaging of `datashuttle` and `terminal_launcher`. Generally you do not need to inspect these files,
+they are pyinstaller stuff.
+- **dist**: Initially generated by pyinstaller, this is the complete distribution. During packaging,
+this file is used for the Pyinstaller dist of `datashuttle` and `terminal_launcher`, then we
+copy a few auxillary files (e.g. license, Wezterm config)). Because we are building two Pyinstaller
+packages into one directory, some care had to be taken to avoid configs. The folders included are:
+1) `datashuttle`: The datashuttle distribution generated by Pyinstaller
+2) `terminal_launcher.exe` and `_internal` is the `terminal_launcher` distribution generated by Pyinstaller.
+3) `_vendored` is our vendor folder, that contains the vendored Wezterm temrinal.
+4) datashuttle license and icon
+- **Output**: This is the output of Inno Setup, containing the installer. This is the final version to be shipped.
+
+
+
+
+## macOS

package/TODO.txt

@@ -0,0 +1 @@
+DONT FORGET TO INCLUDE THE LICENSE

package/datashuttle.spec

@@ -0,0 +1,79 @@
+# main.spec
+# -*- mode: python ; coding: utf-8 -*-
+import os
+import sys
+import platform
+from glob import glob
+from pathlib import Path
+import shutil
+
+# Include .tcss files
+tcss_files = [
+    (f, os.path.join("datashuttle", "tui", "css"))
+    for f in glob("../datashuttle/tui/css/*.tcss")
+]
+
+# Get current conda environment prefix
+env_prefix = sys.prefix
+
+# Detect OS and set rclone path
+
+rclone_src = shutil.which("rclone")
+
+if rclone_src is None:
+    raise FileNotFoundError(
+        "rclone not found in PATH. Ensure it is installed before running PyInstaller."
+    )
+
+binaries = [(rclone_src, ".")]
+
+a = Analysis(
+    ['datashuttle_launcher.py'],  # terminal_launcher
+    pathex=[os.path.abspath('..')],
+    binaries=binaries,
+    datas=tcss_files,
+    hiddenimports=[
+        'datashuttle.tui_launcher',
+        'datashuttle.tui.app',
+        'textual.widgets._tab_pane',
+        'textual.widgets._input',
+        'textual.widgets._tree_control',
+        'rich._unicode_data.unicode17-0-0'
+    ],
+    hookspath=['hooks'],
+    hooksconfig={},
+    runtime_hooks=[],
+    excludes=[],
+    noarchive=False,
+    optimize=0,
+)
+
+pyz = PYZ(a.pure)
+
+exe = EXE(
+    pyz,
+    a.scripts,
+    [],
+    exclude_binaries=True,
+    name='datashuttle',
+    debug=False,
+    bootloader_ignore_signals=False,
+    strip=False,
+    upx=True,
+    console=True,
+    disable_windowed_traceback=False,
+    argv_emulation=False,
+    target_arch=None,
+    codesign_identity=None,
+    entitlements_file=None,
+)
+
+coll = COLLECT(
+    exe,
+    a.binaries,
+    a.datas,
+    strip=False,
+    upx=True,
+    upx_exclude=[],
+    name='datashuttle'
+)

package/datashuttle_launcher.py

@@ -0,0 +1,24 @@
+import sys
+from pathlib import Path
+
+# Add project root to sys.path
+project_root = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(project_root))
+
+# Import main from tui_launcher
+from datashuttle.tui_launcher import main as datashuttle_main
+
+
+def run():
+    # Simulate: datashuttle launch
+    sys.argv = ["datashuttle", "launch"]
+    datashuttle_main()
+
+
+if __name__ == "__main__":
+    try:
+        run()
+    except Exception as e:
+        print(f"\nError: {e}")
+    finally:
+        input("\nPress Enter to exit...")

package/license.txt

@@ -0,0 +1 @@
+hello world