Creating a new Collection model for use in organizing FileGroups.

Replacing Channels and Domains with Collection.

Moving downloading relationships from Channel to Collection.

Created domains.yaml config to manage tagged domain collections.

Adding turn-key upgrade solution in the UI.

Author: lrnselfreliance

.circleci/config.yml

@@ -142,6 +142,66 @@ jobs:
       - run:
           name: Stop React App
           command: pkill -f npm
+  jest-tests:
+    docker:
+      - image: cimg/node:20.13
+    resource_class: medium
+    steps:
+      - checkout
+      - restore_cache:
+          key: app-{{ checksum "app/package-lock.json" }}
+      - run:
+          name: Install Dependencies
+          command: cd app && npm install --legacy-peer-deps
+      - save_cache:
+          key: app-{{ checksum "app/package-lock.json" }}
+          paths:
+            - ./app/node_modules
+      - run:
+          name: Run Jest Tests
+          command: cd app && CI=true npm test -- --watchAll=false --coverage --coverageReporters=text --coverageReporters=lcov
+      - store_artifacts:
+          path: app/coverage
+          destination: coverage-report
+  cypress-e2e-tests:
+    docker:
+      - image: cimg/node:20.18-browsers
+    resource_class: large
+    steps:
+      - checkout
+      - restore_cache:
+          key: app-browsers-{{ checksum "app/package-lock.json" }}
+      - run:
+          name: Install Dependencies
+          command: cd app && npm install --legacy-peer-deps
+      - save_cache:
+          key: app-browsers-{{ checksum "app/package-lock.json" }}
+          paths:
+            - ./app/node_modules
+      - run:
+          name: Start React App in Background
+          command: cd app && DISABLE_ESLINT_PLUGIN=true npm start
+          background: true
+      - run:
+          name: Wait for React App
+          command: |
+            for i in {1..60}; do
+              if curl -s http://localhost:3000 > /dev/null; then
+                echo "React app is ready!"
+                exit 0
+              fi
+              echo "Waiting for React app... ($i/60)"
+              sleep 2
+            done
+            echo "React app failed to start"
+            exit 1
+      - run:
+          name: Run Cypress Tests
+          command: cd app && CI=true npx cypress run
+      - store_artifacts:
+          path: app/cypress/videos
+      - store_artifacts:
+          path: app/cypress/screenshots
 
 workflows:
   wrolpi-api-tests:
@@ -153,3 +213,5 @@ workflows:
   wrolpi-app-test:
     jobs:
       - react-app-start
+      - jest-tests
+      - cypress-e2e-tests

.gitignore

@@ -125,7 +125,7 @@ mapnik.xml
 docker-compose.override.yml
 
 # test directory is used as media directory, we don't want to commit what a user downloads.
-test
+/test
 pg_data
 
 # Directories used to build images
@@ -135,3 +135,5 @@ pg_data
 /pi-gen/*xz
 
 .DS_Store
+app/cypress/screenshots
+app/cypress/videos

.mcp.json

@@ -0,0 +1,12 @@
+{
+  "mcpServers": {
+    "postgres": {
+      "command": "npx",
+      "args": [
+        "-y",
+        "@modelcontextprotocol/server-postgres",
+        "postgresql://postgres:wrolpi@localhost:5432/wrolpi"
+      ]
+    }
+  }
+}

alembic/versions/66407d145b76_.py

@@ -0,0 +1,87 @@
+"""Create collection table for domain collections
+
+Revision ID: 66407d145b76
+Revises: 4f03b9548f6e
+Create Date: 2025-10-26 10:57:16.462524
+
+"""
+import os
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.orm import Session
+
+
+# revision identifiers, used by Alembic.
+revision = '66407d145b76'
+down_revision = '4f03b9548f6e'
+branch_labels = None
+depends_on = None
+
+DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
+
+
+def upgrade():
+    bind = op.get_bind()
+    session = Session(bind=bind)
+
+    # Step 1: Create new collection table (keeping channel table separate for now)
+    op.create_table(
+        'collection',
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('name', sa.String(), nullable=False),
+        sa.Column('description', sa.Text(), nullable=True),
+        sa.Column('kind', sa.String(), nullable=False),
+        sa.Column('directory', sa.Text(), nullable=True),
+        sa.Column('tag_id', sa.Integer(), nullable=True),
+        sa.Column('created_date', sa.DateTime(), server_default=sa.func.now(), nullable=False),
+        sa.Column('item_count', sa.Integer(), nullable=False, server_default='0'),
+        sa.Column('total_size', sa.Integer(), nullable=False, server_default='0'),
+        sa.PrimaryKeyConstraint('id'),
+        sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], name='collection_tag_id_fkey'),
+        sa.UniqueConstraint('directory', name='uq_collection_directory')
+    )
+
+    # Create indexes for collection
+    op.create_index('idx_collection_kind', 'collection', ['kind'], unique=False)
+    op.create_index('idx_collection_item_count', 'collection', ['item_count'], unique=False)
+    op.create_index('idx_collection_total_size', 'collection', ['total_size'], unique=False)
+
+    # Step 2: Create collection_item junction table
+    op.create_table(
+        'collection_item',
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('collection_id', sa.Integer(), nullable=False),
+        sa.Column('file_group_id', sa.Integer(), nullable=False),
+        sa.Column('position', sa.Integer(), nullable=False, server_default='0'),
+        sa.Column('added_date', sa.DateTime(), server_default=sa.func.now(), nullable=False),
+        sa.PrimaryKeyConstraint('id'),
+        sa.ForeignKeyConstraint(['collection_id'], ['collection.id'], ondelete='CASCADE'),
+        sa.ForeignKeyConstraint(['file_group_id'], ['file_group.id'], ondelete='CASCADE'),
+        sa.UniqueConstraint('collection_id', 'file_group_id', name='uq_collection_file_group')
+    )
+
+    # Create indexes for collection_item
+    op.create_index('idx_collection_item_collection_id', 'collection_item', ['collection_id'], unique=False)
+    op.create_index('idx_collection_item_file_group_id', 'collection_item', ['file_group_id'], unique=False)
+    op.create_index('idx_collection_item_position', 'collection_item', ['position'], unique=False)
+    op.create_index('idx_collection_item_collection_position', 'collection_item', ['collection_id', 'position'], unique=False)
+
+    # Ensure table owner in non-docker environments
+    if not DOCKERIZED:
+        session.execute(sa.text('ALTER TABLE public.collection OWNER TO wrolpi'))
+        session.execute(sa.text('ALTER TABLE public.collection_item OWNER TO wrolpi'))
+
+
+def downgrade():
+    # Drop collection_item table and its indexes
+    op.drop_index('idx_collection_item_collection_position', table_name='collection_item')
+    op.drop_index('idx_collection_item_position', table_name='collection_item')
+    op.drop_index('idx_collection_item_file_group_id', table_name='collection_item')
+    op.drop_index('idx_collection_item_collection_id', table_name='collection_item')
+    op.drop_table('collection_item')
+
+    # Drop collection table and its indexes
+    op.drop_index('idx_collection_total_size', table_name='collection')
+    op.drop_index('idx_collection_item_count', table_name='collection')
+    op.drop_index('idx_collection_kind', table_name='collection')
+    op.drop_table('collection')

alembic/versions/add_unique_collection_name_kind.py

@@ -0,0 +1,95 @@
+"""Add unique constraint on collection (name, kind)
+
+This migration:
+1. Removes duplicate collections (keeping the one with most items/archives)
+2. Adds a unique constraint on (name, kind) to prevent future duplicates
+
+Revision ID: add_unique_collection_name_kind
+Revises: migrate_download_to_collection
+Create Date: 2025-11-27
+"""
+import os
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy import text
+from sqlalchemy.orm import Session
+
+# revision identifiers, used by Alembic.
+revision = 'add_unique_collection_name_kind'
+down_revision = 'migrate_download_to_collection'
+branch_labels = None
+depends_on = None
+
+DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
+
+
+def upgrade():
+    bind = op.get_bind()
+    session = Session(bind=bind)
+
+    print("\n" + "=" * 60)
+    print("Add Unique Constraint on Collection (name, kind)")
+    print("=" * 60 + "\n")
+
+    # Step 1: Find and remove duplicate collections
+    print("Step 1: Finding duplicate collections...")
+
+    # Find duplicates - keep the one with the lowest id (first created)
+    duplicates = session.execute(text("""
+        SELECT name, kind, array_agg(id ORDER BY id) as ids
+        FROM collection
+        GROUP BY name, kind
+        HAVING COUNT(*) > 1
+    """)).fetchall()
+
+    if duplicates:
+        print(f"Found {len(duplicates)} sets of duplicate collections")
+        for name, kind, ids in duplicates:
+            keep_id = ids[0]  # Keep the first one (lowest id)
+            remove_ids = ids[1:]  # Remove the rest
+            print(f"  - '{name}' ({kind}): keeping id={keep_id}, removing ids={remove_ids}")
+
+            # Move any archives from duplicate collections to the one we're keeping
+            for remove_id in remove_ids:
+                session.execute(text("""
+                    UPDATE archive SET collection_id = :keep_id
+                    WHERE collection_id = :remove_id
+                """), {'keep_id': keep_id, 'remove_id': remove_id})
+
+                # Move any downloads from duplicate collections
+                session.execute(text("""
+                    UPDATE download SET collection_id = :keep_id
+                    WHERE collection_id = :remove_id
+                """), {'keep_id': keep_id, 'remove_id': remove_id})
+
+                # Move any collection items
+                session.execute(text("""
+                    UPDATE collection_item SET collection_id = :keep_id
+                    WHERE collection_id = :remove_id
+                """), {'keep_id': keep_id, 'remove_id': remove_id})
+
+                # Delete the duplicate collection
+                session.execute(text("""
+                    DELETE FROM collection WHERE id = :remove_id
+                """), {'remove_id': remove_id})
+
+        session.commit()
+        print("Duplicates removed\n")
+    else:
+        print("No duplicate collections found\n")
+
+    # Step 2: Add unique constraint
+    print("Step 2: Adding unique constraint on (name, kind)...")
+    op.create_unique_constraint('uq_collection_name_kind', 'collection', ['name', 'kind'])
+    print("Done\n")
+
+    print("=" * 60)
+    print("Migration Complete")
+    print("=" * 60 + "\n")
+
+    if not DOCKERIZED:
+        session.execute(text('ALTER TABLE public.collection OWNER TO wrolpi'))
+
+
+def downgrade():
+    op.drop_constraint('uq_collection_name_kind', 'collection', type_='unique')

alembic/versions/b43f70f369d0_remove_duplicate_channel_fields_.py

@@ -0,0 +1,73 @@
+"""Remove duplicate Channel fields delegated to Collection
+
+This migration removes fields from Channel that now delegate to Collection:
+- name (now property: channel.name → collection.name)
+- directory (now property: channel.directory → collection.directory)
+- tag_id (now property: channel.tag_id → collection.tag_id)
+
+These fields were already synced to Collection in previous migration.
+
+Revision ID: b43f70f369d0
+Revises: ba98bd360b7a
+Create Date: 2025-11-19 21:48:58.488850
+
+"""
+import os
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy import text
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = 'b43f70f369d0'
+down_revision = 'ba98bd360b7a'
+branch_labels = None
+depends_on = None
+
+DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
+
+
+def upgrade():
+    print("\n" + "="*60)
+    print("Removing Duplicate Fields from Channel")
+    print("="*60 + "\n")
+
+    # Step 1: Drop foreign key constraint on tag_id
+    print("Step 1: Dropping tag_id foreign key constraint...")
+    op.drop_constraint('channel_tag_id_fkey', 'channel', type_='foreignkey')
+    print("✓ Dropped foreign key constraint\n")
+
+    # Step 2: Drop columns (data already in Collection)
+    print("Step 2: Dropping duplicate columns...")
+    op.drop_column('channel', 'name')
+    print("  ✓ Dropped channel.name")
+    op.drop_column('channel', 'directory')
+    print("  ✓ Dropped channel.directory")
+    op.drop_column('channel', 'tag_id')
+    print("  ✓ Dropped channel.tag_id\n")
+
+    print("="*60)
+    print("✓ Channel Cleanup Complete")
+    print("  Channels now delegate name/directory/tag to Collection")
+    print("="*60 + "\n")
+
+
+def downgrade():
+    # Re-add the columns
+    op.add_column('channel', sa.Column('name', sa.String(), nullable=True))
+    op.add_column('channel', sa.Column('directory', sa.String(), nullable=True))
+    op.add_column('channel', sa.Column('tag_id', sa.Integer(), nullable=True))
+
+    # Re-add foreign key
+    op.create_foreign_key('channel_tag_id_fkey', 'channel', 'tag', ['tag_id'], ['id'], ondelete='CASCADE')
+
+    # Restore data from Collection
+    bind = op.get_bind()
+    bind.execute(text("""
+        UPDATE channel
+        SET name = collection.name,
+            directory = collection.directory,
+            tag_id = collection.tag_id
+        FROM collection
+        WHERE channel.collection_id = collection.id
+    """))