Running alembic check in cicd.

Author: lrnselfreliance

.circleci/config.yml

@@ -58,7 +58,11 @@ jobs:
           paths:
             - "venv"
       - run:
+          name: Run Alembic Migrations
           command: ./venv/bin/python ./main.py db upgrade
+      - run:
+          name: Check for Missing Migrations
+          command: ./venv/bin/python ./main.py db check
   api-tests-3-12:
     docker:
       - image: cimg/python:3.12

alembic/env.py

@@ -15,9 +15,20 @@
 
 # add your model's MetaData object here
 # for 'autogenerate' support
-# from myapp import mymodel
-# target_metadata = mymodel.Base.metadata
-target_metadata = None
+from wrolpi.common import Base
+# Import all models so they are registered with the Base metadata
+from wrolpi.files.models import FileGroup, Directory  # noqa: F401
+from wrolpi.files.ebooks import EBook  # noqa: F401
+from wrolpi.tags import Tag  # noqa: F401
+from wrolpi.flags import WROLPiFlag  # noqa: F401
+from wrolpi.downloader import Download  # noqa: F401
+from wrolpi.collections.models import Collection, CollectionItem  # noqa: F401
+from modules.videos.models import Video, Channel  # noqa: F401
+from modules.archive.models import Archive  # noqa: F401
+from modules.zim.models import Zim, TagZimEntry, ZimSubscription  # noqa: F401
+from modules.map.models import MapFile  # noqa: F401
+from modules.inventory.models import Item, Inventory  # noqa: F401
+target_metadata = Base.metadata
 
 
 # other values from the config, defined by the needs of env.py,

alembic/versions/f38cfe4b5cb2_collections_feature.py

@@ -0,0 +1,68 @@
+"""collections feature
+
+Revision ID: f38cfe4b5cb2
+Revises: add_unique_collection_name_kind
+Create Date: 2025-11-29 16:51:34.222412
+
+"""
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision = 'f38cfe4b5cb2'
+down_revision = 'add_unique_collection_name_kind'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('archive', 'collection_id',
+               existing_type=sa.INTEGER(),
+               nullable=True)
+    op.alter_column('file_group', 'mimetype',
+               existing_type=sa.TEXT(),
+               nullable=True)
+    op.alter_column('tag_file', 'tag_id',
+               existing_type=sa.INTEGER(),
+               nullable=False)
+    op.alter_column('tag_file', 'file_group_id',
+               existing_type=sa.BIGINT(),
+               nullable=False)
+    op.alter_column('tag_zim', 'tag_id',
+               existing_type=sa.INTEGER(),
+               nullable=False)
+    op.alter_column('tag_zim', 'zim_id',
+               existing_type=sa.INTEGER(),
+               nullable=False)
+    op.drop_index('video_upload_date_idx', table_name='video')
+    op.drop_column('video', 'duration')
+    op.drop_column('video', 'upload_date')
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('video', sa.Column('upload_date', postgresql.TIMESTAMP(timezone=True), autoincrement=False, nullable=True))
+    op.add_column('video', sa.Column('duration', sa.INTEGER(), autoincrement=False, nullable=True))
+    op.create_index('video_upload_date_idx', 'video', ['upload_date'], unique=False)
+    op.alter_column('tag_zim', 'zim_id',
+               existing_type=sa.INTEGER(),
+               nullable=True)
+    op.alter_column('tag_zim', 'tag_id',
+               existing_type=sa.INTEGER(),
+               nullable=True)
+    op.alter_column('tag_file', 'file_group_id',
+               existing_type=sa.BIGINT(),
+               nullable=True)
+    op.alter_column('tag_file', 'tag_id',
+               existing_type=sa.INTEGER(),
+               nullable=True)
+    op.alter_column('archive', 'collection_id',
+               existing_type=sa.INTEGER(),
+               nullable=False)
+    op.alter_column('file_group', 'mimetype',
+               existing_type=sa.TEXT(),
+               nullable=False)
+    # ### end Alembic commands ###

main.py

@@ -32,7 +32,7 @@
 
 def db_main(args):
     """
-    Handle database migrations.  This uses Alembic, supported commands are "upgrade" and "downgrade".
+    Handle database migrations.  This uses Alembic, supported commands are "upgrade", "downgrade", and "check".
     """
     from alembic.config import Config
     from alembic import command
@@ -48,6 +48,11 @@ def db_main(args):
         command.upgrade(config, 'head')
     elif args.command == 'downgrade':
         command.downgrade(config, '-1')
+    elif args.command == 'check':
+        command.check(config)
+    elif args.command == 'revision':
+        message = getattr(args, 'message', None) or 'auto migration'
+        command.revision(config, message=message, autogenerate=True)
     else:
         print(f'Unknown DB command: {args.command}')
         return 2
@@ -100,7 +105,8 @@ def main():
 
     # DB Parser for running Alembic migrations
     db_parser = sub_commands.add_parser('db')
-    db_parser.add_argument('command', help='Supported commands: upgrade, downgrade')
+    db_parser.add_argument('command', help='Supported commands: upgrade, downgrade, check, revision')
+    db_parser.add_argument('-m', '--message', help='Migration message (for revision command)')
 
     args = parser.parse_args()
 

modules/archive/__init__.py

@@ -190,6 +190,7 @@ def model_archive(file_group: FileGroup, session: Session = None) -> Archive:
 
             # Set collection_id BEFORE adding to session to avoid autoflush constraint violation
             from modules.archive.lib import get_or_create_domain_collection
+
             if file_group.url:
                 # URL is already set on the FileGroup
                 collection = get_or_create_domain_collection(session, file_group.url)
@@ -202,12 +203,8 @@ def model_archive(file_group: FileGroup, session: Session = None) -> Archive:
                     collection = get_or_create_domain_collection(session, url)
                     archive.collection_id = collection.id if collection else None
                 except (RuntimeError, ValueError) as e:
-                    # Could not extract URL from singlefile
-                    if not PYTEST:
-                        # In production, archives must have a URL/collection
-                        raise InvalidArchive(f'Archive has no URL and could not extract from singlefile: {e}') from e
-                    # In tests, allow archives without collections (factory will set it later)
-                    logger.debug(f'Could not extract URL from singlefile (test mode): {e}')
+                    # Could not extract URL from singlefile - archive will have no collection
+                    logger.debug(f'Could not extract URL from singlefile: {e}')
                     archive.collection_id = None
 
             session.add(archive)

modules/inventory/models.py

@@ -23,7 +23,7 @@ class Item(Base, ModelHelper):
     subcategory = Column(String)
     unit = Column(String)
 
-    inventory_id = Column(Integer, ForeignKey('inventory.id'))
+    inventory_id = Column(Integer, ForeignKey('inventory.id', ondelete='CASCADE'))
     inventory = relationship('Inventory', primaryjoin="Item.inventory_id==Inventory.id", back_populates='items')
 
     def __repr__(self):
@@ -35,7 +35,7 @@ class Inventory(Base, ModelHelper):
     __tablename__ = 'inventory'
     id = Column(Integer, primary_key=True)
 
-    name = Column(String, unique=True)
+    name = Column(String, unique=True, nullable=False)
     viewed_at = Column(TZDateTime)
     created_at = Column(TZDateTime, default=now)
     deleted_at = Column(TZDateTime)
@@ -61,6 +61,3 @@ def find_by_name(session: Session, name: str) -> 'Inventory':
         return session.query(Inventory).filter_by(name=name).one()
 
 
-class InventoriesVersion(Base, ModelHelper):
-    __tablename__ = 'inventories_version'
-    version = Column(Integer, primary_key=True)

modules/map/models.py

@@ -10,8 +10,8 @@ class MapFile(Base, ModelHelper):
     __tablename__ = 'map_file'
     id = Column(Integer, primary_key=True)
 
-    path = Column(MediaPathType, unique=True, nullable=False)
-    imported = Column(Boolean, default=False, nullable=False)
+    path = Column(MediaPathType, nullable=False)
+    imported = Column(Boolean, default=False)
     size = Column(BigInteger)
 
     def __repr__(self):

modules/videos/models.py

@@ -2,7 +2,7 @@
 import pathlib
 from typing import Optional, Dict, List
 
-from sqlalchemy import Column, Integer, String, Boolean, JSON, Date, ForeignKey, BigInteger
+from sqlalchemy import Column, Integer, String, Boolean, JSON, Date, ForeignKey, BigInteger, Index, text
 from sqlalchemy.orm import relationship, Session, deferred
 from sqlalchemy.orm.collections import InstrumentedList
 
@@ -28,6 +28,11 @@
 
 class Video(ModelHelper, Base):
     __tablename__ = 'video'
+    __table_args__ = (
+        Index('video_channel_id_idx', 'channel_id'),
+        Index('video_source_id_idx', 'source_id'),
+        Index('video_view_count_idx', 'view_count'),
+    )
     id = Column(Integer, primary_key=True)
 
     source_id = Column(String)  # The id from yt-dlp
@@ -530,18 +535,25 @@ def replace_info_json(self, info_json: dict, clean: bool = True, format_: bool =
 
 class Channel(ModelHelper, Base):
     __tablename__ = 'channel'
+    __table_args__ = (
+        Index('channel_minimum_frequency_idx', 'minimum_frequency'),
+        Index('channel_source_id', 'source_id'),
+        Index('channel_total_size_idx', 'total_size'),
+        Index('channel_video_count_idx', 'video_count'),
+        Index('channel_url_key', 'url', unique=True, postgresql_where=text('url IS NOT NULL')),
+    )
     id = Column(Integer, primary_key=True)
     # name and directory are stored in Collection, accessed via properties
-    url = Column(String, unique=True)  # will only be downloaded if related Download exists.
+    url = Column(String)  # will only be downloaded if related Download exists. Partial unique index on non-NULL values.
     generate_posters = Column(Boolean, default=False)  # generating posters may delete files, and can be slow.
     calculate_duration = Column(Boolean, default=True)  # use ffmpeg to extract duration (slower than info json).
     download_missing_data = Column(Boolean, default=True)  # fetch missing data like `source_id` and video comments.
     source_id = Column(String)  # the ID from the source website.
     refreshed = Column(Boolean, default=False)  # The files in the Channel have been refreshed.
 
     # Columns updated by triggers
-    video_count = Column(Integer, default=0)  # update_channel_video_count
-    total_size = Column(Integer, default=0)  # update_channel_size
+    video_count = Column(Integer, default=0, nullable=False)  # update_channel_video_count
+    total_size = Column(BigInteger, default=0, nullable=False)  # update_channel_size
     minimum_frequency = Column(Integer)  # update_channel_minimum_frequency
 
     info_json = deferred(Column(JSON))

modules/zim/models.py

@@ -6,7 +6,7 @@
 from typing import List, Tuple, OrderedDict as OrderedDictType, Dict, Optional, Set
 
 from libzim import Archive, Searcher, Query, Entry, SuggestionSearcher
-from sqlalchemy import Column, Integer, BigInteger, ForeignKey, Text, tuple_, Boolean
+from sqlalchemy import Column, Integer, BigInteger, ForeignKey, Text, tuple_, Boolean, UniqueConstraint
 from sqlalchemy.orm import relationship, Session
 from sqlalchemy.orm.exc import NoResultFound  # noqa
 
@@ -67,12 +67,15 @@ class Zim(Base, ModelHelper):
     """
 
     __tablename__ = 'zim'
+    __table_args__ = (
+        UniqueConstraint('path', name='zim_path_key'),
+    )
     id = Column(Integer, primary_key=True)
     path: pathlib.Path = Column(MediaPathType, nullable=False)
 
-    file_group_id = Column(BigInteger, ForeignKey('file_group.id', ondelete='CASCADE'), unique=True, nullable=False)
+    file_group_id = Column(BigInteger, ForeignKey('file_group.id', ondelete='CASCADE'), nullable=False)
     file_group: FileGroup = relationship('FileGroup')
-    auto_search = Column(Boolean, nullable=False, default=True)
+    auto_search = Column(Boolean, default=True)
 
     def __repr__(self):
         return f'<Zim id={self.id} file_group_id={self.file_group_id} path={self.path}>'
@@ -353,8 +356,11 @@ def entries_with_tags(cls, tag_names: List[str], session: Session = None) -> Ord
 
 class TagZimEntry(Base):
     __tablename__ = 'tag_zim'
+    __table_args__ = (
+        UniqueConstraint('tag_id', 'zim_id', 'zim_entry', name='tag_zim_tag_id_zim_id_zim_entry_key'),
+    )
 
-    tag_id = Column(Integer, ForeignKey('tag.id', ondelete='CASCADE'), primary_key=True)
+    tag_id = Column(Integer, ForeignKey('tag.id'), primary_key=True)
     tag: Tag = relationship('Tag')
     zim_id = Column(Integer, ForeignKey('zim.id', ondelete='CASCADE'), primary_key=True)
     zim: Zim = relationship('Zim')

modules/zim/test/test_lib.py

@@ -161,10 +161,9 @@ async def test_zim_tags_config(async_client, test_session, test_directory, test_
         ['Tag1', test_zim.path.name, 'two', '2000-01-01T00:00:00.000001+00:00'],
     ]
 
-    # Delete all Tags so they are recreated.
-    test_session.query(tags.Tag).delete()
-    # Delete all TagFileEntry(s), import them again.
+    # Delete all TagZimEntry(s) first (due to FK constraint), then Tags so they are recreated.
     test_session.query(TagZimEntry).delete()
+    test_session.query(tags.Tag).delete()
     test_session.commit()
 
     tags.import_tags_config()