diff --git a/.gitignore b/.gitignore index 79b37fb50..1c0f66014 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,6 @@ *.bin *.gz *_BAK -migrations/ tmp/ logs/ /static/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b842b102..cd451b3b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Included thorough description for wetlab API's update_lab() method [#361](https://github.com/BU-ISCIII/iskylims/pull/361) - Included new API function lab-request-mapping to get LabRequest fields ontology map [#377](https://github.com/BU-ISCIII/iskylims/pull/377) - Improved responses in API create-sample-data by adding ERROR messages and data [#377](https://github.com/BU-ISCIII/iskylims/pull/377) +- Added support for `script-before` and `script-after` hooks in install script. [#389](https://github.com/BU-ISCIII/iskylims/pull/389) +- Committed baseline migration files and added migrations for develop changes [#389](https://github.com/BU-ISCIII/iskylims/pull/389) +- Added developer notes on how to create and manage migration files [#389](https://github.com/BU-ISCIII/iskylims/pull/389) +- Added documentation describing migration scripts and their related versions [#389](https://github.com/BU-ISCIII/iskylims/pull/389) +- Enabled Docker internal networking for local test installation [#389](https://github.com/BU-ISCIII/iskylims/pull/389) +- Opened Docker network to allow localhost MySQL connection when required [#389](https://github.com/BU-ISCIII/iskylims/pull/389) #### Fixes @@ -59,6 +65,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Adapted update_lab serializer call to new serializer update method [#361](https://github.com/BU-ISCIII/iskylims/pull/361) - Leave missing submitting_fields as empty string instead of crashing in wetlab.api.create_sample_data [#363](https://github.com/BU-ISCIII/iskylims/pull/363) - Fixed wetlab API create-sample-data error when submitting_institution fields were not provided [#377](https://github.com/BU-ISCIII/iskylims/pull/377) +- Fixed incorrect git revision propagation to Dockerfile during build [#389](https://github.com/BU-ISCIII/iskylims/pull/389) +- Fixed configuration file accessibility from within Docker container [#389](https://github.com/BU-ISCIII/iskylims/pull/389) +- Fixed disk utilization check to correctly resolve application folder path [#389](https://github.com/BU-ISCIII/iskylims/pull/389) +- Fixed incorrect application folder path resolution in crontab scripts [#389](https://github.com/BU-ISCIII/iskylims/pull/389) +- Fixed samplesheet parsing error [#389](https://github.com/BU-ISCIII/iskylims/pull/389) #### Changed @@ -71,10 +82,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Improved query performance and excluded rejected/archived services from ongoing list. (#299) - Updated sample metadata fields with standardized ontology mappings and schema alignment.[#358](https://github.com/BU-ISCIII/iskylims/pull/358) - API create-sample-data Lab data mapping moved to core_config.LAB_REQUEST_ONTOLOGY_MAP [#377](https://github.com/BU-ISCIII/iskylims/pull/377) +- Renamed `docker-compose.yml` to `docker-compose.test.yml` and `docker-compose.prod.yml` for test clarity [#389](https://github.com/BU-ISCIII/iskylims/pull/389) +- Refactored Docker runtime handling when path is outside repository [#389](https://github.com/BU-ISCIII/iskylims/pull/389) +- Updated `docker_install.sh` with multiple reliability improvements [#389](https://github.com/BU-ISCIII/iskylims/pull/389) +- Updated upgrade scripts documentation to include execution order information and docker upgrade clarifications [#389](https://github.com/BU-ISCIII/iskylims/pull/389) #### Removed - Dummy fix in usage line [#271](https://github.com/BU-ISCIII/iskylims/pull/271) +- Removed migrations from `.gitignore` and app-level `.gitignore` to ensure version control of schema changes [#389](https://github.com/BU-ISCIII/iskylims/pull/389) #### Requirements diff --git a/Dockerfile b/Dockerfile index 2ada0e511..f85a57ba8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,7 +30,7 @@ ENV PATH="/usr/sbin/cron:$PATH" # Set default install type ARG INSTALL_TYPE=dep ARG GIT_REVISION=main -ARG INSTALL_CONF=conf/docker_install_settings.txt +ARG INSTALL_CONF=conf/docker_test_settings.txt # Execute the dependency stage only; app migrations run when the container is up. RUN /bin/bash install.sh --install dep --git_revision $GIT_REVISION --conf $INSTALL_CONF --skip_apache_restart diff --git a/LEAME.md b/LEAME.md index 78f0ed28e..a4602a470 100644 --- a/LEAME.md +++ b/LEAME.md @@ -18,6 +18,7 @@ De acuerdo con la infraestructura existente, la secuenciacion se realiza en un i - [Contenedor local de pruebas](#contenedor-local-de-pruebas) - [Contenedor de produccion](#contenedor-de-produccion) - [Actualizacion del despliegue Docker](#actualizacion-del-despliegue-docker) + - [Actualizacion del despliegue Docker v3.0.0 a 3.1.0](#actualizacion-del-despliegue-docker-v300-a-310) - [Despliegue bare-metal (Ubuntu/CentOS)](#despliegue-bare-metal-ubuntucentos) - [Instalacion](#instalacion) - [Requisitos previos](#requisitos-previos) @@ -70,6 +71,8 @@ Levanta el sistema completo (base de datos, Samba y app) con fixtures y datos de bash docker_install.sh --test ``` +Esto usa `docker-compose.test.yml` por defecto. + Puedes personalizar los valores por defecto: - `--demo_data /ruta/a/iskylims_demo_data.tar.gz` para reutilizar un archivo local (si no, se descarga). @@ -116,6 +119,22 @@ bash docker_install.sh --install_conf conf/my_prod_settings.txt --action upgrade La actualizacion reconstruye/reinicia el contenedor y ejecuta `install.sh` dentro del contenedor, que regenera migraciones, las aplica con `--fake-initial` y evita cargar superusuario/datos demo/prueba. +### Actualizacion del despliegue Docker v3.0.0 a 3.1.0 + +Antes de actualizar, asegurate de tener una copia completa de la base de datos y de los datos en volumenes que uses. Confirma que `conf/my_prod_settings.txt` tenga el host/usuario/password de la base de datos de produccion, la URL/IP del servidor, correo y ajustes de logging usados por el contenedor. + +Para 3.0.0 -> 3.1.0, exporta primero el mapeo de LibraryPool y luego ejecuta la actualizacion con scripts pre/post: + +```bash +mysql --user= --password= --host= --port= iskylims \ + -e "SELECT id, run_process_id_id FROM wetlab_library_pool" \ + > /tmp/library_pool_run_process.tsv + +bash docker_install.sh --install_conf conf/my_prod_settings.txt --action upgrade \ + --script_before convert_rawtop_counter_to_int \ + --script_after library_pool_to_many_relation,/tmp/library_pool_run_process.tsv +``` + ## Despliegue bare-metal (Ubuntu/CentOS) ### Instalacion @@ -233,6 +252,15 @@ bash install.sh --upgrade app --git_revision main --tables # ejemplo ejecutando un script de migracion en la actualizacion bash install.sh --upgrade app --script migrate_optional_values --git_revision main --tables + +# 3.0.0 -> 3.1.0 (scripts de datos pre/post) +mysql --user= --password= --host= --port= iskylims \ + -e "SELECT id, run_process_id_id FROM wetlab_library_pool" \ + > /tmp/library_pool_run_process.tsv + +bash install.sh --upgrade app --git_revision main \ + --script_before convert_rawtop_counter_to_int \ + --script_after library_pool_to_many_relation,/tmp/library_pool_run_process.tsv ``` O ejecuta todo en un unico comando: diff --git a/README.md b/README.md index aae7fb7d5..e8c15a42a 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,9 @@ Application servers run web applications for bioinformatics analysis (GALAXY), t - [Local test stack](#local-test-stack) - [Production container](#production-container) - [Upgrade docker deployment](#upgrade-docker-deployment) + - [Upgrade docker deployment v3.0.0 to 3.1.0](#upgrade-docker-deployment-v300-to-310) + - [Back up first](#back-up-first) + - [Refresh code and settings](#refresh-code-and-settings) - [Bare-metal deployment (Ubuntu/CentOS)](#bare-metal-deployment-ubuntucentos) - [Install](#install) - [Prerequisites](#prerequisites) @@ -26,15 +29,17 @@ Application servers run web applications for bioinformatics analysis (GALAXY), t - [Prepare the database](#prepare-the-database) - [Configure install\_settings.txt](#configure-install_settingstxt) - [Run install.sh](#run-installsh) - - [Upgrade (3.0.x to 3.1.x)](#upgrade-30x-to-31x) - - [Back up first](#back-up-first) - - [Refresh code and settings](#refresh-code-and-settings) + - [Upgrade (3.0.0 to 3.1.0)](#upgrade-300-to-310) + - [Back up first](#back-up-first-1) + - [Refresh code and settings](#refresh-code-and-settings-1) - [Run upgrade steps requiring root](#run-upgrade-steps-requiring-root) - [Run upgrade steps without root](#run-upgrade-steps-without-root) - [What to do if something fails](#what-to-do-if-something-fails) - [Final configuration steps](#final-configuration-steps) - [SAMBA configurarion](#samba-configurarion) - [Email verification](#email-verification) + - [Developer notes](#developer-notes) + - [Django migrations workflow](#django-migrations-workflow) - [Configure Apache server](#configure-apache-server) - [Verification of the installation](#verification-of-the-installation) - [iSkyLIMS documentation](#iskylims-documentation) @@ -71,6 +76,8 @@ Bring up a full test stack (database, Samba, app) plus fixtures and demo data: bash docker_install.sh --test ``` +This uses `docker-compose.test.yml` by default. + Defaults can be customised: - `--demo_data /path/to/iskylims_demo_data.tar.gz` to reuse a local demo archive (otherwise it is downloaded). @@ -117,6 +124,38 @@ bash docker_install.sh --install_conf conf/my_prod_settings.txt --action upgrade The upgrade path rebuilds/restarts the container and runs `install.sh` inside the app container, which regenerates migrations, applies them with `--fake-initial`, and skips superuser/demo/test data loading. +### Upgrade docker deployment v3.0.0 to 3.1.0 + +#### Back up first + +- Full backup of the `iskylims` database. +- Full backup of the logs folder and the documents folder. + +For 3.0.0 -> 3.1.0, export the LibraryPool mapping first, then run the upgrade with pre/post scripts: + +```bash +mysql --user= --password= --host= --port= iskylims \ + -e "SELECT id, run_process_id_id FROM wetlab_library_pool" \ + > /tmp/library_pool_run_process.tsv +``` + +#### Refresh code and settings + +```bash +cd /iskylims +git pull +cp conf/docker_production_settings.txt myprod_settings.txt +sudo nano myprod_settings.txt +``` + +Ensure the file uses Linux-friendly encoding (UTF-8/ASCII) if you edit it on Windows. + +```bash +bash docker_install.sh --install_conf my_prod_settings.txt --action upgrade \ + --script_before convert_rawtop_counter_to_int \ + --script_after library_pool_to_many_relation,/tmp/library_pool_run_process.tsv +``` + ## Bare-metal deployment (Ubuntu/CentOS) ### Install @@ -185,7 +224,7 @@ sudo bash install.sh --install full --git_revision main --tables - If Apache is managed elsewhere, skip the automatic restart with `--skip_apache_restart`. -### Upgrade (3.0.x to 3.1.x) +### Upgrade (3.0.0 to 3.1.0) Follow these steps to move from version 3.0.0 to the 3.1.x series. @@ -196,8 +235,9 @@ Follow these steps to move from version 3.0.0 to the 3.1.x series. - If you use library pools, export them before upgrading: ```bash - mysql --user= --password= --host= --port= iskylims \ - -e "SELECT * FROM wetlab_library_pool" > /backup_lib_pool.sql +mysql --user= --password= --host= --port= iskylims \ + -e "SELECT id, run_process_id_id FROM wetlab_library_pool" \ + > /tmp/library_pool_run_process.tsv ``` #### Refresh code and settings @@ -227,19 +267,9 @@ Upgrade the application code and database: ```bash # with library pool restore -bash install.sh --upgrade app --script /backup_lib_pool.sql --git_revision main --tables - -# without library pool restore -bash install.sh --upgrade app --git_revision main --tables - -# example running a migration script during upgrade -bash install.sh --upgrade app --script migrate_optional_values --git_revision main --tables -``` - -Or run everything in one go: - -```bash -sudo bash install.sh --upgrade full --git_revision main --tables +bash install.sh --upgrade app --git_revision main \ + --script_before convert_rawtop_counter_to_int \ + --script_after library_pool_to_many_relation,/tmp/library_pool_run_process.tsv ``` Upgrades regenerate migrations and apply them with `--fake-initial` so existing tables remain intact, matching the Docker workflow. @@ -277,6 +307,19 @@ mysql -u iskylims -p -h dmysqlps.isciiides.es iskylims < /home/dadmin/backup_pro - Go to Configuration -> Email configuration - Fill the form with the needed params for your email configuration and try to send a test email. +## Developer notes + +### Django migrations workflow + +Migrations are committed to the repo. Do not run `makemigrations` during install/upgrade. + +Baseline + upgrade flow for new releases: + +1. Generate baseline migrations from the last stable tag (example 3.0.0). +2. Commit the baseline migrations. +3. Generate new migrations on `develop` for schema changes and commit them. +4. Upgrades run `migrate --fake-initial` once to align existing tables, then `migrate` to apply the new migration files. + ### Configure Apache server Copy the apache configuration file according to your distribution inside the apache configutation directory and rename it to iskylims.conf diff --git a/UPGRADE_SCRIPTS.md b/UPGRADE_SCRIPTS.md new file mode 100644 index 000000000..919af840f --- /dev/null +++ b/UPGRADE_SCRIPTS.md @@ -0,0 +1,36 @@ +# Upgrade scripts + +This file lists data migration scripts and the version range they apply to. +Run them with: + +```bash +python manage.py runscript +``` + +## 3.0.0 -> 3.1.0 + +- Run order: + 1. Export LibraryPool run_process_id data (before migrations). + 2. Run `convert_rawtop_counter_to_int` before migrations. + 3. Run migrations. + 4. Run `library_pool_to_many_relation` after migrations with the exported file. + +- Export example: + ```bash + mysql -u -p -h -D \ + -e "SELECT id, run_process_id_id FROM wetlab_library_pool" \ + > /tmp/library_pool_run_process.tsv + ``` + +- `wetlab/scripts/convert_rawtop_counter_to_int.py` (script name: `convert_rawtop_counter_to_int`) +- `wetlab/scripts/library_pool_to_many_relation.py` (script name: `library_pool_to_many_relation`) + +## 2.3.0 -> 3.0.0 + +- `core/scripts/rename_app_name.py` (script name: `rename_app_name`) +- `core/scripts/migrate_sample_type.py` (script name: `migrate_sample_type`) +- `core/scripts/migrate_optional_values.py` (script name: `migrate_optional_values`) + +## 2.3.0 -> 2.3.1 + +- `drylab/scripts/drylab_service_state_migration.py` (script name: `drylab_service_state_migration`) diff --git a/clinic/.gitignore b/clinic/.gitignore deleted file mode 100644 index 87884db5e..000000000 --- a/clinic/.gitignore +++ /dev/null @@ -1,113 +0,0 @@ -## Custom -*.xml -*.bin -migrations/ -tmp/ -logs/ -tests/ - - -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -*.egg-info/ -.installed.cfg -*.egg -MANIFEST - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -.hypothesis/ -.pytest_cache/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py -db.sqlite3 - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# pyenv -.python-version - -# celery beat schedule file -celerybeat-schedule - -# SageMath parsed files -*.sage.py - -# Environments -.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ diff --git a/clinic/migrations/0001_initial.py b/clinic/migrations/0001_initial.py new file mode 100644 index 000000000..2de933cee --- /dev/null +++ b/clinic/migrations/0001_initial.py @@ -0,0 +1,307 @@ +# Generated by Django 4.2.25 on 2026-02-11 16:17 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ("core", "0001_initial"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name="ClinicSampleRequest", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("entry_order", models.CharField(blank=True, max_length=8, null=True)), + ( + "confirmation_code", + models.CharField(blank=True, max_length=80, null=True), + ), + ("priority", models.IntegerField(blank=True, null=True)), + ("comments", models.CharField(blank=True, max_length=255, null=True)), + ("service_date", models.DateTimeField(blank=True, null=True)), + ("generated_at", models.DateTimeField(auto_now_add=True)), + ], + ), + migrations.CreateModel( + name="ClinicSampleState", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("clinic_state", models.CharField(max_length=20)), + ], + ), + migrations.CreateModel( + name="ConfigSetting", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("configuration_name", models.CharField(max_length=80)), + ( + "configuration_value", + models.CharField(blank=True, max_length=255, null=True), + ), + ("generated_at", models.DateTimeField(auto_now_add=True)), + ], + ), + migrations.CreateModel( + name="FamilyRelatives", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("relationship", models.CharField(max_length=255)), + ], + ), + migrations.CreateModel( + name="PatientData", + fields=[ + ( + "patien_core", + models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + primary_key=True, + serialize=False, + to="core.patientcore", + ), + ), + ("address", models.CharField(blank=True, max_length=255, null=True)), + ("phone", models.CharField(blank=True, max_length=20, null=True)), + ("email", models.CharField(blank=True, max_length=50, null=True)), + ("birthday", models.DateTimeField(blank=True, null=True)), + ("smoker", models.CharField(blank=True, max_length=20, null=True)), + ( + "notification_preference", + models.CharField(blank=True, max_length=20, null=True), + ), + ("comments", models.CharField(blank=True, max_length=255, null=True)), + ], + ), + migrations.CreateModel( + name="ServiceUnits", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("service_unit_name", models.CharField(max_length=80)), + ], + ), + migrations.CreateModel( + name="ResultParameterValue", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("parameterValue", models.CharField(max_length=255)), + ("generated_at", models.DateTimeField(auto_now_add=True)), + ( + "clinicSample_id", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="clinic.clinicsamplerequest", + ), + ), + ( + "parameter_id", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="core.protocolparameters", + ), + ), + ], + ), + migrations.CreateModel( + name="PatientHistory", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("entry_date", models.DateTimeField(blank=True, null=True)), + ("description", models.CharField(max_length=255)), + ( + "patient_core", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="core.patientcore", + ), + ), + ], + ), + migrations.CreateModel( + name="Family", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("family_id", models.CharField(max_length=50)), + ("relative_1", models.CharField(max_length=50)), + ("relative_2", models.CharField(max_length=50)), + ("affected", models.BooleanField()), + ( + "family_comments", + models.CharField(blank=True, max_length=255, null=True), + ), + ("generated_at", models.DateTimeField(auto_now_add=True)), + ( + "family_relatives_id", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="clinic.familyrelatives", + ), + ), + ( + "patien_core_id", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="core.patientcore", + ), + ), + ], + ), + migrations.CreateModel( + name="Doctor", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("doctor_name", models.CharField(max_length=80)), + ( + "service_unit_id", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="clinic.serviceunits", + ), + ), + ], + ), + migrations.AddField( + model_name="clinicsamplerequest", + name="clinic_sample_state", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="clinic.clinicsamplestate", + ), + ), + migrations.AddField( + model_name="clinicsamplerequest", + name="doctor_id", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="clinic.doctor", + ), + ), + migrations.AddField( + model_name="clinicsamplerequest", + name="patient_core", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="core.patientcore", + ), + ), + migrations.AddField( + model_name="clinicsamplerequest", + name="sample_core", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="core.samples" + ), + ), + migrations.AddField( + model_name="clinicsamplerequest", + name="sample_request_user", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + ), + ), + migrations.AddField( + model_name="clinicsamplerequest", + name="service_unit_id", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="clinic.serviceunits", + ), + ), + ] diff --git a/clinic/migrations/__init__.py b/clinic/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/conf/docker_production_settings.txt b/conf/docker_production_settings.txt index 643586d2c..c92824596 100644 --- a/conf/docker_production_settings.txt +++ b/conf/docker_production_settings.txt @@ -15,7 +15,7 @@ PYTHON_BIN_PATH='python3' DB_USER='django' DB_PASS='djangopass' DB_NAME='iskylims' -DB_SERVER_IP='prod-db-host' # hostname or IP of the production DB +DB_SERVER_IP='host.docker.internal' # hostname or IP of the production DB (use host.docker.internal for host DB) DB_PORT=3306 ### Settings required for sending emails diff --git a/conf/docker_install_settings.txt b/conf/docker_test_settings.txt similarity index 100% rename from conf/docker_install_settings.txt rename to conf/docker_test_settings.txt diff --git a/core/.gitignore b/core/.gitignore deleted file mode 100644 index f5e99c72e..000000000 --- a/core/.gitignore +++ /dev/null @@ -1,103 +0,0 @@ -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class -migrations/ -tests/ - -# C extensions -*.so - -# Distribution / packaging -.Python -env/ -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -*.egg-info/ -.installed.cfg -*.egg - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -.hypothesis/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# pyenv -.python-version - -# celery beat schedule file -celerybeat-schedule - -# SageMath parsed files -*.sage.py - -# dotenv -.env - -# virtualenv -.venv -venv/ -ENV/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ diff --git a/core/migrations/0001_initial.py b/core/migrations/0001_initial.py new file mode 100644 index 000000000..6deb3f11e --- /dev/null +++ b/core/migrations/0001_initial.py @@ -0,0 +1,1149 @@ +# Generated by Django 4.2.25 on 2026-02-11 16:17 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name="City", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("city_name", models.CharField(max_length=80)), + ("geo_loc_latitude", models.CharField(max_length=80)), + ("geo_loc_longitude", models.CharField(max_length=80)), + ("apps_name", models.CharField(max_length=40, null=True)), + ], + options={ + "db_table": "core_city", + }, + ), + migrations.CreateModel( + name="CommercialKits", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("name", models.CharField(max_length=150)), + ("provider", models.CharField(max_length=30)), + ("cat_number", models.CharField(blank=True, max_length=40, null=True)), + ( + "description", + models.CharField(blank=True, max_length=255, null=True), + ), + ("generated_at", models.DateTimeField(auto_now_add=True, null=True)), + ], + options={ + "db_table": "core_commercial_kits", + }, + ), + migrations.CreateModel( + name="Contact", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("contact_name", models.CharField(max_length=80)), + ("contact_mail", models.CharField(max_length=40, null=True)), + ], + options={ + "db_table": "core_contact", + }, + ), + migrations.CreateModel( + name="LabRequest", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("lab_name", models.CharField(max_length=80)), + ("lab_name_coding", models.CharField(max_length=50)), + ("lab_unit", models.CharField(max_length=50)), + ("lab_contact_name", models.CharField(max_length=50)), + ("lab_phone", models.CharField(max_length=20)), + ("lab_email", models.CharField(max_length=70)), + ("address", models.CharField(max_length=255)), + ("apps_name", models.CharField(max_length=40, null=True)), + ( + "lab_city", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="core.city", + ), + ), + ], + options={ + "db_table": "core_lab_request", + }, + ), + migrations.CreateModel( + name="MoleculeType", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("molecule_type", models.CharField(max_length=30)), + ("apps_name", models.CharField(max_length=40, null=True)), + ], + options={ + "db_table": "core_molecule_type", + }, + ), + migrations.CreateModel( + name="MoleculeUsedFor", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("used_for", models.CharField(max_length=50)), + ("apps_name", models.CharField(max_length=50)), + ("massive_use", models.BooleanField(default=False)), + ], + options={ + "db_table": "core_molecule_used_for", + }, + ), + migrations.CreateModel( + name="OntologyMap", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("label", models.CharField(max_length=255)), + ("ontology", models.CharField(blank=True, max_length=50, null=True)), + ], + options={ + "db_table": "core_ontology_map", + }, + ), + migrations.CreateModel( + name="PatientCore", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("patient_name", models.CharField(max_length=255, null=True)), + ("patient_surname", models.CharField(max_length=255, null=True)), + ("patient_code", models.CharField(max_length=255, null=True)), + ], + options={ + "db_table": "core_patient_core", + }, + ), + migrations.CreateModel( + name="PatientProjects", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("project_name", models.CharField(max_length=50)), + ( + "project_manager", + models.CharField(blank=True, max_length=50, null=True), + ), + ( + "project_contact", + models.CharField(blank=True, max_length=50, null=True), + ), + ( + "project_description", + models.CharField(blank=True, max_length=255, null=True), + ), + ("apps_name", models.CharField(max_length=40)), + ("generated_at", models.DateTimeField(auto_now_add=True)), + ], + options={ + "db_table": "core_patient_projects", + }, + ), + migrations.CreateModel( + name="PatientSex", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("sex", models.CharField(max_length=16)), + ], + options={ + "db_table": "core_patient_sex", + }, + ), + migrations.CreateModel( + name="SampleProjectFieldClassification", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("classification_name", models.CharField(max_length=80)), + ("classification_display", models.CharField(max_length=100)), + ("generated_at", models.DateTimeField(auto_now_add=True)), + ], + options={ + "db_table": "core_sample_projects_field_classification", + }, + ), + migrations.CreateModel( + name="SampleProjects", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("sample_project_name", models.CharField(max_length=255)), + ( + "sample_project_manager", + models.CharField(blank=True, max_length=50, null=True), + ), + ( + "sample_project_contact", + models.CharField(blank=True, max_length=250, null=True), + ), + ( + "sample_project_description", + models.CharField(blank=True, max_length=255, null=True), + ), + ("apps_name", models.CharField(max_length=255)), + ("generated_at", models.DateTimeField(auto_now_add=True)), + ], + options={ + "db_table": "core_sample_projects", + }, + ), + migrations.CreateModel( + name="SampleProjectsFields", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("sample_project_field_name", models.CharField(max_length=80)), + ( + "sample_project_field_description", + models.CharField(blank=True, max_length=400, null=True), + ), + ("sample_project_field_order", models.IntegerField()), + ("sample_project_field_used", models.BooleanField()), + ("sample_project_field_type", models.CharField(max_length=20)), + ( + "sample_project_option_list", + models.CharField(blank=True, max_length=255, null=True), + ), + ("sample_project_searchable", models.BooleanField(default=False)), + ("generated_at", models.DateTimeField(auto_now_add=True)), + ( + "sample_project_field_classification_id", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="core.sampleprojectfieldclassification", + ), + ), + ( + "sample_projects_id", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="sample_project_fields", + to="core.sampleprojects", + ), + ), + ], + options={ + "db_table": "core_sample_projects_fields", + }, + ), + migrations.CreateModel( + name="SampleType", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("sample_type", models.CharField(max_length=50)), + ("apps_name", models.CharField(max_length=50)), + ( + "mandatory_fields", + models.CharField(blank=True, max_length=300, null=True), + ), + ("generated_at", models.DateTimeField(auto_now_add=True, null=True)), + ], + options={ + "db_table": "core_sample_type", + }, + ), + migrations.CreateModel( + name="SequencingPlatform", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("platform_name", models.CharField(max_length=30)), + ("company_name", models.CharField(max_length=30)), + ("sequencing_technology", models.CharField(max_length=30)), + ], + options={ + "db_table": "core_sequencing_platform", + }, + ), + migrations.CreateModel( + name="Species", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("species_name", models.CharField(max_length=50)), + ( + "ref_genome_name", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "ref_genome_size", + models.CharField(blank=True, max_length=100, null=True), + ), + ( + "ref_genome_id", + models.CharField(blank=True, max_length=255, null=True), + ), + ("apps_name", models.CharField(max_length=50, null=True)), + ("generated_at", models.DateTimeField(auto_now_add=True, null=True)), + ], + options={ + "db_table": "core_species", + }, + ), + migrations.CreateModel( + name="StateInCountry", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("state_name", models.CharField(max_length=80)), + ("apps_name", models.CharField(max_length=40, null=True)), + ], + options={ + "db_table": "core_state_in_country", + }, + ), + migrations.CreateModel( + name="StatesForMolecule", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("molecule_state_name", models.CharField(max_length=50)), + ], + options={ + "db_table": "core_states_for_molecule", + }, + ), + migrations.CreateModel( + name="StatesForSample", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("sample_state_name", models.CharField(max_length=50)), + ], + options={ + "db_table": "core_states_for_sample", + }, + ), + migrations.CreateModel( + name="UserLotCommercialKits", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("uses_number", models.IntegerField(default=0, null=True)), + ("chip_lot", models.CharField(max_length=50)), + ("latest_used_date", models.DateTimeField(blank=True, null=True)), + ("expiration_date", models.DateField()), + ("run_out", models.BooleanField(default=False)), + ("generated_at", models.DateTimeField(auto_now_add=True, null=True)), + ( + "based_commercial", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="core.commercialkits", + ), + ), + ( + "user", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + ), + ), + ], + options={ + "db_table": "core_user_lot_commercial_kits", + }, + ), + migrations.CreateModel( + name="SequencingConfiguration", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("configuration_name", models.CharField(max_length=255)), + ( + "platform_id", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="core.sequencingplatform", + ), + ), + ], + options={ + "db_table": "core_sequencing_configuration", + }, + ), + migrations.CreateModel( + name="SequencerInLab", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("sequencer_name", models.CharField(max_length=255)), + ( + "sequencer_description", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "sequencer_location", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "sequencer_serial_number", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "sequencer_state", + models.CharField(blank=True, max_length=50, null=True), + ), + ("sequencer_operation_start", models.DateField(blank=True, null=True)), + ("sequencer_operation_end", models.DateField(blank=True, null=True)), + ( + "sequencer_number_lanes", + models.CharField(blank=True, max_length=5, null=True), + ), + ( + "platform_id", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="core.sequencingplatform", + ), + ), + ], + options={ + "db_table": "core_sequencer_in_lab", + }, + ), + migrations.CreateModel( + name="SamplesProjectsTableOptions", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("option_value", models.CharField(max_length=120)), + ( + "sample_project_field", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="opt_value_prop", + to="core.sampleprojectsfields", + ), + ), + ], + options={ + "db_table": "core_sample_projects_table_options", + }, + ), + migrations.CreateModel( + name="Samples", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "sample_name", + models.CharField( + max_length=255, null=True, verbose_name="Sample Name" + ), + ), + ( + "sample_location", + models.CharField( + blank=True, + max_length=255, + null=True, + verbose_name="Sample location", + ), + ), + ( + "sample_entry_date", + models.DateTimeField( + blank=True, null=True, verbose_name="Sample defined date" + ), + ), + ( + "collection_sample_date", + models.DateTimeField( + blank=True, null=True, verbose_name="Sample collection date" + ), + ), + ( + "unique_sample_id", + models.CharField( + max_length=8, null=True, verbose_name="Unique sample id" + ), + ), + ( + "sample_code_id", + models.CharField( + max_length=60, null=True, verbose_name="Sample code id" + ), + ), + ( + "reused_number", + models.IntegerField( + default=0, verbose_name="Number of type reused" + ), + ), + ( + "sequencing_date", + models.DateTimeField( + blank=True, null=True, verbose_name="Sequencing date" + ), + ), + ( + "completed_date", + models.DateTimeField( + blank=True, null=True, verbose_name="Completion date" + ), + ), + ( + "generated_at", + models.DateTimeField( + auto_now_add=True, verbose_name="Generated at" + ), + ), + ( + "only_recorded", + models.BooleanField( + blank=True, + default=False, + null=True, + verbose_name="Only recorded?", + ), + ), + ( + "lab_request", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="core.labrequest", + verbose_name="Laboratory", + ), + ), + ( + "patient_core", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="core.patientcore", + verbose_name="Patient Code ID", + ), + ), + ( + "sample_project", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="core.sampleprojects", + verbose_name="Sample Project", + ), + ), + ( + "sample_state", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="core.statesforsample", + verbose_name="Sample state", + ), + ), + ( + "sample_type", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="core.sampletype", + verbose_name="Sample type", + ), + ), + ( + "sample_user", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + verbose_name="Username", + ), + ), + ( + "species", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="core.species", + verbose_name="Species", + ), + ), + ], + options={ + "db_table": "core_samples", + }, + ), + migrations.CreateModel( + name="SampleProjectsFieldsValue", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "sample_project_field_value", + models.CharField(blank=True, max_length=255, null=True), + ), + ("generated_at", models.DateTimeField(auto_now_add=True)), + ( + "sample_id", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="project_values", + to="core.samples", + verbose_name="Sample Name", + ), + ), + ( + "sample_project_field_id", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="core.sampleprojectsfields", + ), + ), + ], + options={ + "db_table": "core_sample_projects_fields_value", + }, + ), + migrations.AddField( + model_name="sampleprojectfieldclassification", + name="sample_projects_id", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="core.sampleprojects", + ), + ), + migrations.CreateModel( + name="ProtocolType", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("protocol_type", models.CharField(max_length=40)), + ("apps_name", models.CharField(max_length=40)), + ( + "molecule", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="core.moleculetype", + ), + ), + ], + options={ + "db_table": "core_protocol_type", + }, + ), + migrations.CreateModel( + name="Protocols", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("name", models.CharField(max_length=40)), + ( + "description", + models.CharField(blank=True, max_length=160, null=True), + ), + ( + "type", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="core.protocoltype", + ), + ), + ], + options={ + "db_table": "core_protocols", + }, + ), + migrations.CreateModel( + name="ProtocolParameters", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("parameter_name", models.CharField(max_length=255)), + ( + "parameter_description", + models.CharField(blank=True, max_length=400, null=True), + ), + ("parameter_order", models.IntegerField()), + ("parameter_used", models.BooleanField()), + ("parameter_type", models.CharField(default="string", max_length=20)), + ( + "parameter_option_values", + models.CharField(blank=True, max_length=400, null=True), + ), + ( + "parameter_max_value", + models.CharField(blank=True, max_length=50, null=True), + ), + ( + "parameter_min_value", + models.CharField(blank=True, max_length=50, null=True), + ), + ( + "protocol_id", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="core.protocols" + ), + ), + ], + options={ + "db_table": "core_protocol_parameters", + }, + ), + migrations.CreateModel( + name="PatientProjectsFields", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("project_field_name", models.CharField(max_length=50)), + ( + "project_field_description", + models.CharField(blank=True, max_length=400, null=True), + ), + ("project_field_order", models.IntegerField()), + ("project_field_used", models.BooleanField()), + ( + "patient_projects_id", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="core.patientprojects", + ), + ), + ], + options={ + "db_table": "core_patient_projects_fields", + }, + ), + migrations.CreateModel( + name="PatientProjectFieldValue", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("project_field_value", models.CharField(max_length=255)), + ("generated_at", models.DateTimeField(auto_now_add=True)), + ( + "patient_core_id", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="core.patientcore", + ), + ), + ( + "project_field_id", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="core.patientprojectsfields", + ), + ), + ], + options={ + "db_table": "core_patient_project_field_value", + }, + ), + migrations.AddField( + model_name="patientcore", + name="patient_projects", + field=models.ManyToManyField(blank=True, to="core.patientprojects"), + ), + migrations.AddField( + model_name="patientcore", + name="patient_sex", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="core.patientsex", + ), + ), + migrations.CreateModel( + name="MoleculePreparation", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("molecule_code_id", models.CharField(max_length=255)), + ("extraction_type", models.CharField(max_length=50)), + ("molecule_extraction_date", models.DateTimeField(null=True)), + ("reused_number", models.IntegerField(default=0)), + ( + "used_for_massive_sequencing", + models.BooleanField(blank=True, null=True), + ), + ("generated_at", models.DateTimeField(auto_now_add=True)), + ( + "molecule_type", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="core.moleculetype", + ), + ), + ( + "molecule_used_for", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="core.moleculeusedfor", + ), + ), + ( + "molecule_user", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "protocol_used", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="core.protocols" + ), + ), + ( + "sample", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="core.samples" + ), + ), + ( + "state", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="core.statesformolecule", + ), + ), + ( + "user_lot_kit_id", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="core.userlotcommercialkits", + ), + ), + ], + options={ + "db_table": "core_molecule_preparation", + }, + ), + migrations.CreateModel( + name="MoleculeParameterValue", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("parameter_value", models.CharField(max_length=255)), + ("generated_at", models.DateTimeField(auto_now_add=True)), + ( + "molecule_id", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="core.moleculepreparation", + ), + ), + ( + "molecule_parameter_id", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="core.protocolparameters", + ), + ), + ], + options={ + "db_table": "core_molecule_parameter_value", + }, + ), + migrations.AddField( + model_name="commercialkits", + name="platform_kits", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="core.sequencingplatform", + ), + ), + migrations.AddField( + model_name="commercialkits", + name="protocol_kits", + field=models.ManyToManyField(blank=True, to="core.protocols"), + ), + migrations.AddField( + model_name="city", + name="belongs_to_state", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="core.stateincountry", + ), + ), + ] diff --git a/core/migrations/0002_rename_molecule_used_for_moleculepreparation_sample_continues_on_and_more.py b/core/migrations/0002_rename_molecule_used_for_moleculepreparation_sample_continues_on_and_more.py new file mode 100644 index 000000000..9d3be354a --- /dev/null +++ b/core/migrations/0002_rename_molecule_used_for_moleculepreparation_sample_continues_on_and_more.py @@ -0,0 +1,102 @@ +# Generated by Django 4.2.25 on 2026-02-11 16:33 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("core", "0001_initial"), + ] + + operations = [ + migrations.RenameField( + model_name="moleculepreparation", + old_name="molecule_used_for", + new_name="sample_continues_on", + ), + migrations.RenameField( + model_name="sampleprojectsfields", + old_name="sample_project_searchable", + new_name="sample_project_downloadable", + ), + migrations.AddField( + model_name="city", + name="geo_loc_city_cod", + field=models.CharField(blank=True, max_length=10, null=True), + ), + migrations.AddField( + model_name="labrequest", + name="autonom_cod", + field=models.CharField(blank=True, max_length=20, null=True), + ), + migrations.AddField( + model_name="labrequest", + name="center_class_code", + field=models.CharField(blank=True, max_length=20, null=True), + ), + migrations.AddField( + model_name="labrequest", + name="dep_func", + field=models.CharField(blank=True, max_length=80, null=True), + ), + migrations.AddField( + model_name="labrequest", + name="lab_code_1", + field=models.CharField(blank=True, max_length=20, null=True, unique=True), + ), + migrations.AddField( + model_name="labrequest", + name="lab_code_2", + field=models.CharField(blank=True, max_length=20, null=True), + ), + migrations.AddField( + model_name="labrequest", + name="lab_function", + field=models.CharField(blank=True, max_length=120, null=True), + ), + migrations.AddField( + model_name="labrequest", + name="lab_geo_loc_latitude", + field=models.CharField(blank=True, max_length=30, null=True), + ), + migrations.AddField( + model_name="labrequest", + name="lab_geo_loc_longitude", + field=models.CharField(blank=True, max_length=30, null=True), + ), + migrations.AddField( + model_name="labrequest", + name="post_code", + field=models.CharField(blank=True, max_length=10, null=True), + ), + migrations.AddField( + model_name="protocolparameters", + name="parameter_download", + field=models.BooleanField(blank=True, default=False, null=True), + ), + migrations.AddField( + model_name="stateincountry", + name="geo_loc_state_cod", + field=models.CharField( + blank=True, + help_text="Geographic code of the CCAA", + max_length=10, + null=True, + ), + ), + migrations.AddField( + model_name="statesformolecule", + name="molecule_state_display", + field=models.CharField(blank=True, max_length=80, null=True), + ), + migrations.AlterField( + model_name="labrequest", + name="lab_name", + field=models.CharField(max_length=100), + ), + migrations.AlterModelTable( + name="moleculeusedfor", + table="core_sample_continues_on", + ), + ] diff --git a/core/migrations/__init__.py b/core/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/core/scripts/migrate_optional_values.py b/core/scripts/migrate_optional_values.py index 31eca01ea..3dcc9fdeb 100644 --- a/core/scripts/migrate_optional_values.py +++ b/core/scripts/migrate_optional_values.py @@ -1,6 +1,7 @@ import core.models """ + Upgrade: 2.3.0 -> 3.0.0 The script is applicable for the upgrade from 2.3.0 to 3.0.0. Because the new version changes the way that optional values are stored. Instead of having a field "sample_project_option_list" now values are diff --git a/core/scripts/migrate_sample_type.py b/core/scripts/migrate_sample_type.py index d97f34745..df8ee8543 100644 --- a/core/scripts/migrate_sample_type.py +++ b/core/scripts/migrate_sample_type.py @@ -1,6 +1,7 @@ import core.models """ + Upgrade: 2.3.0 -> 3.0.0 The script is applicable for the upgrade from 2.3.0 to 3.0.0. Because the new version changes the value that is stored now is the field name and not the number and instead of optional values now are the diff --git a/core/scripts/rename_app_name.py b/core/scripts/rename_app_name.py index 05eb1b922..549928907 100644 --- a/core/scripts/rename_app_name.py +++ b/core/scripts/rename_app_name.py @@ -1,6 +1,7 @@ import core.models """ + Upgrade: 2.3.0 -> 3.0.0 The script is applicable for the upgrade from 2.3.0 to 3.0.0. Because the application in iSkylims have been renamed, this required that some tables where was indicated the application name must be diff --git a/django_utils/.gitignore b/django_utils/.gitignore deleted file mode 100644 index ed8eee433..000000000 --- a/django_utils/.gitignore +++ /dev/null @@ -1,103 +0,0 @@ -# Byte-compiled / optimized / DLL files -__migrations__/ -migrations/ -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -env/ -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -*.egg-info/ -.installed.cfg -*.egg - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover -.hypothesis/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# pyenv -.python-version - -# celery beat schedule file -celerybeat-schedule - -# SageMath parsed files -*.sage.py - -# dotenv -.env - -# virtualenv -.venv -venv/ -ENV/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -# mypy -.mypy_cache/ diff --git a/django_utils/migrations/0001_initial.py b/django_utils/migrations/0001_initial.py new file mode 100644 index 000000000..421f9be63 --- /dev/null +++ b/django_utils/migrations/0001_initial.py @@ -0,0 +1,115 @@ +# Generated by Django 4.2.25 on 2026-02-11 16:17 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name="Center", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("center_name", models.CharField(max_length=50, verbose_name="Center")), + ( + "center_abbr", + models.CharField(max_length=25, verbose_name="Acronym"), + ), + ], + options={ + "db_table": "utils_center", + }, + ), + migrations.CreateModel( + name="ClassificationArea", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("classification_area_name", models.CharField(max_length=80)), + ( + "classification_area_description", + models.CharField(blank=True, max_length=255, null=True), + ), + ], + options={ + "db_table": "utils_classification_area", + }, + ), + migrations.CreateModel( + name="Profile", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "profile_position", + models.CharField(max_length=50, verbose_name="Position"), + ), + ( + "profile_area", + models.CharField(max_length=50, verbose_name="Area / Unit"), + ), + ( + "profile_extension", + models.CharField(max_length=5, verbose_name="Phone extension"), + ), + ( + "profile_center", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="django_utils.center", + verbose_name="Center", + ), + ), + ( + "profile_classification_area", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="django_utils.classificationarea", + ), + ), + ( + "profile_user_id", + models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + related_name="profile", + to=settings.AUTH_USER_MODEL, + ), + ), + ], + options={ + "db_table": "utils_profile", + }, + ), + ] diff --git a/django_utils/migrations/__init__.py b/django_utils/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 1af9f79de..f7dfe2a1a 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -7,7 +7,7 @@ services: build: context: . args: - INSTALL_TYPE: ${INSTALL_TYPE:-app} + INSTALL_TYPE: ${INSTALL_TYPE:-dep} GIT_REVISION: ${GIT_REVISION:-main} INSTALL_CONF: ${INSTALL_CONF:-conf/docker_production_settings.txt} ports: @@ -17,3 +17,5 @@ services: volumes: - /etc/localtime:/etc/localtime:ro - /usr/share/zoneinfo:/usr/share/zoneinfo + extra_hosts: + - "host.docker.internal:host-gateway" diff --git a/docker-compose.yml b/docker-compose.test.yml similarity index 87% rename from docker-compose.yml rename to docker-compose.test.yml index c461959f4..f0fabbb9e 100644 --- a/docker-compose.yml +++ b/docker-compose.test.yml @@ -38,7 +38,12 @@ services: command: '-S -s "ngs_data;/mnt/test_ngs_data;yes;yes;no;samba_user;none;none;ngs data samba share" -u "samba_user;sambapasswd" -p' app: - build: . + build: + context: . + args: + INSTALL_TYPE: ${INSTALL_TYPE:-dep} + GIT_REVISION: ${GIT_REVISION:-main} + INSTALL_CONF: ${INSTALL_CONF:-conf/docker_test_settings.txt} container_name: iskylims_app ports: - "8001:8001" diff --git a/docker_install.sh b/docker_install.sh index f700f3832..9510b4f60 100644 --- a/docker_install.sh +++ b/docker_install.sh @@ -6,7 +6,7 @@ usage() { cat << EOF This script installs and upgrades the iskylims app. -Usage : $0 [--demo_data] [--install_type] [--git_revision] [--compose_file] [--install_conf] [--action] [--script] [--test] +Usage : $0 [--demo_data] [--install_type] [--git_revision] [--compose_file] [--install_conf] [--action] [--script] [--script_before] [--script_after] [--test] Optional input data: --demo_data | Provide already downloaded demo data from Zenodo --install_type | Specify the installation type for iSkyLIMS (default: full) @@ -14,7 +14,9 @@ Usage : $0 [--demo_data] [--install_type] [--git_revision] [--compose_file] [--i --compose_file | docker compose file to use (overrides default) --install_conf | Settings file consumed during docker image build (mandatory for production) --action | install (default) or upgrade, to control DB initialisation steps - --script | Run a Django migration script (can be repeated) + --script | Run a Django migration script after migrations (can be repeated) + --script_before | Run a Django migration script before migrations (can be repeated) + --script_after | Run a Django migration script after migrations (can be repeated) --skip_demo_data | Skip downloading/copying demo data to samba container --skip_test_data | Skip loading test fixtures (test/test_data.json) --test | Use development/test compose file and sample data @@ -56,6 +58,8 @@ do --install_conf) set -- "$@" -s ;; --action) set -- "$@" -a ;; --script) set -- "$@" -m ;; + --script_before) set -- "$@" -b ;; + --script_after) set -- "$@" -f ;; --skip_demo_data) set -- "$@" -n ;; --skip_test_data) set -- "$@" -t ;; --test) set -- "$@" -p ;; @@ -73,15 +77,18 @@ demo_data=false git_revision="main" compose_file="" install_conf="" +install_conf_container="" skip_demo_data="" skip_test_data="" mode="production" action="install" run_script=false +run_script_before=false migration_script=() +migration_script_before=() # PARSE VARIABLE ARGUMENTS WITH getopts -options=":d:i:g:c:s:a:m:vhntp" +options=":d:i:g:c:s:a:m:b:f:vhntp" while getopts $options opt; do case $opt in d) @@ -107,6 +114,14 @@ while getopts $options opt; do run_script=true migration_script+=("$OPTARG") ;; + b) + run_script_before=true + migration_script_before+=("$OPTARG") + ;; + f) + run_script=true + migration_script+=("$OPTARG") + ;; n) skip_demo_data=true ;; @@ -143,10 +158,10 @@ shift $((OPTIND-1)) if [ "$mode" = "test" ]; then if [ -z "$compose_file" ]; then - compose_file="docker-compose.yml" + compose_file="docker-compose.test.yml" fi if [ -z "$install_conf" ]; then - install_conf="conf/docker_install_settings.txt" + install_conf="conf/docker_test_settings.txt" fi else if [ -z "$compose_file" ]; then @@ -191,11 +206,24 @@ if [ ! -f "$install_conf" ]; then fi repo_root="$(pwd)" +temp_install_conf="" if [[ "$install_conf" = /* ]] && [[ "$install_conf" != "$repo_root/"* ]]; then - install_conf_copy="conf/docker_runtime_settings.txt" - echo "Copying $install_conf into repository as $install_conf_copy for Docker build/runtime." - cp "$install_conf" "$install_conf_copy" - install_conf="$install_conf_copy" + temp_install_conf="$repo_root/.tmp_docker_install_conf_$$.txt" + echo "Copying $install_conf into temporary file $temp_install_conf for Docker build/runtime." + cp "$install_conf" "$temp_install_conf" + install_conf="$temp_install_conf" + cleanup_temp_conf() { + if [ -n "$temp_install_conf" ] && [ -f "$temp_install_conf" ]; then + rm -f "$temp_install_conf" + fi + } + trap cleanup_temp_conf EXIT +fi + +if [[ "$install_conf" = "$repo_root/"* ]]; then + install_conf_container="${install_conf#$repo_root/}" +else + install_conf_container="$install_conf" fi service_exists() { @@ -214,27 +242,53 @@ ensure_app_running() { fi } -echo "Deploying containers (compose file: $compose_file) with INSTALL_TYPE="dep" and GIT_REVISION=$git_revision..." -docker compose -f "$compose_file" build --no-cache --build-arg INSTALL_TYPE="dep" --build-arg GIT_REVISION="$git_revision" --build-arg INSTALL_CONF="$install_conf" +echo "Deploying containers (compose file: $compose_file) with INSTALL_TYPE=dep and GIT_REVISION=$git_revision..." +INSTALL_TYPE="dep" GIT_REVISION="$git_revision" INSTALL_CONF="$install_conf_container" \ + docker compose -f "$compose_file" build --no-cache \ + --build-arg INSTALL_TYPE="dep" \ + --build-arg GIT_REVISION="$git_revision" \ + --build-arg INSTALL_CONF="$install_conf_container" docker compose -f "$compose_file" up -d echo "Waiting 20 seconds for starting database and web services..." sleep 20 ensure_app_running -script_args="" +host_install_conf_path="$install_conf" +if [[ "$host_install_conf_path" != /* ]]; then + host_install_conf_path="$repo_root/$host_install_conf_path" +fi + +container_install_conf_path="$install_conf_container" +if [[ "$container_install_conf_path" != /* ]]; then + container_install_conf_path="/srv/iskylims/$container_install_conf_path" +fi + +if ! docker exec -it iskylims_app test -f "$container_install_conf_path"; then + echo "Copying install configuration into container at $container_install_conf_path" + docker cp "$host_install_conf_path" "iskylims_app:$container_install_conf_path" +fi + +script_args_before="" +if [ "$run_script_before" = true ]; then + for val in "${migration_script_before[@]}"; do + script_args_before+=" --script_before $(printf '%q' "$val")" + done +fi + +script_args_after="" if [ "$run_script" = true ]; then for val in "${migration_script[@]}"; do - script_args+=" --script $(printf '%q' "$val")" + script_args_after+=" --script_after $(printf '%q' "$val")" done fi if [ "$action" = "upgrade" ]; then echo "Running install.sh upgrade inside the container" - docker exec -it iskylims_app bash -c "cd /srv/iskylims && bash install.sh --upgrade app --git_revision \"$git_revision\" --conf \"$install_conf\" --skip_apache_restart$script_args" + docker exec -it iskylims_app bash -c "cd /srv/iskylims && bash install.sh --upgrade app --git_revision \"$git_revision\" --conf \"$install_conf_container\" --skip_apache_restart$script_args_before$script_args_after" else echo "Running install.sh install inside the container" - docker exec -it iskylims_app bash -c "cd /srv/iskylims && bash install.sh --install app --git_revision \"$git_revision\" --conf \"$install_conf\" --skip_apache_restart$script_args" + docker exec -it iskylims_app bash -c "cd /srv/iskylims && bash install.sh --install app --git_revision \"$git_revision\" --conf \"$install_conf_container\" --skip_apache_restart$script_args_before$script_args_after" fi if ! docker exec -it iskylims_app test -f /opt/iskylims/manage.py; then diff --git a/drylab/.gitignore b/drylab/.gitignore deleted file mode 100644 index 781e59dfd..000000000 --- a/drylab/.gitignore +++ /dev/null @@ -1,67 +0,0 @@ -# Custom -*.save* -*.swp -drylab_samba_conf.py -drylab_email_conf.py - -# Django migrations -__migrations__/ -migrations/ - -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] - -# C extensions -*.so - -# Distribution / packaging -.Python -env/ -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -*.egg-info/ -.installed.cfg -*.egg - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*,cover - -# Translations -*.mo -*.pot - -# Django stuff: -*.log - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ diff --git a/drylab/migrations/0001_initial.py b/drylab/migrations/0001_initial.py new file mode 100644 index 000000000..5b271eca7 --- /dev/null +++ b/drylab/migrations/0001_initial.py @@ -0,0 +1,579 @@ +# Generated by Django 4.2.25 on 2026-02-11 16:17 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import mptt.fields + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ("core", "0001_initial"), + ("wetlab", "0001_initial"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name="AvailableService", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "avail_service_description", + models.CharField(max_length=200, verbose_name="Available services"), + ), + ("service_in_use", models.BooleanField(default=True)), + ("service_id", models.CharField(blank=True, max_length=40, null=True)), + ( + "description", + models.CharField(blank=True, max_length=200, null=True), + ), + ("lft", models.PositiveIntegerField(editable=False)), + ("rght", models.PositiveIntegerField(editable=False)), + ("tree_id", models.PositiveIntegerField(db_index=True, editable=False)), + ("level", models.PositiveIntegerField(editable=False)), + ( + "parent", + mptt.fields.TreeForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to="drylab.availableservice", + ), + ), + ], + options={ + "verbose_name": "AvailableService", + "verbose_name_plural": "AvailableServices", + "db_table": "drylab_available_service", + "ordering": ["tree_id", "lft"], + }, + ), + migrations.CreateModel( + name="ConfigSetting", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("configuration_name", models.CharField(max_length=80)), + ( + "configuration_value", + models.CharField(blank=True, max_length=255, null=True), + ), + ("generated_at", models.DateTimeField(auto_now_add=True)), + ], + options={ + "db_table": "drylab_config_setting", + }, + ), + migrations.CreateModel( + name="Pipelines", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("pipeline_name", models.CharField(max_length=50)), + ("pipeline_version", models.CharField(max_length=10)), + ("pipeline_in_use", models.BooleanField(default=True)), + ( + "pipeline_file", + models.FileField( + blank=True, null=True, upload_to="drylab/pipelinesFiles" + ), + ), + ( + "pipeline_url", + models.CharField(blank=True, max_length=200, null=True), + ), + ( + "pipeline_description", + models.CharField(blank=True, max_length=500, null=True), + ), + ("generated_at", models.DateTimeField(auto_now_add=True)), + ( + "user_name", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + ), + ), + ], + options={ + "db_table": "drylab_pipelines", + }, + ), + migrations.CreateModel( + name="Resolution", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "resolution_number", + models.CharField( + max_length=255, null=True, verbose_name="Resolution name" + ), + ), + ( + "resolution_estimated_date", + models.DateField( + null=True, verbose_name=" Estimated resolution date" + ), + ), + ( + "resolution_date", + models.DateField(auto_now_add=True, verbose_name="Resolution date"), + ), + ("resolution_queued_date", models.DateField(blank=True, null=True)), + ( + "resolution_in_progress_date", + models.DateField(blank=True, null=True), + ), + ("resolution_delivery_date", models.DateField(blank=True, null=True)), + ( + "resolution_notes", + models.TextField( + blank=True, + max_length=1000, + null=True, + verbose_name="Resolution notes", + ), + ), + ( + "resolution_full_number", + models.CharField( + blank=True, + max_length=255, + null=True, + verbose_name="Acronym Name", + ), + ), + ( + "resolution_pdf_file", + models.FileField( + blank=True, null=True, upload_to="documents/drylab/resolutions" + ), + ), + ( + "available_services", + models.ManyToManyField(blank=True, to="drylab.availableservice"), + ), + ( + "resolution_assigned_user", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="groups+", + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "resolution_pipelines", + models.ManyToManyField(blank=True, to="drylab.pipelines"), + ), + ], + options={ + "db_table": "drylab_resolution", + }, + ), + migrations.CreateModel( + name="ResolutionStates", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("state_value", models.CharField(max_length=50)), + ( + "state_display", + models.CharField(blank=True, max_length=80, null=True), + ), + ( + "description", + models.CharField(blank=True, max_length=255, null=True), + ), + ], + options={ + "db_table": "drylab_resolution_states", + }, + ), + migrations.CreateModel( + name="Service", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "service_center", + models.CharField( + max_length=50, null=True, verbose_name="Sequencing center" + ), + ), + ( + "service_request_number", + models.CharField( + max_length=80, null=True, verbose_name="Service ID" + ), + ), + ("service_request_int", models.CharField(max_length=80, null=True)), + ( + "service_run_specs", + models.CharField( + blank=True, + max_length=10, + null=True, + verbose_name="Run specifications", + ), + ), + ( + "service_status", + models.CharField( + choices=[ + ("recorded", "Recorded"), + ("approved", "Approved"), + ("rejected", "Rejected"), + ("queued", "Queued"), + ("in_progress", "In Progress"), + ("delivered", "Delivered"), + ("archived", "Archived"), + ], + max_length=15, + verbose_name="Service status", + ), + ), + ( + "service_notes", + models.TextField( + blank=True, + max_length=2048, + null=True, + verbose_name="Service Notes", + ), + ), + ( + "service_created_date", + models.DateField(auto_now_add=True, null=True), + ), + ("service_approved_date", models.DateField(blank=True, null=True)), + ("service_rejected_date", models.DateField(blank=True, null=True)), + ("service_delivered_date", models.DateField(blank=True, null=True)), + ( + "service_available_service", + mptt.fields.TreeManyToManyField( + to="drylab.availableservice", verbose_name="AvailableServices" + ), + ), + ( + "service_project_names", + models.ManyToManyField( + blank=True, to="wetlab.projects", verbose_name="User's projects" + ), + ), + ( + "service_sequencing_platform", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="core.sequencingplatform", + ), + ), + ], + options={ + "db_table": "drylab_service", + }, + ), + migrations.CreateModel( + name="ServiceState", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("state_value", models.CharField(max_length=50)), + ( + "state_display", + models.CharField(blank=True, max_length=80, null=True), + ), + ( + "description", + models.CharField(blank=True, max_length=255, null=True), + ), + ("show_in_stats", models.BooleanField(default=False)), + ], + options={ + "db_table": "drylab_service_state", + }, + ), + migrations.CreateModel( + name="UploadServiceFile", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("upload_file", models.FileField(upload_to="drylab/service_files")), + ( + "upload_file_name", + models.CharField(blank=True, max_length=255, null=True), + ), + ("uploaded_at", models.DateTimeField(auto_now_add=True)), + ( + "upload_service", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="drylab.service", + ), + ), + ], + options={ + "db_table": "drylab_upload_service_file", + }, + ), + migrations.AddField( + model_name="service", + name="service_state", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="drylab.servicestate", + verbose_name="Service State", + ), + ), + migrations.AddField( + model_name="service", + name="service_user_id", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + ), + ), + migrations.CreateModel( + name="ResolutionParameters", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("resolution_parameter", models.CharField(max_length=50)), + ("resolution_param_value", models.CharField(max_length=80)), + ( + "resolution_param_notes", + models.CharField(blank=True, max_length=200, null=True), + ), + ("generated_at", models.DateTimeField(auto_now_add=True)), + ( + "resolution", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="drylab.resolution", + ), + ), + ], + options={ + "db_table": "drylab_resolution_parameters", + }, + ), + migrations.AddField( + model_name="resolution", + name="resolution_service_id", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="resolutions", + to="drylab.service", + ), + ), + migrations.AddField( + model_name="resolution", + name="resolution_state", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="drylab.resolutionstates", + ), + ), + migrations.CreateModel( + name="RequestedSamplesInServices", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("sample_key", models.CharField(blank=True, max_length=15, null=True)), + ("sample_name", models.CharField(blank=True, max_length=50, null=True)), + ( + "sample_path", + models.CharField(blank=True, max_length=250, null=True), + ), + ( + "run_name_key", + models.CharField(blank=True, max_length=15, null=True), + ), + ("run_name", models.CharField(blank=True, max_length=50, null=True)), + ("project_key", models.CharField(blank=True, max_length=15, null=True)), + ( + "project_name", + models.CharField(blank=True, max_length=50, null=True), + ), + ("only_recorded_sample", models.BooleanField(default=False)), + ("generated_at", models.DateField(auto_now_add=True)), + ( + "samples_in_service", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="samples", + to="drylab.service", + ), + ), + ], + options={ + "db_table": "drylab_request_samples_in_services", + }, + ), + migrations.CreateModel( + name="ParameterPipeline", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("parameter_name", models.CharField(max_length=80)), + ( + "parameter_value", + models.CharField(blank=True, max_length=200, null=True), + ), + ( + "parameter_type", + models.CharField(blank=True, max_length=20, null=True), + ), + ( + "parameter_pipeline", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="drylab.pipelines", + ), + ), + ], + options={ + "db_table": "drylab_parameter_pipeline", + }, + ), + migrations.CreateModel( + name="Delivery", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "delivery_notes", + models.TextField(blank=True, max_length=1000, null=True), + ), + ("execution_start_date", models.DateField(blank=True, null=True)), + ("execution_end_date", models.DateField(blank=True, null=True)), + ( + "execution_time", + models.CharField(blank=True, max_length=80, null=True), + ), + ( + "permanent_used_space", + models.CharField(blank=True, max_length=80, null=True), + ), + ( + "temporary_used_space", + models.CharField(blank=True, max_length=80, null=True), + ), + ( + "delivery_resolution_id", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="delivery", + to="drylab.resolution", + ), + ), + ( + "pipelines_in_delivery", + models.ManyToManyField(blank=True, to="drylab.pipelines"), + ), + ], + options={ + "db_table": "drylab_delivery", + }, + ), + ] diff --git a/drylab/migrations/0002_remove_service_service_status.py b/drylab/migrations/0002_remove_service_service_status.py new file mode 100644 index 000000000..e36b7be5d --- /dev/null +++ b/drylab/migrations/0002_remove_service_service_status.py @@ -0,0 +1,17 @@ +# Generated by Django 4.2.25 on 2026-02-11 16:33 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("drylab", "0001_initial"), + ] + + operations = [ + migrations.RemoveField( + model_name="service", + name="service_status", + ), + ] diff --git a/drylab/migrations/__init__.py b/drylab/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/drylab/scripts/drylab_service_state_migration.py b/drylab/scripts/drylab_service_state_migration.py index 7ea4c29d0..17b7336f4 100644 --- a/drylab/scripts/drylab_service_state_migration.py +++ b/drylab/scripts/drylab_service_state_migration.py @@ -1,6 +1,7 @@ from drylab.models import Service, ServiceState """ + Upgrade: 2.3.0 -> 2.3.1 The script is applicable for the upgrade from 2.3.0 to 2.3.1. Service state that was defined as option choice in models,is replaced in version 2.3.1 as a primary key in SampleState table, given in this way diff --git a/install.sh b/install.sh index 08304acb7..a85c9a4be 100644 --- a/install.sh +++ b/install.sh @@ -15,7 +15,9 @@ usage : $0 --upgrade --git_revision --conf --conf | Select custom configuration file. Default: ./install_settings.txt --tables | Load the first inital tables (from conf folder) --skip_tables | Skip loading initial tables (even during install) - --script | Run a migration script. + --script | Run a migration script after migrations. + --script_before | Run a migration script before migrations. + --script_after | Run a migration script after migrations (same as --script). --ren_app | Rename apps required for the upgrade migration to 3.0.0 --docker | Deprecated. Use --skip_apache_restart to avoid Apache checks/restart. @@ -35,6 +37,9 @@ Examples: Make adjustments for apps renaming in upgrade 2.3.0 to 2.3.1 $0 --upgrade full --ren_app --script --tables + + Upgrade running pre/post migration scripts: + $0 --upgrade app --script_before --script_after EOF } @@ -315,7 +320,7 @@ rename_apps_if_needed() { if [ -d "$INSTALL_PATH/iSkyLIMS_core" ]; then echo "Changing app dir names in $INSTALL_PATH..." rm -rf $INSTALL_PATH/.git $INSTALL_PATH/.github $INSTALL_PATH/.gitignore \ - $INSTALL_PATH/.Rhistory $INSTALL_PATH/docker-compose.yml $INSTALL_PATH/docker_iskylims_install.sh \ + $INSTALL_PATH/.Rhistory $INSTALL_PATH/docker-compose.test.yml $INSTALL_PATH/docker_iskylims_install.sh \ $INSTALL_PATH/Dockerfile $INSTALL_PATH/install.sh $INSTALL_PATH/install_settings.txt mv $INSTALL_PATH/iSkyLIMS_core $INSTALL_PATH/core mv $INSTALL_PATH/iSkyLIMS_wetlab $INSTALL_PATH/wetlab @@ -455,12 +460,27 @@ install_system_packages() { # run_django_deploy: execute makemigrations/migrate and optional fixture/superuser steps. run_django_deploy() { local mode="${1:-install}" - echo "Generating Django migrations" - python manage.py makemigrations $MIGRATION_MODULES + if [ "$run_script_before" = true ]; then + for val in "${migration_script_before[@]}"; do + if [[ $val = *","* ]]; then + parameters=(${val//,/ }) + echo "Running pre-migration script: ${parameters[0]}" + ./manage.py runscript ${parameters[0]} --script-args ${parameters[1]} + echo "Done pre-migration script: ${parameters[0]}" + else + echo "Running pre-migration script: $val" + ./manage.py runscript $val + echo "Done pre-migration script: $val" + fi + done + fi if [ "$mode" = "upgrade" ]; then echo "Applying migrations in fake-initial mode" python manage.py migrate --noinput --fake-initial + # Second pass ensures non-initial migrations are applied after fake-initial. + echo "Applying migrations" + python manage.py migrate --noinput else echo "Applying migrations" python manage.py migrate --noinput @@ -476,13 +496,13 @@ run_django_deploy() { for val in "${migration_script[@]}"; do if [[ $val = *","* ]]; then parameters=(${val//,/ }) - echo "Running migration script: ${parameters[0]}" + echo "Running post-migration script: ${parameters[0]}" ./manage.py runscript ${parameters[0]} --script-args ${parameters[1]} - echo "Done migration script: ${parameters[0]}" + echo "Done post-migration script: ${parameters[0]}" else - echo "Running migration script: $val" + echo "Running post-migration script: $val" ./manage.py runscript $val - echo "Done migration script: $val" + echo "Done post-migration script: $val" fi done fi @@ -651,7 +671,7 @@ upgrade_application_files() { echo "Copying files to installation folder" rsync -rlv conf/ $INSTALL_PATH/conf/ rsync -rlv --fuzzy --delay-updates --delete-delay \ - --exclude "logs" --exclude "documents" --exclude "migrations" --exclude "__pycache__" \ + --exclude "logs" --exclude "documents" --exclude "__pycache__" \ README.md LICENSE test conf $REQUIRED_MODULES $INSTALL_PATH cd $INSTALL_PATH @@ -759,6 +779,9 @@ do --install) set -- "$@" -i ;; --upgrade) set -- "$@" -u ;; --script) set -- "$@" -s ;; + --script_before) set -- "$@" -p ;; + --script_after) set -- "$@" -o ;; + --script_prev) set -- "$@" -p ;; --tables) set -- "$@" -t ;; --skip_tables) set -- "$@" -b ;; --git_revision) set -- "$@" -g ;; @@ -788,10 +811,13 @@ docker=false prefilled_tables="conf/first_install_tables.json" restart_apache=true run_script=false +run_script_before=false +migration_script=() +migration_script_before=() skip_tables=false # PARSE VARIABLE ARGUMENTS WITH getops -options=":c:s:i:u:r:g:tdbkvha" +options=":c:s:i:u:r:g:tdbkvhao:p:" while getopts $options opt; do case $opt in i ) @@ -820,6 +846,14 @@ while getopts $options opt; do run_script=true migration_script+=("$OPTARG") ;; + p ) + run_script_before=true + migration_script_before+=("$OPTARG") + ;; + o ) + run_script=true + migration_script+=("$OPTARG") + ;; t ) tables=true ;; diff --git a/test/test_data.json b/test/test_data.json index f81d46aad..700109f1b 100644 --- a/test/test_data.json +++ b/test/test_data.json @@ -15856,7 +15856,7 @@ "model": "wetlab.sambaconnectiondata", "pk": 1, "fields": { - "samba_folder_name": "", + "samba_folder_name": "Runs", "domain": "", "host_name": "samba", "ip_server": "", diff --git a/wetlab/.gitignore b/wetlab/.gitignore deleted file mode 100644 index b3b95c969..000000000 --- a/wetlab/.gitignore +++ /dev/null @@ -1,68 +0,0 @@ -## Custom -*.xml -*.bin -migrations/ -tmp/ -logs/ -tests/ - -## Cusotomized script - -script/private*.py -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] - -# C extensions -*.so - -# Distribution / packaging -.Python -env/ -build/ -develop-eggs/ -dist/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -*.egg-info/ -.installed.cfg -*.egg - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*,cover - -# Translations -*.mo -*.pot - -# Django stuff: -*.log - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ diff --git a/wetlab/config.py b/wetlab/config.py index a69a9f222..6f789af5b 100644 --- a/wetlab/config.py +++ b/wetlab/config.py @@ -864,7 +864,7 @@ # ######################## Configuration test errors ##################################### ERROR_NOT_FOLDER_RUN_TEST_WAS_FOUND = [ "Unable to run the configuration test", - "Run test folder was found on remote server", + "Run test folder was not found on remote server", ] ERROR_NO_RUN_TEST_WAS_CREATED = [ "Unable to continue with configuration testing", diff --git a/wetlab/migrations/0001_initial.py b/wetlab/migrations/0001_initial.py new file mode 100644 index 000000000..b2d84ab8b --- /dev/null +++ b/wetlab/migrations/0001_initial.py @@ -0,0 +1,1149 @@ +# Generated by Django 4.2.25 on 2026-02-11 16:17 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ("core", "0001_initial"), + ("django_utils", "0001_initial"), + ] + + operations = [ + migrations.CreateModel( + name="AdditionaKitsLibPrepare", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("kit_name", models.CharField(max_length=255)), + ( + "description", + models.CharField(blank=True, max_length=400, null=True), + ), + ("kit_order", models.IntegerField()), + ("kit_used", models.BooleanField()), + ("generated_at", models.DateTimeField(auto_now_add=True)), + ( + "commercial_kit_id", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="core.commercialkits", + ), + ), + ( + "protocol_id", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="core.protocols" + ), + ), + ( + "register_user", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + ), + ), + ], + options={ + "db_table": "wetlab_lib_additional_kits_lib_prepare", + }, + ), + migrations.CreateModel( + name="CollectionIndexKit", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("collection_index_name", models.CharField(max_length=125)), + ("version", models.CharField(max_length=80, null=True)), + ("plate_extension", models.CharField(max_length=125, null=True)), + ("adapter_1", models.CharField(max_length=125, null=True)), + ("adapter_2", models.CharField(max_length=125, null=True)), + ( + "collection_index_file", + models.FileField( + max_length=500, upload_to="wetlab/collection_index_kits/" + ), + ), + ("generated_at", models.DateTimeField(auto_now_add=True, null=True)), + ], + options={ + "db_table": "wetlab_collection_index_kit", + }, + ), + migrations.CreateModel( + name="ConfigSetting", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("configuration_name", models.CharField(max_length=80)), + ( + "configuration_value", + models.CharField(blank=True, max_length=255, null=True), + ), + ("generated_at", models.DateTimeField(auto_now_add=True)), + ], + options={ + "db_table": "wetlab_config_setting", + }, + ), + migrations.CreateModel( + name="LibPrepareStates", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("lib_prep_state", models.CharField(max_length=50)), + ], + options={ + "db_table": "wetlab_lib_prepare_states", + }, + ), + migrations.CreateModel( + name="LibraryKit", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("library_name", models.CharField(max_length=125)), + ("generated_at", models.DateTimeField(auto_now_add=True)), + ], + options={ + "db_table": "wetlab_library_kit", + }, + ), + migrations.CreateModel( + name="PoolStates", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("pool_state", models.CharField(max_length=50)), + ], + options={ + "db_table": "wetlab_pool_states", + }, + ), + migrations.CreateModel( + name="Projects", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "base_space_library", + models.CharField(blank=True, max_length=45, null=True), + ), + ("project_name", models.CharField(max_length=45)), + ( + "library_kit", + models.CharField(blank=True, max_length=125, null=True), + ), + ( + "base_space_file", + models.CharField(blank=True, max_length=255, null=True), + ), + ("generated_at", models.DateTimeField(auto_now_add=True)), + ("project_run_date", models.DateField(blank=True, null=True)), + ( + "library_kit_id", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="wetlab.librarykit", + ), + ), + ], + options={ + "db_table": "wetlab_projects", + }, + ), + migrations.CreateModel( + name="RunConfigurationTest", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("run_test_name", models.CharField(max_length=80)), + ("run_test_folder", models.CharField(max_length=200)), + ], + options={ + "db_table": "wetlab_lib_run_configuration_test", + }, + ), + migrations.CreateModel( + name="RunErrors", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("error_code", models.CharField(max_length=10)), + ("error_text", models.CharField(max_length=255)), + ], + options={ + "db_table": "wetlab_run_errors", + }, + ), + migrations.CreateModel( + name="RunProcess", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("run_name", models.CharField(max_length=45)), + ( + "sample_sheet", + models.FileField( + blank=True, null=True, upload_to="wetlab/sample_sheet/" + ), + ), + ("generated_at", models.DateTimeField(auto_now_add=True)), + ("run_date", models.DateField(blank=True, null=True)), + ("run_finish_date", models.DateTimeField(blank=True, null=True)), + ("bcl2fastq_finish_date", models.DateTimeField(blank=True, null=True)), + ("run_completed_date", models.DateTimeField(blank=True, null=True)), + ( + "run_forced_continue_on_error", + models.BooleanField(blank=True, default=False, null=True), + ), + ( + "index_library", + models.CharField(blank=True, max_length=85, null=True), + ), + ("samples", models.CharField(blank=True, max_length=45)), + ("use_space_img_mb", models.CharField(blank=True, max_length=10)), + ("use_space_fasta_mb", models.CharField(blank=True, max_length=10)), + ("use_space_other_mb", models.CharField(blank=True, max_length=10)), + ( + "center_requested_by", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="django_utils.center", + ), + ), + ( + "reagent_kit", + models.ManyToManyField(blank=True, to="core.userlotcommercialkits"), + ), + ( + "run_error", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="wetlab.runerrors", + ), + ), + ], + options={ + "db_table": "wetlab_run_process", + }, + ), + migrations.CreateModel( + name="RunStates", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("run_state_name", models.CharField(max_length=50)), + ], + options={ + "db_table": "wetlab_run_states", + }, + ), + migrations.CreateModel( + name="SambaConnectionData", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "samba_folder_name", + models.CharField(blank=True, max_length=80, null=True), + ), + ("domain", models.CharField(blank=True, max_length=80, null=True)), + ("host_name", models.CharField(blank=True, max_length=80, null=True)), + ("ip_server", models.CharField(blank=True, max_length=20, null=True)), + ("port_server", models.CharField(blank=True, max_length=10, null=True)), + ( + "remote_server_name", + models.CharField(blank=True, max_length=80, null=True), + ), + ( + "shared_folder_name", + models.CharField(blank=True, max_length=80, null=True), + ), + ("user_id", models.CharField(blank=True, max_length=80, null=True)), + ( + "user_password", + models.CharField(blank=True, max_length=20, null=True), + ), + ("is_direct_tcp", models.BooleanField(default=True)), + ("ntlm_used", models.BooleanField(default=True)), + ], + options={ + "db_table": "wetlab_samba_connection_data", + }, + ), + migrations.CreateModel( + name="RunningParameters", + fields=[ + ( + "run_name_id", + models.OneToOneField( + on_delete=django.db.models.deletion.CASCADE, + primary_key=True, + serialize=False, + to="wetlab.runprocess", + ), + ), + ("run_id", models.CharField(max_length=255)), + ("experiment_name", models.CharField(max_length=255)), + ( + "rta_version", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "system_suite_version", + models.CharField(blank=True, max_length=255, null=True), + ), + ("library_id", models.CharField(blank=True, max_length=255, null=True)), + ("chemistry", models.CharField(blank=True, max_length=255, null=True)), + ( + "run_start_date", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "analysis_workflow_type", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "run_management_type", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "planned_read1_cycles", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "planned_read2_cycles", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "planned_index1_read_cycles", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "planned_index2_read_cycles", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "application_version", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "num_tiles_per_swath", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "image_channel", + models.CharField(blank=True, max_length=255, null=True), + ), + ("flowcell", models.CharField(blank=True, max_length=255, null=True)), + ( + "image_dimensions", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "flowcell_layout", + models.CharField(blank=True, max_length=255, null=True), + ), + ], + options={ + "db_table": "wetlab_running_parameters", + }, + ), + migrations.CreateModel( + name="StatsRunSummary", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("level", models.CharField(max_length=20)), + ("yield_total", models.CharField(max_length=10)), + ("projected_total_yield", models.CharField(max_length=10)), + ("aligned", models.CharField(max_length=10)), + ("error_rate", models.CharField(max_length=10)), + ("intensity_cycle", models.CharField(max_length=10)), + ("bigger_q30", models.CharField(max_length=10)), + ("generated_at", models.DateTimeField(auto_now_add=True)), + ("stats_summary_run_date", models.DateField(null=True)), + ( + "runprocess_id", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="wetlab.runprocess", + ), + ), + ], + options={ + "db_table": "wetlab_stats_run_summary", + }, + ), + migrations.CreateModel( + name="StatsRunRead", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("read", models.CharField(max_length=10)), + ("lane", models.CharField(max_length=10)), + ("tiles", models.CharField(max_length=10)), + ("density", models.CharField(max_length=40)), + ("cluster_pf", models.CharField(max_length=40)), + ("phas_prephas", models.CharField(max_length=40)), + ("reads", models.CharField(max_length=40)), + ("reads_pf", models.CharField(max_length=40)), + ("q30", models.CharField(max_length=40)), + ("yields", models.CharField(max_length=40)), + ("cycles_err_rated", models.CharField(max_length=40)), + ("aligned", models.CharField(max_length=40)), + ("error_rate", models.CharField(max_length=40)), + ("error_rate_35", models.CharField(max_length=40)), + ("error_rate_50", models.CharField(max_length=40)), + ("error_rate_75", models.CharField(max_length=40)), + ("error_rate_100", models.CharField(max_length=40)), + ("intensity_cycle", models.CharField(max_length=40)), + ("generated_at", models.DateTimeField(auto_now_add=True)), + ("stats_read_run_date", models.DateField(null=True)), + ( + "runprocess_id", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="wetlab.runprocess", + ), + ), + ], + options={ + "db_table": "wetlab_stats_run_read", + }, + ), + migrations.CreateModel( + name="StatsLaneSummary", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("default_all", models.CharField(max_length=40, null=True)), + ("lane", models.CharField(max_length=10)), + ("pf_cluster", models.CharField(max_length=64)), + ("percent_lane", models.CharField(max_length=64)), + ("perfect_barcode", models.CharField(max_length=64)), + ("one_mismatch", models.CharField(max_length=64)), + ("yield_mb", models.CharField(max_length=64)), + ("bigger_q30", models.CharField(max_length=64)), + ("mean_quality", models.CharField(max_length=64)), + ("generated_at", models.DateTimeField(auto_now_add=True)), + ( + "project_id", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="wetlab.projects", + ), + ), + ( + "runprocess_id", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="wetlab.runprocess", + ), + ), + ], + options={ + "db_table": "wetlab_stats_lane_summary", + }, + ), + migrations.CreateModel( + name="StatsFlSummary", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("default_all", models.CharField(max_length=40, null=True)), + ("flow_raw_cluster", models.CharField(max_length=40)), + ("flow_pf_cluster", models.CharField(max_length=40)), + ("flow_yield_mb", models.CharField(max_length=40)), + ("sample_number", models.CharField(max_length=40)), + ("generated_at", models.DateTimeField(auto_now_add=True)), + ( + "project_id", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="wetlab.projects", + ), + ), + ( + "runprocess_id", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="wetlab.runprocess", + ), + ), + ], + options={ + "db_table": "wetlab_stats_fl_summary", + }, + ), + migrations.CreateModel( + name="SamplesInProject", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("sample_name", models.CharField(max_length=255)), + ("barcode_name", models.CharField(max_length=255)), + ("pf_clusters", models.CharField(max_length=55)), + ("percent_in_project", models.CharField(max_length=25)), + ("yield_mb", models.CharField(max_length=55)), + ("quality_q30", models.CharField(max_length=55)), + ("mean_quality", models.CharField(max_length=55)), + ("generated_at", models.DateTimeField(auto_now_add=True)), + ( + "project_id", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="wetlab.projects", + ), + ), + ( + "run_process_id", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="wetlab.runprocess", + ), + ), + ( + "sample_in_core", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="core.samples", + ), + ), + ( + "user_id", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + ), + ), + ], + options={ + "db_table": "wetlab_samples_in_project", + }, + ), + migrations.AddField( + model_name="runprocess", + name="state", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="state_of_run", + to="wetlab.runstates", + ), + ), + migrations.AddField( + model_name="runprocess", + name="state_before_error", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="wetlab.runstates", + ), + ), + migrations.AddField( + model_name="runprocess", + name="used_sequencer", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="core.sequencerinlab", + ), + ), + migrations.CreateModel( + name="RawTopUnknowBarcodes", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("lane_number", models.CharField(max_length=4)), + ("top_number", models.CharField(max_length=4)), + ("count", models.CharField(max_length=40)), + ("sequence", models.CharField(max_length=40)), + ("generated_at", models.DateTimeField(auto_now_add=True)), + ( + "runprocess_id", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="wetlab.runprocess", + ), + ), + ], + options={ + "db_table": "wetlab_raw_top_unknown_barcodes", + }, + ), + migrations.CreateModel( + name="RawDemuxStats", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("default_all", models.CharField(max_length=40, null=True)), + ("raw_yield", models.CharField(max_length=255)), + ("raw_yield_q30", models.CharField(max_length=255)), + ("raw_quality", models.CharField(max_length=255)), + ("pf_yield", models.CharField(max_length=255)), + ("pf_yield_q30", models.CharField(max_length=255)), + ("pf_quality_score", models.CharField(max_length=255)), + ("generated_at", models.DateTimeField(auto_now_add=True)), + ( + "project_id", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="wetlab.projects", + ), + ), + ( + "runprocess_id", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="wetlab.runprocess", + ), + ), + ], + options={ + "db_table": "wetlab_raw_demux_stats", + }, + ), + migrations.AddField( + model_name="projects", + name="run_process", + field=models.ManyToManyField(to="wetlab.runprocess"), + ), + migrations.AddField( + model_name="projects", + name="user_id", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + ), + ), + migrations.CreateModel( + name="LibUserSampleSheet", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "sample_sheet", + models.FileField(upload_to="wetlab/sample_sheets_lib_prep/"), + ), + ("generated_at", models.DateTimeField(auto_now_add=True, null=True)), + ("application", models.CharField(blank=True, max_length=70, null=True)), + ("instrument", models.CharField(blank=True, max_length=70, null=True)), + ("adapter_1", models.CharField(blank=True, max_length=70, null=True)), + ("adapter_2", models.CharField(blank=True, max_length=70, null=True)), + ("assay", models.CharField(blank=True, max_length=70, null=True)), + ("reads", models.CharField(blank=True, max_length=10, null=True)), + ("confirmed_used", models.BooleanField(default=False)), + ("iem_version", models.CharField(blank=True, max_length=5, null=True)), + ( + "collection_index_kit_id", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="wetlab.collectionindexkit", + ), + ), + ( + "register_user", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "sequencing_configuration", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="core.sequencingconfiguration", + ), + ), + ], + options={ + "db_table": "wetlab_lib_user_samplesheet", + }, + ), + migrations.CreateModel( + name="LibraryPool", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("pool_name", models.CharField(max_length=50)), + ("sample_number", models.IntegerField(default=0)), + ("pool_code_id", models.CharField(blank=True, max_length=50)), + ("adapter", models.CharField(blank=True, max_length=50, null=True)), + ("paired_end", models.CharField(blank=True, max_length=10, null=True)), + ("generated_at", models.DateTimeField(auto_now_add=True)), + ( + "platform", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="core.sequencingplatform", + ), + ), + ( + "pool_state", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="wetlab.poolstates", + ), + ), + ( + "register_user", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "run_process_id", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="wetlab.runprocess", + ), + ), + ], + options={ + "db_table": "wetlab_library_pool", + "ordering": ("pool_name",), + }, + ), + migrations.CreateModel( + name="LibPrepare", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "lib_prep_code_id", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "user_sample_id", + models.CharField(blank=True, max_length=100, null=True), + ), + ( + "project_in_samplesheet", + models.CharField(blank=True, max_length=80, null=True), + ), + ( + "sample_plate", + models.CharField(blank=True, max_length=50, null=True), + ), + ("sample_well", models.CharField(blank=True, max_length=20, null=True)), + ( + "index_plate_well", + models.CharField(blank=True, max_length=20, null=True), + ), + ("i7_index_id", models.CharField(blank=True, max_length=25, null=True)), + ("i7_index", models.CharField(blank=True, max_length=30, null=True)), + ("i5_index_id", models.CharField(blank=True, max_length=25, null=True)), + ("i5_index", models.CharField(blank=True, max_length=30, null=True)), + ( + "genome_folder", + models.CharField(blank=True, max_length=180, null=True), + ), + ("manifest", models.CharField(blank=True, max_length=80, null=True)), + ("reused_number", models.IntegerField(default=0)), + ("unique_id", models.CharField(blank=True, max_length=16, null=True)), + ( + "user_in_samplesheet", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "samplename_in_samplesheet", + models.CharField(blank=True, max_length=255, null=True), + ), + ( + "prefix_protocol", + models.CharField(blank=True, max_length=25, null=True), + ), + ( + "lib_prep_state", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="wetlab.libpreparestates", + ), + ), + ( + "molecule_id", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="core.moleculepreparation", + ), + ), + ("pools", models.ManyToManyField(blank=True, to="wetlab.librarypool")), + ( + "protocol_id", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="core.protocols", + ), + ), + ( + "register_user", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "sample_id", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="core.samples", + ), + ), + ( + "user_lot_kit_id", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="core.userlotcommercialkits", + ), + ), + ( + "user_sample_sheet", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="wetlab.libusersamplesheet", + ), + ), + ], + options={ + "db_table": "wetlab_lib_prepare", + "ordering": ("lib_prep_code_id",), + }, + ), + migrations.CreateModel( + name="LibParameterValue", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("parameter_value", models.CharField(max_length=255)), + ("generated_at", models.DateTimeField(auto_now_add=True)), + ( + "library_id", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="wetlab.libprepare", + ), + ), + ( + "parameter_id", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="core.protocolparameters", + ), + ), + ], + options={ + "db_table": "wetlab_lib_parameter_value", + }, + ), + migrations.CreateModel( + name="GraphicsStats", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("folder_run_graphic", models.CharField(max_length=255)), + ("cluster_count_graph", models.CharField(max_length=255)), + ("flowcell_graph", models.CharField(max_length=255)), + ("intensity_by_cycle_graph", models.CharField(max_length=255)), + ("heatmap_graph", models.CharField(max_length=255)), + ("histogram_graph", models.CharField(max_length=255)), + ("sample_qc_graph", models.CharField(max_length=255)), + ("generated_at", models.DateTimeField(auto_now_add=True, null=True)), + ( + "runprocess_id", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="wetlab.runprocess", + ), + ), + ], + options={ + "db_table": "wetlab_graphics_stats", + }, + ), + migrations.CreateModel( + name="CollectionIndexValues", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("default_well", models.CharField(max_length=10, null=True)), + ("index_7", models.CharField(max_length=25, null=True)), + ("i_7_seq", models.CharField(max_length=25, null=True)), + ("index_5", models.CharField(max_length=25, null=True)), + ("i_5_seq", models.CharField(max_length=25, null=True)), + ( + "collection_index_kit_id", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="wetlab.collectionindexkit", + ), + ), + ], + options={ + "db_table": "wetlab_collection_index_values", + }, + ), + migrations.CreateModel( + name="AdditionalUserLotKit", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("value", models.CharField(max_length=255)), + ("generated_at", models.DateTimeField(auto_now_add=True)), + ( + "additional_lot_kits", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="wetlab.additionakitslibprepare", + ), + ), + ( + "lib_prep_id", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="wetlab.libprepare", + ), + ), + ( + "user_lot_kit_id", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="core.userlotcommercialkits", + ), + ), + ], + options={ + "db_table": "wetlab_lib_additional_user_lot_kit", + }, + ), + ] diff --git a/wetlab/migrations/0002_remove_librarypool_run_process_id_and_more.py b/wetlab/migrations/0002_remove_librarypool_run_process_id_and_more.py new file mode 100644 index 000000000..46552de57 --- /dev/null +++ b/wetlab/migrations/0002_remove_librarypool_run_process_id_and_more.py @@ -0,0 +1,92 @@ +# Generated by Django 4.2.25 on 2026-02-11 16:33 + +from django.db import migrations, models + + +def drop_librarypool_run_process_id(apps, schema_editor): + with schema_editor.connection.cursor() as cursor: + cursor.execute(""" + SELECT 1 + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'wetlab_library_pool' + AND COLUMN_NAME = 'run_process_id_id' + """) + if cursor.fetchone() is None: + return + + cursor.execute(""" + SELECT CONSTRAINT_NAME + FROM information_schema.KEY_COLUMN_USAGE + WHERE TABLE_SCHEMA = DATABASE() + AND TABLE_NAME = 'wetlab_library_pool' + AND COLUMN_NAME = 'run_process_id_id' + AND REFERENCED_TABLE_NAME IS NOT NULL + """) + for (constraint_name,) in cursor.fetchall(): + cursor.execute( + f"ALTER TABLE wetlab_library_pool DROP FOREIGN KEY `{constraint_name}`" + ) + + cursor.execute("ALTER TABLE wetlab_library_pool DROP COLUMN run_process_id_id") + + +class Migration(migrations.Migration): + + dependencies = [ + ("wetlab", "0001_initial"), + ] + + operations = [ + migrations.RunSQL( + """ + UPDATE wetlab_raw_top_unknown_barcodes + SET count = NULL + WHERE count IS NOT NULL AND count NOT REGEXP '^[0-9]+$' + """, + reverse_sql=migrations.RunSQL.noop, + ), + migrations.SeparateDatabaseAndState( + database_operations=[ + migrations.RunPython( + drop_librarypool_run_process_id, migrations.RunPython.noop + ), + ], + state_operations=[ + migrations.RemoveField( + model_name="librarypool", + name="run_process_id", + ), + ], + ), + migrations.AddField( + model_name="runprocess", + name="library_pool", + field=models.ManyToManyField(blank=True, to="wetlab.librarypool"), + ), + migrations.AddField( + model_name="runstates", + name="description", + field=models.CharField(blank=True, max_length=255, null=True), + ), + migrations.AddField( + model_name="runstates", + name="show_in_stats", + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name="runstates", + name="state_display", + field=models.CharField(blank=True, max_length=80, null=True), + ), + migrations.AlterField( + model_name="libprepare", + name="prefix_protocol", + field=models.CharField(blank=True, max_length=50, null=True), + ), + migrations.AlterField( + model_name="rawtopunknowbarcodes", + name="count", + field=models.IntegerField(), + ), + ] diff --git a/wetlab/migrations/__init__.py b/wetlab/migrations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/wetlab/scripts/convert_rawtop_counter_to_int.py b/wetlab/scripts/convert_rawtop_counter_to_int.py index 94c8fb6a3..54a80fa8d 100644 --- a/wetlab/scripts/convert_rawtop_counter_to_int.py +++ b/wetlab/scripts/convert_rawtop_counter_to_int.py @@ -2,7 +2,8 @@ def run(): - """The script implemted the issue #158 Unable to convert barcode count to + """Upgrade: 3.0.0 -> 3.1.0 + The script implemted the issue #158 Unable to convert barcode count to integer, to convert the counter that are in a string format, separated by "," to int. """ diff --git a/wetlab/scripts/library_pool_to_many_relation.py b/wetlab/scripts/library_pool_to_many_relation.py index 67cbd1c21..925a08014 100644 --- a/wetlab/scripts/library_pool_to_many_relation.py +++ b/wetlab/scripts/library_pool_to_many_relation.py @@ -2,12 +2,16 @@ def run(f_name): - """This script is part of the issue "#180,when deleting run , pool and + """Upgrade: 3.0.0 -> 3.1.0 + This script is part of the issue "#180,when deleting run , pool and library_preparations are also deleted" on Class LibraryPool. - The first part of the script fetch the existing data on run_process_id - and create a file containing the pk of the libraryPool instance and - the pk of the run_process_id - The second part of the script fetch the data from the file and add the + This script expects a file created externally that contains the pk of the + LibraryPool instance and the pk of the run_process_id. + Export example: + mysql -u -p -h -D \ + -e "SELECT id, run_process_id_id FROM wetlab_library_pool" \ + > /tmp/library_pool_run_process.tsv + The script fetches the data from the file and adds the run_process_pk to the run_process field of the LibraryPool instance """ if hasattr(wetlab.models.LibraryPool, "run_process"): diff --git a/wetlab/utils/crontab_process.py b/wetlab/utils/crontab_process.py index 53b2c262b..67e5c613d 100644 --- a/wetlab/utils/crontab_process.py +++ b/wetlab/utils/crontab_process.py @@ -47,7 +47,7 @@ def get_run_disk_utilization(conn, run_folder, experiment_name): full_path_run = os.path.join(application_folder, run_folder) try: - get_full_list = conn.listPath(shared_folder, run_folder) + get_full_list = conn.listPath(shared_folder, full_path_run) except Exception: string_message = experiment_name + " : Unable to get the folder " + run_folder wetlab.utils.common.logging_errors(string_message, True, False) @@ -70,7 +70,7 @@ def get_run_disk_utilization(conn, run_folder, experiment_name): "%s : Starting getting disk space utilization for Data Folder", experiment_name, ) - dir_data = os.path.join(run_folder, "Data") + dir_data = os.path.join(full_path_run, "Data") data_dir_size = get_size_dir(dir_data, conn, shared_folder) elif item_list.filename == "Images": @@ -2874,9 +2874,9 @@ def process_and_store_raw_demux_project_data( experiment_name + " : Created project name " + project - + "Because it was not store" + + " because it's a new project and it was not store before." ) - wetlab.utils.common.logging_warnings(string_message, True) + wetlab.utils.common.logging_warnings(string_message, False) logger.info("%s : Processing demultiplexing raw project data", experiment_name) for project in parsed_data.keys(): diff --git a/wetlab/utils/crontab_update_run.py b/wetlab/utils/crontab_update_run.py index c5453847c..3d6618e70 100644 --- a/wetlab/utils/crontab_update_run.py +++ b/wetlab/utils/crontab_update_run.py @@ -772,6 +772,11 @@ def manage_run_in_processed_run_state(conn, run_process_objs): .last() .get_run_folder() ) + root_run_folder = os.path.join( + "/", + wetlab.utils.crontab_process.get_samba_application_shared_folder(), + run_folder, + ) # delete existing information to avoid having duplicated tables wetlab.utils.crontab_process.delete_existing_run_metrics_table_processed( run_process_obj, experiment_name @@ -781,7 +786,7 @@ def manage_run_in_processed_run_state(conn, run_process_objs): wetlab.utils.common.get_samba_atribute_data( conn, wetlab.utils.crontab_process.get_samba_shared_folder(), - run_folder, + root_run_folder, "create_time", ) ) @@ -804,7 +809,7 @@ def manage_run_in_processed_run_state(conn, run_process_objs): # Get run metric files run_metric_files = wetlab.utils.crontab_process.get_run_metric_files( - conn, run_folder, experiment_name + conn, root_run_folder, experiment_name ) if "ERROR" in run_metric_files: diff --git a/wetlab/utils/samplesheet.py b/wetlab/utils/samplesheet.py index a232c0a02..3a10cdb5d 100644 --- a/wetlab/utils/samplesheet.py +++ b/wetlab/utils/samplesheet.py @@ -99,7 +99,7 @@ def file_read_to_dictionary( if not line or re.match("^,+$", line): # Empty/Filler/Artifact lines with no info continue - if section in wetlab.config.TABULAR_DATA_SECTIONS_SAMPLE_SHEET: + if section in wetlab.config.TABULAR_DATA_SECTIONS_SAMPLE_SHEET.values(): # Data is tabular; append as list of lists until next header if not isinstance(samplesheet[section], list): samplesheet[section] = []