Introduce Docker Compose env file

.circleci/config.yml

@@ -1,8 +1,8 @@
 version: 2
 jobs:
-  py-3.8: &build-template
+  py-3.10: &build-template
     docker:
-      - image: qlands/odktools_circleci:20220727
+      - image: qlands/odktools_circleci:20240210
       - image: docker.elastic.co/elasticsearch/elasticsearch:7.14.0
         environment:
           - bootstrap.memory_lock=true
@@ -34,15 +34,31 @@ jobs:
       - run:          
           name: Run servers
           command: |
+            sudo chown -R redis /etc/redis
+            sudo chgrp -R redis /etc/redis            
+            sudo chown -R redis /var/log/redis
+            sudo chgrp -R redis /var/log/redis            
+            sudo chown -R redis /var/lib/redis
+            sudo chgrp -R redis /var/lib/redis
             sudo /etc/init.d/redis-server start
-            sudo service mongodb start
+            sudo mongod --fork --pidfilepath /var/run/mongod.pid --logpath /var/log/mongodb/mongod.log --config /etc/mongod.conf
             sudo service mosquitto start
             tail -f /dev/null
           background: true
       - run:
           # Install dependencies with --user to enable easier caching
           name: Install dependencies
           command: |
+            sudo chown -R circleci /opt/formshare_repository
+            sudo chown -R circleci /opt/formshare_log
+            sudo chown -R circleci /opt/formshare_celery
+            sudo chown -R circleci /opt/formshare_gunicorn
+            sudo chown -R circleci /opt/formshare_config            
+            sudo chgrp -R circleci /opt/formshare_repository
+            sudo chgrp -R circleci /opt/formshare_log
+            sudo chgrp -R circleci /opt/formshare_celery
+            sudo chgrp -R circleci /opt/formshare_gunicorn
+            sudo chgrp -R circleci /opt/formshare_config
             pip install -r requirements.txt --user
             pip install codecov --user
             /home/circleci/.local/bin/codecov
@@ -100,4 +116,4 @@ workflows:
   version: 2
   build:
     jobs:
-      - py-3.8
+      - py-3.10

.gitignore

@@ -21,6 +21,7 @@ lib64/
 parts/
 sdist/
 var/
+formshare/static/ephemeral/*.js
 wheels/
 *.egg-info/
 .installed.cfg
@@ -30,6 +31,7 @@ MANIFEST
 tmp/
 .directory
 docker/
+tryouts/
 
 # PyInstaller
 #  Usually these files are written by a python script from a template

Apache_configuration/module_changes.sh

@@ -0,0 +1,16 @@
+cd /etc/apache2/mods-enabled
+ln -s ../mods-available/cache.load .
+ln -s ../mods-available/cache_socache.load .
+ln -s ../mods-available/headers.load .
+ln -s ../mods-available/lbmethod_byrequests.load .
+rm mpm_event.conf
+rm mpm_event.load
+ln -s ../mods-available/mpm_prefork.conf .
+ln -s ../mods-available/mpm_prefork.load .
+ln -s ../mods-available/proxy.conf .
+ln -s ../mods-available/proxy.load .
+ln -s ../mods-available/proxy_http.load .
+ln -s ../mods-available/rewrite.load .
+ln -s ../mods-available/socache_shmcb.load .
+ln -s ../mods-available/ssl.conf .
+ln -s ../mods-available/ssl.load .

CITATION.cff

@@ -0,0 +1,9 @@
+cff-version: 1.1.0
+message: "If you use this software, please cite it as below."
+authors:
+  - family-names: Carlos
+    given-names: Quiros
+    orcid: https://orcid.org/0000-0002-9485-9961
+title: qlands/FormShare: Stable 2.21.0
+version: 2.21.0
+date-released: 2022-12-14

CONTRIBUTING.md

@@ -0,0 +1,28 @@
+# How to contribute to FormShare
+
+The best way to contribute to FormShare is by testing it and posting issues or new features. If you can fix things or can write new features do the following:
+
+1. Fork FormShare
+2. Clone your fork in your local computer
+3. Create a branch for your fix **based on the master branch**
+4. Create the fix, commit the code, and push the branch to your forked repository
+5. Create a pull request
+
+We also appreciate and need translation files.
+
+## Localization
+
+FormShare comes out of the box in English, Spanish, French, and Portuguese. It uses Babel for translation and you can help us by creating new translations or by correcting an existing one.
+
+To generate a new translation:
+
+```
+$ cd formshare
+$ python setup.py init_catalog -l [new_language_ISO_639-1_code]
+$ python setup.py extract_messages
+$ python setup.py update_catalog
+```
+
+The translation files (.po) are available at formshare/locale/[language-code]/LC_MESSAGES. You can edit a .po file with tools like [PoEdit](https://poedit.net/download), [Lokalize](https://userbase.kde.org/Lokalize), [GTranslator](https://gitlab.gnome.org/GNOME/gtranslator), or a simple text editor. Once the translation is done you can send us the updated or new .po file as an issue and we will add it to FormShare.
+
+## 

README.md

@@ -1,4 +1,4 @@
-[![CircleCI](https://circleci.com/gh/qlands/FormShare/tree/master-2.0.svg?style=shield&circle-token=:circle-token)](https://circleci.com/gh/qlands/FormShare)
+[![CircleCI](https://circleci.com/gh/qlands/FormShare/tree/master-2.0.svg?style=shield)](https://circleci.com/gh/qlands/FormShare)
 [![Codecov](https://codecov.io/github/qlands/FormShare/branch/master-2.0/graph/badge.svg)](https://app.codecov.io/gh/qlands/FormShare/commits?page=1)
 [![Black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/python/black)
 
@@ -25,59 +25,87 @@ FormShare **is for organizations** to install it in their server or cloud servic
 
 ## Features
 
-**Current features**
-
-- Case management (Longitudinal data collection) (Version >= 2.8.0)
-  - Using the Official ODK Collect App
-  - Intelligent work flow using Official ODK standards: 
-    - Case creator forms will create cases.
-    - Follow-up forms will attach information to each case.
-    - Deactivate forms will deactivate cases. For example, a household that decides to exit a longitudinal study will not appear in follow-up forms after deactivation.
-    - Activate forms will activate cases again.  For example, a household that decides to re-enter a longitudinal study will appear again in follow-up forms after activation.
-    - Move information from case creator forms into follow-up, deactivation and activation forms. For example, the sex of a participant (e. g., female) could be used in follow-up case forms to ask specific questions according to sex (e. g., if female, do they have access to reproductive health services since our last visit?)
-- Support multiple time zones:
-  - FormShare's: This is the time zone of the Linux server running FormShare.
-  - User: This is the time zone of a user logged into FormShare.
-  - Project: Each project can have a different time zone. This should be the time zone where submissions happen.
-  - Assistant: This is the time zone of an assistant logged into FormShare.
-  - Partner: This is the time zone of a partner logged into FormShare.
-
-- User accounts and management
-- Group-based user permissions
-- Projects to organize users, permissions, and forms
-- Separate access for data collectors and data cleaners. This is useful when dealing with hundreds of data collectors that do not need a FormShare user access 
-- Separate access for partners: Partners are trusted individuals outside your organization that require access to products and other resources in FormShare to collaborate with you. For example, a professor at a university (a partner in your project) requiring a KML export to match GPS points with weather data. 
-- User collaborations at project level, e. g., you can allow a colleague to maintain certain aspects of your project.
-- Form and submission management
-  - With easy setup of ODK Collect Projects using QR images
-  - With support for form version updates
-  - With testing and production stages
-  - Showing structural changes between incremental versions.
-  - With form and submission multimedia or data attachments
-  - With a table preview of submission data (even with thousand or millions of records) allowing in-table edits and recording any changes made to the data
-- OData live data feed for analysis with tools like Excel and Power BI. **With all CRUD operation supported (e. g., update)**. You can even use [Excel](https://github.com/qlands/MrBot-OData-Add-In) to clean data.
-- Extensibility system, e. g., You can write extensions to connect FormShare with Microsoft 365 authentication system
-- Documentation for running on AWS using Docker
-- Data cleaning API integration with R, STATA, or SPSS
-- Real-time map visualization of geo-referenced submissions at project and form level
-- Fill out forms using the web browser through [Enketo](https://github.com/qlands/formshare_enketo_plugin)
-- Download form attachments
-- Filtering submissions by submission metadata (e. g., date and time received on server)
-- Parse submissions into database tables. FormShare will create a MySQL data repository to store your data
-  - The repository is controlled with a primary key e. g., Farmed ID
-  - Duplicated submissions go to a cleaning pipe-line system that makes it easier to compare submissions and decide what to do with the duplicates
-- Private and publishable data downloads in Excel and CSV formats
-  - Mark data fields as sensitive to exclude them from publishable products
-  - Version control on products, e. g., FormShare will let you know if a data export does not have the latest submissions or changes in the database
-  - Make publishable products publicly available knowing that sensitive fields are excluded automatically 
-- Download geo-referenced information in KML format
-
-**Short-term features:**
-
-- Graph visualization with dashboards.
-- Connecting data fields with ontological variables. This is useful when comparing variables across studies even if variable names are different
-- Real-time data aggregation (pull data from different forms into one common data bucket). This is useful when dealing with slightly different forms for different geographies but where that certain fields could be aggregated into a common pot for analysis
-- Real-time data cleaning scripts using R
+### Case management (Longitudinal data collection) (Version >= 2.8.0)
+
+- Using the Official ODK Collect App.
+- Intelligent workflow using Official ODK standards: 
+  - Case creator forms will create cases.
+  - Follow-up forms will attach information to each case.
+  - Deactivate forms will deactivate cases. For example, a household that decides to exit a longitudinal study will not appear in follow-up forms after deactivation.
+  - Activate forms will activate cases again.  For example, a household that decides to re-enter a longitudinal study will appear again in follow-up forms after activation.
+  - Move information from case creator forms into follow-up, deactivation, and activation forms. For example, the sex of a participant (e. g., female) could be used in follow-up case forms to ask specific questions according to sex (e. g., if female, do they have access to reproductive health services since our last visit?)
+
+### Support for multiple time zones
+
+FormShare can produce dates and times in the following time zones:
+
+- FormShare: This is the time zone of the Linux server running FormShare.
+- User: This is the time zone of a user logged into FormShare.
+- Project: Each project can have a different time zone. This should be the time zone where submissions happen.
+- Assistant: This is the time zone of an assistant logged into FormShare.
+- Partner: This is the time zone of a partner logged into FormShare.
+
+### User and project management
+
+- User accounts and management.
+- Group-based user permissions.
+- Projects to organize users, permissions, and forms.
+- Separate access for data collectors and data cleaners. This is useful when dealing with hundreds of data collectors that do not need a FormShare user's account.
+- Separate access for partners: Partners are trusted individuals outside your organization that require access to products and other resources in FormShare to collaborate with you. For example, a professor at a university (a partner in your project) requires a KML export to match GPS points with weather data. 
+- User collaborations at a project level, e.g., you can allow a colleague to maintain certain aspects of your project.
+
+### Form and submission management
+
+- Easy setup of ODK Collect Projects using configurable QR images.
+- Support for form version updates.
+- With testing and production stages.
+- Showing structural changes between incremental versions.
+- With form and submission multimedia or data attachments.
+- Table preview of submission data (even with thousands or millions of records) allowing in-table edits and recording any changes made to the data.
+- Fill out forms using the web browser through [Enketo](https://github.com/qlands/formshare_enketo_plugin).
+- Real-time map visualization of geo-referenced submissions at project and form levels.
+- Download form attachments.
+
+### Database management and interoperability 
+
+- Parse submissions into database tables. FormShare will create a MySQL data repository to store your data.
+  - The repository is controlled with a primary key e.g., Farmed ID.
+  - Duplicated submissions go to a cleaning pipeline system that makes it easier to compare submissions and decide what to do with the duplicates.
+- Filtering submissions by submission metadata (e.g., date and time received on server).
+- Data cleaning API integration with R, STATA, or SPSS.
+- OData real-time data feed for analysis with tools like Excel and Power BI. **With all CRUD operations supported (e. g., update)**. You can even use [Excel](https://github.com/qlands/MrBot-OData-Add-In) to clean data.
+- Data dictionary with personal information protection.
+
+### Product management
+
+- Private and publishable data downloads in Excel, CSV, and JSON formats.
+  - Publishable products exclude protected fields (personal information protection) automatically.
+  - Version control on products, e.g., FormShare will let you know if a data export does not have the latest submissions or changes in the database.
+- Download geo-referenced information in KML format.
+
+### Performance, concurrency, and parallelism
+
+Though FormShare with the default settings can handle a load that would fit most organizations, it can be configured to handle any load. FormShare has been tested, using [JMeter](https://jmeter.apache.org/), under extreme traffic (1000 parallel submissions ) in an eight years old laptop with 16 GB of RAM and an [i7-4712HQ](https://www.intel.com/content/www/us/en/products/sku/78932/intel-core-i74712hq-processor-6m-cache-up-to-3-30-ghz/specifications.html).
+
+- Each submission is transactional. This means that a submission is either processed completely or thoroughly discarded for ODK Collect to re-send it.
+- Under extreme traffic,  FormShare can store a complex survey like [RHoMIS](https://www.rhomis.org/) at 18 submissions per second with zero failures.
+- Data exports support concurrent processing. A survey like RHoMIS with 100,000 submissions and millions of rows would take less than a minute to export to JSON or CSV.
+- The performance of the user interface or the data cleaning interface is not affected by the number of submissions.
+
+### Analytics
+
+- Real-time analytics from R using the [FormShare R Package](https://cran.r-project.org/web/packages/FormShare/index.html).
+- Real-time analytics from platforms like [Tableau](https://public.tableau.com/en-us/s/) and [Power BI](https://powerbi.microsoft.com/en-us/) using the [FormShare OData extension](https://github.com/qlands/formshare_odata_plugin).
+
+### Extensibility and others
+
+- Extensibility system, e.g., You can write extensions to connect FormShare with Microsoft 365 authentication system. See a list of extensions [below](#Customization-and-Extension).
+- Documentation for running on AWS using Docker.
+- Data fields tagged with ontological codes. This is useful when comparing variables across studies even if variable names are different.
+
+### Future features
+
+- Real-time data aggregation (pull data from different forms into one common data bucket). This is useful when dealing with slightly different forms for different geographies but where that certain fields could be aggregated into a common pot for analysis.
 
 ScreenShot
 ----------
@@ -86,11 +114,13 @@ ScreenShot
 
 Releases
 ------------
-The current stable release is 2.17.0 and it is available [here](https://github.com/qlands/FormShare/tree/stable-2.17.0) 
+The current stable release is 2.33.0 and it is available [here](https://github.com/qlands/FormShare/tree/stable-2.33.0) 
 
-The database signature for stable 2.17.0 is 0e3e192833a9
+The database signature for stable 2.33.0 is 1903afedb92d
 
-The Docker image for stable 2.17.0 is 20220728
+The Docker image for stable 2.33.0 is 20240213
+
+Requires ODKTools [2.10](https://github.com/qlands/odktools/tree/stable-2.10)
 
 Installation
 ------------
@@ -110,9 +140,9 @@ sudo apt-get install -y docker-compose
 
 # Get the Docker Compose file
 cd /opt
-sudo mkdir formshare_docker_compose_20220728
-cd formshare_docker_compose_20220728
-sudo wget https://raw.githubusercontent.com/qlands/FormShare/stable-2.17.0/docker_compose/docker-compose.yml
+sudo mkdir formshare_docker_compose_20240213
+cd formshare_docker_compose_20240213
+sudo wget https://raw.githubusercontent.com/qlands/FormShare/stable-2.33.0/docker_compose/docker-compose.yml
 
 # Make the directory structure for FormShare
 sudo mkdir /opt/formshare
@@ -129,27 +159,22 @@ mkdir /opt/formshare/formshare_odata_webapps
 mkdir /opt/formshare/elasticsearch
 mkdir /opt/formshare/elasticsearch/esdata
 mkdir /opt/formshare/elasticsearch/esdata2
-mkdir /opt/formshare/elasticsearch/esdata3
 sudo chmod -R g+w /opt/formshare
 
 # Set enough memory for Elasticsearch
 sudo sysctl -w vm.max_map_count=262144
 echo 'vm.max_map_count=262144' | sudo tee -a /etc/sysctl.d/60-vm-max_map_count.conf
 
 # Download all the required Docker Images
-cd /opt/formshare_docker_compose_20220728
+cd /opt/formshare_docker_compose_20240213
 sudo docker-compose pull
 
-# Edit the docker-compose.yml file to set the MySQL root and FormShare admin passwords
-sudo nano /opt/formshare_docker_compose_20220728/docker-compose.yml
-# Press Alt+Shit+3 to show the line numbers in Nano
-
-Edit line 10: Change the root password from "my_secure_password" to your password
-Edit line 67: Change the root password from "my_secure_password" to the same password of line 10
-Edit line 68: Change the admin user name (optional)
-Edit line 69: Change the admin email address
-Edit line 70: Change the admin user password from "my_secure_password" to your password
-Edit line 75: Change the IP address for the IP address of the machine running the Docker service
+# Set environment variables. Make sure to change the following:
+# MYSQL_PASSWORD
+# FORMSHARE_ADMIN_EMAIL
+# FORMSHARE_ADMIN_PASSWORD
+sudo wget -O .env https://raw.githubusercontent.com/qlands/FormShare/stable-2.33.0/docker_compose/env.example
+sudo nano .env
 
 # Save the file with Ctlr+o Enter . Exit with Ctrl+x
 
@@ -189,7 +214,7 @@ sudo service apache2 start
 # Subsequent start will take about 2 minutes. You can check the status with "sudo docker stats". 
 # FormShare will be ready for usage when the container reaches more than 500 kB of MEM USAGE
 # This is the only two commands you need to start FormShare after a server restart
-cd /opt/formshare_docker_compose_20220728
+cd /opt/formshare_docker_compose_20240213
 sudo docker-compose up -d
 
 # Browse to FormShare
@@ -465,4 +490,4 @@ The translation files (.po) are available at formshare/locale/[language-code]/LC
 
 FormShare is released under the terms of the GNU Affero General Public License. 
 
-The plug-in mechanism since it is based on PyUtilib is covered by a [BSD type of license](https://github.com/PyUtilib/pyutilib/blob/master/LICENSE.txt).
+The plug-in mechanism since it is based on PyUtilib is covered by a [BSD type of license](https://github.com/PyUtilib/pyutilib/blob/master/LICENSE.txt).

alembic/env.py

@@ -3,9 +3,8 @@
 from logging.config import fileConfig
 
 from alembic import context
-from sqlalchemy import engine_from_config, pool
-
 from formshare.models.meta import Base
+from sqlalchemy import engine_from_config, pool
 
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.

alembic/versions/019967069cf0_added_table_of_project_files.py

@@ -5,6 +5,7 @@
 Create Date: 2018-11-18 08:39:19.845537
 
 """
+
 import sqlalchemy as sa
 
 from alembic import op