diff --git a/.gitignore b/.gitignore index fd9d0df39..314a123f4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ # Ignore everything in the .venv folder .venv/ .vscode/settings.json +.pnpm-store/ \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index 11f6a2d01..e3c0ffe24 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -25,5 +25,16 @@ "backend/server/backend/lib/python3.12/site-packages/django/contrib/sites/locale", "backend/server/backend/lib/python3.12/site-packages/rest_framework/templates/rest_framework/docs/langs" ], - "i18n-ally.keystyle": "nested" + "i18n-ally.keystyle": "nested", + "sqltools.connections": [ + { + "previewLimit": 50, + "server": "localhost", + "port": 5432, + "driver": "PostgreSQL", + "name": "default", + "database": "adventurelog", + "username": "admin" + } + ] } diff --git a/Makefile b/Makefile new file mode 100644 index 000000000..6c653fe72 --- /dev/null +++ b/Makefile @@ -0,0 +1,64 @@ +# Makefile for AdventureLog project +DOCKER_COMPOSE = docker compose -f docker-compose.yml +DOCKER_COMPOSE_TRAEFIK = docker compose -f docker-compose-traefik.yaml + +.PHONY: help +help: ## Display this help + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n\nTargets:\n"} /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 }' $(MAKEFILE_LIST) + +all: dev ## Build all services (alias for dev) + +download-countries: ## Download countries + @cd backend/server && mkdir -p media + @cd backend/server && python manage.py download-countries --force + @echo "Countries downloaded" + +download-cities: ## Download cities + @cd backend/server && mkdir -p media/cities_data + @cd backend/server && python manage.py cities --import=country + # @cd backend/server && python manage.py cities --import=region + # @cd backend/server && python manage.py cities --import=city + @cd backend/server && python manage.py cities --import=alt_name + @echo "Cities downloaded" + +import-translations: ## Import translations + @cd backend/server && python manage.py get-translations + @echo "Translations Imported" + +dev-db: dev ## Start development database + @if [ ! "$$(docker ps -q -f name=adventurelog-development)" ]; then \ + if [ "$$(docker ps -aq -f status=exited -f name=adventurelog-development)" ]; then \ + docker rm adventurelog-development; \ + fi; \ + docker run --name adventurelog-development \ + -e POSTGRES_USER=admin \ + -e POSTGRES_PASSWORD=admin \ + -e POSTGRES_DB=adventurelog \ + -p 5432:5432 \ + -d postgis/postgis:15-3.3; \ + echo "Development database started. Please wait a few seconds for it to initialize."; \ + echo "Please update the backend/.env file with these credentials:"; \ + echo "PGHOST=localhost"; \ + echo "PGDATABASE=adventurelog"; \ + echo "PGUSER=admin"; \ + echo "PGPASSWORD=admin"; \ + else \ + echo "Development database is already running."; \ + fi + +web: dev ## Start web service + @cd frontend && pnpm dev + +django: dev ## Start Django server + @cd backend/server && python manage.py migrate + @cd frontend && pnpm django + +dev: ## Setup Development Environment + @[ -f backend/server/.env ] || cp backend/server/.env.example backend/server/.env + @[ -f frontend/.env ] || cp frontend/.env.example frontend/.env + @cd frontend && pnpm install + @[ -d .venv ] || python -m venv .venv + . .venv/bin/activate + @pip install -r backend/server/requirements.txt + + diff --git a/backend/Dockerfile b/backend/Dockerfile index aa0f9f413..fdcfdb58f 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -22,7 +22,7 @@ RUN pip install --upgrade pip \ && pip install -r requirements.txt # Create necessary directories -RUN mkdir -p /code/static /code/media +RUN mkdir -p /code/static /code/media /code/media/cities_data # RUN mkdir -p /code/staticfiles /code/media # Copy the Django project code into the Docker image diff --git a/backend/entrypoint.sh b/backend/entrypoint.sh index 9ca926cc6..9f42038c1 100644 --- a/backend/entrypoint.sh +++ b/backend/entrypoint.sh @@ -51,8 +51,6 @@ else: EOF fi - -# Sync the countries and world travel regions # Sync the countries and world travel regions python manage.py download-countries if [ $? -eq 137 ]; then @@ -60,6 +58,17 @@ if [ $? -eq 137 ]; then exit 1 fi +# Sync the translations if $COUNTRY_TRANSLATIONS is true +if [ -n "$COUNTRY_TRANSLATIONS" -a "$COUNTRY_TRANSLATIONS" = "true" ]; then + echo "Syncing translations for countries..." + # Get the translations for all countries + python manage.py cities --import=country + # Get the translations for all alt names + python manage.py cities --import=alt_name + # Get the translations for all countries + python manage.py get-translations +fi + cat /code/adventurelog.txt # Start gunicorn diff --git a/backend/server/.env.example b/backend/server/.env.example index 598aeb733..81894e0c3 100644 --- a/backend/server/.env.example +++ b/backend/server/.env.example @@ -13,6 +13,8 @@ FRONTEND_URL='http://localhost:3000' EMAIL_BACKEND='console' +COUNTRY_TRANSLATIONS=True + # EMAIL_BACKEND='email' # EMAIL_HOST='smtp.gmail.com' # EMAIL_USE_TLS=False diff --git a/backend/server/main/settings.py b/backend/server/main/settings.py index dd099a1c2..4c7043142 100644 --- a/backend/server/main/settings.py +++ b/backend/server/main/settings.py @@ -61,6 +61,7 @@ 'users', 'integrations', 'django.contrib.gis', + 'cities', # 'achievements', # Not done yet, will be added later in a future update # 'widget_tweaks', # 'slippers', @@ -167,6 +168,7 @@ MEDIA_URL = '/media/' MEDIA_ROOT = BASE_DIR / 'media' # This path must match the NGINX root STATICFILES_DIRS = [BASE_DIR / 'static'] +CITIES_DATA_DIR = MEDIA_ROOT / 'cities_data' STORAGES = { "staticfiles": { @@ -304,4 +306,12 @@ # ADVENTURELOG_CDN_URL = getenv('ADVENTURELOG_CDN_URL', 'https://cdn.adventurelog.app') # https://github.com/dr5hn/countries-states-cities-database/tags -COUNTRY_REGION_JSON_VERSION = 'v2.5' \ No newline at end of file +COUNTRY_REGION_JSON_VERSION = 'v2.5' + +# English, Spanish, French, German, Italian, Chinese, Dutch, Swedish +CITIES_LOCALES = ['en', 'es', 'fr', 'de', 'it', 'zh', 'nl', 'sv', 'LANGUAGES'] +CITIES_POSTAL_CODES = [] +CITIES_PLUGINS = [ + # Reduce memory usage when importing large datasets (e.g. "allCountries.zip") + 'cities.plugin.reset_queries.Plugin', +] \ No newline at end of file diff --git a/backend/server/requirements.txt b/backend/server/requirements.txt index dcd01250f..fe906f481 100644 --- a/backend/server/requirements.txt +++ b/backend/server/requirements.txt @@ -21,4 +21,5 @@ icalendar==6.1.0 ijson==3.3.0 tqdm==4.67.1 overpy==0.7 -publicsuffix2==2.20191221 \ No newline at end of file +publicsuffix2==2.20191221 +django-cities==0.6.2 \ No newline at end of file diff --git a/backend/server/worldtravel/management/commands/download-countries.py b/backend/server/worldtravel/management/commands/download-countries.py index f5c5702d9..056c56833 100644 --- a/backend/server/worldtravel/management/commands/download-countries.py +++ b/backend/server/worldtravel/management/commands/download-countries.py @@ -92,6 +92,7 @@ def handle(self, **options): country_capital = country['capital'] longitude = round(float(country['longitude']), 6) if country['longitude'] else None latitude = round(float(country['latitude']), 6) if country['latitude'] else None + translations = country['translations'] processed_country_codes.add(country_code) @@ -102,6 +103,7 @@ def handle(self, **options): country_obj.capital = country_capital country_obj.longitude = longitude country_obj.latitude = latitude + country_obj.translations = translations countries_to_update.append(country_obj) else: country_obj = Country( @@ -110,7 +112,8 @@ def handle(self, **options): subregion=country_subregion, capital=country_capital, longitude=longitude, - latitude=latitude + latitude=latitude, + translations=translations ) countries_to_create.append(country_obj) @@ -213,7 +216,7 @@ def handle(self, **options): batch = countries_to_update[i:i + batch_size] for i in tqdm(range(0, len(countries_to_update), batch_size), desc="Updating countries"): batch = countries_to_update[i:i + batch_size] - Country.objects.bulk_update(batch, ['name', 'subregion', 'capital', 'longitude', 'latitude']) + Country.objects.bulk_update(batch, ['name', 'subregion', 'capital', 'longitude', 'latitude', 'translations']) for i in tqdm(range(0, len(regions_to_update), batch_size), desc="Updating regions"): batch = regions_to_update[i:i + batch_size] diff --git a/backend/server/worldtravel/management/commands/get-translations.py b/backend/server/worldtravel/management/commands/get-translations.py new file mode 100644 index 000000000..1f8d2b59f --- /dev/null +++ b/backend/server/worldtravel/management/commands/get-translations.py @@ -0,0 +1,17 @@ +from main.settings import CITIES_LOCALES +from worldtravel.models import Country +from django.core.management.base import BaseCommand + +class Command(BaseCommand): + help = 'Get translations for all countries' + + def handle(self, *args, **options): + countries = Country.objects.all() + countries_to_update = [] + for country in countries: + updated = country.get_translations(CITIES_LOCALES) + if updated: + countries_to_update.append(country) + # Bulk update the translations + Country.objects.bulk_update(countries_to_update, ['translations']) + print(f"Updated translations for {len(countries_to_update)} countries") \ No newline at end of file diff --git a/backend/server/worldtravel/migrations/0016_country_translations.py b/backend/server/worldtravel/migrations/0016_country_translations.py new file mode 100644 index 000000000..cbd0a4533 --- /dev/null +++ b/backend/server/worldtravel/migrations/0016_country_translations.py @@ -0,0 +1,18 @@ +# Generated by Django 5.0.8 on 2025-01-13 17:50 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('worldtravel', '0015_city_insert_id_country_insert_id_region_insert_id'), + ] + + operations = [ + migrations.AddField( + model_name='country', + name='translations', + field=models.JSONField(default=dict, blank=True), + ), + ] diff --git a/backend/server/worldtravel/models.py b/backend/server/worldtravel/models.py index 6c7ebb883..f2f116de3 100644 --- a/backend/server/worldtravel/models.py +++ b/backend/server/worldtravel/models.py @@ -2,7 +2,7 @@ from django.contrib.auth import get_user_model from django.core.exceptions import ValidationError from django.contrib.gis.db import models as gis_models - +from cities.models import Country as CityCountry, City as CityCity, Region as CityRegion User = get_user_model() @@ -18,13 +18,31 @@ class Country(models.Model): longitude = models.DecimalField(max_digits=9, decimal_places=6, null=True, blank=True) latitude = models.DecimalField(max_digits=9, decimal_places=6, null=True, blank=True) insert_id = models.UUIDField(unique=False, blank=True, null=True) - + translations = models.JSONField(default=dict, blank=True) + class Meta: verbose_name = "Country" verbose_name_plural = "Countries" def __str__(self): return self.name + + def get_translations(self, languages: list[str])->bool: + # get the translations for the country + translations = self.translations + try: + # get the preferred alt names for the country + alt_names = CityCountry.objects.get(code=self.country_code).alt_names.filter(language_code__in=languages, is_preferred=True) + for alt_name in alt_names: + translations[alt_name.language_code] = alt_name.name + + if self.translations != translations: + self.translations = translations + return True + return False + except CityCountry.DoesNotExist: + print(f"Country {self.name} ({self.country_code}) not found in cities.models.Country") + return False class Region(models.Model): id = models.CharField(primary_key=True) diff --git a/backend/server/worldtravel/serializers.py b/backend/server/worldtravel/serializers.py index 99c737982..7252a09f0 100644 --- a/backend/server/worldtravel/serializers.py +++ b/backend/server/worldtravel/serializers.py @@ -29,7 +29,7 @@ def get_num_visits(self, obj): class Meta: model = Country fields = '__all__' - read_only_fields = ['id', 'name', 'country_code', 'subregion', 'flag_url', 'num_regions', 'num_visits', 'longitude', 'latitude', 'capital'] + read_only_fields = ['id', 'name', 'country_code', 'subregion', 'flag_url', 'num_regions', 'num_visits', 'longitude', 'latitude', 'capital', 'translations'] class RegionSerializer(serializers.ModelSerializer): diff --git a/documentation/docs/configuration/updating.md b/documentation/docs/configuration/updating.md index 85fec59b2..026d7d911 100644 --- a/documentation/docs/configuration/updating.md +++ b/documentation/docs/configuration/updating.md @@ -22,3 +22,13 @@ Once you are in the container run the following command to resync the region dat ```bash python manage.py download-countries --force ``` + +## Updating the Country Translations + +If you would like to get translations for country names, you can run the following command. This will get the translations for all countries and save them to the database. + +```bash +python manage.py cities --import=country +python manage.py cities --import=alt_name # This takes a while to run, around 20 - 30 minutes, but only needs to be done once +python manage.py get-translations +``` \ No newline at end of file diff --git a/frontend/src/lib/components/CountryCard.svelte b/frontend/src/lib/components/CountryCard.svelte index 3894631f7..811a70d93 100644 --- a/frontend/src/lib/components/CountryCard.svelte +++ b/frontend/src/lib/components/CountryCard.svelte @@ -1,13 +1,41 @@