diff --git a/app.py b/app.py index c576ac6..bcb8b36 100644 --- a/app.py +++ b/app.py @@ -18,14 +18,17 @@ id = config[0][0] pwd = config[0][1] host = config[0][2] -engine = create_engine('postgresql://%s:%s@%s/dvf2'%(id, pwd, host)) +#engine = create_engine('postgresql://%s:%s@%s/dvf2'%(id, pwd, host)) +#table = "public.dvf" +engine = create_engine('mysql+pymysql://%s:%s@%s/dvf2'%(id, pwd, host)) +table = "dvf" # Chargement des natures de culture plus besoin @app.route('/api/dates2') def dates(): - dateMin = pd.read_sql("""SELECT min(date_mutation) FROM public.dvf """, engine) - dateMax = pd.read_sql("""SELECT max(date_mutation) FROM public.dvf """, engine) + dateMin = pd.read_sql("SELECT min(date_mutation) FROM " + table, engine) + dateMax = pd.read_sql("SELECT max(date_mutation) FROM " + table, engine) return '{"min": "' + str(dateMin['min'][0]) + '", "max": "' + str(dateMax['max'][0]) + '"}' @@ -56,7 +59,7 @@ def send_donneesgeo(path): @app.route('/api/mutations2///from=&to=') def get_mutations2(commune, sectionPrefixee, dateminimum, datemaximum): - mutations = pd.read_sql("""SELECT * FROM public.dvf WHERE code_commune = %(code)s AND section_prefixe = %(sectionPrefixee)s AND date_mutation >= %(datemin)s AND date_mutation <= %(datemax)s """, engine, params = {"code": commune, "sectionPrefixee" : sectionPrefixee, "datemin": dateminimum, "datemax": datemaximum}) + mutations = pd.read_sql("SELECT * FROM " + table + " WHERE code_commune = %(code)s AND section_prefixe = %(sectionPrefixee)s AND date_mutation >= %(datemin)s AND date_mutation <= %(datemax)s ", engine, params = {"code": commune, "sectionPrefixee" : sectionPrefixee, "datemin": dateminimum, "datemax": datemaximum}) mutations = mutations.applymap(str) # Str pour éviter la conversion des dates en millisecondes. nbMutations = len(mutations.id_mutation.unique()) @@ -67,7 +70,7 @@ def get_mutations2(commune, sectionPrefixee, dateminimum, datemaximum): @app.route('/api/mutations3//') def get_mutations3(commune, sectionPrefixee): - mutations = pd.read_sql("""SELECT * FROM public.dvf WHERE code_commune = %(code)s AND section_prefixe = %(sectionPrefixee)s""", engine, params = {"code": commune, "sectionPrefixee" : sectionPrefixee}) + mutations = pd.read_sql("SELECT * FROM " + table + " WHERE code_commune = %(code)s AND section_prefixe = %(sectionPrefixee)s", engine, params = {"code": commune, "sectionPrefixee" : sectionPrefixee}) mutations = mutations.applymap(str) # Str pour éviter la conversion des dates en millisecondes. json_mutations = '{"mutations": ' + mutations.to_json(orient = 'records') + '}' return json_mutations @@ -75,7 +78,7 @@ def get_mutations3(commune, sectionPrefixee): @app.route('/api/parcelles2//from=&to=') def get_parcelle(parcelle, dateminimum, datemaximum): - mutations = pd.read_sql("""SELECT * FROM public.dvf WHERE id_parcelle = %(code)s AND date_mutation >= %(datemin)s AND date_mutation <= %(datemax)s ;""", + mutations = pd.read_sql("SELECT * FROM " + table + " WHERE id_parcelle = %(code)s AND date_mutation >= %(datemin)s AND date_mutation <= %(datemax)s ;", engine, params = {"code": parcelle, "datemin": dateminimum, "datemax": datemaximum}) mutations = mutations.sort_values(by=['date_mutation'], ascending = False) @@ -99,7 +102,7 @@ def get_parcelle(parcelle, dateminimum, datemaximum): infos = infos.to_json(orient = 'records') # Mutations liées - mutations_liees = pd.read_sql("""SELECT * FROM public.dvf WHERE id_mutation = %(id_mutation)s AND id_parcelle<> %(parcelle)s;""", + mutations_liees = pd.read_sql("SELECT * FROM " + table + " WHERE id_mutation = %(id_mutation)s AND id_parcelle<> %(parcelle)s;", engine, params = {"id_mutation" : mutationIndex, "parcelle" : parcelle}) mutations_liees['type_local'].replace('Local industriel. commercial ou assimilé', 'Local industriel commercial ou assimilé', inplace = True) diff --git a/db/mysql_alter_table.sql b/db/mysql_alter_table.sql new file mode 100644 index 0000000..9d493cf --- /dev/null +++ b/db/mysql_alter_table.sql @@ -0,0 +1,7 @@ +-- Update après chargement + +-- Ajout de colonne + +UPDATE dvf SET section_prefixe = substr(id_parcelle, 6, 5) ; +UPDATE dvf SET nature_culture = 'Terrain à bâtir' WHERE code_nature_culture = 'AB'; +UPDATE dvf SET nature_culture_speciale = 'Abreuvoirs' WHERE code_nature_culture_speciale = 'ABREU'; \ No newline at end of file diff --git a/db/mysql_build_db.sh b/db/mysql_build_db.sh new file mode 100644 index 0000000..dd43918 --- /dev/null +++ b/db/mysql_build_db.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +# exit when any command fails +set -e +set -x + +# Script de creation de la base de donnees MySQL +DIR=$(echo $(dirname $0)) +cd $DIR + + +sudo mysql -e "GRANT ALL PRIVILEGES ON dvf2.* TO 'postgres'@'localhost' IDENTIFIED BY 'postgres';" +echo 1 +sudo mysql -e "CREATE DATABASE IF NOT EXISTS dvf2;" +echo 2 +cat mysql_create_table.sql | sudo mysql dvf2 +echo 3 +sudo mysql dvf2 -e "show tables;" + +# Chargement des données +DATADIR="data" +mkdir -p $DATADIR +YEARS="2014 2015 2016 2017 2018" + +for YEAR in $(echo $YEARS | tr " " "\n"); do + FILE=$DATADIR"/full_"$YEAR".csv" + GZ_FILE=$FILE".gz" + echo "[[ wget "$GZ_FILE + [ ! -f $GZ_FILE ] && wget -r -np -nH --cut-dirs 5 https://cadastre.data.gouv.fr/data/etalab-dvf/latest/csv/$YEAR/full.csv.gz -O $GZ_FILE + + echo "[[ gz "$GZ_FILE" => "$FILE + [ ! -f $FILE ] && gunzip -k -f $GZ_FILE +done + + +SECURE_FILE_PRIV="/var/lib/mysql-files/" + +#Chargement des données +DATAPATH=$( cd $DATADIR ; pwd -P ) +for YEAR in $(echo $YEARS | tr " " "\n"); do + FILE="full_"$YEAR".csv" + echo "[[ $FILE => mysql" + sudo cp $DATAPATH"/"$FILE $SECURE_FILE_PRIV"/"$FILE + sudo mysql dvf2 -e "SET SESSION sql_mode = ''; LOAD DATA INFILE '$SECURE_FILE_PRIV/$FILE' INTO TABLE dvf COLUMNS TERMINATED BY ',' LINES TERMINATED BY '\n' IGNORE 1 LINES + (id_mutation,date_mutation,numero_disposition,nature_mutation,valeur_fonciere,adresse_numero,adresse_suffixe,adresse_nom_voie,adresse_code_voie,code_postal,code_commune,nom_commune,code_departement,ancien_code_commune,ancien_nom_commune,id_parcelle,ancien_id_parcelle,numero_volume,lot1_numero,lot1_surface_carrez,lot2_numero,lot2_surface_carrez,lot3_numero,lot3_surface_carrez,lot4_numero,lot4_surface_carrez,lot5_numero,lot5_surface_carrez,nombre_lots,code_type_local,type_local,surface_reelle_bati,nombre_pieces_principales,code_nature_culture,nature_culture,code_nature_culture_speciale,nature_culture_speciale,surface_terrain,longitude,latitude);" + sudo rm $SECURE_FILE_PRIV"/"$FILE +done + +cat mysql_alter_table.sql | sudo mysql dvf2 + diff --git a/db/mysql_create_table.sql b/db/mysql_create_table.sql new file mode 100644 index 0000000..31c975f --- /dev/null +++ b/db/mysql_create_table.sql @@ -0,0 +1,58 @@ +DROP TABLE IF EXISTS `dvf`; + +CREATE TABLE `dvf` ( + `id` int(11) NOT NULL AUTO_INCREMENT, + `id_mutation` varchar(255) DEFAULT '', + `date_mutation` date DEFAULT NULL, + `numero_disposition` int(11) DEFAULT NULL, + `nature_mutation` varchar(255) DEFAULT '', + `valeur_fonciere` decimal(12,3) DEFAULT 0.0, + `adresse_numero` int(11) DEFAULT NULL, + `adresse_suffixe` varchar(255) DEFAULT '', + `adresse_nom_voie` varchar(255) DEFAULT '', + `adresse_code_voie` varchar(255) DEFAULT '', + `code_postal` varchar(255) DEFAULT '', + `code_commune` varchar(255) DEFAULT '', + `nom_commune` varchar(255) DEFAULT '', + `code_departement` varchar(255) DEFAULT '', + `ancien_code_commune` varchar(255) DEFAULT '', + `ancien_nom_commune` varchar(255) DEFAULT '', + `id_parcelle` varchar(255) DEFAULT '', + `ancien_id_parcelle` varchar(255) DEFAULT '', + `numero_volume` varchar(255) DEFAULT '', + `lot1_numero` varchar(255) DEFAULT '', + `lot1_surface_carrez` decimal(9,2) DEFAULT 0.0, + `lot2_numero` varchar(255) DEFAULT '', + `lot2_surface_carrez` decimal(9,2) DEFAULT 0.0, + `lot3_numero` varchar(255) DEFAULT '', + `lot3_surface_carrez` decimal(9,2) DEFAULT 0.0, + `lot4_numero` varchar(255) DEFAULT '', + `lot4_surface_carrez` decimal(9,2) DEFAULT 0.0, + `lot5_numero` varchar(255) DEFAULT '', + `lot5_surface_carrez` decimal(9,2) DEFAULT 0.0, + `nombre_lots` int(11) DEFAULT NULL, + `code_type_local` varchar(255) DEFAULT '', + `type_local` varchar(255) DEFAULT '', + `surface_reelle_bati` decimal(9,2) DEFAULT 0.0, + `nombre_pieces_principales` int(11) DEFAULT NULL, + `code_nature_culture` varchar(255) DEFAULT '', + `nature_culture` varchar(255) DEFAULT '', + `code_nature_culture_speciale` varchar(255) DEFAULT '', + `nature_culture_speciale` varchar(255) DEFAULT '', + `surface_terrain` decimal(12,3) DEFAULT 0.0, + `longitude` decimal(9,7) DEFAULT 0.0, + `latitude` decimal(9,7) DEFAULT 0.0, + `section_prefixe` varchar(255) DEFAULT '', + PRIMARY KEY (`id`), + KEY `idx_sectionPrefixe` (`section_prefixe`), + KEY `idx_commune` (`code_commune`), + KEY `idx_date` (`date_mutation`), + KEY `idx_parcelle` (`id_parcelle`), + KEY `idx_section_commune` (`code_commune`,`section_prefixe`), + KEY `id_mutation` (`id_mutation`) +) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8; +-- ) ENGINE=MyISAM AUTO_INCREMENT=1 DEFAULT CHARSET=utf8; + + + + diff --git a/mysql_README.md b/mysql_README.md new file mode 100644 index 0000000..6102531 --- /dev/null +++ b/mysql_README.md @@ -0,0 +1 @@ +sudo apt install python3-pymysql diff --git a/static/js/index.js b/static/js/index.js index c6cf169..9af7dca 100644 --- a/static/js/index.js +++ b/static/js/index.js @@ -338,32 +338,29 @@ function resetDepartement() { } } +function getSelectValue(id) { + var e = document.getElementById(id); + return e.options[e.selectedIndex].value; +} + function selectionnerDepartement() { // L'utilisateur a cliqué sur la liste déroulante des départements - var e = document.getElementById("departements"); - var sonCode = e.options[e.selectedIndex].value; - entrerDansDepartement(sonCode); + entrerDansDepartement(getSelectValue("departements")); }; function selectionnerCommune() { // L'utilisateur a cliqué sur la liste déroulante des communes - var e = document.getElementById("communes"); - var sonCode = e.options[e.selectedIndex].value; - entrerDansCommune(sonCode); + entrerDansCommune(getSelectValue("communes")); } function selectionnerSection() { // L'utilisateur a cliqué sur la liste déroulante des sections - var e = document.getElementById("sections"); - var newIdSection = e.options[e.selectedIndex].value; - entrerDansSection(newIdSection); + entrerDansSection(getSelectValue("sections")); } function selectionnerParcelle() { // L'utilisateur a cliqué sur la liste déroulante des sections - var e = document.getElementById("parcelles"); - var sonCode = e.options[e.selectedIndex].value; - entrerDansParcelle(sonCode); + entrerDansParcelle(getSelectValue("parcelles")); } function filledCommunesOptions(feature) { @@ -426,6 +423,11 @@ function onSectionClicked(event) { entrerDansSection(newIdSection); } + +function setLocationHash(hash) { + if (!parent.location.hash.includes(hash)){ parent.location.hash = hash } +} + function entrerDansMutation(sonIndex) { vue.mutationIndex = sonIndex; @@ -435,11 +437,17 @@ function entrerDansMutation(sonIndex) { codesParcelles.push(parcelleLiee); } } + var codesParcelle = codesParcelles[0]; + if(codesParcelle == "") return; + console.log("Nous entrons dans la mutation " + codesParcelle); + + setLocationHash(getSelectValue("departements") + "-" + getSelectValue("communes") + "-" + getSelectValue("sections") + "-" + codesParcelle); mutationsFilter() } function entrerDansSection(newIdSection) { + if(newIdSection == "") return; if (idSection) { resetSection() } @@ -448,6 +456,8 @@ function entrerDansSection(newIdSection) { resetParcelle() } + console.log("Nous entrons dans la section " + newIdSection); + setLocationHash(getSelectValue("departements") + "-" + getSelectValue("communes") + "-" + newIdSection); idSection = newIdSection; $.when( // Charge la couche géographique @@ -466,11 +476,13 @@ function entrerDansSection(newIdSection) { parcellesFilter() fit(parcelles) vue.section = true + document.getElementById("parcelles").dispatchEvent(new Event('change')); } ); } function entrerDansCommune(sonCode) { + if(sonCode == "") return; if (codeCommune) { resetCommune() } @@ -480,6 +492,8 @@ function entrerDansCommune(sonCode) { } console.log("Nous entrons dans la commune " + sonCode); + + setLocationHash(getSelectValue("departements") + "-" + sonCode); codeCommune = sonCode; getSections(codeCommune).then( function (data) { @@ -497,11 +511,13 @@ function entrerDansCommune(sonCode) { vue.commune = { code: sonCode }; + document.getElementById("sections").dispatchEvent(new Event('change')); } ); } function entrerDansDepartement(sonCode) { + if(sonCode == "") return; if (codeDepartement) { resetDepartement() } @@ -509,6 +525,7 @@ function entrerDansDepartement(sonCode) { // Vide l'interface codeDepartement = sonCode; console.log('Nous entrons dans le département ' + codeDepartement); + setLocationHash(codeDepartement); // Charge les communes getCommunes(codeDepartement).then(afficherCommunesDepartement) } @@ -523,6 +540,8 @@ function afficherCommunesDepartement(data) { resetSourcesData(['sections', 'parcelles']) fit(communes) + + document.getElementById("communes").dispatchEvent(new Event('change')); } function onCityClicked(event) { @@ -543,6 +562,43 @@ function toggleLeftBar() { vue.fold_left = !vue.fold_left; } +var DeepLink = { + recurisveSelect: function() { + var selectIdValues = DeepLink.selectIdValues; + var ids = DeepLink.selectIds; + var id = ids.shift(); + var nextId = ids[0]; + if (id != null) { + var value = selectIdValues[id]; + var elem = document.getElementById(id); + elem.removeEventListener('change', DeepLink.recurisveSelect); + if (nextId) { + var nextElem = document.getElementById(nextId); + nextElem.addEventListener('change', DeepLink.recurisveSelect); + } + if (value != "" && elem.value != value) { + elem.value = value; + elem.dispatchEvent(new Event('change')); + } + + } + }, + + handle: function(){ + var bashItems = parent.location.hash.replace(/.*#/g, '').split('-'); + var selectIdValues = {}; + if(bashItems[0]){ selectIdValues.departements = bashItems[0]; } + if(bashItems[1]){ selectIdValues.communes = bashItems[1]; } + if(bashItems[2]){ selectIdValues.sections = bashItems[2]; } + if(bashItems[3]){ selectIdValues.parcelles = bashItems[3]; } + + DeepLink.selectIdValues = selectIdValues; + DeepLink.selectIds = Object.keys(selectIdValues); + DeepLink.recurisveSelect() + } +}; + + // C'est le code qui est appelé au début (sans que personne ne clique) (function () { @@ -574,6 +630,8 @@ function toggleLeftBar() { map.addLayer(departementsLayer) map.addLayer(departementsContoursLayer) map.setPaintProperty(departementsContoursLayer.id, 'line-color', vue.mapStyle === 'ortho' ? '#fff' : '#000') + + DeepLink.handle() }) } })