Skip to content

Commit 2d40d4a

Browse files
kriomkriom
committed
script for mysql
1 parent 2e97b17 commit 2d40d4a

File tree

3 files changed

+115
-0
lines changed

3 files changed

+115
-0
lines changed

‎db/mysql_alter_table.sql‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
-- Update après chargement
2+
3+
-- Ajout de colonne
4+
5+
UPDATE dvf SET section_prefixe = substr(id_parcelle, 6, 5) ;
6+
UPDATE dvf SET nature_culture = 'Terrain à bâtir' WHERE code_nature_culture = 'AB';
7+
UPDATE dvf SET nature_culture_speciale = 'Abreuvoirs' WHERE code_nature_culture_speciale = 'ABREU';

‎db/mysql_build_db.sh‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
#!/bin/bash
2+
3+
# exit when any command fails
4+
set -e
5+
set -x
6+
7+
# Script de creation de la base de donnees MySQL
8+
DIR=$(echo $(dirname $0))
9+
cd $DIR
10+
11+
12+
sudo mysql -e "GRANT ALL PRIVILEGES ON dvf2.* TO 'postgres'@'localhost' IDENTIFIED BY 'postgres';"
13+
echo 1
14+
sudo mysql -e "CREATE DATABASE IF NOT EXISTS dvf2;"
15+
echo 2
16+
cat mysql_create_table.sql | sudo mysql dvf2
17+
echo 3
18+
sudo mysql dvf2 -e "show tables;"
19+
20+
# Chargement des données
21+
DATADIR="data"
22+
mkdir -p $DATADIR
23+
YEARS="2014 2015 2016 2017 2018"
24+
25+
for YEAR in $(echo $YEARS | tr " " "\n"); do
26+
FILE=$DATADIR"/full_"$YEAR".csv"
27+
GZ_FILE=$FILE".gz"
28+
echo "[[ wget "$GZ_FILE
29+
[ ! -f $GZ_FILE ] && wget -r -np -nH --cut-dirs 5 https://cadastre.data.gouv.fr/data/etalab-dvf/latest/csv/$YEAR/full.csv.gz -O $GZ_FILE
30+
31+
echo "[[ gz "$GZ_FILE" => "$FILE
32+
[ ! -f $FILE ] && gunzip -k -f $GZ_FILE
33+
done
34+
35+
36+
SECURE_FILE_PRIV="/var/lib/mysql-files/"
37+
38+
#Chargement des données
39+
DATAPATH=$( cd $DATADIR ; pwd -P )
40+
for YEAR in $(echo $YEARS | tr " " "\n"); do
41+
FILE="full_"$YEAR".csv"
42+
echo "[[ $FILE => mysql"
43+
sudo cp $DATAPATH"/"$FILE $SECURE_FILE_PRIV"/"$FILE
44+
sudo mysql dvf2 -e "SET SESSION sql_mode = ''; LOAD DATA INFILE '$SECURE_FILE_PRIV/$FILE' INTO TABLE dvf COLUMNS TERMINATED BY ',' LINES TERMINATED BY '\n' IGNORE 1 LINES
45+
(id_mutation,date_mutation,numero_disposition,nature_mutation,valeur_fonciere,adresse_numero,adresse_suffixe,adresse_nom_voie,adresse_code_voie,code_postal,code_commune,nom_commune,code_departement,ancien_code_commune,ancien_nom_commune,id_parcelle,ancien_id_parcelle,numero_volume,lot1_numero,lot1_surface_carrez,lot2_numero,lot2_surface_carrez,lot3_numero,lot3_surface_carrez,lot4_numero,lot4_surface_carrez,lot5_numero,lot5_surface_carrez,nombre_lots,code_type_local,type_local,surface_reelle_bati,nombre_pieces_principales,code_nature_culture,nature_culture,code_nature_culture_speciale,nature_culture_speciale,surface_terrain,longitude,latitude);"
46+
sudo rm $SECURE_FILE_PRIV"/"$FILE
47+
done
48+
49+
cat mysql_alter_table.sql | sudo mysql dvf2
50+

‎db/mysql_create_table.sql‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
DROP TABLE IF EXISTS `dvf`;
2+
3+
CREATE TABLE `dvf` (
4+
`id` int(11) NOT NULL AUTO_INCREMENT,
5+
`id_mutation` varchar(255) DEFAULT '',
6+
`date_mutation` date DEFAULT NULL,
7+
`numero_disposition` int(11) DEFAULT NULL,
8+
`nature_mutation` varchar(255) DEFAULT '',
9+
`valeur_fonciere` decimal(12,3) DEFAULT 0.0,
10+
`adresse_numero` int(11) DEFAULT NULL,
11+
`adresse_suffixe` varchar(255) DEFAULT '',
12+
`adresse_nom_voie` varchar(255) DEFAULT '',
13+
`adresse_code_voie` varchar(255) DEFAULT '',
14+
`code_postal` varchar(255) DEFAULT '',
15+
`code_commune` varchar(255) DEFAULT '',
16+
`nom_commune` varchar(255) DEFAULT '',
17+
`code_departement` varchar(255) DEFAULT '',
18+
`ancien_code_commune` varchar(255) DEFAULT '',
19+
`ancien_nom_commune` varchar(255) DEFAULT '',
20+
`id_parcelle` varchar(255) DEFAULT '',
21+
`ancien_id_parcelle` varchar(255) DEFAULT '',
22+
`numero_volume` varchar(255) DEFAULT '',
23+
`lot1_numero` varchar(255) DEFAULT '',
24+
`lot1_surface_carrez` decimal(9,2) DEFAULT 0.0,
25+
`lot2_numero` varchar(255) DEFAULT '',
26+
`lot2_surface_carrez` decimal(9,2) DEFAULT 0.0,
27+
`lot3_numero` varchar(255) DEFAULT '',
28+
`lot3_surface_carrez` decimal(9,2) DEFAULT 0.0,
29+
`lot4_numero` varchar(255) DEFAULT '',
30+
`lot4_surface_carrez` decimal(9,2) DEFAULT 0.0,
31+
`lot5_numero` varchar(255) DEFAULT '',
32+
`lot5_surface_carrez` decimal(9,2) DEFAULT 0.0,
33+
`nombre_lots` int(11) DEFAULT NULL,
34+
`code_type_local` varchar(255) DEFAULT '',
35+
`type_local` varchar(255) DEFAULT '',
36+
`surface_reelle_bati` decimal(9,2) DEFAULT 0.0,
37+
`nombre_pieces_principales` int(11) DEFAULT NULL,
38+
`code_nature_culture` varchar(255) DEFAULT '',
39+
`nature_culture` varchar(255) DEFAULT '',
40+
`code_nature_culture_speciale` varchar(255) DEFAULT '',
41+
`nature_culture_speciale` varchar(255) DEFAULT '',
42+
`surface_terrain` decimal(12,3) DEFAULT 0.0,
43+
`longitude` decimal(9,7) DEFAULT 0.0,
44+
`latitude` decimal(9,7) DEFAULT 0.0,
45+
`section_prefixe` varchar(255) DEFAULT '',
46+
PRIMARY KEY (`id`),
47+
KEY `idx_sectionPrefixe` (`section_prefixe`),
48+
KEY `idx_commune` (`code_commune`),
49+
KEY `idx_date` (`date_mutation`),
50+
KEY `idx_parcelle` (`id_parcelle`),
51+
KEY `idx_section_commune` (`code_commune`,`section_prefixe`),
52+
KEY `id_mutation` (`id_mutation`)
53+
) ENGINE=InnoDB AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
54+
-- ) ENGINE=MyISAM AUTO_INCREMENT=1 DEFAULT CHARSET=utf8;
55+
56+
57+
58+

0 commit comments

Comments
 (0)