From b7a623bc2c903595534169ffd8241210cae1855e Mon Sep 17 00:00:00 2001 From: W Date: Sun, 12 Mar 2017 23:01:59 +0100 Subject: [PATCH 1/6] Fail if DB_DIR is not a directory If DB_DIR is not a directory, which may happen when dispatcher is not yet started, refuse to work further as nothing good will come from it. --- src/bin/apply_osc_to_db.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/bin/apply_osc_to_db.sh b/src/bin/apply_osc_to_db.sh index a9743ee66..ed8a306a1 100755 --- a/src/bin/apply_osc_to_db.sh +++ b/src/bin/apply_osc_to_db.sh @@ -31,6 +31,11 @@ if [[ ! ${EXEC_DIR:0:1} == "/" ]]; then DB_DIR=`$EXEC_DIR/dispatcher --show-dir` +if [[ ! -d $DB_DIR ]] ; then + echo "Can't find DB_DIR. Returned value: $DB_DIR" + exit 1; +fi + REPLICATE_DIR="$1" if [[ ! ${REPLICATE_DIR:0:1} == "/" ]]; then { From decf5d3541865d26d44aa37b02eed4520671adf1 Mon Sep 17 00:00:00 2001 From: W Date: Sun, 12 Mar 2017 23:03:46 +0100 Subject: [PATCH 2/6] Implement auto mode for fetch_osc When auto is set as REPLICATION_ID, try to guess the latest/greatest file that is present locally. --- src/bin/fetch_osc.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/bin/fetch_osc.sh b/src/bin/fetch_osc.sh index 24d6495c5..fa5ecc416 100755 --- a/src/bin/fetch_osc.sh +++ b/src/bin/fetch_osc.sh @@ -38,6 +38,17 @@ if [[ ! -d $LOCAL_DIR ]]; }; fi +if [[ $REPLICATE_ID == "auto" ]] ; then + REPLICATE_ID=`find $LOCAL_DIR -type f -name '*state.txt' | sort | tail -n 1 | sed -e "s#^$LOCAL_DIR##" -e 's#[./]##g' -e 's#statetxt##' -e 's/^0\+//g'` + if [[ "x$REPLICATE_ID" == "x" ]] ; then + REPLICATE_ID=`cat $DB_DIR/replicate_id` + fi + if [[ "x$REPLICATE_ID" == "x" ]] ; then + echo "Could not determine REPLICATE_ID. Exiting" + exit 1 + fi +fi + # $1 - remote source # $2 - local destination fetch_file() From f381eb7e729cec3f3f674dc7fe40abb7884fbd19 Mon Sep 17 00:00:00 2001 From: W Date: Sun, 12 Mar 2017 23:59:13 +0100 Subject: [PATCH 3/6] Add missing $DB_DIR --- src/bin/fetch_osc.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/bin/fetch_osc.sh b/src/bin/fetch_osc.sh index fa5ecc416..37cbb5c45 100755 --- a/src/bin/fetch_osc.sh +++ b/src/bin/fetch_osc.sh @@ -40,6 +40,14 @@ fi if [[ $REPLICATE_ID == "auto" ]] ; then REPLICATE_ID=`find $LOCAL_DIR -type f -name '*state.txt' | sort | tail -n 1 | sed -e "s#^$LOCAL_DIR##" -e 's#[./]##g' -e 's#statetxt##' -e 's/^0\+//g'` + EXEC_DIR=`dirname $BASH_SOURCE` + DB_DIR=`$EXEC_DIR/dispatcher --show-dir` + + if [[ ! -d $DB_DIR ]] ; then + echo "Can't find DB_DIR. Returned value: $DB_DIR" + exit 1; + fi + if [[ "x$REPLICATE_ID" == "x" ]] ; then REPLICATE_ID=`cat $DB_DIR/replicate_id` fi From ae5c2e985a919501478aa7b49f03412bd6eb02da Mon Sep 17 00:00:00 2001 From: W Date: Sun, 12 Mar 2017 23:59:13 +0100 Subject: [PATCH 4/6] Add missing $DB_DIR --- src/bin/fetch_osc.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/bin/fetch_osc.sh b/src/bin/fetch_osc.sh index fa5ecc416..8d8e73f4f 100755 --- a/src/bin/fetch_osc.sh +++ b/src/bin/fetch_osc.sh @@ -40,6 +40,15 @@ fi if [[ $REPLICATE_ID == "auto" ]] ; then REPLICATE_ID=`find $LOCAL_DIR -type f -name '*state.txt' | sort | tail -n 1 | sed -e "s#^$LOCAL_DIR##" -e 's#[./]##g' -e 's#statetxt##' -e 's/^0\+//g'` + + EXEC_DIR="`dirname $0`/" + DB_DIR=`$EXEC_DIR/dispatcher --show-dir` + + if [[ ! -d $DB_DIR ]] ; then + echo "Can't find DB_DIR. Returned value: $DB_DIR" + exit 1; + fi + if [[ "x$REPLICATE_ID" == "x" ]] ; then REPLICATE_ID=`cat $DB_DIR/replicate_id` fi From b1ca502fa1918c135abb62e8c5036c625a5353d8 Mon Sep 17 00:00:00 2001 From: W Date: Thu, 16 Mar 2017 22:51:17 +0100 Subject: [PATCH 5/6] Add desired_cpu_load startup parameter Instead of generating areas in 3 seconds intervals, check how much time it took to generate the areas and sleep to reach average `desired_cpu_load`. If you set it to: 1 - script will sleep 99x more than it works (so average cpu load is 1%) 50 - script will sleep the same time as it works (so average cpu load is 50%) 100 - script will sleep only 3 seconds (hardcoded minimum) giving average cpu load 100% --- src/bin/rules_loop.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/bin/rules_loop.sh b/src/bin/rules_loop.sh index c506b7bcf..fccc947ea 100755 --- a/src/bin/rules_loop.sh +++ b/src/bin/rules_loop.sh @@ -19,11 +19,12 @@ if [[ -z $1 ]]; then { - echo Usage: $0 database_dir + echo "Usage: $0 database_dir [desired_cpu_load]" exit 0 }; fi +CPU_LOAD=${2:-100} DB_DIR=$1 EXEC_DIR="`dirname $0`/" @@ -37,8 +38,14 @@ pushd "$EXEC_DIR" while [[ true ]]; do { + START=$(date +%s) echo "`date '+%F %T'`: update started" >>$DB_DIR/rules_loop.log ./osm3s_query --progress --rules <$DB_DIR/rules/areas.osm3s echo "`date '+%F %T'`: update finished" >>$DB_DIR/rules_loop.log - sleep 3 + WORK_TIME=$(( $(date +%s) - START )) + SLEEP_TIME=$(( WORK_TIME * 100 / CPU_LOAD - WORK_TIME)) + # let SLEEP_TIME be at least 3 seconds + SLEEP_TIME=$(( SLEEP_TIME < 3 ? 3 : SLEEP_TIME)) + echo "It took $WORK_TIME to run the loop. Desired load is: ${CPU_LOAD}%. Sleeping: $SLEEP_TIME" + sleep $SLEEP_TIME }; done From 302f9f135b7d795b1403720f536430549986ce44 Mon Sep 17 00:00:00 2001 From: Wiktor Niesiobedzki Date: Mon, 18 Sep 2017 10:49:27 +0200 Subject: [PATCH 6/6] Do not use empty state files for REPLICATE_ID guessing --- src/bin/fetch_osc.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bin/fetch_osc.sh b/src/bin/fetch_osc.sh index 8d8e73f4f..ef5a4b078 100755 --- a/src/bin/fetch_osc.sh +++ b/src/bin/fetch_osc.sh @@ -39,7 +39,7 @@ if [[ ! -d $LOCAL_DIR ]]; fi if [[ $REPLICATE_ID == "auto" ]] ; then - REPLICATE_ID=`find $LOCAL_DIR -type f -name '*state.txt' | sort | tail -n 1 | sed -e "s#^$LOCAL_DIR##" -e 's#[./]##g' -e 's#statetxt##' -e 's/^0\+//g'` + REPLICATE_ID=`find $LOCAL_DIR -type f -name '*state.txt' -not -size 0 | sort | tail -n 1 | sed -e "s#^$LOCAL_DIR##" -e 's#[./]##g' -e 's#statetxt##' -e 's/^0\+//g'` EXEC_DIR="`dirname $0`/" DB_DIR=`$EXEC_DIR/dispatcher --show-dir`