diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..8c271f4d7 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +benchmark/wiki-dumps/1000.xml filter=lfs diff=lfs merge=lfs -text diff --git a/.lfsconfig b/.lfsconfig new file mode 100644 index 000000000..2cf1bce1b --- /dev/null +++ b/.lfsconfig @@ -0,0 +1,2 @@ +[lfs] + fetchexclude = * diff --git a/.travis.yml b/.travis.yml index db243e585..24d9ef705 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,30 @@ language: java +matrix: + include: + - script: + - travis_wait 20 mvn --errors --batch-mode clean package + + after_failure: + - ls -al + - export SSHPASS=$FLAME_PASS + - sshpass -e scp -v -o stricthostkeychecking=no runtime/oom.hprof flamestream@marnikitta.com:~/$TRAVIS_BUILD_ID.hprof + - before_install: + - git lfs pull --include benchmark/wiki-dumps/1000.xml + - sudo apt-get update + - sudo apt-get install software-properties-common -y + - sudo apt-add-repository ppa:ansible/ansible -y + - sudo apt-get update + - sudo apt-get install ansible -y + services: docker + script: + - ln -s ../../../../wiki-dumps/1000.xml benchmark/ansible/roles/flamestream-bench/files/wiki.xml + - timeout 8m ./run_benchmarks.sh + after_script: + - docker exec worker1 ls /opt/flamestream/ + - docker exec worker1 cat /opt/flamestream/worker.log + - docker exec worker2 ls /opt/flamestream/ + - docker exec worker2 cat /opt/flamestream/worker.log + - docker exec worker2 cat /opt/flamestream/bench.log sudo: required @@ -20,11 +46,3 @@ env: - JAVA_OPTS="-Xmx256m -Xms256m" install: true - -script: - - travis_wait 20 mvn --errors --batch-mode clean package - -after_failure: - - ls -al - - export SSHPASS=$FLAME_PASS - - sshpass -e scp -v -o stricthostkeychecking=no runtime/oom.hprof flamestream@marnikitta.com:~/$TRAVIS_BUILD_ID.hprof diff --git a/benchmark/ansible/group_vars/all.yml b/benchmark/ansible/group_vars/all.yml index 35be35337..7159ea35c 100644 --- a/benchmark/ansible/group_vars/all.yml +++ b/benchmark/ansible/group_vars/all.yml @@ -1,7 +1,7 @@ rate: 50 guarantees: AT_MOST_ONCE millis_between_commits: 50 -validator: Wiki10000Validator +validator: Wiki1000Validator base_dir: /opt flamestream_dir: "{{ base_dir }}/flamestream" diff --git a/benchmark/ansible/roles/flamestream-bench/tasks/main.yml b/benchmark/ansible/roles/flamestream-bench/tasks/main.yml index 7372fa137..3d80ba34c 100644 --- a/benchmark/ansible/roles/flamestream-bench/tasks/main.yml +++ b/benchmark/ansible/roles/flamestream-bench/tasks/main.yml @@ -14,9 +14,6 @@ src: bench.conf.j2 dest: "{{ flamestream_dir }}/bench.conf" -- name: Clean shm - shell: rm -rf /dev/shm/aeron* - - name: Start the flamestream bench shell: > java diff --git a/benchmark/ansible/roles/flamestream-destroy/tasks/main.yml b/benchmark/ansible/roles/flamestream-destroy/tasks/main.yml index 927fe7e0f..90fb3da96 100644 --- a/benchmark/ansible/roles/flamestream-destroy/tasks/main.yml +++ b/benchmark/ansible/roles/flamestream-destroy/tasks/main.yml @@ -7,12 +7,6 @@ path: "{{ flamestream_dir }}/data/" state: absent -- name: Fetch replay times - fetch: - src: /tmp/elements.cnt - dest: results/{{ lookup("pipe", "git rev-parse --short HEAD") }}/{{ groups['workers'] | length }}/{{ rate }}/flamestream/rate/{{ inventory_hostname }}.cnt - flat: true - - name: Fetch worker logs fetch: src: "{{ flamestream_dir }}/worker.log" diff --git a/benchmark/ansible/roles/flamestream-worker/tasks/main.yml b/benchmark/ansible/roles/flamestream-worker/tasks/main.yml index d01855070..073da3647 100644 --- a/benchmark/ansible/roles/flamestream-worker/tasks/main.yml +++ b/benchmark/ansible/roles/flamestream-worker/tasks/main.yml @@ -12,5 +12,5 @@ src: worker.json.j2 dest: "{{ flamestream_dir }}/worker.json" -- name: Run flamstream +- name: Run flamestream shell: "cd {{ flamestream_dir }} && bash entrypoint.sh restart worker.json" diff --git a/benchmark/wiki-dumps/.keep b/benchmark/wiki-dumps/.keep new file mode 100644 index 000000000..e69de29bb diff --git a/benchmark/wiki-dumps/1000.xml b/benchmark/wiki-dumps/1000.xml new file mode 100644 index 000000000..fd3d1d34f --- /dev/null +++ b/benchmark/wiki-dumps/1000.xml @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17e4c0588df92392a0fb98915738785e6a2203c0b89abf39cac91acac9072f59 +size 51414148 diff --git a/run_benchmarks.sh b/run_benchmarks.sh old mode 100644 new mode 100755 index a177c8717..d8ae28a5b --- a/run_benchmarks.sh +++ b/run_benchmarks.sh @@ -38,14 +38,14 @@ copy_flink_artifacts() { } docker_compose_up() { - sudo docker-compose -f "$DOCKER_COMPOSE" up -d + docker-compose -f "$DOCKER_COMPOSE" up -d } local_bench() { docker_compose_up \ && package \ && copy_worker_artifacts \ - && sudo ansible-playbook -v -i "${ANSIBLE_HOME}/local.yml" "${ANSIBLE_HOME}/flamestream.yml" + && ansible-playbook -v -i "${ANSIBLE_HOME}/local.yml" "${ANSIBLE_HOME}/flamestream.yml" } remote_bench() { @@ -67,4 +67,4 @@ remote_flink_bench() { && ansible-playbook -v -i "${ANSIBLE_HOME}/aws.yml" "${ANSIBLE_HOME}/flink.yml" } -[[ "$0" == "$BASH_SOURCE" ]] && remote_bench +[[ "$0" == "$BASH_SOURCE" ]] && local_bench diff --git a/runtime/src/main/java/com/spbsu/flamestream/runtime/WorkerApplication.java b/runtime/src/main/java/com/spbsu/flamestream/runtime/WorkerApplication.java index 8bd3a4dcb..0bff88960 100644 --- a/runtime/src/main/java/com/spbsu/flamestream/runtime/WorkerApplication.java +++ b/runtime/src/main/java/com/spbsu/flamestream/runtime/WorkerApplication.java @@ -77,6 +77,8 @@ public void run() { final Map props = new HashMap<>(); props.put("akka.remote.artery.canonical.hostname", host.host()); props.put("akka.remote.artery.canonical.port", String.valueOf(host.port())); + props.put("akka.remote.artery.bind.hostname", "0.0.0.0"); + props.put("akka.remote.artery.bind.port", String.valueOf(host.port())); try { final File shm = new File(("/dev/shm")); if (shm.exists() && shm.isDirectory()) { diff --git a/runtime/src/main/resources/logback.xml b/runtime/src/main/resources/logback.xml index 695319a64..eb2d7f03a 100644 --- a/runtime/src/main/resources/logback.xml +++ b/runtime/src/main/resources/logback.xml @@ -24,7 +24,7 @@ - + diff --git a/runtime/src/test/resources/logback-test.xml b/runtime/src/test/resources/logback-test.xml index 15e04d86b..380cb2eb3 100644 --- a/runtime/src/test/resources/logback-test.xml +++ b/runtime/src/test/resources/logback-test.xml @@ -8,7 +8,7 @@ - +