nexmark.Query8

.gitmodules

@@ -0,0 +1,4 @@
+[submodule "examples/nexmark"]
+	path = examples/nexmark
+	url = https://github.com/nexmark/nexmark/
+	branch = master

benchmark/Puppetfile

@@ -0,0 +1,11 @@
+mod 'my', local: true
+mod 'geoffwilliams-filemagic', '0.6.1'
+mod 'puppet-python', '3.0.1'
+mod 'puppet-squid', '2.2.2'
+mod 'puppet-alternatives', '3.0.0'
+mod 'puppetlabs-apt', '7.6.0'
+mod 'puppetlabs-concat', '6.1.0'
+mod 'puppetlabs-sshkeys_core'
+mod 'puppetlabs-stdlib', '6.0.0'
+mod 'puppetlabs-vcsrepo', '3.0.0'
+mod 'saz-ssh', '6.2.0'

benchmark/ansible/flamestream.yml

@@ -1,28 +1,19 @@
 ---
-- name: Install Python
-  hosts: all
-  gather_facts: false
-  tasks:
-    - name: Install Python 2.x
-      raw: test -e /usr/bin/python || (apt update && apt install -y python-simplejson)
-      register: test
-      changed_when: test.stdout
-
-- name: Prepare cluster
+- name: Deliver flamestream artifacts
   hosts: all
   roles:
-    - { role: common-packages }
+    - { role: flamestream-common  }
+
+- name: Stop workers
+  hosts: workers
+  tasks:
+    - shell: "cd {{ flamestream_dir }} && bash entrypoint.sh stop"
 
 - name: Setup zookeeper
   hosts: manager
   roles:
     - { role: zookeeper  }
 
-- name: Deliver flamestream artifacts
-  hosts: all
-  roles:
-    - { role: flamestream-common  }
-
 - hosts: workers
   tasks:
     - file:
@@ -38,33 +29,3 @@
   tasks:
     - wait_for:
         path: /tmp/flame_stream
-
-#- name: Pause
-#  hosts: bench
-#  tasks:
-#    - pause:
-
-- name: Run the benchmark
-  hosts: bench
-  roles:
-    - { role: flamestream-bench , bench_class: com.spbsu.flamestream.example.benchmark.BenchStand }
-
-- name: Destroy flamestream
-  hosts: workers
-  roles:
-    - { role: flamestream-destroy }
-
-- name: Destroy zookeeper
-  hosts: manager
-  roles:
-    - { role: zookeeper-destroy }
-
-- hosts: all
-  tasks:
-    - name: Fetch traces
-      fetch:
-        src: /tmp/trace.csv
-        dest: "{{ results_dir }}/flamestream/traces/{{ inventory_hostname }}.csv"
-        flat: true
-  tags:
-    - lat

benchmark/ansible/group_vars/all.yml

@@ -1,18 +1,18 @@
-rate: 50
+local_acker_flush_delay_in_millis: 10
 guarantees: AT_MOST_ONCE
-millis_between_commits: 50
+millis_between_commits: 0
 validator: Wiki1000Validator
-base_dir: /opt
+base_dir: "{{ ansible_env.HOME }}"
 
 flamestream_dir: "{{ base_dir }}/flamestream"
 zookeeper_dir: "{{ base_dir }}/zookeeper"
 bench_dir: "{{ base_dir }}/flamestream"
 flink_dir: "{{ base_dir }}/flink"
-tarball_dir: /opt/src
+tarball_dir: "{{ base_dir }}/tarballs"
 
 results_dir: results/{{ lookup("pipe", "git rev-parse --short HEAD") }}/{{ groups['workers'] | length }}/{{ rate }}/{{ guarantees }}/{{ millis_between_commits }}
 
-zookeeper_version: 3.5.8
+zookeeper_version: 3.6.2
 zookeeper_client_port: 2181
 
 flamestream_worker_port: 5412

benchmark/ansible/roles/flamestream-bench/tasks/main.yml

@@ -17,7 +17,7 @@
 - name: Start the flamestream bench
   shell: >
     java
-    -Daeron.term.buffer.length=4194304 -Daeron.mtu.length=16384 -Xms500m -Xmx500m -XX:+PrintGC -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -XX:+HeapDumpOnOutOfMemoryError
+    -Daeron.term.buffer.length=4194304 -Daeron.mtu.length=16384 -Xms500m -Xmx1500m -Xlog:gc,gc+cpu=info::utc -XX:+HeapDumpOnOutOfMemoryError
     -cp '{{ flamestream_dir }}/lib/*'
     {{ bench_class }}
     {{ flamestream_dir }}/bench.conf {{ flamestream_dir }}/deployer.conf > {{ flamestream_dir }}/bench.log 2>&1

benchmark/ansible/roles/flamestream-bench/templates/bench.conf.j2

@@ -1,11 +1,11 @@
 benchmark {
   bench-host = "{{ groups['bench'][0] }}"
-  input-host = "{{ groups['input'][0] }}"
+  input-host = "{{ groups['workers'][0] }}"
   bench-source-port = 4567
   bench-sink-port = 5678
 
   wiki-dump-path = "{{ flamestream_dir }}/wiki.xml"
   sleep-between-docs-ms = {{ rate }}
-  worker-id = "{{ groups['worker'][0] }}"
+  worker-id = "{{ groups['workers'][0] }}"
   validator = com.spbsu.flamestream.example.benchmark.validators.{{ validator }}
 }

benchmark/ansible/roles/flamestream-worker/tasks/main.yml

@@ -18,3 +18,5 @@
     DEFAULT_MINIMAL_TIME: 0
     MAX_ELEMENTS_IN_GRAPH: 100
     MILLIS_BETWEEN_COMMITS: "{{ millis_between_commits }}"
+    ACKERS_NUMBER: 1
+    ACKER_WINDOW: 10

benchmark/ansible/roles/zookeeper/tasks/main.yml

@@ -25,11 +25,18 @@
     src: zoo.cfg.j2
     dest: "{{ zookeeper_dir }}/conf/zoo.cfg"
 
-- name: Restart zk
-  shell: "{{ zookeeper_dir }}/bin/zkServer.sh restart"
+- name: Stop zk
+  shell: "{{ zookeeper_dir }}/bin/zkServer.sh stop"
 
-- name: Clean graphs
-  shell: "{{ zookeeper_cli }} rmr /graph"
+- file:
+    state: absent
+    path: "{{ zookeeper_dir }}/data/"
+
+- name: Start zk
+  shell: "{{ zookeeper_dir }}/bin/zkServer.sh start"
+
+- command: "{{ zookeeper_dir }}/bin/zkServer.sh status"
+  retries: 5
+  register: result
+  until: result is not failed
 
-- name: Clean configs
-  shell: "{{ zookeeper_cli }} rmr /config"

benchmark/ansible/roles/zookeeper/vars/main.yml

@@ -1,6 +1,6 @@
 ---
 zookeeper_dir: "{{ base_dir }}/zookeeper"
-zookeeper_url: http://www.apache.org/dist/zookeeper/zookeeper-{{ zookeeper_version }}/zookeeper-{{ zookeeper_version }}.tar.gz
+zookeeper_url: http://www.apache.org/dist/zookeeper/zookeeper-{{ zookeeper_version }}/apache-zookeeper-{{ zookeeper_version }}-bin.tar.gz
 tick_time: 3000
 zookeeper_data_dir: "{{ zookeeper_dir }}/data"
-zookeeper_cli: "{{ zookeeper_dir }}/bin/zkCli.sh -server localhost:{{ zookeeper_client_port }}"
+zookeeper_cli: "{{ zookeeper_dir }}/bin/zkCli.sh -server localhost:{{ zookeeper_client_port }}"

benchmark/aws/remote.tf

@@ -0,0 +1,102 @@
+variable "access_key" {}
+variable "secret_key" {}
+variable "key_pair" {}
+variable "cluster_size" {
+  default = 11
+}
+variable "region" {
+  default = "us-east-2"
+}
+
+provider "aws" {
+  access_key = "${var.access_key}"
+  secret_key = "${var.secret_key}"
+  region = "${var.region}"
+}
+
+resource "aws_key_pair" "mega-benchmarker" {
+  key_name = "mega-benchmarker"
+  public_key = "${var.key_pair}"
+}
+
+resource "aws_vpc" "workers" {
+  cidr_block = "10.0.0.0/24"
+}
+
+resource "aws_internet_gateway" "workers" {
+  vpc_id = "${aws_vpc.workers.id}"
+}
+
+resource "aws_subnet" "workers" {
+  vpc_id = "${aws_vpc.workers.id}"
+  cidr_block = "10.0.0.0/24"
+  map_public_ip_on_launch = true
+}
+
+resource "aws_route_table" "workers" {
+  vpc_id = "${aws_vpc.workers.id}"
+
+  route {
+    cidr_block = "0.0.0.0/0"
+    gateway_id = "${aws_internet_gateway.workers.id}"
+  }
+}
+
+resource "aws_route_table_association" "public" {
+  subnet_id = "${aws_subnet.workers.id}"
+  route_table_id = "${aws_route_table.workers.id}"
+}
+
+resource "aws_security_group" "workers" {
+  vpc_id = "${aws_vpc.workers.id}"
+
+  ingress {
+    from_port = 0
+    to_port = 0
+    protocol = "-1"
+    cidr_blocks = [
+      "${aws_vpc.workers.cidr_block}"]
+  }
+
+  ingress {
+    from_port = 8081
+    to_port = 8081
+    protocol = "tcp"
+    cidr_blocks = [
+      "0.0.0.0/0"]
+  }
+
+  ingress {
+    from_port = 22
+    to_port = 22
+    protocol = "tcp"
+    cidr_blocks = [
+      "0.0.0.0/0"]
+  }
+
+  egress {
+    protocol = -1
+    from_port = 0
+    to_port = 0
+    cidr_blocks = [
+      "0.0.0.0/0"]
+  }
+}
+
+resource "aws_instance" "worker" {
+  count = "${var.cluster_size}"
+  ami = "ami-965e6bf3"
+  instance_type = "t2.small"
+  key_name = "${aws_key_pair.mega-benchmarker.key_name}"
+  subnet_id = "${aws_subnet.workers.id}"
+  vpc_security_group_ids = [
+    "${aws_security_group.workers.id}"]
+  associate_public_ip_address = true
+}
+
+output "public_ips" {
+  value = "${join(", ", aws_instance.worker.*.public_ip)}"
+}
+output "private_ips" {
+  value = "${join(", ", aws_instance.worker.*.private_ip)}"
+}