WIP

Author: chrisplo

Makefile

@@ -9,7 +9,7 @@ DEFAULT_DOCKER_VERSION := 1.12.6
 V2PLUGIN_DOCKER_VERSION := 1.13.1
 SHELL := /bin/bash
 # TODO: contivmodel should be removed once its code passes golint and misspell
-EXCLUDE_DIRS := bin docs Godeps scripts vagrant vendor install contivmodel
+EXCLUDE_DIRS := bin docs Godeps scripts vagrant vendor install contivmodel venv
 PKG_DIRS := $(filter-out $(EXCLUDE_DIRS),$(subst /,,$(sort $(dir $(wildcard */)))))
 TO_BUILD := ./netplugin/ ./netmaster/ ./netctl/netctl/ ./mgmtfn/k8splugin/contivk8s/ ./mgmtfn/mesosplugin/netcontiv/
 HOST_GOBIN := `if [ -n "$$(go env GOBIN)" ]; then go env GOBIN; else dirname $$(which go); fi`
@@ -131,9 +131,18 @@ update:
 start:
 	CONTIV_DOCKER_VERSION="$${CONTIV_DOCKER_VERSION:-$(DEFAULT_DOCKER_VERSION)}" CONTIV_NODE_OS=${CONTIV_NODE_OS} vagrant up
 
+# ===================================================================
+# kubernetes helper targets
+
+k8s-check-python-deps:
+	@command -v pip >/dev/null \
+		|| echo No pip available, make sure netaddr and parse packages are installed
+	@command -v pip >/dev/null && pip show -q netaddr && pip show -q parse \
+		|| (echo parse and netaddr python packages are required; exit 1)
+
 # ===================================================================
 # kubernetes cluster bringup/cleanup targets
-k8s-cluster:
+k8s-cluster: k8s-check-python-deps
 	cd vagrant/k8s/ && CONTIV_K8S_USE_KUBEADM=1 ./setup_cluster.sh
 
 k8s-l3-cluster:
@@ -147,11 +156,24 @@ k8s-l3-destroy:
 
 # ===================================================================
 # kubernetes test targets
-k8s-test: k8s-cluster
-	cd vagrant/k8s/ && vagrant ssh k8master -c 'bash -lc "cd /opt/gopath/src/github.com/contiv/netplugin && make run-build"'
-	cd $(GOPATH)/src/github.com/contiv/netplugin/scripts/python && PYTHONIOENCODING=utf-8 ./createcfg.py -scheduler 'k8s' -binpath contiv/bin -install_mode 'kubeadm'
-	CONTIV_K8S_USE_KUBEADM=1 CONTIV_NODES=3 go test -v -timeout 540m ./test/systemtests -check.v -check.abort -check.f $(K8S_SYSTEM_TESTS_TO_RUN)
-	cd vagrant/k8s && vagrant destroy -f
+k8s-build: FIRST_MASTER_HOSTNAME:=k8master
+k8s-build: first_master_make_targets:="compile archive"
+k8s-build: export VAGRANT_CWD=$(PWD)/vagrant/k8s
+k8s-build: TO_BUILD := netplugin netmaster mgmtfn test/systemtests
+k8s-build: make-on-first-master-dep
+
+# for k8s-test, run-build will be run on the node instead of local
+k8s-test: FIRST_MASTER_HOSTNAME:=k8master
+k8s-test: first_master_make_targets:=run-build
+k8s-test: export VAGRANT_CWD:=$(PWD)/vagrant/k8s
+k8s-test: export CONTIV_K8S_USE_KUBEADM:=1 CONTIV_NODES:=3 PYTHONIOENCODING:=utf-8
+k8s-test: k8s-cluster make-on-first-master-dep
+k8s-test-now:
+	cd $(GOPATH)/src/github.com/contiv/netplugin/scripts/python \
+		&& ./createcfg.py -scheduler 'k8s' -binpath contiv/bin -install_mode 'kubeadm'
+	go test -v -timeout 540m ./test/systemtests -check.v -check.abort -check.f \
+		$(K8S_SYSTEM_TESTS_TO_RUN)
+	#cd vagrant/k8s && vagrant destroy -f
 
 k8s-l3-test: k8s-l3-cluster
 	cd vagrant/k8s/ && vagrant ssh k8master -c 'bash -lc "cd /opt/gopath/src/github.com/contiv/netplugin && make run-build"'
@@ -204,7 +226,7 @@ integ-test: stop clean start ssh-build
 ubuntu-tests:
 	CONTIV_NODE_OS=ubuntu make clean build unit-test system-test stop
 
-system-test:start
+system-test: start
 	@echo "system-test: running the following system tests:" $(SYSTEM_TESTS_TO_RUN)
 	cd $(GOPATH)/src/github.com/contiv/netplugin/scripts/python && PYTHONIOENCODING=utf-8 ./createcfg.py
 	go test -v -timeout 480m ./test/systemtests -check.v -check.abort -check.f $(SYSTEM_TESTS_TO_RUN)
@@ -293,7 +315,7 @@ host-plugin-create:
 host-plugin-update: host-plugin-remove unarchive host-plugin-create
 # same behavior as host-plugin-update but runs locally with docker 1.13+
 plugin-update: tar
-	$(call make-on-node1, host-plugin-update)
+	$(call make-on-first-master, host-plugin-update)
 
 # cleanup all containers, recreate and start the v2plugin on all hosts
 # uses the latest compiled binaries
@@ -315,21 +337,23 @@ host-pluginfs-unpack:
 		--exclude=etc/terminfo/v/vt220
 
 # Runs make targets on the first netplugin vagrant node
-# this is used as a macro like $(call make-on-node1, compile checks)
-make-on-node1 = vagrant ssh netplugin-node1 -c '\
+# this is used as a macro like $(call make-on-first-master, compile checks)
+
+FIRST_MASTER_HOSTNAME ?= netplugin-node1
+make-on-first-master = vagrant ssh $(FIRST_MASTER_HOSTNAME) -c '\
 	bash -lc "source /etc/profile.d/envvar.sh \
 	&& cd /opt/gopath/src/github.com/contiv/netplugin && make $(1)"'
 
-# Calls macro make-on-node1 but can be used as a dependecy by setting
-# the variable "node1-make-targets"
-make-on-node1-dep:
-	$(call make-on-node1, $(node1-make-targets))
+# Calls macro make-on-first-master but can be used as a dependecy by setting
+# the variable "first_master_make_targets"
+make-on-first-master-dep:
+	$(call make-on-first-master, $(first_master_make_targets))
 
 # assumes the v2plugin archive is available, installs the v2plugin and resets
 # everything on the vm to clean state
 v2plugin-install:
 	@echo Installing v2plugin
-	$(call make-on-node1, install-shell-completion host-pluginfs-unpack \
+	$(call make-on-first-master, install-shell-completion host-pluginfs-unpack \
 		host-plugin-restart host-swarm-restart)
 
 # Just like demo-v2plugin except builds are done locally and cached
@@ -341,8 +365,8 @@ demo-v2plugin-from-local: tar host-pluginfs-create start v2plugin-install
 # then creates and enables v2plugin
 demo-v2plugin: export CONTIV_DOCKER_VERSION ?= $(V2PLUGIN_DOCKER_VERSION)
 demo-v2plugin: export CONTIV_DOCKER_SWARM := swarm_mode
-demo-v2plugin: node1-make-targets := host-pluginfs-create
-demo-v2plugin: ssh-build make-on-node1-dep v2plugin-install
+demo-v2plugin: first_master_make_targets := host-pluginfs-create
+demo-v2plugin: ssh-build make-on-first-master-dep v2plugin-install
 
 # release a v2 plugin from the VM
 host-plugin-release: tar host-pluginfs-create host-pluginfs-unpack host-plugin-create

mgmtfn/k8splugin/driver.go

@@ -404,7 +404,7 @@ func addPod(w http.ResponseWriter, r *http.Request, vars map[string]string) (int
 	ep, err := createEP(epReq)
 	if err != nil {
 		log.Errorf("Error creating ep. Err: %v", err)
-		setErrorResp(&resp, "Error creating EP", err)
+		setErrorResp(&resp, "Error creating EP '"+epReq.Name+"'", err)
 		return resp, err
 	}
 

scripts/netContain/scripts/contivNet.sh

@@ -131,7 +131,6 @@ if [ $netmaster == true ]; then
 		sleep 5
 	done
 elif [ $netplugin == true ]; then
-	echo "Starting netplugin"
     if [[ "$cluster_store" =~ ^etcd://.+ ]]; then
         store_arg="--etcd-endpoints $(echo $cluster_store | sed s/etcd/http/)"
     elif [[ "$cluster_store" =~ ^consul://.+ ]]; then
@@ -146,7 +145,10 @@ elif [ $netplugin == true ]; then
 			if [ "$vlan_if" != "" ]; then
 				vlan_if_param="--vlan-if"
 			fi
+            echo "Starting netplugin $(date)"
+            set -x
 			/contiv/bin/netplugin $debug $store_arg $vtep_ip_param $vtep_ip $vlan_if_param $vlan_if --plugin-mode $plugin || true
+            set +x
 			echo "CRITICAL : Netplugin has exited. Trying to respawn in 5s"
 		fi
 		sleep 5

state/etcdstatedriver.go

@@ -127,7 +127,7 @@ func (d *EtcdStateDriver) Read(key string) ([]byte, error) {
 		}
 
 		if client.IsKeyNotFound(err) {
-			return []byte{}, core.Errorf("key not found")
+			return []byte{}, core.Errorf("key '%s' not found", key)
 		}
 
 		if err.Error() == client.ErrClusterUnavailable.Error() {

test/systemtests/How-to-Run.md

@@ -1,3 +1,5 @@
+# A guide to running netplugin systemtests on Vagrant and Baremetal platforms:
+
 Current framework can run system-tests for
 
 ```
@@ -9,7 +11,6 @@ Baremetal-
 Swarm -- ACI
 Swarm -- Non-ACI
 ```
-A guide to running netplugin systemtests on Vagrant and Baremetal platforms:
 
 Customize the example JSON file `netplugin/systemtests/cfg.json.example` according to your environment and rename it to `netplugin/systemtests/cfg.json`. A typical file for vagrant with swarm looks like:
 ```
@@ -46,15 +47,33 @@ Customize the example JSON file `netplugin/systemtests/cfg.json.example` accordi
 ]
 ```
 
-Testing with Vagrant:
+
+### Testing with Vagrant:
 
 * Make a suitable JSON file on your local machine (inside the systemtests directory).
 * From the netplugin directory of your machine (outside the vagrant nodes), run:
 
 ```
   make system-test
 ```
-Testing with Baremetal with Swarm:
+
+
+### Testing k8s with Vagrant:
+
+To run all the k8s system tests:
+
+```
+make k8s-test
+```
+
+To work on a single system test, such as TestNetworkAddDeleteNoGatewayVLAN:
+
+```
+  make K8S_SYSTEM_TESTS_TO_RUN=TestNetworkAddDeleteNoGatewayVLAN start k8s-test
+```
+
+
+### Testing with Baremetal with Swarm:
 
 For ACI testing , We need to have connectivity to APIC and ACI Fabric Switches from Baremetal VMs and Hosts.
 * You need to complete Pre-requisites, Step 1, Step 2, Step3 metioned here : https://github.com/contiv/demo/tree/master/net
@@ -84,7 +103,9 @@ godep go test -v -timeout 240m ./systemtests -check.v -check.f "TestACI"
 
 	This will run all the test functions which have the string TestACI
 ```
-Troubleshooting
+
+
+### Troubleshooting
 
 * First delete all netmaster, netctl, netplugin, contivk8s binaries from $GOBIN directory from all Nodes in the Cluster
 * You can perform following steps to clear etcd states

test/systemtests/kubeadm_test.go

@@ -9,9 +9,10 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/Sirupsen/logrus"
 	"os"
 	"time"
+
+	"github.com/Sirupsen/logrus"
 )
 
 type kubePod struct {
@@ -618,7 +619,9 @@ func (k *kubePod) runCommandUntilNoNetmasterError() error {
 	}
 
 	processCheckCmd := `kubectl -n kube-system exec ` + podName + ` -- pgrep netmaster`
-	return k8sMaster.runCommandUntilNoError(processCheckCmd)
+	err = k8sMaster.runCommandUntilNoError(processCheckCmd)
+	logrus.Infof("netmaster status check complete on: %s", k.node.Name())
+	return err
 }
 
 func (k *kubePod) runCommandUntilNoNetpluginError() error {
@@ -633,7 +636,8 @@ func (k *kubePod) runCommandUntilNoNetpluginError() error {
 	}
 
 	processCheckCmd := `kubectl -n kube-system exec ` + podName + ` -- pgrep netplugin`
-	return k8sMaster.runCommandUntilNoError(processCheckCmd)
+	err = k8sMaster.runCommandUntilNoError(processCheckCmd)
+	return err
 }
 
 func (k *kubePod) rotateNetmasterLog() error {

test/systemtests/network_test.go

@@ -259,9 +259,13 @@ func (s *systemtestSuite) testNetworkAddDeleteNoGateway(c *C, encap string) {
 
 		for _, name := range netNames {
 			var err error
-			// There seem to be a docker bug in creating external connectivity if we run
-			// containers in parallel. So, running it serially for this test
-			containers[name], err = s.runContainersSerial(numContainer, false, name, "", nil)
+			if s.basicInfo.Scheduler == "k8s" {
+				containers[name], err = s.runContainers(numContainer, false, name, "", nil, nil)
+			} else {
+				// There seem to be a docker bug in creating external connectivity if we run
+				// containers in parallel. So, running it serially for this test
+				containers[name], err = s.runContainersSerial(numContainer, false, name, "", nil)
+			}
 			c.Assert(err, IsNil)
 		}
 

test/systemtests/util_test.go

@@ -186,7 +186,7 @@ func (s *systemtestSuite) runContainers(num int, withService bool, networkName s
 	for i := 0; i < num; i++ {
 		go func(i int) {
 			nodeNum := i % len(s.nodes)
-			var name string
+			var name, cname string
 
 			mutex.Lock()
 			if len(names) > 0 {
@@ -197,6 +197,9 @@ func (s *systemtestSuite) runContainers(num int, withService bool, networkName s
 
 			if name == "" {
 				name = fmt.Sprintf("%s-srv%d-%d", strings.Replace(networkName, "/", "-", -1), i, nodeNum)
+				cname = name
+			} else {
+				cname = fmt.Sprintf("%s-%d", name, i)
 			}
 
 			var serviceName string
@@ -205,7 +208,6 @@ func (s *systemtestSuite) runContainers(num int, withService bool, networkName s
 				serviceName = name
 			}
 
-			cname := fmt.Sprintf("%s-%d", name, i)
 			spec := containerSpec{
 				imageName:   "contiv/alpine",
 				networkName: networkName,
@@ -1249,8 +1251,8 @@ func (s *systemtestSuite) SetUpSuiteVagrant(c *C) {
 			c.Assert(s.vagrant.Setup(false, []string{"CONTIV_L3=1 VAGRANT_CWD=" + topDir + "/src/github.com/contiv/netplugin/vagrant/k8s/"}, contivNodes), IsNil)
 
 			// Sleep to give enough time for the netplugin pods to come up
-			logrus.Infof("Sleeping for 1 minute for pods to come up")
-			time.Sleep(time.Minute)
+			//logrus.Infof("Sleeping for 1 minute for pods to come up")
+			//			time.Sleep(time.Minute)
 
 		case swarmScheduler:
 			c.Assert(s.vagrant.Setup(false, append([]string{"CONTIV_NODES=3 CONTIV_L3=1"}, s.basicInfo.SwarmEnv), contivNodes+contivL3Nodes), IsNil)
@@ -1276,8 +1278,8 @@ func (s *systemtestSuite) SetUpSuiteVagrant(c *C) {
 			c.Assert(s.vagrant.Setup(false, []string{"VAGRANT_CWD=" + topDir + "/src/github.com/contiv/netplugin/vagrant/k8s/"}, contivNodes), IsNil)
 
 			// Sleep to give enough time for the netplugin pods to come up
-			logrus.Infof("Sleeping for 1 minute for pods to come up")
-			time.Sleep(time.Minute)
+			//logrus.Infof("Sleeping for 1 minute for pods to come up")
+			//time.Sleep(time.Minute)
 
 		case swarmScheduler:
 			c.Assert(s.vagrant.Setup(false, append([]string{}, s.basicInfo.SwarmEnv), contivNodes), IsNil)