Run kube services under /system.slice

Author: Luke Addison

puppet/modules/kubernetes/manifests/kubelet.pp

@@ -30,7 +30,7 @@
     default  => 'cgroupfs',
   },
   String $cgroup_root = '/',
-  Optional[String] $cgroup_kube_name = '/podruntime.slice',
+  Optional[String] $cgroup_kube_name = undef,
   Optional[String] $cgroup_kube_reserved_memory = '256Mi',
   Optional[String] $cgroup_kube_reserved_cpu = '10m',
   Optional[String] $cgroup_system_name = '/system.slice',

puppet/modules/kubernetes/spec/classes/kubelet_spec.rb

@@ -211,6 +211,7 @@
 
       context 'with both cpu and memory a supplied' do
         let(:params) { {
+          "cgroup_kube_name" => "/podruntime.slice",
           "cgroup_#{cgroup_type}_reserved_cpu"    => '100m',
           "cgroup_#{cgroup_type}_reserved_memory" => '128Mi',
         }}
@@ -221,6 +222,7 @@
 
       context 'with only cpu supplied' do
         let(:params) { {
+          "cgroup_kube_name" => "/podruntime.slice",
           "cgroup_#{cgroup_type}_reserved_cpu"    => '100m',
           "cgroup_#{cgroup_type}_reserved_memory" => nil,
         }}
@@ -231,6 +233,7 @@
 
       context 'with only memory supplied' do
         let(:params) { {
+          "cgroup_kube_name" => "/podruntime.slice",
           "cgroup_#{cgroup_type}_reserved_cpu"    => nil,
           "cgroup_#{cgroup_type}_reserved_memory" => '128Mi',
         }}
@@ -241,6 +244,7 @@
 
       context 'with nothing supplied' do
         let(:params) { {
+          "cgroup_kube_name" => "/podruntime.slice",
           "cgroup_#{cgroup_type}_reserved_cpu"    => nil,
           "cgroup_#{cgroup_type}_reserved_memory" => nil,
         }}

puppet/modules/kubernetes/templates/kube-apiserver.service.erb

@@ -4,7 +4,12 @@ Documentation=https://github.com/GoogleCloudPlatform/kubernetes
 <%= scope.function_template(['kubernetes/_systemd_unit.erb']) %>
 
 [Service]
-Slice=podruntime.slice
+<%
+  if scope['kubernetes::kubelet::cgroup_kube_name']
+  @cgroup_kube_basename = scope.call_function('regsubst', [scope['kubernetes::kubelet::cgroup_kube_name'], '^\/', ''])
+-%>
+  Slice=<%= @cgroup_kube_basename %>
+<% end -%>
 User=<%= scope['kubernetes::user'] %>
 Group=<%= scope['kubernetes::group'] %>
 <%- if scope['kubernetes::_service_account_key_file'] and scope['kubernetes::service_account_key_generate'] -%>

puppet/modules/kubernetes/templates/kube-controller-manager.service.erb

@@ -4,7 +4,12 @@ Documentation=https://github.com/GoogleCloudPlatform/kubernetes
 <%= scope.function_template(['kubernetes/_systemd_unit.erb']) %>
 
 [Service]
-Slice=podruntime.slice
+<%
+  if scope['kubernetes::kubelet::cgroup_kube_name']
+  @cgroup_kube_basename = scope.call_function('regsubst', [scope['kubernetes::kubelet::cgroup_kube_name'], '^\/', ''])
+-%>
+  Slice=<%= @cgroup_kube_basename %>
+<% end -%>
 User=<%= scope['kubernetes::user'] %>
 Group=<%= scope['kubernetes::group'] %>
 <%- if scope['kubernetes::_service_account_key_file'] and scope['kubernetes::service_account_key_generate'] -%>

puppet/modules/kubernetes/templates/kube-proxy.service.erb

@@ -10,7 +10,17 @@ ExecStartPre=/sbin/sysctl -w net.bridge.bridge-nf-call-ip6tables=1
 ExecStart=<%= scope['kubernetes::_dest_dir'] %>/proxy \
   --v=<%= scope['kubernetes::log_level'] %> \
   --cluster-cidr=<%= scope['kubernetes::pod_network'] %> \
-  --resource-container=podruntime.slice \
+<%
+  if scope['kubernetes::kubelet::cgroup_kube_name']
+  @cgroup_kube_basename = scope.call_function('regsubst', [scope['kubernetes::kubelet::cgroup_kube_name'], '^\/', ''])
+-%>
+  --resource-container=<%= @cgroup_kube_basename %> \
+<%
+  elsif scope['kubernetes::kubelet::cgroup_system_name']
+  @cgroup_system_basename = scope.call_function('regsubst', [scope['kubernetes::kubelet::cgroup_system_name'], '^\/', ''])
+-%>
+  --resource-container=<%= @cgroup_system_basename %> \
+<% end -%>
 <% if @kubeconfig_path -%>
   --kubeconfig=<%= @kubeconfig_path %> \
 <% end -%>

puppet/modules/kubernetes/templates/kube-scheduler.service.erb

@@ -4,7 +4,12 @@ Documentation=https://github.com/GoogleCloudPlatform/kubernetes
 <%= scope.function_template(['kubernetes/_systemd_unit.erb']) %>
 
 [Service]
-Slice=podruntime.slice
+<%
+  if scope['kubernetes::kubelet::cgroup_kube_name']
+  @cgroup_kube_basename = scope.call_function('regsubst', [scope['kubernetes::kubelet::cgroup_kube_name'], '^\/', ''])
+-%>
+  Slice=<%= @cgroup_kube_basename %>
+<% end -%>
 User=<%= scope['kubernetes::user'] %>
 Group=<%= scope['kubernetes::group'] %>
 ExecStart=<%= scope['kubernetes::_dest_dir'] %>/scheduler \

puppet/modules/kubernetes/templates/kubelet.service.erb

@@ -4,7 +4,12 @@ Documentation=https://github.com/GoogleCloudPlatform/kubernetes
 <%= scope.function_template(['kubernetes/_systemd_unit.erb']) %>
 
 [Service]
-Slice=podruntime.slice
+<%
+  if @cgroup_kube_name
+  @cgroup_kube_basename = scope.call_function('regsubst', [@cgroup_kube_name, '^\/', ''])
+-%>
+Slice=<%= @cgroup_kube_basename %>
+<% end -%>
 WorkingDirectory=<%= @kubelet_dir %>
 <% if @cloud_provider == 'aws' -%>
 # prevent metadata service access on AWS
@@ -73,16 +78,16 @@ ExecStart=<%= scope['kubernetes::_dest_dir'] %>/kubelet \
   --cgroup-driver=<%= @cgroup_driver %> \
   --cgroup-root=<%= @cgroup_root %> \
 <% if @cgroup_kube_name -%>
-  --kube-reserved-cgroup=<%= @cgroup_kube_name %> \
-  --runtime-cgroups=<%= @cgroup_kube_name %> \
-  --kubelet-cgroups=<%= @cgroup_kube_name %> \
 <%
     # build kube reserved command line
     @cgroup_kube_reserved = []
     @cgroup_kube_reserved << "cpu=#{@cgroup_kube_reserved_cpu}" unless @cgroup_kube_reserved_cpu.nil? or @cgroup_kube_reserved_cpu == 'nil'
     @cgroup_kube_reserved << "memory=#{@cgroup_kube_reserved_memory}" unless @cgroup_kube_reserved_memory.nil? or @cgroup_kube_reserved_memory == 'nil'
     if @cgroup_kube_reserved.length > 0
 -%>
+  --kube-reserved-cgroup=<%= @cgroup_kube_name %> \
+  --runtime-cgroups=<%= @cgroup_kube_name %> \
+  --kubelet-cgroups=<%= @cgroup_kube_name %> \
   "--kube-reserved=<%= @cgroup_kube_reserved.join(',') %>" \
 <% end -%>
 <% end -%>

puppet/modules/site_module/manifests/docker_config.pp

@@ -3,10 +3,17 @@
     ensure  => file,
     content => template('site_module/docker.erb'),
   }
-  file { '/etc/systemd/system/docker.service.d':
-    ensure  => directory,
-  } -> file { '/etc/systemd/system/docker.service.d/10-slice.conf':
-    ensure  => directory,
-    content => '[Service]\nSlice=podruntime.slice\n',
+
+  if $kubernetes::kubelet::cgroup_kube_name {
+
+    $cgroup_kube_basename = regsubst( $kubernetes::kubelet::cgroup_kube_name, '^\/', '')
+
+    file { '/etc/systemd/system/docker.service.d':
+      ensure  => directory,
+    } -> file { '/etc/systemd/system/docker.service.d/10-slice.conf':
+      ensure  => directory,
+      content => "[Service]\nSlice=${cgroup_kube_basename}\n",
+    }
+
   }
 }