Add a docker image loader TOSCA component

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## UNRELEASED
 
+### ENHANCEMENTS
+
+* Add a TOSCA component to load a docker image from a tar archive ([GH-140](https://github.com/ystia/forge/issues/140))
+
 ### BUG FIXES
 
 * Docker installation requires apt-transport-https to work properly on Debian based systems ([GH-147](https://github.com/ystia/forge/issues/147))

org/ystia/docker/ansible/playbooks/create.yaml

@@ -12,13 +12,18 @@
 
   - set_fact:
       # The baseurl of the RedHat/CentOS docker repository.
-      redhat_docker_repository: "{{ REPOSITORY_URL  if (REPOSITORY_URL != '') else ('https://download.docker.com/linux/centos/7/$basearch/stable') }}"
+      redhat_docker_repository: "{{ REPOSITORY_URL  }}"
       redhat_docker_gpgkey: "{{ DOCKER_GPGKEY if (DOCKER_GPGKEY != '') else ('https://download.docker.com/linux/centos/gpg') }}"
       # The baseurl of the Debian/Ubuntu docker repository.
-      debian_docker_repository: "{{ REPOSITORY_URL if (REPOSITORY_URL != '') else ('deb https://download.docker.com/linux/ubuntu xenial stable') }}"
+      debian_docker_repository: "{{ REPOSITORY_URL }}"
       debian_docker_gpgkey: "{{ DOCKER_GPGKEY if (DOCKER_GPGKEY != '') else ('https://download.docker.com/linux/ubuntu/gpg') }}"
       docker_package_name:  "docker-ce{{ '-'  if (DOCKER_VERSION != '') else ('') }}{{ DOCKER_VERSION if (DOCKER_VERSION != '') else ('') }}"
 
+  - set_fact:
+      redhat_docker_repository: "https://download.docker.com/linux/centos/{{ ansible_distribution_major_version }}/$basearch/stable"
+      debian_docker_repository: "deb https://download.docker.com/linux/ubuntu {{ ansible_distribution_release }} stable"
+    when: REPOSITORY_URL == ""
+
   - name: Add Docker Yum repository
     yum_repository:
       name: docker

org/ystia/docker/ansible/types.yml

@@ -11,6 +11,7 @@ metadata:
   template_version: 3.0.0-SNAPSHOT
   template_author: Ystia
 
+description: Docker types
 imports:
   - tosca-normative-types:1.0.0-ALIEN20
   - yorc-types:1.1.0

org/ystia/docker/containers/generic/playbooks/create.yaml

@@ -8,6 +8,9 @@
 - name: Install python requirements
   hosts: all
   become: true
+  environment:
+    # Adding /usr/local/bin needed by virtualenv on CentOS 8
+    PATH: "{{ ansible_env.PATH }}:/usr/local/bin"
   tasks:
   - name: Get python version
     python_requirements_info:
@@ -16,20 +19,52 @@
   - name: Get python major version
     set_fact:
       python_major_version: "{{pri.python_version | replace('\n', '') | regex_replace('^(\\d+).*', '\\1') }}"
-  - name: Install pip version compatible with python 2
-    easy_install:
-      name: pip<21.0
+  - name: Set python version for pip
+    set_fact:
+      python_pip_pkg: "python3-pip"
+      pip_cmd: "pip3"
+    when: python_major_version != "2"
+  - name: Set python 2 version for pip
+    set_fact:
+      python_pip_pkg: "python-pip"
+      pip_cmd: "pip"
     when: python_major_version == "2"
-  - name: Install pip
-    easy_install:
-      name: pip
-      state: latest  
+  - name: RedHat - install prerequisites
+    yum:
+      name:
+        - "{{ python_pip_pkg }}"
+      state: present
+      update_cache: yes
+    when: ansible_os_family == 'RedHat'
+  - name: Debian - install prerequisites
+    apt:
+      name:
+        - "{{ python_pip_pkg }}"
+      state: present
+      update_cache: yes
+    register: apt_res
+    retries: 5
+    delay: 15
+    until: apt_res is success
+    when: ansible_os_family == 'Debian'
+  - name: Install latest Pip version
+    pip:
+      name: "pip"
+      state: latest
+      executable: "{{pip_cmd}}"
     when: python_major_version != "2"
+  - name: Install latest Pip version compatible with python 2
+    pip:
+      name: "pip<21.0"
+      state: latest
+      executable: "{{pip_cmd}}"
+    when: python_major_version == "2"
 
   - name: Install 'virtualenv' package
     pip:
       name: virtualenv
       state: latest
+      executable: "{{pip_cmd}}"
 
   - name: Check docker python dependencies
     shell: >
@@ -38,7 +73,9 @@
 
   - name: Install 'docker' python package
     pip:
-      name: docker
+      name:
+        - docker
+        - six
       virtualenv: /usr/local/docker-py-env
 
 - name: Create Docker Container

org/ystia/docker/containers/generic/playbooks/delete.yaml

@@ -11,8 +11,14 @@
   vars:
     ansible_python_interpreter: "/usr/local/docker-py-env/bin/python"
   tasks:
-  - name: delete docker container
+  - name: Delete docker container
     docker_container: 
       name: "{{CONTAINER_ID}}"
       state: absent
+    when: STARTED_CONTAINER_ID is not defined or STARTED_CONTAINER_ID == ""
+  - name: Delete started docker container
+    docker_container:
+      name: "{{STARTED_CONTAINER_ID}}"
+      state: absent
+    when: STARTED_CONTAINER_ID is defined and STARTED_CONTAINER_ID != ""
 

org/ystia/docker/containers/generic/playbooks/docker_container_tasks.yaml

@@ -30,14 +30,17 @@
     name: "{{NODE}}-{{INSTANCE}}"
     auto_remove: "{{AUTO_REMOVE}}"
     cleanup: "{{CLEANUP}}"
+    command: "{{COMMAND}}"
     # cpus: "{{CPU_SHARE}}"
     detach: "{{DETACH}}"
     env: "{{DOCKER_ENV}}"
     exposed_ports: "{{DOCKER_EXP_PORTS}}"
+    hostname: "{{HOSTNAME}}"
     image: "{{IMAGE}}"
     keep_volumes: "{{KEEP_VOLUMES}}"
     memory: "{{MEM_SHARE_LIMIT}}"
     memory_reservation: "{{MEM_SHARE}}"
+    network_mode: "{{NETWORK_MODE}}"
     published_ports: "{{DOCKER_PUB_PORT}}"
     restart_policy: "{{RESTART_POLICY}}"
     shm_size: "{{SHM_SIZE}}"

org/ystia/docker/containers/generic/playbooks/start.yaml

@@ -7,6 +7,7 @@
 - name: Start Docker Container
   hosts: all
   become: true
+  # When the container fails, go on to retrieve outputs
   vars:
     ansible_python_interpreter: "/usr/local/docker-py-env/bin/python"
     DOCKER_ENV: {}
@@ -15,8 +16,24 @@
     DOCKER_VOLUMES: []
     DOCKER_STATE: "started"
   tasks:
-  - import_tasks: docker_container_tasks.yaml
-  - name: Get container output
+  - name: Start container
+    import_tasks: docker_container_tasks.yaml
+    no_log: true
+    ignore_errors: yes
+  - name: Store container id (needed if it was directly started or config changed between create and start)
+    set_fact:
+      STARTED_CONTAINER_ID: "{{docker_res.ansible_facts.docker_container['Id']}}"
+    when: docker_res.ansible_facts is defined
+  - name: Get container output (truncated)
     debug:
-      msg: "{{ docker_res.ansible_facts.docker_container.Output }}"
-    when: not DETACH|bool
+      msg: "{{ docker_res.ansible_facts.docker_container.Output[-20000:] }}"
+    when: not DETACH|bool and docker_res.ansible_facts is defined
+  - name: Get error 
+    debug:
+      msg: "{{ docker_res.msg[-20000:] }}"
+    when: docker_res.ansible_facts is not defined
+  - name: Check status
+    debug:
+      msg: "Check container exit code"
+    failed_when: docker_res.ansible_facts is not defined or docker_res.ansible_facts.docker_container.State.ExitCode != 0
+

org/ystia/docker/containers/generic/playbooks/stop.yaml

@@ -15,4 +15,10 @@
     docker_container: 
       name: "{{CONTAINER_ID}}"
       state: stopped
+    when: STARTED_CONTAINER_ID is not defined or STARTED_CONTAINER_ID == ""
+  - name: Stop started docker container
+    docker_container: 
+      name: "{{STARTED_CONTAINER_ID}}"
+      state: stopped
+    when: STARTED_CONTAINER_ID is defined and STARTED_CONTAINER_ID != ""
 

org/ystia/docker/containers/generic/types.yml

@@ -11,7 +11,7 @@ metadata:
   template_version: 3.0.0-SNAPSHOT
   template_author: Ystia
 
-
+description: Docker container types
 
 imports:
   - tosca-normative-types:1.0.0-ALIEN20
@@ -47,6 +47,10 @@ node_types:
           List of additional container ports which informs Docker that the container listens on the specified network ports at runtime.
           If the port is already exposed using EXPOSE in a Dockerfile, it does not need to be exposed again.
         required: false
+      hostname:
+        type: string
+        description: Container hostname
+        required: false
       image:
         type: string
         description: >
@@ -59,6 +63,13 @@ node_types:
         description: >
           Retain volumes associated with a removed container.
         default: true
+      network_mode:
+        type: string
+        description: >
+          Connect the container to a network.
+          Possible values: bridge, none, host, <network-name>|<network-id>
+          or container:<name|id> to reuse another container's network stack
+        required: false
       published_ports:
         type: list
         entry_schema:
@@ -102,21 +113,26 @@ node_types:
         required: false
     attributes:
       container_id: { get_operation_output: [SELF, Standard, create, CONTAINER_ID] }
+      started_container_id: { get_operation_output: [SELF, Standard, create, STARTED_CONTAINER_ID] }
     interfaces:
       Standard:
         inputs:
           # Will be empty for create operation but not a big deal
           CONTAINER_ID: { get_attribute: [SELF, container_id] }
+          STARTED_CONTAINER_ID: { get_attribute: [SELF, started_container_id] }
           AUTO_REMOVE: { get_property: [SELF, auto_remove] }
           CLEANUP: { get_property: [SELF, cleanup] }
+          COMMAND: { get_property: [SELF, docker_run_cmd] }
           CPU_SHARE: { get_property: [SELF, cpu_share] }
           DETACH: { get_property: [SELF, detach] }
           ENV_VARS: { get_property: [SELF, docker_env_vars] }
           EXPOSED_PORTS: { get_property: [SELF, exposed_ports] }
+          HOSTNAME: { get_property: [SELF, hostname] }
           IMAGE: { get_property: [SELF, image] }
           KEEP_VOLUMES: { get_property: [SELF, keep_volumes] }
           MEM_SHARE: { get_property: [SELF, mem_share] }
           MEM_SHARE_LIMIT: { get_property: [SELF, mem_share_limit] }
+          NETWORK_MODE: { get_property: [SELF, network_mode] }
           PUBLISHED_PORTS: { get_property: [SELF, published_ports] }
           RESTART_POLICY: { get_property: [SELF, restart_policy] }
           SHM_SIZE: { get_property: [SELF, shm_size] }

org/ystia/docker/images/playbooks/create.yaml

@@ -0,0 +1,72 @@
+#
+# Ystia Forge
+# Copyright (C) 2018 Bull S. A. S. - Bull, Rue Jean Jaures, B.P.68, 78340, Les Clayes-sous-Bois, France.
+# Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+#
+- name: Install ansible docker_image requirements
+  hosts: all
+  become: true
+  tasks:
+  - name: Get python version
+    python_requirements_info:
+    register: pri
+    failed_when: pri == None or pri.python_version == None or pri.python_version == ''
+  - name: Get python major version
+    set_fact:
+      python_major_version: "{{pri.python_version | replace('\n', '') | regex_replace('^(\\d+).*', '\\1') }}"
+  - name: Set python version for pip
+    set_fact:
+      python_pip_pkg: "python3-pip"
+      pip_cmd: "pip3"
+    when: python_major_version != "2"
+  - name: Set python 2 version for pip
+    set_fact:
+      python_pip_pkg: "python-pip"
+      pip_cmd: "pip"
+    when: python_major_version == "2"
+  - name: RedHat - install prerequisites
+    yum:
+      name:
+        - "{{ python_pip_pkg }}"
+      state: present
+      update_cache: yes
+    when: ansible_os_family == 'RedHat'
+  - name: Debian - install prerequisites
+    apt:
+      name:
+        - "{{ python_pip_pkg }}"
+      state: present
+      update_cache: yes
+    register: apt_res
+    retries: 5
+    delay: 15
+    until: apt_res is success
+    when: ansible_os_family == 'Debian'
+  - name: Install latest Pip version
+    pip:
+      name: "pip"
+      state: latest
+      executable: "{{pip_cmd}}"
+    when: python_major_version != "2"
+  - name: Install latest Pip version compatible with python 2
+    pip:
+      name: "pip<21.0"
+      state: latest
+      executable: "{{pip_cmd}}"
+    when: python_major_version == "2"
+  - name: Add user to docker group
+    user:
+      name: "{{USER}}"
+      groups: docker
+      append: yes
+    when: USER != "root"
+  - name: Install 'docker' python package
+    pip:
+      name: docker
+      # In user directory to avoid potential issues with distutils
+      # see issue https://github.com/pypa/pip/issues/5247
+      extra_args: --user
+      executable: "{{pip_cmd}}"
+    become_user: "{{USER}}"
+    become_method: sudo
+

org/ystia/docker/images/playbooks/load_archive.yaml

@@ -0,0 +1,40 @@
+#
+# Ystia Forge
+# Copyright (C) 2020 Bull S. A. S. - Bull, Rue Jean Jaures, B.P.68, 78340, Les Clayes-sous-Bois, France.
+# Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.
+#
+- name: Load Docker image from tar archive
+  hosts: all
+  tasks:
+  - name: Get archive PATH property
+    set_fact:
+      archive_path: "{{PATH}}"
+    failed_when: PATH == ""
+  - name: Load tar archive
+    docker_image:
+      name: "{{NAME}}"
+      tag: "{{TAG}}"
+      push: "{{PUSH}}"
+      repository: "{{REPOSITORY}}"
+      load_path: "{{archive_path}}"
+      source: load
+      force_source: "{{FORCE_LOAD}}"
+    become: true
+    become_user: "{{USER}}"
+    become_method: sudo
+    register: result
+  - name: Fail if the image specified does not correpond to the archive specified
+    debug:
+      msg: "Checking if image {{NAME}} {{TAG}} is in tar archive {{PATH}}"
+    failed_when: result.image == None
+  - name: Check if an image was loaded
+    debug:
+      msg: "Image {{NAME}} {{TAG}} already loaded"
+    when: result.image == {}
+  - name: Intialize fact used in opration output
+    set_fact:
+      REPO_TAGS: []
+  - name: Set repo tags output if returned by docker_image
+    set_fact:
+      REPO_TAGS: "{{result.image.RepoTags | to_json}}"
+    when: result.image != None and result.image != {}