Merge pull request 'Handle case of absent instance_config.yml gracefully' (#13) from improved-failure-handling into main

Reviewed-on: #13

.drone.yml

@@ -3,28 +3,22 @@ kind: pipeline
 name: linters
 steps:
   - name: tox -e linters
-    image: python:3.8-buster
+    image: python:3.9-buster
     commands:
-      - pip install tox==3.14.6
+      - pip install tox
       - tox -e linters
-trigger:
-  ref:
-    - "refs/heads/master"
 
 ---
 kind: pipeline
 name: packaging
 steps:
   - name: tox -e packaging
-    image: python:3.8-buster
+    image: python:3.9-buster
     commands:
-      - pip install tox==3.14.6
+      - pip install tox
       - tox -e packaging
 depends_on:
   - linters
-trigger:
-  ref:
-    - "refs/heads/master"
 
 ---
 kind: pipeline
@@ -33,13 +27,10 @@ steps:
   - name: tox -e py36
     image: python:3.6-buster
     commands:
-      - pip install tox==3.14.6
+      - pip install tox
       - tox -e py36
 depends_on:
   - linters
-trigger:
-  ref:
-    - "refs/heads/master"
 
 ---
 kind: pipeline
@@ -48,13 +39,10 @@ steps:
   - name: tox -e py37
     image: python:3.7-buster
     commands:
-      - pip install tox==3.14.6
+      - pip install tox
       - tox -e py37
 depends_on:
   - linters
-trigger:
-  ref:
-    - "refs/heads/master"
 
 ---
 kind: pipeline
@@ -63,42 +51,37 @@ steps:
   - name: tox -e py38
     image: python:3.8-buster
     commands:
-      - pip install tox==3.14.6
+      - pip install tox
       - tox -e py38
 depends_on:
   - linters
-trigger:
-  ref:
-    - "refs/heads/master"
 
 ---
 kind: pipeline
 name: devel
 steps:
   - name: tox -e devel
-    image: python:3.8-buster
+    image: python:3.9-buster
     failure: ignore
     commands:
-      - pip install tox==3.14.6
+      - pip install tox
       - tox -e devel
 depends_on:
   - linters
-trigger:
-  ref:
-    - "refs/heads/master"
 
 ---
 kind: pipeline
 name: integration
 steps:
   - name: molecule test
-    image: python:3.8-buster
+    image: python:3.9-buster
     environment:
       MOLECULE_NO_LOG: false
       HCLOUD_TOKEN:
         from_secret: HCLOUD_TOKEN
     commands:
       - pip install -e .
+      - pip install "ansible>=2.10, <2.11" netaddr
       - export INSTANCE_UUID=$(openssl rand -hex 5)
       - cd integration && molecule test
 depends_on:
@@ -107,6 +90,3 @@ depends_on:
   - py36
   - py37
   - py38
-trigger:
-  ref:
-    - "refs/heads/master"

CHANGELOG.md

@@ -7,9 +7,41 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [1.2.1] - 2021-06-02
+
+### Fixed
+
+- Remove async task handling for network deletion ([#30](https://github.com/ansible-community/molecule-hetznercloud/pull/30), credit @ggggut)
+
+## [1.2.0] - 2021-06-02
+
+### Added
+
+- Allow to create networks during test runs ([#29](https://github.com/ansible-community/molecule-hetznercloud/pull/29), thanks @ggggut!)
+
+## [1.1.0] - 2021-03-30
+
+## Changed
+
+- Relaxed bounds on Molecule to allow all versions less than `v4` ([#27](https://github.com/ansible-community/molecule-hetznercloud/pull/27))
+
+## [1.0.0] - 2021-01-06
+
+This is a major release with breaking changes for your schema and support for a
+new major version of Molecule. If you use the `volumes:` key in your
+`molecule.yml` then this change will break your configuration. Please see the
+section on "Volume Handling" in the README.md on how to upgrade successfully.
+You will now need to install Ansible yourself as Molecule does not do it for
+you. If there are any other breaking changes, please report them on the issue
+tracker so that we can mention them here.
+
+- Support Python 3.9.
+- Support Molecule 3.2.1
+- Add volume creation and clean up handling
+
 ## [0.2.2] - 2020-06-15
 
-## Fixed
+### Fixed
 
 - Point to an open issue tracker
 

README.md

@@ -21,6 +21,22 @@ useful, please support my maintenance work financially through my
 Sponsor profile](https://github.com/sponsors/decentral1se). I do not receive
 any financial support from RedHat or Hetzner Cloud for this work.
 
+## Install
+
+```bash
+$ pip install molecule-hetznercloud
+```
+
+If you're looking for a container approach, see [ansible-community/toolset](https://github.com/ansible-community/toolset).
+
+## Upgrade
+
+Please see the [CHANGELOG.md](./CHANGELOG.md) for migration guides.
+
+```bash
+$ pip install --upgrade molecule-hetznercloud
+```
+
 ## Usage
 
 You need to expose a `HCLOUD_TOKEN` environment variable in your environment.
@@ -31,10 +47,9 @@ Find out more about how to get one of those [over here](https://docs.hetzner.clo
 $ export HCLOUD_TOKEN=mycoolapitoken
 ```
 
-Then install the required Python package.
+Then create a role using the driver plugin.
 
 ```bash
-$ pip install molecule-hetznercloud
 $ molecule init role myrolename -d hetznercloud
 ```
 
@@ -72,6 +87,74 @@ $ export MOLECULE_NO_LOG=False  # not so verbose, helpful
 $ export MOLECULE_DEBUG=True  # very verbose, last ditch effort
 ```
 
+## Volume Handling
+
+> **WARNING**: this feature appears to be broke. See [#24](https://github.com/ansible-community/molecule-hetznercloud/issues/24) for more
+
+It is possible to have the driver manage volumes during the test run. You can
+add the following stanza to your Molecule configuration to have Molecule create
+this volume for the managed VPS. This volume will be cleaned up after use.
+
+```yaml
+platforms:
+  - name: instance
+    server_type: cx11
+    image: debian-10
+    volumes:
+      - name: "molecule-hetznercloud-volume-1-${INSTANCE_UUID}"
+        location: /foo/bar
+      - name: "molecule-hetznercloud-volume-2-${INSTANCE_UUID}"
+        size: 20
+```
+
+Supported keys are:
+
+- **name** (required): name of volume
+- **size** (optional, default: `10GB`): size of volume
+- **location** (optional, default: `omitted`): path for volume
+
+## Network Creation
+
+This Driver is able to generate networks and subnetworks during the test run.
+This can be useful for cluster tests. You can create networks with the
+following snippet:
+
+```yaml
+platforms:
+  - name: instance1
+    server_type: cx11
+    image: debian-10
+    networks:
+      test-network:
+        ip_range: 10.10.0.0/16
+        subnet:
+          ip: 10.10.10.1/24
+          type: cloud
+          network_zone: eu-central
+      test-network-2:
+        ip_range: 10.20.0.0/16
+        subnet:
+          ip: 10.20.10.1/24
+  - name: instance2
+    server_type: cx11
+    image: debian-10
+    networks:
+      test-network:
+        subnet:
+          ip: 10.10.10.2/24
+```
+
+The networks **ip_range** is only important for creating. If you have multiple
+hosts, it is okay to only define **ip_range** once. The supported keys are:
+
+- **networks**
+    - **ip_range** (required): ip range of network (usually `/16`)
+
+- **subnet**
+    - **ip** (required): ip that should be assigned to host (also generates subnetwork) - prefix mandatory
+    - **type** (optional, default: `cloud`): type of subnetwork
+    - **network_zone** (optional, default: `eu-central`): network zone of subnetwork
+
 ## Only use `molecule.yml` for configuration
 
 It is being worked on that it is possible to remove all the files except the
@@ -100,7 +183,7 @@ See [CHANGELOG.md](./CHANGELOG.md).
 
 ## Contact
 
-- Ping @decentral1se on the `#ansible-molecule` channel on [Freenode](https://webchat.freenode.net).
+- Ping @decentral1se on the `#ansible-molecule` channel on [Libera](https://libera.chat/).
 
 ## License
 
@@ -119,12 +202,12 @@ $ tox -v
 
 ### Integration
 
-Only doable by [Autonomic Cooperative](https://autonomic.zone/) members.
-
-```bash
-$ sudo apt install -y direnv
-$ cp .envrc.sample .envrc
-$ direnv allow
-$ pip install -e .
-$ cd integration-test-role && molecule test
+```
+git clone https://github.com/ansible-community/molecule-hetznercloud.git
+cd molecule-hetznercloud
+python3 -m venv .venv && source .venv/bin/activate
+pip install -e . "ansible<4" netaddr
+export INSTANCE_UUID=$(openssl rand -hex 5)
+export HCLOUD_TOKEN=YOURKEY
+cd integration && molecule test
 ```

integration/molecule/default/molecule.yml

@@ -7,6 +7,21 @@ platforms:
   - name: "molecule-hetznercloud-${INSTANCE_UUID}"
     server_type: cx11
     image: debian-10
+    # https://github.com/ansible-community/molecule-hetznercloud/issues/24
+    # volumes:
+    # - name: "molecule-hetznercloud-volume-1-${INSTANCE_UUID}"
+    # - name: "molecule-hetznercloud-volume-2-${INSTANCE_UUID}"
+    networks:
+      molecule-hetznercloud-network-1:
+        ip_range: 10.10.0.0/16
+        subnet:
+          ip: 10.10.10.1/24
+          type: cloud
+          network_zone: eu-central
+      molecule-hetznercloud-network-2:
+        ip_range: 10.20.0.0/16
+        subnet:
+          ip: 10.20.10.1/24
 provisioner:
   name: ansible
 verifier:

molecule_hetznercloud/cookiecutter/scenario/driver/hetznercloud/{{cookiecutter.molecule_directory}}/{{cookiecutter.scenario_name}}/INSTALL.rst

@@ -5,7 +5,7 @@ Hetzner Cloud plugin installation guide
 Requirements
 ============
 
-* Ansible >= 2.9
+* Ansible >= 2.10
 * ``HCLOUD_TOKEN`` exposed in your environment
 
 Install

molecule_hetznercloud/cookiecutter/scenario/driver/hetznercloud/{{cookiecutter.molecule_directory}}/{{cookiecutter.scenario_name}}/create.yml

@@ -33,7 +33,6 @@
         server_type: "{{ item.server_type }}"
         ssh_keys:
           - "{{ ssh_key_name }}"
-        volumes: "{{ item.volumes | default(omit) }}"
         image: "{{ item.image }}"
         location: "{{ item.location | default(omit) }}"
         datacenter: "{{ item.datacenter | default(omit) }}"
@@ -53,6 +52,73 @@
       retries: 300
       with_items: "{{ server.results }}"
 
+    - name: Create volume(s)
+      hcloud_volume:
+        name: "{{ item.volumes.name | default(item.name) }}"
+        server: "{{ item.name }}"
+        location: "{{ item.volumes.location | default(omit) }}"
+        size: "{{ item.volumes.size | default(10) }}"
+        api_token: "{{ lookup('env', 'HCLOUD_TOKEN') }}"
+        state: "present"
+      with_items: "{{ molecule_yml.platforms }}"
+      when:
+        - item.volumes is defined
+        - item.volumes.create | default(False) | bool
+      register: volumes
+      async: 7200
+      poll: 0
+
+    - name: Wait for volume(s) creation to complete
+      async_status:
+        jid: "{{ item.ansible_job_id }}"
+      register: hetzner_volumes
+      until: hetzner_volumes.finished
+      retries: 300
+      when: volumes.changed
+      with_items: "{{ volumes.results }}"
+
+    - name: Create private network(s)
+      hcloud_network:
+        name: "{{ item.name }}"
+        ip_range: "{{ item.ip_range | default(omit) }}"
+        api_token: "{{ lookup('env', 'HCLOUD_TOKEN') }}"
+        state: "present"
+      loop: "{{ molecule_yml.platforms|molecule_get_hetznercloud_networks('networks') }}"
+      register: networks
+      async: 7200
+      poll: 0
+
+    - name: Wait for network(s) creation to complete
+      async_status:
+        jid: "{{ item.ansible_job_id }}"
+      register: hetzner_networks
+      until: hetzner_networks.finished
+      retries: 300
+      when:
+        - networks is defined
+        - networks.changed
+      with_items: "{{ networks.results }}"
+
+    - name: Create private subnetwork(s)
+      hcloud_subnetwork:
+        network: "{{ item.network_name }}"
+        ip_range: "{{ item.ip|ipaddr('network/prefix') }}"
+        network_zone: "{{ item.network_zone | default('eu-central') }}"
+        type: "{{ item.type | default('cloud') }}"
+        api_token: "{{ lookup('env', 'HCLOUD_TOKEN') }}"
+        state: "present"
+      loop: "{{ molecule_yml.platforms|molecule_get_hetznercloud_networks('subnetworks') }}"
+      register: subnetworks
+
+    - name: Attach Server to Subnetwork(s)
+      hcloud_server_network:
+        network: "{{ item.network_name }}"
+        server: "{{ item.server_name }}"
+        ip: "{{ item.ip|ipaddr('address') }}"
+        api_token: "{{ lookup('env', 'HCLOUD_TOKEN') }}"
+        state: "present"
+      loop: "{{ molecule_yml.platforms|molecule_get_hetznercloud_networks('subnetworks') }}"
+
     - name: Populate instance config dict
       set_fact:
         instance_conf_dict: {
@@ -61,7 +127,9 @@
           'address': "{{ item.hcloud_server.ipv4_address }}",
           'user': "{{ ssh_user }}",
           'port': "{{ ssh_port }}",
-          'identity_file': "{{ ssh_path }}", }
+          'identity_file': "{{ ssh_path }}",
+          'volumes': "{{ item.item.item.volumes | default({}) }}",
+          'networks': "{{ item.item.item.networks | default({}) | dict2items(key_name='name', value_name='data') }}", }
       with_items: "{{ hetzner_jobs.results }}"
       register: instance_config_dict
       when: server.changed | bool
@@ -73,7 +141,10 @@
 
     - name: Dump instance config
       copy:
-        content: "{{ instance_conf | to_json | from_json | molecule_to_yaml | molecule_header }}"
+        content: |
+          # Molecule managed
+
+          {{ instance_conf | to_json | from_json | to_yaml }}
         dest: "{{ molecule_instance_config }}"
       when: server.changed | bool
 
@@ -83,5 +154,5 @@
         host: "{{ item.address }}"
         search_regex: SSH
         delay: 10
-      with_items: "{{ lookup('file', molecule_instance_config) | molecule_from_yaml }}"
+      with_items: "{{ lookup('file', molecule_instance_config) | from_yaml }}"
 {%- endraw %}

molecule_hetznercloud/cookiecutter/scenario/driver/hetznercloud/{{cookiecutter.molecule_directory}}/{{cookiecutter.scenario_name}}/destroy.yml

@@ -10,7 +10,7 @@
       block:
         - name: Populate instance config from file
           set_fact:
-            instance_conf: "{{ lookup('file', molecule_instance_config) | molecule_from_yaml }}"
+            instance_conf: "{{ lookup('file', molecule_instance_config) | from_yaml }}"
             skip_instances: false
       rescue:
         - name: Populate instance config when file missing
@@ -37,6 +37,37 @@
       retries: 300
       with_items: "{{ server.results }}"
 
+    - name: Destroy volume(s)
+      hcloud_volume:
+        name: "{{ item.volumes.name | default(item.instance) }}"
+        server: "{{ item.instance }}"
+        api_token: "{{ lookup('env', 'HCLOUD_TOKEN') }}"
+        state: "absent"
+      register: volumes
+      with_items: "{{ instance_conf }}"
+      when:
+        - item.volumes is defined
+        - item.volumes.create | default(False) | bool
+      async: 7200
+      poll: 0
+
+    - name: Wait for volume(s) deletion to complete
+      async_status:
+        jid: "{{ item.ansible_job_id }}"
+      register: hetzner_volumes
+      until: hetzner_volumes.finished
+      retries: 300
+      when: volumes.changed
+      with_items: "{{ volumes.results }}"
+
+    - name: Destroy network(s)
+      hcloud_network:
+        name: "{{ item.1.name }}"
+        api_token: "{{ lookup('env', 'HCLOUD_TOKEN') }}"
+        state: absent
+      register: networks
+      loop: "{{ instance_conf|subelements('networks', skip_missing=True) }}"
+
     - name: Remove registered SSH key
       hcloud_ssh_key:
         name: "{{ instance_conf[0].ssh_key_name }}"
@@ -52,7 +83,10 @@
 
     - name: Dump instance config
       copy:
-        content: "{{ instance_conf | molecule_to_yaml | molecule_header }}"
+        content: |
+          # Molecule managed
+
+          {{ instance_conf | to_json | from_json | to_yaml }}
         dest: "{{ molecule_instance_config }}"
       when: server.changed | bool
 {%- endraw %}

molecule_hetznercloud/cookiecutter/scenario/driver/hetznercloud/{{cookiecutter.molecule_directory}}/{{cookiecutter.scenario_name}}/molecule.yml

@@ -7,6 +7,19 @@ platforms:
   - name: "{{ cookiecutter.role_name }}"
     server_type: cx11
     image: debian-10
+    volumes:
+      - name: "molecule-hetznercloud-volume-1"
+    networks:
+      molecule-hetznercloud-network-1:
+        ip_range: 10.10.0.0/16
+        subnet:
+          ip: 10.10.10.1/24
+          type: cloud
+          network_zone: eu-central
+      molecule-hetznercloud-network-2:
+        ip_range: 10.20.0.0/16
+        subnet:
+          ip: 10.20.10.1/24
 provisioner:
   name: ansible
 lint: |

molecule_hetznercloud/playbooks/create.yml

@@ -32,7 +32,6 @@
         server_type: "{{ item.server_type }}"
         ssh_keys:
           - "{{ ssh_key_name }}"
-        volumes: "{{ item.volumes | default(omit) }}"
         image: "{{ item.image }}"
         location: "{{ item.location | default(omit) }}"
         datacenter: "{{ item.datacenter | default(omit) }}"
@@ -52,6 +51,72 @@
       retries: 300
       with_items: "{{ server.results }}"
 
+    - name: Create volume(s)
+      hcloud_volume:
+        name: "{{ item.1.name | default(item.0.name) }}"
+        server: "{{ item.0.name }}"
+        location: "{{ item.1.location | default(omit) }}"
+        size: "{{ item.1.size | default(10) }}"
+        api_token: "{{ lookup('env', 'HCLOUD_TOKEN') }}"
+        state: "present"
+      loop: "{{ molecule_yml.platforms|subelements('volumes', skip_missing=True)}}"
+      register: volumes
+      async: 7200
+      poll: 0
+
+    - name: Wait for volume(s) creation to complete
+      async_status:
+        jid: "{{ item.ansible_job_id }}"
+      register: hetzner_volumes
+      until: hetzner_volumes.finished
+      retries: 300
+      when:
+        - volumes is defined
+        - volumes.changed
+      with_items: "{{ volumes.results }}"
+
+    - name: Create private network(s)
+      hcloud_network:
+        name: "{{ item.name }}"
+        ip_range: "{{ item.ip_range | default(omit) }}"
+        api_token: "{{ lookup('env', 'HCLOUD_TOKEN') }}"
+        state: "present"
+      loop: "{{ molecule_yml.platforms|molecule_get_hetznercloud_networks('networks') }}"
+      register: networks
+      async: 7200
+      poll: 0
+
+    - name: Wait for network(s) creation to complete
+      async_status:
+        jid: "{{ item.ansible_job_id }}"
+      register: hetzner_networks
+      until: hetzner_networks.finished
+      retries: 300
+      when:
+        - networks is defined
+        - networks.changed
+      with_items: "{{ networks.results }}"
+
+    - name: Create private subnetwork(s)
+      hcloud_subnetwork:
+        network: "{{ item.network_name }}"
+        ip_range: "{{ item.ip|ipaddr('network/prefix') }}"
+        network_zone: "{{ item.network_zone | default('eu-central') }}"
+        type: "{{ item.type | default('cloud') }}"
+        api_token: "{{ lookup('env', 'HCLOUD_TOKEN') }}"
+        state: "present"
+      loop: "{{ molecule_yml.platforms|molecule_get_hetznercloud_networks('subnetworks') }}"
+      register: subnetworks
+
+    - name: Attach Server to Subnetwork(s)
+      hcloud_server_network:
+        network: "{{ item.network_name }}"
+        server: "{{ item.server_name }}"
+        ip: "{{ item.ip|ipaddr('address') }}"
+        api_token: "{{ lookup('env', 'HCLOUD_TOKEN') }}"
+        state: "present"
+      loop: "{{ molecule_yml.platforms|molecule_get_hetznercloud_networks('subnetworks') }}"
+
     - name: Populate instance config dict
       set_fact:
         instance_conf_dict:
@@ -62,6 +127,8 @@
             "user": "{{ ssh_user }}",
             "port": "{{ ssh_port }}",
             "identity_file": "{{ ssh_path }}",
+            "volumes": "{{ item.item.item.volumes | default({}) }}",
+            "networks": "{{ item.item.item.networks | default({}) | dict2items(key_name='name', value_name='data') }}",
           }
       with_items: "{{ hetzner_jobs.results }}"
       register: instance_config_dict
@@ -74,7 +141,10 @@
 
     - name: Dump instance config
       copy:
-        content: "{{ instance_conf | to_json | from_json | molecule_to_yaml | molecule_header }}"
+        content: |
+          # Molecule managed
+
+          {{ instance_conf | to_json | from_json | to_yaml }}
         dest: "{{ molecule_instance_config }}"
       when: server.changed | bool
 
@@ -84,4 +154,4 @@
         host: "{{ item.address }}"
         search_regex: SSH
         delay: 10
-      with_items: "{{ lookup('file', molecule_instance_config) | molecule_from_yaml }}"
+      with_items: "{{ lookup('file', molecule_instance_config) | from_yaml }}"

molecule_hetznercloud/playbooks/destroy.yml

@@ -6,16 +6,16 @@
   no_log: "{{ molecule_no_log }}"
   tasks:
     - name: Populate the instance config
-      block:
-        - name: Populate instance config from file
       set_fact:
-            instance_conf: "{{ lookup('file', molecule_instance_config) | molecule_from_yaml }}"
+        instance_conf: "{{ lookup('file', molecule_instance_config, errors='warn') | from_yaml }}"
         skip_instances: false
-      rescue:
+      register: instance_config_lookup
+
     - name: Populate instance config when file missing
       set_fact:
         instance_conf: {}
         skip_instances: true
+      when: not instance_config_lookup.ansible_facts.instance_conf
 
     - name: Destroy molecule instance(s)
       hcloud_server:
@@ -36,6 +36,34 @@
       retries: 300
       with_items: "{{ server.results }}"
 
+    - name: Destroy volume(s)
+      hcloud_volume:
+        name: "{{ item.volumes.name | default(item.instance) }}"
+        server: "{{ item.instance }}"
+        api_token: "{{ lookup('env', 'HCLOUD_TOKEN') }}"
+        state: absent
+      register: volumes
+      with_items: "{{ instance_conf }}"
+      async: 7200
+      poll: 0
+
+    - name: Wait for volume(s) deletion to complete
+      async_status:
+        jid: "{{ item.ansible_job_id }}"
+      register: hetzner_volumes
+      until: hetzner_volumes.finished
+      retries: 300
+      when: volumes.changed
+      with_items: "{{ volumes.results }}"
+
+    - name: Destroy network(s)
+      hcloud_network:
+        name: "{{ item.1.name }}"
+        api_token: "{{ lookup('env', 'HCLOUD_TOKEN') }}"
+        state: absent
+      register: networks
+      loop: "{{ instance_conf|subelements('networks', skip_missing=True) }}"
+
     - name: Remove registered SSH key
       hcloud_ssh_key:
         name: "{{ instance_conf[0].ssh_key_name }}"
@@ -51,6 +79,9 @@
 
     - name: Dump instance config
       copy:
-        content: "{{ instance_conf | molecule_to_yaml | molecule_header }}"
+        content: |
+          # Molecule managed
+
+          {{ instance_conf | to_json | from_json | to_yaml }}
         dest: "{{ molecule_instance_config }}"
       when: server.changed | bool

molecule_hetznercloud/playbooks/filter_plugins/get_hetznercloud_networks.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+"""Usage:"""
+""" loop: "{{ molecule_yml.platforms|molecule_get_hetznercloud_networks('networks') }}" """  # noqa
+""" loop: "{{ molecule_yml.platforms|molecule_get_hetznercloud_networks('subnetworks') }}" """  # noqa
+
+
+def merge_two_dicts(x, y):
+    z = x.copy()
+    z.update(y)
+    return z
+
+
+def get_hetznercloud_networks(data, request):
+    network_list = {}
+    subnetwork_list = []
+
+    if request == "networks":
+
+        for platform in data:
+            if "networks" in platform:
+                for network_name, values in platform["networks"].items():
+                    del values["subnet"]
+                    values["name"] = network_name
+                    if network_name in network_list:
+                        network_list[network_name] = merge_two_dicts(
+                            network_list[network_name], values
+                        )
+                    else:
+                        network_list[network_name] = values
+
+        return [x for x in network_list.values()]
+
+    elif request == "subnetworks":
+
+        for platform in data:
+            name = platform["name"]
+            if "networks" in platform:
+                for network_name, values in platform["networks"].items():
+                    values["name"] = network_name
+                    if "subnet" in values:
+                        values["subnet"]["server_name"] = name
+                        values["subnet"]["network_name"] = network_name
+                        subnetwork_list.append(values["subnet"])
+
+        return subnetwork_list
+
+
+class FilterModule(object):
+    """Core Molecule filter plugins."""
+
+    def filters(self):
+        return {
+            "molecule_get_hetznercloud_networks": get_hetznercloud_networks,
+        }

molecule_hetznercloud/test/conftest.py

@@ -12,13 +12,14 @@ LOG = logger.get_logger(__name__)
 
 @pytest.helpers.register
 def run_command(cmd, env=os.environ, log=True):
+    if cmd.__class__.__name__ == "Command":
         if log:
             cmd = _rebake_command(cmd, env)
         cmd = cmd.bake(_truncate_exc=False)
-    return util.run_command(cmd)
+    return util.run_command(cmd, env=env)
 
 
-def _rebake_command(cmd, env, out=LOG.out, err=LOG.error):
+def _rebake_command(cmd, env, out=LOG.info, err=LOG.error):
     return cmd.bake(_env=env, _out=out, _err=err)
 
 

molecule_hetznercloud/test/functional/conftest.py

@@ -1,13 +1,10 @@
 import os
-import pkg_resources
 import shutil
 
 import pexpect
+import pkg_resources
 import pytest
-import sh
-
-from molecule import logger
-from molecule import util
+from molecule import logger, util
 
 from ..conftest import change_dir_to
 
@@ -37,9 +34,8 @@ def with_scenario(request, scenario_to_test, driver_name, scenario_name, skip_te
         yield
         if scenario_name:
             msg = "CLEANUP: Destroying instances for all scenario(s)"
-            LOG.out(msg)
-            options = {"driver_name": driver_name, "all": True}
-            cmd = sh.molecule.bake("destroy", **options)
+            LOG.info(msg)
+            cmd = ["molecule", "destroy", "--driver-name", driver_name, "--all"]
             pytest.helpers.run_command(cmd)
 
 
@@ -63,56 +59,56 @@ def skip_test(request, driver_name):
 
 @pytest.helpers.register
 def idempotence(scenario_name):
-    options = {"scenario_name": scenario_name}
-    cmd = sh.molecule.bake("create", **options)
+    cmd = ["molecule", "create", "--scenario-name", scenario_name]
     pytest.helpers.run_command(cmd)
 
-    options = {"scenario_name": scenario_name}
-    cmd = sh.molecule.bake("converge", **options)
+    cmd = ["molecule", "converge", "--scenario_name", scenario_name]
     pytest.helpers.run_command(cmd)
 
-    options = {"scenario_name": scenario_name}
-    cmd = sh.molecule.bake("idempotence", **options)
+    cmd = ["molecule", "--scenario_name", scenario_name]
     pytest.helpers.run_command(cmd)
 
 
 @pytest.helpers.register
 def init_role(temp_dir, driver_name):
-    role_directory = os.path.join(temp_dir.strpath, "test-init")
-
-    cmd = sh.molecule.bake(
-        "init", "role", {"driver-name": driver_name, "role-name": "test-init"}
-    )
+    cmd = ["molecule", "init", "role", "test-init", "--driver-name", driver_name]
     pytest.helpers.run_command(cmd)
+
+    role_directory = os.path.join(temp_dir.strpath, "test-init")
     pytest.helpers.metadata_lint_update(role_directory)
 
     with change_dir_to(role_directory):
-        options = {"all": True}
-        cmd = sh.molecule.bake("test", **options)
+        cmd = ["molecule", "test", "--all"]
         pytest.helpers.run_command(cmd)
 
 
 @pytest.helpers.register
 def init_scenario(temp_dir, driver_name):
-    role_directory = os.path.join(temp_dir.strpath, "test-init")
-    cmd = sh.molecule.bake(
-        "init", "role", {"driver-name": driver_name, "role-name": "test-init"}
-    )
+    cmd = ["molecule", "init", "role", "test-init", "--driver-name", driver_name]
     pytest.helpers.run_command(cmd)
+
+    role_directory = os.path.join(temp_dir.strpath, "test-init")
     pytest.helpers.metadata_lint_update(role_directory)
 
     with change_dir_to(role_directory):
         molecule_directory = pytest.helpers.molecule_directory()
         scenario_directory = os.path.join(molecule_directory, "test-scenario")
 
-        options = {"scenario_name": "test-scenario", "role_name": "test-init"}
-        cmd = sh.molecule.bake("init", "scenario", **options)
+        cmd = [
+            "molecule",
+            "init",
+            "scenario",
+            "test-scenario",
+            "--role-name",
+            "test-init",
+            "--driver-name",
+            driver_name,
+        ]
         pytest.helpers.run_command(cmd)
 
         assert os.path.isdir(scenario_directory)
 
-        options = {"scenario_name": "test-scenario", "all": True}
-        cmd = sh.molecule.bake("test", **options)
+        cmd = ["molecule", "test", "--scenario-name", "test-scenario", "--all"]
         pytest.helpers.run_command(cmd)
 
 
@@ -125,13 +121,13 @@ def metadata_lint_update(role_directory):
     shutil.copy(ansible_lint_src, role_directory)
 
     with change_dir_to(role_directory):
-        cmd = sh.ansible_lint.bake(".")
+        cmd = ["ansible-lint", "."]
     pytest.helpers.run_command(cmd)
 
 
 @pytest.helpers.register
 def list(x):
-    cmd = sh.molecule.bake("list")
+    cmd = ["molecule", "list"]
     out = pytest.helpers.run_command(cmd, log=False)
     out = out.stdout.decode("utf-8")
     out = util.strip_ansi_color(out)
@@ -142,10 +138,9 @@ def list(x):
 
 @pytest.helpers.register
 def list_with_format_plain(x):
-    cmd = sh.molecule.bake("list", {"format": "plain"})
-    out = pytest.helpers.run_command(cmd, log=False)
-    out = out.stdout.decode("utf-8")
-    out = util.strip_ansi_color(out)
+    cmd = ["molecule", "list", "--format", "plain"]
+    result = util.run_command(cmd)
+    out = util.strip_ansi_color(result.stdout)
 
     for l in x.splitlines():
         assert l in out
@@ -153,12 +148,10 @@ def list_with_format_plain(x):
 
 @pytest.helpers.register
 def login(login_args, scenario_name="default"):
-    options = {"scenario_name": scenario_name}
-    cmd = sh.molecule.bake("destroy", **options)
+    cmd = ["molecule", "destroy", "--scenario-name", scenario_name]
     pytest.helpers.run_command(cmd)
 
-    options = {"scenario_name": scenario_name}
-    cmd = sh.molecule.bake("create", **options)
+    cmd = ["molecule", "create", "--scenario-name", scenario_name]
     pytest.helpers.run_command(cmd)
 
     for instance, regexp in login_args:
@@ -175,31 +168,26 @@ def login(login_args, scenario_name="default"):
 
 @pytest.helpers.register
 def test(driver_name, scenario_name="default", parallel=False):
-    options = {
-        "scenario_name": scenario_name,
-        "all": scenario_name is None,
-        "parallel": parallel,
-    }
+    cmd = ["molecule", "test", "--scenario-name", scenario_name]
 
-    if driver_name == "delegated":
-        options = {"scenario_name": scenario_name}
+    if driver_name != "delegated":
+        if scenario_name is None:
+            cmd.append("--all")
+        if parallel:
+            cmd.append("--parallel")
 
-    cmd = sh.molecule.bake("test", **options)
     pytest.helpers.run_command(cmd)
 
 
 @pytest.helpers.register
 def verify(scenario_name="default"):
-    options = {"scenario_name": scenario_name}
-    cmd = sh.molecule.bake("create", **options)
+    cmd = ["molecule", "create", "--scenario-name", scenario_name]
     pytest.helpers.run_command(cmd)
 
-    options = {"scenario_name": scenario_name}
-    cmd = sh.molecule.bake("converge", **options)
+    cmd = ["molecule", "converge", "--scenario-name", scenario_name]
     pytest.helpers.run_command(cmd)
 
-    options = {"scenario_name": scenario_name}
-    cmd = sh.molecule.bake("verify", **options)
+    cmd = ["molecule", "verify", "--scenario-name", scenario_name]
     pytest.helpers.run_command(cmd)
 
 

molecule_hetznercloud/test/resources/playbooks/hetznercloud/create.yml

@@ -23,6 +23,7 @@
       hcloud_ssh_key:
         name: "{{ ssh_key_name }}"
         public_key: "{{ generated_ssh_key.public_key }}"
+        api_token: "{{ lookup('env', 'HCLOUD_TOKEN') }}"
         state: present
 
     - name: Create molecule instance(s)
@@ -31,7 +32,6 @@
         server_type: "{{ item.server_type }}"
         ssh_keys:
           - "{{ ssh_key_name }}"
-        volumes: "{{ item.volumes | default(omit) }}"
         image: "{{ item.image }}"
         location: "{{ item.location | default(omit) }}"
         datacenter: "{{ item.datacenter | default(omit) }}"
@@ -51,19 +51,84 @@
       retries: 300
       with_items: "{{ server.results }}"
 
-    # Mandatory configuration for Molecule to function.
+    - name: Create volume(s)
+      hcloud_volume:
+        name: "{{ item.volumes.name | default(item.name) }}"
+        server: "{{ item.name }}"
+        location: "{{ item.volumes.location | default(omit) }}"
+        size: "{{ item.volumes.size | default(10) }}"
+        api_token: "{{ lookup('env', 'HCLOUD_TOKEN') }}"
+        state: "present"
+      with_items: "{{ molecule_yml.platforms }}"
+      when:
+        - item.volumes is defined
+        - item.volumes.create | default(False) | bool
+      register: volumes
+      async: 7200
+      poll: 0
+
+    - name: Wait for volume(s) creation to complete
+      async_status:
+        jid: "{{ item.ansible_job_id }}"
+      register: hetzner_volumes
+      until: hetzner_volumes.finished
+      retries: 300
+      when: volumes.changed
+      with_items: "{{ volumes.results }}"
+
+    - name: Create private network(s)
+      hcloud_network:
+        name: "{{ item.name }}"
+        ip_range: "{{ item.ip_range | default(omit) }}"
+        api_token: "{{ lookup('env', 'HCLOUD_TOKEN') }}"
+        state: "present"
+      loop: "{{ molecule_yml.platforms|molecule_get_hetznercloud_networks('networks') }}"
+      register: networks
+      async: 7200
+      poll: 0
+
+    - name: Wait for network(s) creation to complete
+      async_status:
+        jid: "{{ item.ansible_job_id }}"
+      register: hetzner_networks
+      until: hetzner_networks.finished
+      retries: 300
+      when:
+        - networks is defined
+        - networks.changed
+      with_items: "{{ networks.results }}"
+
+    - name: Create private subnetwork(s)
+      hcloud_subnetwork:
+        network: "{{ item.network_name }}"
+        ip_range: "{{ item.ip|ipaddr('network/prefix') }}"
+        network_zone: "{{ item.network_zone | default('eu-central') }}"
+        type: "{{ item.type | default('cloud') }}"
+        api_token: "{{ lookup('env', 'HCLOUD_TOKEN') }}"
+        state: "present"
+      loop: "{{ molecule_yml.platforms|molecule_get_hetznercloud_networks('subnetworks') }}"
+      register: subnetworks
+
+    - name: Attach Server to Subnetwork(s)
+      hcloud_server_network:
+        network: "{{ item.network_name }}"
+        server: "{{ item.server_name }}"
+        ip: "{{ item.ip|ipaddr('address') }}"
+        api_token: "{{ lookup('env', 'HCLOUD_TOKEN') }}"
+        state: "present"
+      loop: "{{ molecule_yml.platforms|molecule_get_hetznercloud_networks('subnetworks') }}"
 
     - name: Populate instance config dict
       set_fact:
-        instance_conf_dict:
-          {
-            "instance": "{{ item.hcloud_server.name }}",
-            "ssh_key_name": "{{ ssh_key_name }}",
-            "address": "{{ item.hcloud_server.ipv4_address }}",
-            "user": "{{ ssh_user }}",
-            "port": "{{ ssh_port }}",
-            "identity_file": "{{ ssh_path }}",
-          }
+        instance_conf_dict: {
+          'instance': "{{ item.hcloud_server.name }}",
+          'ssh_key_name': "{{ ssh_key_name }}",
+          'address': "{{ item.hcloud_server.ipv4_address }}",
+          'user': "{{ ssh_user }}",
+          'port': "{{ ssh_port }}",
+          'identity_file': "{{ ssh_path }}",
+          'volumes': "{{ item.item.item.volumes | default({}) }}",
+          'networks': "{{ item.item.item.networks | default({}) | dict2items(key_name='name', value_name='data') }}", }
       with_items: "{{ hetzner_jobs.results }}"
       register: instance_config_dict
       when: server.changed | bool
@@ -75,7 +140,10 @@
 
     - name: Dump instance config
       copy:
-        content: "{{ instance_conf | to_json | from_json | molecule_to_yaml | molecule_header }}"
+        content: |
+          # Molecule managed
+
+          {{ instance_conf | to_json | from_json | to_yaml }}
         dest: "{{ molecule_instance_config }}"
       when: server.changed | bool
 
@@ -85,4 +153,4 @@
         host: "{{ item.address }}"
         search_regex: SSH
         delay: 10
-      with_items: "{{ lookup('file', molecule_instance_config) | molecule_from_yaml }}"
+      with_items: "{{ lookup('file', molecule_instance_config) | from_yaml }}"

molecule_hetznercloud/test/resources/playbooks/hetznercloud/destroy.yml

@@ -9,7 +9,7 @@
       block:
         - name: Populate instance config from file
           set_fact:
-            instance_conf: "{{ lookup('file', molecule_instance_config) | molecule_from_yaml }}"
+            instance_conf: "{{ lookup('file', molecule_instance_config) | from_yaml }}"
             skip_instances: false
       rescue:
         - name: Populate instance config when file missing
@@ -36,15 +36,45 @@
       retries: 300
       with_items: "{{ server.results }}"
 
+    - name: Destroy volume(s)
+      hcloud_volume:
+        name: "{{ item.volumes.name | default(item.instance) }}"
+        server: "{{ item.instance }}"
+        api_token: "{{ lookup('env', 'HCLOUD_TOKEN') }}"
+        state: "absent"
+      register: volumes
+      with_items: "{{ instance_conf }}"
+      when:
+        - item.volumes is defined
+        - item.volumes.create | default(False) | bool
+      async: 7200
+      poll: 0
+
+    - name: Wait for volume(s) deletion to complete
+      async_status:
+        jid: "{{ item.ansible_job_id }}"
+      register: hetzner_volumes
+      until: hetzner_volumes.finished
+      retries: 300
+      when: volumes.changed
+      with_items: "{{ volumes.results }}"
+
+    - name: Destroy network(s)
+      hcloud_network:
+        name: "{{ item.1.name }}"
+        api_token: "{{ lookup('env', 'HCLOUD_TOKEN') }}"
+        state: absent
+      register: networks
+      loop: "{{ instance_conf|subelements('networks', skip_missing=True) }}"
+
     - name: Remove registered SSH key
       hcloud_ssh_key:
         name: "{{ instance_conf[0].ssh_key_name }}"
+        api_token: "{{ lookup('env', 'HCLOUD_TOKEN') }}"
         state: absent
       when:
         - not skip_instances
-        - instance_conf  # must contain at least one instance
-
-    # Mandatory configuration for Molecule to function.
+        - instance_conf | length  # must contain at least one instance
 
     - name: Populate instance config
       set_fact:
@@ -52,6 +82,9 @@
 
     - name: Dump instance config
       copy:
-        content: "{{ instance_conf | molecule_to_yaml | molecule_header }}"
+        content: |
+          # Molecule managed
+
+          {{ instance_conf | to_json | from_json | to_yaml }}
         dest: "{{ molecule_instance_config }}"
       when: server.changed | bool

molecule_hetznercloud/test/unit/cookiecutter/test_molecule.py

@@ -1,8 +1,6 @@
 import os
 
 import pytest
-import sh
-
 from molecule import util
 from molecule.command.init import base
 from molecule.model import schema_v3
@@ -57,5 +55,5 @@ def test_drivers(
 
     assert {} == schema_v3.validate(data)
 
-    cmd = sh.yamllint.bake("-s", _molecule_file)
+    cmd = ["yamllint", "-s", _molecule_file]
     pytest.helpers.run_command(cmd)

molecule_hetznercloud/test/unit/driver/test_hetznercloud.py

@@ -1,7 +1,6 @@
 import os
 
 import pytest
-
 from molecule import config
 from molecule_hetznercloud import driver
 
@@ -15,13 +14,6 @@ def test_hetznercloud_config_gives_config_object(hetznercloud_instance):
     assert isinstance(hetznercloud_instance._config, config.Config)
 
 
-def test_hetznercloud_testinfra_options_property(hetznercloud_instance):
-    assert {
-        "connection": "ansible",
-        "ansible-inventory": hetznercloud_instance._config.provisioner.inventory_file,
-    } == hetznercloud_instance.testinfra_options
-
-
 def test_hetznercloud_name_property(hetznercloud_instance):
     assert "hetznercloud" == hetznercloud_instance.name
 

molecule_hetznercloud/test/unit/model/v2/test_platforms_section.py

@@ -10,7 +10,7 @@ def _model_platform_hetznercloud_section_data():
             {
                 "name": "instance",
                 "server_type": "",
-                "volumes": [""],
+                "volumes": [],
                 "image": "",
                 "location": "",
                 "datacenter": "",
@@ -35,7 +35,7 @@ def _model_platforms_hetznercloud_errors_section_data():
             {
                 "name": 0,
                 "server_type": 0,
-                "volumes": {},
+                "volumes": [],
                 "image": 0,
                 "location": 0,
                 "datacenter": 0,

setup.cfg

@@ -30,7 +30,10 @@ classifiers =
     Natural Language :: English
     Operating System :: OS Independent
     Programming Language :: Python :: 3
+    Programming Language :: Python :: 3.6
+    Programming Language :: Python :: 3.7
     Programming Language :: Python :: 3.8
+    Programming Language :: Python :: 3.9
     Topic :: System :: Systems Administration
     Topic :: Utilities
 keywords =
@@ -49,23 +52,23 @@ packages = find:
 include_package_data = True
 zip_safe = False
 setup_requires =
-    setuptools_scm >= 3.5.0
-    setuptools_scm_git_archive >= 1.1
+    setuptools_scm
+    setuptools_scm_git_archive
 install_requires =
-    hcloud >= 1.6.3, < 2
-    molecule >= 3.0.4, <= 3.1
+    hcloud >= 1.10.0, < 2
+    molecule >= 3.2.1, < 4
     pyyaml >= 5.3.1, < 6
 
 [options.extras_require]
 test =
-    hcloud >= 1.6.3, < 2
+    hcloud >= 1.10.0, < 2
     mock >= 4.0.2, < 5
-    pytest-cov >= 2.8.1, < 3
+    pytest-cov >= 2.10.1, < 3
     pytest-helpers-namespace >= 2019.1.8, < 2020
-    pytest-mock >= 3.1.0, < 4
+    pytest-mock >= 3.5.0, < 4
     pytest-verbose-parametrize>=1.7.0, < 2
-    pytest-xdist>=1.31.0, < 2
-    pytest>=5.4.1, < 6
+    pytest-xdist>=2.2.0, < 3
+    pytest>=6.2.1, < 7
 
 [options.packages.find]
 where = .

tox.ini

@@ -1,12 +1,10 @@
 # For more information about tox, see https://tox.readthedocs.io/en/latest/
 [tox]
-minversion = 3.14.0
 envlist =
     linters
     packaging
-    py{36,37,38}
+    py{36,37,38,39}
     devel
-
 skipsdist = True
 skip_missing_interpreters = False
 isolated_build = True
@@ -19,12 +17,11 @@ setenv =
     PYTHONDONTWRITEBYTECODE=1
     PYTEST_ADDOPTS=molecule_hetznercloud/test/unit/ --cov={toxinidir}/molecule_hetznercloud/ --no-cov-on-fail {env:PYTEST_ADDOPTS:-n auto}
 deps =
-    ansible>=2.9,<2.10
+    ansible>=2.10,<2.11
 extras =
     test
 commands =
     python -m pytest {posargs}
-
 whitelist_externals =
     bash
     twine
@@ -34,7 +31,7 @@ whitelist_externals =
 [testenv:linters]
 commands =
     python -m pre_commit run {posargs:--all}
-deps = pre-commit>=1.18.1
+deps = pre-commit
 skip_install = true
 usedevelop = false
 
@@ -42,9 +39,9 @@ usedevelop = false
 usedevelop = false
 skip_install = true
 deps =
-    collective.checkdocs >= 0.2
-    pep517 >= 0.5.0
-    twine >= 2.0.0
+    collective.checkdocs
+    pep517
+    twine
 commands =
     bash -c "rm -rf {toxinidir}/dist/ && mkdir -p {toxinidir}/dist/"
     python -m pep517.build \