Merge pull request 'Handle case of absent instance_config.yml gracefully' (#13) from improved-failure-handling into main

Reviewed-on: #13

.drone.yml

@@ -81,7 +81,7 @@ steps:
         from_secret: HCLOUD_TOKEN
     commands:
       - pip install -e .
-      - pip install "ansible>=2.10, <2.11"
+      - pip install "ansible>=2.10, <2.11" netaddr
       - export INSTANCE_UUID=$(openssl rand -hex 5)
       - cd integration && molecule test
 depends_on:

CHANGELOG.md

@@ -7,11 +7,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [1.2.1] - 2021-06-02
+
+### Fixed
+
+- Remove async task handling for network deletion ([#30](https://github.com/ansible-community/molecule-hetznercloud/pull/30), credit @ggggut)
+
+## [1.2.0] - 2021-06-02
+
+### Added
+
+- Allow to create networks during test runs ([#29](https://github.com/ansible-community/molecule-hetznercloud/pull/29), thanks @ggggut!)
+
 ## [1.1.0] - 2021-03-30
 
 ## Changed
 
-- Relaxed bounds on Molecule to allow allow all versions < v4 ([#27](https://github.com/ansible-community/molecule-hetznercloud/pull/27))
+- Relaxed bounds on Molecule to allow all versions less than `v4` ([#27](https://github.com/ansible-community/molecule-hetznercloud/pull/27))
 
 ## [1.0.0] - 2021-01-06
 
@@ -29,7 +41,7 @@ tracker so that we can mention them here.
 
 ## [0.2.2] - 2020-06-15
 
-## Fixed
+### Fixed
 
 - Point to an open issue tracker
 

README.md

@@ -27,6 +27,8 @@ any financial support from RedHat or Hetzner Cloud for this work.
 $ pip install molecule-hetznercloud
 ```
 
+If you're looking for a container approach, see [ansible-community/toolset](https://github.com/ansible-community/toolset).
+
 ## Upgrade
 
 Please see the [CHANGELOG.md](./CHANGELOG.md) for migration guides.
@@ -87,20 +89,22 @@ $ export MOLECULE_DEBUG=True  # very verbose, last ditch effort
 
 ## Volume Handling
 
-It is possible to have the driver manage volumes during the test run.
+> **WARNING**: this feature appears to be broke. See [#24](https://github.com/ansible-community/molecule-hetznercloud/issues/24) for more
 
-You can add the following stanza to your Molecule configuration to have
+It is possible to have the driver manage volumes during the test run. You can
-Molecule create this volume for the managed VPS. This volume will be cleaned up
+add the following stanza to your Molecule configuration to have Molecule create
-after use (**Please note**: there is a bug raised against clean-up right now,
+this volume for the managed VPS. This volume will be cleaned up after use.
-see [#24](https://github.com/ansible-community/molecule-hetznercloud/issues/24)
-for more).
 
 ```yaml
-volumes:
+platforms:
-  - name: "molecule-hetznercloud-volume-1-${INSTANCE_UUID}"
+  - name: instance
-    location: /foo/bar
+    server_type: cx11
-  - name: "molecule-hetznercloud-volume-2-${INSTANCE_UUID}"
+    image: debian-10
-    size: 20
+    volumes:
+      - name: "molecule-hetznercloud-volume-1-${INSTANCE_UUID}"
+        location: /foo/bar
+      - name: "molecule-hetznercloud-volume-2-${INSTANCE_UUID}"
+        size: 20
 ```
 
 Supported keys are:
@@ -109,6 +113,48 @@ Supported keys are:
 - **size** (optional, default: `10GB`): size of volume
 - **location** (optional, default: `omitted`): path for volume
 
+## Network Creation
+
+This Driver is able to generate networks and subnetworks during the test run.
+This can be useful for cluster tests. You can create networks with the
+following snippet:
+
+```yaml
+platforms:
+  - name: instance1
+    server_type: cx11
+    image: debian-10
+    networks:
+      test-network:
+        ip_range: 10.10.0.0/16
+        subnet:
+          ip: 10.10.10.1/24
+          type: cloud
+          network_zone: eu-central
+      test-network-2:
+        ip_range: 10.20.0.0/16
+        subnet:
+          ip: 10.20.10.1/24
+  - name: instance2
+    server_type: cx11
+    image: debian-10
+    networks:
+      test-network:
+        subnet:
+          ip: 10.10.10.2/24
+```
+
+The networks **ip_range** is only important for creating. If you have multiple
+hosts, it is okay to only define **ip_range** once. The supported keys are:
+
+- **networks**
+    - **ip_range** (required): ip range of network (usually `/16`)
+
+- **subnet**
+    - **ip** (required): ip that should be assigned to host (also generates subnetwork) - prefix mandatory
+    - **type** (optional, default: `cloud`): type of subnetwork
+    - **network_zone** (optional, default: `eu-central`): network zone of subnetwork
+
 ## Only use `molecule.yml` for configuration
 
 It is being worked on that it is possible to remove all the files except the
@@ -137,7 +183,7 @@ See [CHANGELOG.md](./CHANGELOG.md).
 
 ## Contact
 
-- Ping @decentral1se on the `#ansible-molecule` channel on [Freenode](https://webchat.freenode.net).
+- Ping @decentral1se on the `#ansible-molecule` channel on [Libera](https://libera.chat/).
 
 ## License
 
@@ -156,12 +202,12 @@ $ tox -v
 
 ### Integration
 
-Only doable by [Autonomic Cooperative](https://autonomic.zone/) members.
+```
-
+git clone https://github.com/ansible-community/molecule-hetznercloud.git
-```bash
+cd molecule-hetznercloud
-$ sudo apt install -y direnv
+python3 -m venv .venv && source .venv/bin/activate
-$ cp .envrc.sample .envrc
+pip install -e . "ansible<4" netaddr
-$ direnv allow
+export INSTANCE_UUID=$(openssl rand -hex 5)
-$ pip install -e . ansible
+export HCLOUD_TOKEN=YOURKEY
-$ cd integration && molecule test
+cd integration && molecule test
 ```

integration/molecule/default/molecule.yml

@@ -7,9 +7,21 @@ platforms:
   - name: "molecule-hetznercloud-${INSTANCE_UUID}"
     server_type: cx11
     image: debian-10
-    volumes:
+    # https://github.com/ansible-community/molecule-hetznercloud/issues/24
-      - name: "molecule-hetznercloud-volume-1-${INSTANCE_UUID}"
+    # volumes:
-      # - name: "molecule-hetznercloud-volume-2-${INSTANCE_UUID}"
+    # - name: "molecule-hetznercloud-volume-1-${INSTANCE_UUID}"
+    # - name: "molecule-hetznercloud-volume-2-${INSTANCE_UUID}"
+    networks:
+      molecule-hetznercloud-network-1:
+        ip_range: 10.10.0.0/16
+        subnet:
+          ip: 10.10.10.1/24
+          type: cloud
+          network_zone: eu-central
+      molecule-hetznercloud-network-2:
+        ip_range: 10.20.0.0/16
+        subnet:
+          ip: 10.20.10.1/24
 provisioner:
   name: ansible
 verifier:

molecule_hetznercloud/cookiecutter/scenario/driver/hetznercloud/{{cookiecutter.molecule_directory}}/{{cookiecutter.scenario_name}}/create.yml

@@ -77,6 +77,48 @@
       when: volumes.changed
       with_items: "{{ volumes.results }}"
 
+    - name: Create private network(s)
+      hcloud_network:
+        name: "{{ item.name }}"
+        ip_range: "{{ item.ip_range | default(omit) }}"
+        api_token: "{{ lookup('env', 'HCLOUD_TOKEN') }}"
+        state: "present"
+      loop: "{{ molecule_yml.platforms|molecule_get_hetznercloud_networks('networks') }}"
+      register: networks
+      async: 7200
+      poll: 0
+
+    - name: Wait for network(s) creation to complete
+      async_status:
+        jid: "{{ item.ansible_job_id }}"
+      register: hetzner_networks
+      until: hetzner_networks.finished
+      retries: 300
+      when:
+        - networks is defined
+        - networks.changed
+      with_items: "{{ networks.results }}"
+
+    - name: Create private subnetwork(s)
+      hcloud_subnetwork:
+        network: "{{ item.network_name }}"
+        ip_range: "{{ item.ip|ipaddr('network/prefix') }}"
+        network_zone: "{{ item.network_zone | default('eu-central') }}"
+        type: "{{ item.type | default('cloud') }}"
+        api_token: "{{ lookup('env', 'HCLOUD_TOKEN') }}"
+        state: "present"
+      loop: "{{ molecule_yml.platforms|molecule_get_hetznercloud_networks('subnetworks') }}"
+      register: subnetworks
+
+    - name: Attach Server to Subnetwork(s)
+      hcloud_server_network:
+        network: "{{ item.network_name }}"
+        server: "{{ item.server_name }}"
+        ip: "{{ item.ip|ipaddr('address') }}"
+        api_token: "{{ lookup('env', 'HCLOUD_TOKEN') }}"
+        state: "present"
+      loop: "{{ molecule_yml.platforms|molecule_get_hetznercloud_networks('subnetworks') }}"
+
     - name: Populate instance config dict
       set_fact:
         instance_conf_dict: {
@@ -86,7 +128,8 @@
           'user': "{{ ssh_user }}",
           'port': "{{ ssh_port }}",
           'identity_file': "{{ ssh_path }}",
-          'volumes': "{{ item.item.item.volumes | default({}) }}", }
+          'volumes': "{{ item.item.item.volumes | default({}) }}",
+          'networks': "{{ item.item.item.networks | default({}) | dict2items(key_name='name', value_name='data') }}", }
       with_items: "{{ hetzner_jobs.results }}"
       register: instance_config_dict
       when: server.changed | bool

molecule_hetznercloud/cookiecutter/scenario/driver/hetznercloud/{{cookiecutter.molecule_directory}}/{{cookiecutter.scenario_name}}/destroy.yml

@@ -60,6 +60,14 @@
       when: volumes.changed
       with_items: "{{ volumes.results }}"
 
+    - name: Destroy network(s)
+      hcloud_network:
+        name: "{{ item.1.name }}"
+        api_token: "{{ lookup('env', 'HCLOUD_TOKEN') }}"
+        state: absent
+      register: networks
+      loop: "{{ instance_conf|subelements('networks', skip_missing=True) }}"
+
     - name: Remove registered SSH key
       hcloud_ssh_key:
         name: "{{ instance_conf[0].ssh_key_name }}"

molecule_hetznercloud/cookiecutter/scenario/driver/hetznercloud/{{cookiecutter.molecule_directory}}/{{cookiecutter.scenario_name}}/molecule.yml

@@ -7,6 +7,19 @@ platforms:
   - name: "{{ cookiecutter.role_name }}"
     server_type: cx11
     image: debian-10
+    volumes:
+      - name: "molecule-hetznercloud-volume-1"
+    networks:
+      molecule-hetznercloud-network-1:
+        ip_range: 10.10.0.0/16
+        subnet:
+          ip: 10.10.10.1/24
+          type: cloud
+          network_zone: eu-central
+      molecule-hetznercloud-network-2:
+        ip_range: 10.20.0.0/16
+        subnet:
+          ip: 10.20.10.1/24
 provisioner:
   name: ansible
 lint: |

molecule_hetznercloud/playbooks/create.yml

@@ -75,6 +75,48 @@
         - volumes.changed
       with_items: "{{ volumes.results }}"
 
+    - name: Create private network(s)
+      hcloud_network:
+        name: "{{ item.name }}"
+        ip_range: "{{ item.ip_range | default(omit) }}"
+        api_token: "{{ lookup('env', 'HCLOUD_TOKEN') }}"
+        state: "present"
+      loop: "{{ molecule_yml.platforms|molecule_get_hetznercloud_networks('networks') }}"
+      register: networks
+      async: 7200
+      poll: 0
+
+    - name: Wait for network(s) creation to complete
+      async_status:
+        jid: "{{ item.ansible_job_id }}"
+      register: hetzner_networks
+      until: hetzner_networks.finished
+      retries: 300
+      when:
+        - networks is defined
+        - networks.changed
+      with_items: "{{ networks.results }}"
+
+    - name: Create private subnetwork(s)
+      hcloud_subnetwork:
+        network: "{{ item.network_name }}"
+        ip_range: "{{ item.ip|ipaddr('network/prefix') }}"
+        network_zone: "{{ item.network_zone | default('eu-central') }}"
+        type: "{{ item.type | default('cloud') }}"
+        api_token: "{{ lookup('env', 'HCLOUD_TOKEN') }}"
+        state: "present"
+      loop: "{{ molecule_yml.platforms|molecule_get_hetznercloud_networks('subnetworks') }}"
+      register: subnetworks
+
+    - name: Attach Server to Subnetwork(s)
+      hcloud_server_network:
+        network: "{{ item.network_name }}"
+        server: "{{ item.server_name }}"
+        ip: "{{ item.ip|ipaddr('address') }}"
+        api_token: "{{ lookup('env', 'HCLOUD_TOKEN') }}"
+        state: "present"
+      loop: "{{ molecule_yml.platforms|molecule_get_hetznercloud_networks('subnetworks') }}"
+
     - name: Populate instance config dict
       set_fact:
         instance_conf_dict:
@@ -86,6 +128,7 @@
             "port": "{{ ssh_port }}",
             "identity_file": "{{ ssh_path }}",
             "volumes": "{{ item.item.item.volumes | default({}) }}",
+            "networks": "{{ item.item.item.networks | default({}) | dict2items(key_name='name', value_name='data') }}",
           }
       with_items: "{{ hetzner_jobs.results }}"
       register: instance_config_dict

molecule_hetznercloud/playbooks/destroy.yml

@@ -6,16 +6,16 @@
   no_log: "{{ molecule_no_log }}"
   tasks:
     - name: Populate the instance config
-      block:
+      set_fact:
-        - name: Populate instance config from file
+        instance_conf: "{{ lookup('file', molecule_instance_config, errors='warn') | from_yaml }}"
-          set_fact:
+        skip_instances: false
-            instance_conf: "{{ lookup('file', molecule_instance_config) | from_yaml }}"
+      register: instance_config_lookup
-            skip_instances: false
+
-      rescue:
+    - name: Populate instance config when file missing
-        - name: Populate instance config when file missing
+      set_fact:
-          set_fact:
+        instance_conf: {}
-            instance_conf: {}
+        skip_instances: true
-            skip_instances: true
+      when: not instance_config_lookup.ansible_facts.instance_conf
 
     - name: Destroy molecule instance(s)
       hcloud_server:
@@ -56,6 +56,14 @@
       when: volumes.changed
       with_items: "{{ volumes.results }}"
 
+    - name: Destroy network(s)
+      hcloud_network:
+        name: "{{ item.1.name }}"
+        api_token: "{{ lookup('env', 'HCLOUD_TOKEN') }}"
+        state: absent
+      register: networks
+      loop: "{{ instance_conf|subelements('networks', skip_missing=True) }}"
+
     - name: Remove registered SSH key
       hcloud_ssh_key:
         name: "{{ instance_conf[0].ssh_key_name }}"

molecule_hetznercloud/playbooks/filter_plugins/get_hetznercloud_networks.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+"""Usage:"""
+""" loop: "{{ molecule_yml.platforms|molecule_get_hetznercloud_networks('networks') }}" """  # noqa
+""" loop: "{{ molecule_yml.platforms|molecule_get_hetznercloud_networks('subnetworks') }}" """  # noqa
+
+
+def merge_two_dicts(x, y):
+    z = x.copy()
+    z.update(y)
+    return z
+
+
+def get_hetznercloud_networks(data, request):
+    network_list = {}
+    subnetwork_list = []
+
+    if request == "networks":
+
+        for platform in data:
+            if "networks" in platform:
+                for network_name, values in platform["networks"].items():
+                    del values["subnet"]
+                    values["name"] = network_name
+                    if network_name in network_list:
+                        network_list[network_name] = merge_two_dicts(
+                            network_list[network_name], values
+                        )
+                    else:
+                        network_list[network_name] = values
+
+        return [x for x in network_list.values()]
+
+    elif request == "subnetworks":
+
+        for platform in data:
+            name = platform["name"]
+            if "networks" in platform:
+                for network_name, values in platform["networks"].items():
+                    values["name"] = network_name
+                    if "subnet" in values:
+                        values["subnet"]["server_name"] = name
+                        values["subnet"]["network_name"] = network_name
+                        subnetwork_list.append(values["subnet"])
+
+        return subnetwork_list
+
+
+class FilterModule(object):
+    """Core Molecule filter plugins."""
+
+    def filters(self):
+        return {
+            "molecule_get_hetznercloud_networks": get_hetznercloud_networks,
+        }

molecule_hetznercloud/test/resources/playbooks/hetznercloud/create.yml

@@ -76,6 +76,48 @@
       when: volumes.changed
       with_items: "{{ volumes.results }}"
 
+    - name: Create private network(s)
+      hcloud_network:
+        name: "{{ item.name }}"
+        ip_range: "{{ item.ip_range | default(omit) }}"
+        api_token: "{{ lookup('env', 'HCLOUD_TOKEN') }}"
+        state: "present"
+      loop: "{{ molecule_yml.platforms|molecule_get_hetznercloud_networks('networks') }}"
+      register: networks
+      async: 7200
+      poll: 0
+
+    - name: Wait for network(s) creation to complete
+      async_status:
+        jid: "{{ item.ansible_job_id }}"
+      register: hetzner_networks
+      until: hetzner_networks.finished
+      retries: 300
+      when:
+        - networks is defined
+        - networks.changed
+      with_items: "{{ networks.results }}"
+
+    - name: Create private subnetwork(s)
+      hcloud_subnetwork:
+        network: "{{ item.network_name }}"
+        ip_range: "{{ item.ip|ipaddr('network/prefix') }}"
+        network_zone: "{{ item.network_zone | default('eu-central') }}"
+        type: "{{ item.type | default('cloud') }}"
+        api_token: "{{ lookup('env', 'HCLOUD_TOKEN') }}"
+        state: "present"
+      loop: "{{ molecule_yml.platforms|molecule_get_hetznercloud_networks('subnetworks') }}"
+      register: subnetworks
+
+    - name: Attach Server to Subnetwork(s)
+      hcloud_server_network:
+        network: "{{ item.network_name }}"
+        server: "{{ item.server_name }}"
+        ip: "{{ item.ip|ipaddr('address') }}"
+        api_token: "{{ lookup('env', 'HCLOUD_TOKEN') }}"
+        state: "present"
+      loop: "{{ molecule_yml.platforms|molecule_get_hetznercloud_networks('subnetworks') }}"
+
     - name: Populate instance config dict
       set_fact:
         instance_conf_dict: {
@@ -85,7 +127,8 @@
           'user': "{{ ssh_user }}",
           'port': "{{ ssh_port }}",
           'identity_file': "{{ ssh_path }}",
-          'volumes': "{{ item.item.item.volumes | default({}) }}", }
+          'volumes': "{{ item.item.item.volumes | default({}) }}",
+          'networks': "{{ item.item.item.networks | default({}) | dict2items(key_name='name', value_name='data') }}", }
       with_items: "{{ hetzner_jobs.results }}"
       register: instance_config_dict
       when: server.changed | bool

molecule_hetznercloud/test/resources/playbooks/hetznercloud/destroy.yml

@@ -59,6 +59,14 @@
       when: volumes.changed
       with_items: "{{ volumes.results }}"
 
+    - name: Destroy network(s)
+      hcloud_network:
+        name: "{{ item.1.name }}"
+        api_token: "{{ lookup('env', 'HCLOUD_TOKEN') }}"
+        state: absent
+      register: networks
+      loop: "{{ instance_conf|subelements('networks', skip_missing=True) }}"
+
     - name: Remove registered SSH key
       hcloud_ssh_key:
         name: "{{ instance_conf[0].ssh_key_name }}"