Terraform init -upgrade flag (#4455)

* Adds optional `-upgrade` flag to terraform init.

This allows Terraform to install provider dependencies into an existing project when the provider constraints change.

* fix transposed documentation keys

* Add integration tests for terraform init

* Revert to validate_certs: yes for general public testing

* skip integration tests on irrelevant platforms

* skip legacy Python versions from CI tests

* add changelog fragment

* Update plugins/modules/cloud/misc/terraform.py

Adds version_added metadata to the new module option.

Co-authored-by: Felix Fontein <felix@fontein.de>

* Change terraform_arch constant to Ansible fact mapping

* correct var typo, clarify task purpose

* Squashed some logic bugs, added override for local Terraform

If `existing_terraform_path` is provided, the playbook will not download Terraform or check its version.

I also tested this on a local system with Terraform installed, and squashed some bugs related to using of an
existing binary.

* revert to previous test behavior for TF install

* readability cleanup

* Update plugins/modules/cloud/misc/terraform.py

Co-authored-by: Felix Fontein <felix@fontein.de>

changelogs/fragments/4455-terraform-provider-upgrade.yml

@@ -0,0 +1,3 @@
+---
+minor_changes:
+  - terraform - adds ``terraform_upgrade`` parameter which allows ``terraform init`` to satisfy new provider constraints in an existing Terraform project (https://github.com/ansible-collections/community.general/issues/4333).

plugins/modules/cloud/misc/terraform.py

@@ -124,6 +124,12 @@
     type: list
     elements: path
     version_added: '0.2.0'
+  provider_upgrade:
+    description:
+      - Allows Terraform init to upgrade providers to versions specified in the project's version constraints.
+    default: false
+    type: bool
+    version_added: 4.8.0
   init_reconfigure:
     description:
       - Forces backend reconfiguration during init.
@@ -266,7 +272,7 @@ def _state_args(state_file):
     return []
 
 
-def init_plugins(bin_path, project_path, backend_config, backend_config_files, init_reconfigure, plugin_paths):
+def init_plugins(bin_path, project_path, backend_config, backend_config_files, init_reconfigure, provider_upgrade, plugin_paths):
     command = [bin_path, 'init', '-input=false']
     if backend_config:
         for key, val in backend_config.items():
@@ -279,6 +285,8 @@ def init_plugins(bin_path, project_path, backend_config, backend_config_files, i
             command.extend(['-backend-config', f])
     if init_reconfigure:
         command.extend(['-reconfigure'])
+    if provider_upgrade:
+        command.extend(['-upgrade'])
     if plugin_paths:
         for plugin_path in plugin_paths:
             command.extend(['-plugin-dir', plugin_path])
@@ -384,6 +392,7 @@ def main():
             overwrite_init=dict(type='bool', default=True),
             check_destroy=dict(type='bool', default=False),
             parallelism=dict(type='int'),
+            provider_upgrade=dict(type='bool', default=False),
         ),
         required_if=[('state', 'planned', ['plan_file'])],
         supports_check_mode=True,
@@ -405,6 +414,7 @@ def main():
     init_reconfigure = module.params.get('init_reconfigure')
     overwrite_init = module.params.get('overwrite_init')
     check_destroy = module.params.get('check_destroy')
+    provider_upgrade = module.params.get('provider_upgrade')
 
     if bin_path is not None:
         command = [bin_path]
@@ -422,7 +432,7 @@ def main():
 
     if force_init:
         if overwrite_init or not os.path.isfile(os.path.join(project_path, ".terraform", "terraform.tfstate")):
-            init_plugins(command[0], project_path, backend_config, backend_config_files, init_reconfigure, plugin_paths)
+            init_plugins(command[0], project_path, backend_config, backend_config_files, init_reconfigure, provider_upgrade, plugin_paths)
 
     workspace_ctx = get_workspace_context(command[0], project_path)
     if workspace_ctx["current"] != workspace:

tests/integration/targets/terraform/.gitignore

@@ -0,0 +1,4 @@
+**/.terraform/*
+*.tfstate
+*.tfstate.*
+.terraform.lock.hcl

tests/integration/targets/terraform/aliases

@@ -0,0 +1,7 @@
+shippable/posix/group1
+skip/windows
+skip/aix
+skip/osx
+skip/macos
+skip/freebsd
+skip/python2

tests/integration/targets/terraform/meta/main.yml

@@ -0,0 +1,3 @@
+dependencies:
+  - setup_pkg_mgr
+  - setup_remote_tmp_dir

tests/integration/targets/terraform/tasks/main.yml

@@ -0,0 +1,70 @@
+---
+
+
+# This block checks and registers Terraform version of the binary found in path.
+
+- name: Check for existing Terraform in path
+  block:
+  - name: Check if terraform is present in path
+    command: "command -v terraform"
+    register: terraform_binary_path
+    ignore_errors: true
+
+  - name: Check Terraform version
+    command: terraform version
+    register: terraform_version_output
+    when: terraform_binary_path.rc == 0
+
+  - name: Set terraform version
+    set_fact:
+      terraform_version_installed: "{{ terraform_version_output.stdout | regex_search('(?!Terraform.*v)([0-9]+\\.[0-9]+\\.[0-9]+)') }}"
+    when: terraform_version_output.changed
+
+# This block handles the tasks of installing the Terraform binary. This happens if there is no existing
+# terraform in $PATH OR version does not match `terraform_version`.
+
+- name: Execute Terraform install tasks
+  block:
+
+  - name: Install Terraform
+    debug:
+      msg: "Installing terraform {{ terraform_version }}, found: {{ terraform_version_installed | default('no terraform binary found') }}."
+
+  - name: Ensure unzip is present
+    ansible.builtin.package:
+      name: unzip
+      state: present
+
+  - name: Install Terraform binary
+    unarchive:
+      src: "{{ terraform_url }}"
+      dest: "{{ remote_tmp_dir }}"
+      mode: 0755
+      remote_src: yes
+      validate_certs: "{{ validate_certs }}"
+
+  when: terraform_version_installed is not defined or terraform_version_installed != terraform_version
+
+# This sets `terraform_binary_path` to coalesced output of first non-empty string in this order:
+# path from the 'Check if terraform is present in path' task, and lastly, the fallback path.
+
+- name: Set path to terraform binary
+  set_fact:
+    terraform_binary_path: "{{ terraform_binary_path.stdout or remote_tmp_dir ~ '/terraform' }}"
+
+- name: Create terraform project directory
+  file:
+    path: "{{ terraform_project_dir }}/{{ item['name'] }}"
+    state: directory
+    mode: 0755
+  loop: "{{ terraform_provider_versions }}"
+  loop_control:
+    index_var: provider_index
+
+- name: Loop over provider upgrade test tasks
+  include_tasks: test_provider_upgrade.yml
+  vars:
+    tf_provider: "{{ terraform_provider_versions[provider_index] }}"
+  loop: "{{ terraform_provider_versions }}"
+  loop_control:
+    index_var: provider_index

tests/integration/targets/terraform/tasks/test_provider_upgrade.yml

@@ -0,0 +1,23 @@
+---
+
+- name: Output terraform provider test project
+  ansible.builtin.template:
+    src: templates/provider_test/main.tf.j2
+    dest: "{{ terraform_project_dir }}/{{ tf_provider['name'] }}/main.tf"
+    force: yes
+  register: terraform_provider_hcl
+
+# The purpose of this task is to init terraform multiple times with different provider module
+# versions, so that we can verify that provider upgrades during init work as intended.
+
+- name: Init Terraform configuration with pinned provider version
+  community.general.terraform:
+    project_path: "{{ terraform_provider_hcl.dest | dirname }}"
+    binary_path: "{{ terraform_binary_path }}"
+    force_init: yes
+    provider_upgrade: "{{ terraform_provider_upgrade }}"
+    state: present
+  register: terraform_init_result
+
+- assert:
+    that: terraform_init_result is not failed

tests/integration/targets/terraform/templates/provider_test/main.tf.j2

@@ -0,0 +1,8 @@
+terraform {
+  required_providers {
+    {{ tf_provider['name'] }} = {
+      source  = "{{ tf_provider['source'] }}"
+      version = "{{ tf_provider['version'] }}"
+    }
+  }
+}

tests/integration/targets/terraform/vars/main.yml

@@ -0,0 +1,37 @@
+---
+
+# Terraform version that will be downloaded
+terraform_version: 1.1.7
+
+# Architecture of the downloaded Terraform release (needs to match target testing platform)
+
+terraform_arch: "{{ ansible_system | lower }}_{{terraform_arch_map[ansible_architecture] }}"
+
+# URL of where the Terraform binary will be downloaded from
+terraform_url: "https://releases.hashicorp.com/terraform/{{ terraform_version }}/terraform_{{ terraform_version }}_{{ terraform_arch }}.zip"
+
+# Controls whether the unarchive task will validate TLS certs of the Terraform binary host
+validate_certs: yes
+
+# Directory where Terraform tests will be created
+terraform_project_dir: "{{ remote_tmp_dir }}/tf_provider_test"
+
+# Controls whether terraform init will use the `-upgrade` flag
+terraform_provider_upgrade: yes
+
+# list of dicts containing Terraform providers that will be tested
+# The null provider is a good candidate, as it's small and has no external dependencies
+terraform_provider_versions: 
+  - name: "null" 
+    source: "hashicorp/null"
+    version: ">=2.0.0, < 3.0.0"
+  - name: "null" 
+    source: "hashicorp/null"
+    version: ">=3.0.0"
+
+# mapping between values returned from ansible_architecture and arch names used by golang builds of Terraform
+# see https://www.terraform.io/downloads
+
+terraform_arch_map:
+  x86_64: amd64
+  arm64: arm64