Include global options in command docs

We get a lot of requests from users to include
global options in a command's help page. This PR
fulfils them by pre-generating .rst files for
global options and synopsis and writing them to
commands' help docs.

An alternative that was considered attempted to
plumb the global arg table from the CLIDriver to
any help command that would need it. However, we
abandoned the approach because it was too invasive
to the existing interfaces and there was no easy
way to apply the changes to customizations.

.changes/next-release/enhancement-docs-19006.json

@@ -0,0 +1,5 @@
+{
+  "type": "enhancement",
+  "category": "docs",
+  "description": "Improve AWS CLI docs to include global options available to service commands."
+}

awscli/bcdoc/docevents.py

@@ -23,6 +23,7 @@
     'doc-option': '.%s.%s',
     'doc-option-example': '.%s.%s',
     'doc-options-end': '.%s',
+    'doc-global-option': '.%s',
     'doc-examples': '.%s',
     'doc-output': '.%s',
     'doc-subitems-start': '.%s',
@@ -74,6 +75,8 @@ def generate_events(session, help_command):
                          arg_name=arg_name, help_command=help_command)
     session.emit('doc-options-end.%s' % help_command.event_class,
                  help_command=help_command)
+    session.emit('doc-global-option.%s' % help_command.event_class,
+                 help_command=help_command)
     session.emit('doc-subitems-start.%s' % help_command.event_class,
                  help_command=help_command)
     if help_command.command_table:

awscli/bcdoc/restdoc.py

@@ -102,6 +102,11 @@ def remove_last_doc_string(self):
             start, end = self._last_doc_string
             del self._writes[start:end]
 
+    def write_from_file(self, filename):
+        with open(filename, 'r') as f:
+            for line in f.readlines():
+                self.writeln(line.strip())
+
 
 class DocumentStructure(ReSTDocument):
     def __init__(self, name, section_names=None, target='man', context=None):

awscli/clidocs.py

@@ -12,6 +12,7 @@
 # language governing permissions and limitations under the License.
 import logging
 import os
+import re
 from botocore import xform_name
 from botocore.model import StringShape
 from botocore.utils import is_json_value_header
@@ -26,6 +27,11 @@
 )
 
 LOG = logging.getLogger(__name__)
+EXAMPLES_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)),
+                            'examples')
+GLOBAL_OPTIONS_FILE = os.path.join(EXAMPLES_DIR, 'global_options.rst')
+GLOBAL_OPTIONS_SYNOPSIS_FILE = os.path.join(EXAMPLES_DIR,
+                                            'global_synopsis.rst')
 
 
 class CLIDocumentEventHandler(object):
@@ -146,6 +152,8 @@ def doc_synopsis_option(self, arg_name, help_command, **kwargs):
 
     def doc_synopsis_end(self, help_command, **kwargs):
         doc = help_command.doc
+        # Append synopsis for global options.
+        doc.write_from_file(GLOBAL_OPTIONS_SYNOPSIS_FILE)
         doc.style.end_codeblock()
         # Reset the documented arg groups for other sections
         # that may document args (the detailed docs following
@@ -183,6 +191,11 @@ def doc_option(self, arg_name, help_command, **kwargs):
         doc.style.dedent()
         doc.style.new_paragraph()
 
+    def doc_global_option(self, help_command, **kwargs):
+        doc = help_command.doc
+        doc.style.h2('Global Options')
+        doc.write_from_file(GLOBAL_OPTIONS_FILE)
+
     def doc_relateditems_start(self, help_command, **kwargs):
         if help_command.related_items:
             doc = help_command.doc
@@ -297,20 +310,10 @@ def doc_synopsis_end(self, help_command, **kwargs):
         doc.style.new_paragraph()
 
     def doc_options_start(self, help_command, **kwargs):
-        doc = help_command.doc
-        doc.style.h2('Options')
+        pass
 
     def doc_option(self, arg_name, help_command, **kwargs):
-        doc = help_command.doc
-        argument = help_command.arg_table[arg_name]
-        doc.writeln('``%s`` (%s)' % (argument.cli_name,
-                                     argument.cli_type_name))
-        doc.include_doc_string(argument.documentation)
-        if argument.choices:
-            doc.style.start_ul()
-            for choice in argument.choices:
-                doc.style.li(choice)
-            doc.style.end_ul()
+        pass
 
     def doc_subitems_start(self, help_command, **kwargs):
         doc = help_command.doc
@@ -348,6 +351,9 @@ def doc_option_example(self, arg_name, help_command, **kwargs):
     def doc_options_end(self, help_command, **kwargs):
         pass
 
+    def doc_global_option(self, help_command, **kwargs):
+        pass
+
     def doc_description(self, help_command, **kwargs):
         doc = help_command.doc
         service_model = help_command.obj
@@ -384,17 +390,8 @@ def doc_description(self, help_command, **kwargs):
         doc.style.h2('Description')
         doc.include_doc_string(operation_model.documentation)
         self._add_webapi_crosslink(help_command)
-        self._add_top_level_args_reference(help_command)
         self._add_note_for_document_types_if_used(help_command)
 
-    def _add_top_level_args_reference(self, help_command):
-        help_command.doc.writeln('')
-        help_command.doc.write("See ")
-        help_command.doc.style.internal_link(
-            title="'aws help'",
-            page='/reference/index'
-        )
-        help_command.doc.writeln(' for descriptions of global parameters.')
 
     def _add_webapi_crosslink(self, help_command):
         doc = help_command.doc
@@ -592,9 +589,6 @@ def doc_output(self, help_command, event_name, **kwargs):
             for member_name, member_shape in output_shape.members.items():
                 self._doc_member(doc, member_name, member_shape, stack=[])
 
-    def doc_options_end(self, help_command, **kwargs):
-        self._add_top_level_args_reference(help_command)
-
 
 class TopicListerDocumentEventHandler(CLIDocumentEventHandler):
     DESCRIPTION = (
@@ -729,3 +723,44 @@ def _line_has_tag(self, line):
 
     def doc_subitems_start(self, help_command, **kwargs):
         pass
+
+
+class GlobalOptionsDocumenter:
+    """Documenter used to pre-generate global options docs."""
+
+    def __init__(self, help_command):
+        self._help_command = help_command
+
+    def _remove_multilines(self, s):
+        return re.sub(r'\n+', '\n', s)
+
+    def doc_global_options(self):
+        help_command = self._help_command
+        for arg in help_command.arg_table:
+            argument = help_command.arg_table.get(arg)
+            help_command.doc.writeln(
+                f"``{argument.cli_name}`` ({argument.cli_type_name})")
+            help_command.doc.style.indent()
+            help_command.doc.style.new_paragraph()
+            help_command.doc.include_doc_string(argument.documentation)
+            if argument.choices:
+                help_command.doc.style.start_ul()
+                for choice in argument.choices:
+                    help_command.doc.style.li(choice)
+                help_command.doc.style.end_ul()
+            help_command.doc.style.dedent()
+            help_command.doc.style.new_paragraph()
+        global_options = help_command.doc.getvalue().decode('utf-8')
+        return self._remove_multilines(global_options)
+
+    def doc_global_synopsis(self):
+        help_command = self._help_command
+        for arg in help_command.arg_table:
+            argument = help_command.arg_table.get(arg)
+            if argument.cli_type_name == 'boolean':
+                arg_synopsis = f"[{argument.cli_name}]"
+            else:
+                arg_synopsis = f"[{argument.cli_name} <value>]"
+            help_command.doc.writeln(arg_synopsis)
+        global_synopsis = help_command.doc.getvalue().decode('utf-8')
+        return self._remove_multilines(global_synopsis)

awscli/customizations/commands.py

@@ -370,7 +370,6 @@ def doc_description(self, help_command, **kwargs):
         self.doc.style.h2('Description')
         self.doc.write(help_command.description)
         self.doc.style.new_paragraph()
-        self._add_top_level_args_reference(help_command)
 
     def doc_synopsis_start(self, help_command, **kwargs):
         if not help_command.synopsis:
@@ -411,12 +410,16 @@ def doc_synopsis_option(self, arg_name, help_command, **kwargs):
             pass
 
     def doc_synopsis_end(self, help_command, **kwargs):
-        if not help_command.synopsis:
+        if not help_command.synopsis and not help_command.command_table:
             super(BasicDocHandler, self).doc_synopsis_end(
                 help_command=help_command, **kwargs)
         else:
             self.doc.style.end_codeblock()
 
+    def doc_global_option(self, help_command, **kwargs):
+        if not help_command.command_table:
+            super().doc_global_option(help_command, **kwargs)
+
     def doc_examples(self, help_command, **kwargs):
         if help_command.examples:
             self.doc.style.h2('Examples')
@@ -438,6 +441,3 @@ def doc_subitems_end(self, help_command, **kwargs):
 
     def doc_output(self, help_command, event_name, **kwargs):
         pass
-
-    def doc_options_end(self, help_command, **kwargs):
-        self._add_top_level_args_reference(help_command)

awscli/examples/global_options.rst

@@ -0,0 +1,72 @@
+``--debug`` (boolean)
+
+  Turn on debug logging.
+
+``--endpoint-url`` (string)
+
+  Override command's default URL with the given URL.
+
+``--no-verify-ssl`` (boolean)
+
+  By default, the AWS CLI uses SSL when communicating with AWS services. For each SSL connection, the AWS CLI will verify SSL certificates. This option overrides the default behavior of verifying SSL certificates.
+
+``--no-paginate`` (boolean)
+
+  Disable automatic pagination.
+
+``--output`` (string)
+
+  The formatting style for command output.
+
+
+  *   json
+
+  *   text
+
+  *   table
+
+
+``--query`` (string)
+
+  A JMESPath query to use in filtering the response data.
+
+``--profile`` (string)
+
+  Use a specific profile from your credential file.
+
+``--region`` (string)
+
+  The region to use. Overrides config/env settings.
+
+``--version`` (string)
+
+  Display the version of this tool.
+
+``--color`` (string)
+
+  Turn on/off color output.
+
+
+  *   on
+
+  *   off
+
+  *   auto
+
+
+``--no-sign-request`` (boolean)
+
+  Do not sign requests. Credentials will not be loaded if this argument is provided.
+
+``--ca-bundle`` (string)
+
+  The CA certificate bundle to use when verifying SSL certificates. Overrides config/env settings.
+
+``--cli-read-timeout`` (int)
+
+  The maximum socket read time in seconds. If the value is set to 0, the socket read will be blocking and not timeout. The default value is 60 seconds.
+
+``--cli-connect-timeout`` (int)
+
+  The maximum socket connect time in seconds. If the value is set to 0, the socket connect will be blocking and not timeout. The default value is 60 seconds.
+

awscli/examples/global_synopsis.rst

@@ -0,0 +1,14 @@
+[--debug]
+[--endpoint-url <value>]
+[--no-verify-ssl]
+[--no-paginate]
+[--output <value>]
+[--query <value>]
+[--profile <value>]
+[--region <value>]
+[--version <value>]
+[--color <value>]
+[--no-sign-request]
+[--ca-bundle <value>]
+[--cli-read-timeout <value>]
+[--cli-connect-timeout <value>]

scripts/make-global-opts-documentation

@@ -0,0 +1,38 @@
+#!/usr/bin/env python
+"""Generate a global options section.
+
+The purpose of this script is to pre-generate
+global options for the help docs.
+The output of this script is loaded and applied to
+every subcommand's help docs.
+
+"""
+
+import os
+
+from awscli.clidriver import create_clidriver
+from awscli.clidocs import (
+    EXAMPLES_DIR, GLOBAL_OPTIONS_FILE,
+    GLOBAL_OPTIONS_SYNOPSIS_FILE, GlobalOptionsDocumenter
+)
+
+
+def main():
+    if not os.path.isdir(EXAMPLES_DIR):
+        os.makedirs(EXAMPLES_DIR)
+    driver = create_clidriver()
+    options_help_command = driver.create_help_command()
+    synopsis_help_command = driver.create_help_command()
+    options_documenter = GlobalOptionsDocumenter(options_help_command)
+    synopsis_documenter = GlobalOptionsDocumenter(synopsis_help_command)
+
+    with open(GLOBAL_OPTIONS_FILE, 'w') as f:
+        for line in options_documenter.doc_global_options():
+            f.write(line)
+    with open(GLOBAL_OPTIONS_SYNOPSIS_FILE, 'w') as f:
+        for line in synopsis_documenter.doc_global_synopsis():
+            f.write(line)
+
+
+if __name__ == "__main__":
+    main()

tests/functional/docs/test_help_output.py

@@ -46,9 +46,9 @@ def test_output(self):
         self.assert_contains('``--no-paginate``')
         # Arg with choices
         self.assert_contains('``--color``')
-        self.assert_contains('* on')
-        self.assert_contains('* off')
-        self.assert_contains('* auto')
+        self.assert_contains('*   on')
+        self.assert_contains('*   off')
+        self.assert_contains('*   auto')
         # Then we should see the services.
         self.assert_contains('* ec2')
         self.assert_contains('* s3api')
@@ -80,22 +80,23 @@ def test_operation_help_output(self):
         self.assert_contains('Launches the specified number of instances')
         self.assert_contains('``--count`` (string)')
 
-    def test_waiter_does_not_have_duplicate_global_params_link(self):
-        self.driver.main(['ec2', 'wait', 'help'])
-        self.assert_contains_with_count(
-            'for descriptions of global parameters', 1)
-
     def test_custom_service_help_output(self):
         self.driver.main(['s3', 'help'])
         self.assert_contains('.. _cli:aws s3:')
         self.assert_contains('high-level S3 commands')
         self.assert_contains('* cp')
 
+    def test_waiter_does_not_have_global_args(self):
+        self.driver.main(['ec2', 'wait', 'help'])
+        self.assert_not_contains('--debug')
+        self.assert_not_contains('Global Options')
+
     def test_custom_operation_help_output(self):
         self.driver.main(['s3', 'ls', 'help'])
         self.assert_contains('.. _cli:aws s3 ls:')
         self.assert_contains('List S3 objects')
         self.assert_contains('--summarize')
+        self.assert_contains('--debug')
 
     def test_topic_list_help_output(self):
         self.driver.main(['help', 'topics'])