Squashed 'manage_externals/' changes from a3b3a0373..82d3b247f

82d3b247f Merge pull request #178 from johnpaulalex/test_doc
3223f49ea Additional documentation of system tests - global variables, method descriptions.
45b7c01c3 Merge pull request #177 from jedwards4b/git_workflow
ace90b2c2 try setting credentials this way
f4d6aa933 try setting credentials this way
1d61a6944 use this to set git credentials
7f9d330e1 use this to set git credentials
5ac731b85 add tmate code
836847be7 get git workflow working
dcd462d71 Merge pull request #176 from jedwards4b/add_github_testing
2d2479e9d Merge pull request #175 from johnpaulalex/fix
711a53fdf add github testing of prs and automatic tagging of main
cfe0f888a fix typos
5665d6140 Fix broken checkout behavior introduced by PR #172.
27909e255 Merge pull request #173 from johnpaulalex/readall
00ad0440b Further tiny refactorings and docs of checkout API (no-op).    Remove unused load_all param in _External.checkout().    Rename _External.checkout_externals() to checkout_subexternals(), to remove the ambiguity about whether the main external pointed to by the _External is itelf checked out (it is not)    Clarify load_all documentation - it’s always recursive, but applies different criteria at each level.    Rename variables in checkout.py (e.g. ext_description)  to match the equivalent code in sourcetree.py.
2ea3d1a3a Merge pull request #172 from johnpaulalex/fixit
43bf8092c Merge pull request #171 from johnpaulalex/docstatus
e6aa7d21e Merge pull request #170 from johnpaulalex/printdir
adbd71557 On checkout, refresh locally installed optional packages regardless of whether -o is passed in.
add074593 Comment tweaks, and fix 'ppath' typo
696527cb8 Document the format of various status dictionaries, and the various paths and path components within an _External.
c677b9403 When processing an external, print out its path in addition to the base filename (to disambiguate all the externals.cfg's)
975d7fd5a Merge pull request #169 from johnpaulalex/docfix_branch
09709e36d Document _Externals.status().  The original comment was apparently copy-pasted from checkout().
1d880e090 Merge pull request #167 from billsacks/fix_svn_on_windows
3510da848 Tweak a unit test to improve coverage
eb7fc1368 Handle the possibility that the URL already ends with '/'
02ea87e3d Fix svn URLs on Windows
b1c02ab54 Merge pull request #165 from gold2718/doc_fix
9f4be8c7b Add documentation about externals = None feature

git-subtree-dir: manage_externals
git-subtree-split: 82d3b247fc8fb3b70458bf2b4570ee6678754c4d

.github/workflows/bumpversion.yml

@@ -0,0 +1,19 @@
+name: Bump version
+on:
+  push:
+    branches:
+      - main
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Bump version and push tag
+        id: tag_version
+        uses: mathieudutour/github-tag-action@v5.5
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          create_annotated_tag: true
+          default_bump: patch
+          dry_run: false
+          tag_prefix: manic-

.github/workflows/tests.yml

@@ -0,0 +1,30 @@
+# This is a workflow to compile the cmeps source without cime
+name: Test Manic
+
+# Controls when the action will run. Triggers the workflow on push or pull request
+# events but only for the master branch
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+  test-manic:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Test Manic
+        run: |
+          pushd test
+          git config --global user.email "devnull@example.com"
+          git config --global user.name "GITHUB tester"
+          git config --global protocol.file.allow always
+          make utest
+          make stest
+          popd
+
+      - name: Setup tmate session
+        if: ${{ failure() }}
+        uses: mxschmitt/action-tmate@v3

README.md

@@ -201,6 +201,11 @@ The root of the source tree will be referred to as `${SRC_ROOT}` below.
     externals description file pointed 'useful_library/sub-xternals.cfg',
     Then the main 'externals' field in the top level repo should point to
     'sub-externals.cfg'.
+    Note that by default, `checkout_externals` will clone an external's
+    submodules. As a special case, the entry, `externals = None`, will
+    prevent this behavior. For more control over which externals are
+    checked out, create an externals file (and see the `from_submodule`
+    configuration entry below).
 
   * from_submodule (True / False) : used to pull the repo_url, local_path,
     and hash properties for this external from the .gitmodules file in

manic/checkout.py

@@ -227,6 +227,12 @@ def commandline_arguments(args=None):
     Now, %(prog)s will process Externals.cfg and also process
     Externals_LIBX.cfg as if it was a sub-external.
 
+    Note that by default, checkout_externals will clone an external's
+    submodules. As a special case, the entry, "externals = None", will
+    prevent this behavior. For more control over which externals are
+    checked out, create an externals file (and see the from_submodule
+    configuration entry below).
+
   * from_submodule (True / False) : used to pull the repo_url, local_path,
     and hash properties for this external from the .gitmodules file in
     this repository. Note that the section name (the entry in square
@@ -332,7 +338,34 @@ def commandline_arguments(args=None):
         options = parser.parse_args()
     return options
 
-
+def _dirty_local_repo_msg(program_name, config_file):
+    return """The external repositories labeled with 'M' above are not in a clean state.
+The following are four options for how to proceed:
+(1) Go into each external that is not in a clean state and issue either a 'git status' or
+    an 'svn status' command (depending on whether the external is managed by git or
+    svn). Either revert or commit your changes so that all externals are in a clean
+    state. (To revert changes in git, follow the instructions given when you run 'git
+    status'.) (Note, though, that it is okay to have untracked files in your working
+    directory.) Then rerun {program_name}.
+(2) Alternatively, you do not have to rely on {program_name}. Instead, you can manually
+    update out-of-sync externals (labeled with 's' above) as described in the
+    configuration file {config_file}. (For example, run 'git fetch' and 'git checkout'
+    commands to checkout the appropriate tags for each external, as given in
+    {config_file}.)
+(3) You can also use {program_name} to manage most, but not all externals: You can specify
+    one or more externals to ignore using the '-x' or '--exclude' argument to
+    {program_name}. Excluding externals labeled with 'M' will allow {program_name} to
+    update the other, non-excluded externals.
+(4) As a last resort, if you are confident that there is no work that needs to be saved
+    from a given external, you can remove that external (via "rm -rf [directory]") and
+    then rerun the {program_name} tool. This option is mainly useful as a workaround for
+    issues with this tool (such as https://github.com/ESMCI/manage_externals/issues/157).
+The external repositories labeled with '?' above are not under version
+control using the expected protocol. If you are sure you want to switch
+protocols, and you don't have any work you need to save from this
+directory, then run "rm -rf [directory]" before rerunning the
+{program_name} tool.
+""".format(program_name=program_name, config_file=config_file)
 # ---------------------------------------------------------------------
 #
 # main
@@ -363,19 +396,25 @@ def main(args):
         load_all = True
 
     root_dir = os.path.abspath(os.getcwd())
-    external_data = read_externals_description_file(root_dir, args.externals)
-    external = create_externals_description(
-        external_data, components=args.components, exclude=args.exclude)
+    model_data = read_externals_description_file(root_dir, args.externals)
+    ext_description = create_externals_description(
+        model_data, components=args.components, exclude=args.exclude)
 
     for comp in args.components:
-        if comp not in external.keys():
+        if comp not in ext_description.keys():
+            # Note we can't print out the list of found externals because
+            # they were filtered in create_externals_description above.
             fatal_error(
                 "No component {} found in {}".format(
                     comp, args.externals))
 
-    source_tree = SourceTree(root_dir, external, svn_ignore_ancestry=args.svn_ignore_ancestry)
-    printlog('Checking status of externals: ', end='')
-    tree_status = source_tree.status()
+    source_tree = SourceTree(root_dir, ext_description, svn_ignore_ancestry=args.svn_ignore_ancestry)
+    if args.components:
+        components_str = 'specified components'
+    else:
+        components_str = 'required & optional components'
+    printlog('Checking local status of ' + components_str + ': ', end='')
+    tree_status = source_tree.status(print_progress=True)
     printlog('')
 
     if args.status:
@@ -390,43 +429,8 @@ def main(args):
             for comp in sorted(tree_status):
                 tree_status[comp].log_status_message(args.verbose)
             # exit gracefully
-            msg = """The external repositories labeled with 'M' above are not in a clean state.
-
-The following are four options for how to proceed:
-
-(1) Go into each external that is not in a clean state and issue either a 'git status' or
-    an 'svn status' command (depending on whether the external is managed by git or
-    svn). Either revert or commit your changes so that all externals are in a clean
-    state. (To revert changes in git, follow the instructions given when you run 'git
-    status'.) (Note, though, that it is okay to have untracked files in your working
-    directory.) Then rerun {program_name}.
-
-(2) Alternatively, you do not have to rely on {program_name}. Instead, you can manually
-    update out-of-sync externals (labeled with 's' above) as described in the
-    configuration file {config_file}. (For example, run 'git fetch' and 'git checkout'
-    commands to checkout the appropriate tags for each external, as given in
-    {config_file}.)
-
-(3) You can also use {program_name} to manage most, but not all externals: You can specify
-    one or more externals to ignore using the '-x' or '--exclude' argument to
-    {program_name}. Excluding externals labeled with 'M' will allow {program_name} to
-    update the other, non-excluded externals.
-
-(4) As a last resort, if you are confident that there is no work that needs to be saved
-    from a given external, you can remove that external (via "rm -rf [directory]") and
-    then rerun the {program_name} tool. This option is mainly useful as a workaround for
-    issues with this tool (such as https://github.com/ESMCI/manage_externals/issues/157).
-
-
-The external repositories labeled with '?' above are not under version
-control using the expected protocol. If you are sure you want to switch
-protocols, and you don't have any work you need to save from this
-directory, then run "rm -rf [directory]" before rerunning the
-{program_name} tool.
-""".format(program_name=program_name, config_file=args.externals)
-
             printlog('-' * 70)
-            printlog(msg)
+            printlog(_dirty_local_repo_msg(program_name, args.externals))
             printlog('-' * 70)
         else:
             if not args.components:

manic/externals_description.py

@@ -71,7 +71,8 @@ def read_externals_description_file(root_dir, file_name):
     root_dir = os.path.abspath(root_dir)
     msg = 'In directory : {0}'.format(root_dir)
     logging.info(msg)
-    printlog('Processing externals description file : {0}'.format(file_name))
+    printlog('Processing externals description file : {0} ({1})'.format(file_name,
+                                                                        root_dir))
 
     file_path = os.path.join(root_dir, file_name)
     if not os.path.exists(file_name):
@@ -281,6 +282,10 @@ def read_gitmodules_file(root_dir, file_name):
 def create_externals_description(
         model_data, model_format='cfg', components=None, exclude=None, parent_repo=None):
     """Create the a externals description object from the provided data
+        
+    components: list of component names to include, None to include all. If a
+                name isn't found, it is silently omitted from the return value.
+    exclude: list of component names to skip.
     """
     externals_description = None
     if model_format == 'dict':
@@ -357,8 +362,9 @@ class ExternalsDescription(dict):
     input value.
 
     """
-    # keywords defining the interface into the externals description data
-    EXTERNALS = 'externals'
+    # keywords defining the interface into the externals description data; these
+    # are brought together by the schema below.
+    EXTERNALS = 'externals'  # path to externals file.
     BRANCH = 'branch'
     SUBMODULE = 'from_submodule'
     HASH = 'hash'
@@ -384,6 +390,8 @@ class ExternalsDescription(dict):
     _V1_BRANCH = 'BRANCH'
     _V1_REQ_SOURCE = 'REQ_SOURCE'
 
+    # Dictionary keys are component names. The corresponding values are laid out
+    # according to this schema.
     _source_schema = {REQUIRED: True,
                       PATH: 'string',
                       EXTERNALS: 'string',
@@ -760,6 +768,8 @@ def __init__(self, model_data, components=None, exclude=None, parent_repo=None):
         """Convert the config data into a standardized dict that can be used to
         construct the source objects
 
+        components: list of component names to include, None to include all.
+        exclude: list of component names to skip.
         """
         ExternalsDescription.__init__(self, parent_repo=parent_repo)
         self._schema_major = 1
@@ -783,6 +793,9 @@ def _remove_metadata(model_data):
 
     def _parse_cfg(self, cfg_data, components=None, exclude=None):
         """Parse a config_parser object into a externals description.
+
+        components: list of component names to include, None to include all.
+        exclude: list of component names to skip.
         """
         def list_to_dict(input_list, convert_to_lower_case=True):
             """Convert a list of key-value pairs into a dictionary.

manic/externals_status.py

@@ -29,16 +29,16 @@ class ExternalStatus(object):
     transactions (e.g. add, remove, rename, untracked files).
 
     """
-    DEFAULT = '-'
+    # sync_state and clean_state can be one of the following:
+    DEFAULT = '-'  # aka not set yet.
     UNKNOWN = '?'
     EMPTY = 'e'
-    MODEL_MODIFIED = 's'  # a.k.a. out-of-sync
-    DIRTY = 'M'
-
-    STATUS_OK = ' '
+    MODEL_MODIFIED = 's'  # repo version != externals (sync_state only)
+    DIRTY = 'M'       # repo is dirty (clean_state only)
+    STATUS_OK = ' '   # repo is clean/matches externals.
     STATUS_ERROR = '!'
 
-    # source types
+    # source_type can be one of the following:
     OPTIONAL = 'o'
     STANDALONE = 's'
     MANAGED = ' '
@@ -55,19 +55,21 @@ def __init__(self):
     def log_status_message(self, verbosity):
         """Write status message to the screen and log file
         """
-        self._default_status_message()
+        printlog(self._default_status_message())
         if verbosity >= VERBOSITY_VERBOSE:
-            self._verbose_status_message()
+            printlog(self._verbose_status_message())
         if verbosity >= VERBOSITY_DUMP:
-            self._dump_status_message()
+            printlog(self._dump_status_message())
+
+    def __repr__(self):
+        return self._default_status_message()
 
     def _default_status_message(self):
         """Return the default terse status message string
         """
-        msg = '{sync}{clean}{src_type} {path}'.format(
+        return '{sync}{clean}{src_type} {path}'.format(
             sync=self.sync_state, clean=self.clean_state,
             src_type=self.source_type, path=self.path)
-        printlog(msg)
 
     def _verbose_status_message(self):
         """Return the verbose status message string
@@ -82,14 +84,12 @@ def _verbose_status_message(self):
         if self.sync_state != self.STATUS_OK:
             sync_str = '{current} --> {expected}'.format(
                 current=self.current_version, expected=self.expected_version)
-        msg = '        {clean}, {sync}'.format(clean=clean_str, sync=sync_str)
-        printlog(msg)
+        return '        {clean}, {sync}'.format(clean=clean_str, sync=sync_str)
 
     def _dump_status_message(self):
         """Return the dump status message string
         """
-        msg = indent_string(self.status_output, 12)
-        printlog(msg)
+        return indent_string(self.status_output, 12)
 
     def safe_to_update(self):
         """Report if it is safe to update a repository. Safe is defined as:

manic/repository_factory.py

@@ -15,6 +15,7 @@ def create_repository(component_name, repo_info, svn_ignore_ancestry=False):
     """Determine what type of repository we have, i.e. git or svn, and
     create the appropriate object.
 
+    Can return None (e.g. if protocol is 'externals_only').
     """
     protocol = repo_info[ExternalsDescription.PROTOCOL].lower()
     if protocol == 'git':

manic/repository_svn.py

@@ -43,10 +43,15 @@ def __init__(self, component_name, repo, ignore_ancestry=False):
         """
         Repository.__init__(self, component_name, repo)
         self._ignore_ancestry = ignore_ancestry
+        if self._url.endswith('/'):
+            # there is already a '/' separator in the URL; no need to add another
+            url_sep = ''
+        else:
+            url_sep = '/'
         if self._branch:
-            self._url = os.path.join(self._url, self._branch)
+            self._url = self._url + url_sep + self._branch
         elif self._tag:
-            self._url = os.path.join(self._url, self._tag)
+            self._url = self._url + url_sep + self._tag
         else:
             msg = "DEV_ERROR in svn repository. Shouldn't be here!"
             fatal_error(msg)