Skip to content

Commit

Permalink
Build: adapt sync_repository_task and sync_versions
Browse files Browse the repository at this point in the history
  • Loading branch information
@humitos
humitos committed Mar 8, 2022
1 parent 5bfe10e commit d0edfaf
Show file tree
Hide file tree
Showing 2 changed files with 35 additions and 98 deletions.
63 changes: 21 additions & 42 deletions readthedocs/projects/tasks/builds.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,8 +100,6 @@ class SyncRepositoryTask(SyncRepositoryMixin, Task):
RepositoryError,
)

# TODO: adapt this task to make it work. It's currently untested.

def before_start(self, task_id, args, kwargs):
log.info('Running task.', name=self.name)

Expand Down Expand Up @@ -146,13 +144,12 @@ def after_return(self, status, retval, task_id, args, kwargs, einfo):
clean_build(self.data.version)

def execute(self):
# TODO: initialize the doc builder here
self.data.builder = DocumentationBuilder()

environment = self.data.environment_class(
project=self.data.project,
version=self.data.version,
environment=self.get_vcs_env_vars(),
environment={
"GIT_TERMINAL_PROMPT": 0,
},
# Do not try to save commands on the db because they are just for
# sync repository
record=False,
Expand All @@ -163,26 +160,25 @@ def execute(self):
sender=self.data.version,
environment=environment,
)
self.update_versions_from_repository(environment)

def update_versions_from_repository(self, environment):
"""
Update Read the Docs versions from VCS repository.
vcs_repository = self.data.project.vcs_repo(
version=self.data.version.slug,
environment=environment,
verbose_name=self.data.version.verbose_name,
version_type=self.data.version.type,
)
if any(
[
not vcs_repository.supports_lsremote,
not self.data.project.has_feature(Feature.VCS_REMOTE_LISTING),
]
):
log.info("Syncing repository via full clone.")
vcs_repository.update()
else:
log.info("Syncing repository via remote listing.")

Depending if the VCS backend supports remote listing, we just list its branches/tags
remotely or we do a full clone and local listing of branches/tags.
"""
version_repo = self.get_vcs_repo(environment)
if any([
not version_repo.supports_lsremote,
not self.data.project.has_feature(Feature.VCS_REMOTE_LISTING),
]):
log.info('Syncing repository via full clone.')
self.sync_repo(environment)
else:
log.info('Syncing repository via remote listing.')
# TODO: needs to figure it out how to do this `sync_versions` here
self.sync_versions(version_repo)
self.sync_versions(vcs_repository)


@app.task(
Expand Down Expand Up @@ -559,10 +555,7 @@ def execute(self):

# Sync tags/branches from VCS repository into Read the Docs' `Version`
# objects in the database
#
# FIXME: review why I had to disable this for now.
log.error("Not triggering `sync_versions` because it's broken")
# self.sync_versions()
self.sync_versions(self.data.builder.vcs_repository)

self.update_build(state=BUILD_STATE_INSTALLING)
self.data.builder.setup_environment()
Expand Down Expand Up @@ -597,20 +590,6 @@ def get_build(build_pk):
for key, val in build.items() if key not in private_keys
}

# TODO: can we name this `clonning`
# def setup_vcs(self, environment):
# """
# Update the checkout of the repo to make sure it's the latest.

# This also syncs versions in the DB.
# """
# self.update_build(state=BUILD_STATE_CLONING)
# self.sync_repo(environment)

# commit = self.data.build_commit or self.get_vcs_repo(environment).commit
# if commit:
# self.data.build['commit'] = commit

# NOTE: this can be just updated on `self.data.build['']` and sent once the
# build has finished to reduce API calls.
def set_valid_clone(self):
Expand Down
70 changes: 14 additions & 56 deletions readthedocs/projects/tasks/mixins.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,48 +44,7 @@ def get_version(version_pk):
version_data = api_v2.version(version_pk).get()
return APIVersion(**version_data)

# def get_vcs_repo(self, environment):
# """
# Get the VCS object of the current project.

# All VCS commands will be executed using `environment`.
# """
# version_repo = self.data.project.vcs_repo(
# version=self.data.version.slug,
# environment=environment,
# verbose_name=self.data.version.verbose_name,
# version_type=self.data.version.type
# )
# return version_repo

# def sync_repo(self, environment):
# """Update the project's repository and hit ``sync_versions`` API."""
# # Make Dirs
# if not os.path.exists(self.data.project.doc_path):
# os.makedirs(self.data.project.doc_path)

# if not self.data.project.vcs_class():
# raise RepositoryError(
# _('Repository type "{repo_type}" unknown').format(
# repo_type=self.data.project.repo_type,
# ),
# )

# # Get the actual code on disk
# log.info(
# 'Checking out version.',
# version_identifier=self.data.version.identifier,
# )
# version_repo = self.get_vcs_repo(environment)

# self.data.builder.checkout()
# # version_repo.update()

# self.sync_versions(version_repo)
# identifier = self.data.build_commit or self.data.version.identifier
# version_repo.checkout(identifier)

def sync_versions(self, version_repo):
def sync_versions(self, vcs_repository):
"""
Update tags/branches via a Celery task.
Expand All @@ -97,31 +56,29 @@ def sync_versions(self, version_repo):
# NOTE: `sync_versions` should receive `tags` and `branches` already
# and just validate them trigger the task. All the other logic should
# be done by the DocumentationBuilder or the VCS backend. We should not
# check this here and do not depend on ``version_repo``.
# check this here and do not depend on ``vcs_repository``.

tags = None
branches = None
if (
version_repo.supports_lsremote and
not version_repo.repo_exists() and
self.data.project.has_feature(Feature.VCS_REMOTE_LISTING)
vcs_repository.supports_lsremote
and not vcs_repository.repo_exists()
and self.data.project.has_feature(Feature.VCS_REMOTE_LISTING)
):
# Do not use ``ls-remote`` if the VCS does not support it or if we
# have already cloned the repository locally. The latter happens
# when triggering a normal build.
branches, tags = version_repo.lsremote
log.info('Remote versions.', branches=branches, tags=tags)
branches, tags = vcs_repository.lsremote

branches_data = []
tags_data = []

if (
version_repo.supports_tags and
not self.data.project.has_feature(Feature.SKIP_SYNC_TAGS)
if vcs_repository.supports_tags and not self.data.project.has_feature(
Feature.SKIP_SYNC_TAGS
):
# Will be an empty list if we called lsremote and had no tags returned
if tags is None:
tags = version_repo.tags
tags = vcs_repository.tags
tags_data = [
{
'identifier': v.identifier,
Expand All @@ -130,13 +87,12 @@ def sync_versions(self, version_repo):
for v in tags
]

if (
version_repo.supports_branches and
not self.data.project.has_feature(Feature.SKIP_SYNC_BRANCHES)
if vcs_repository.supports_branches and not self.data.project.has_feature(
Feature.SKIP_SYNC_BRANCHES
):
# Will be an empty list if we called lsremote and had no branches returned
if branches is None:
branches = version_repo.branches
branches = vcs_repository.branches
branches_data = [
{
'identifier': v.identifier,
Expand All @@ -145,6 +101,8 @@ def sync_versions(self, version_repo):
for v in branches
]

log.debug("Synchronizing versions.", branches=branches, tags=tags)

self.validate_duplicate_reserved_versions(
tags_data=tags_data,
branches_data=branches_data,
Expand Down

0 comments on commit d0edfaf

Please sign in to comment.