diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 000000000..28c073947 --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,3 @@ +Release type: minor + +Replace `pytz` with native alternatives. diff --git a/pelican/contents.py b/pelican/contents.py index c979dd0a1..6df6e57c3 100644 --- a/pelican/contents.py +++ b/pelican/contents.py @@ -7,7 +7,13 @@ from html import unescape from urllib.parse import unquote, urljoin, urlparse, urlunparse -import pytz +from datetime import timezone + +try: + import zoneinfo +except ModuleNotFoundError: + import backports.zoneinfo + from pelican.plugins import signals from pelican.settings import DEFAULT_CONFIG @@ -120,9 +126,12 @@ def __init__(self, content, metadata=None, settings=None, self.date_format = self.date_format[1] # manage timezone - default_timezone = settings.get('TIMEZONE', 'UTC') - timezone = getattr(self, 'timezone', default_timezone) - self.timezone = pytz.timezone(timezone) + default_timezone = settings.get("TIMEZONE", "UTC") + timezone = getattr(self, "timezone", default_timezone) + try: + self.timezone = zoneinfo.ZoneInfo(timezone) + except NameError: + self.timezone = backports.zoneinfo.ZoneInfo(timezone) if hasattr(self, 'date'): self.date = set_date_tzinfo(self.date, timezone) @@ -525,7 +534,7 @@ def __init__(self, *args, **kwargs): if self.date.tzinfo is None: now = datetime.datetime.now() else: - now = datetime.datetime.utcnow().replace(tzinfo=pytz.utc) + now = datetime.datetime.utcnow().replace(tzinfo=timezone.utc) if self.date > now: self.status = 'draft' diff --git a/pelican/tools/pelican_quickstart.py b/pelican/tools/pelican_quickstart.py index 2d5629efa..efa4c83fd 100755 --- a/pelican/tools/pelican_quickstart.py +++ b/pelican/tools/pelican_quickstart.py @@ -7,7 +7,15 @@ from jinja2 import Environment, FileSystemLoader -import pytz +try: + import zoneinfo +except ModuleNotFoundError: + import backports.zoneinfo + +try: + import tzlocal +except ModuleNotFoundError: + pass try: import readline # NOQA @@ -15,62 +23,57 @@ pass try: - import tzlocal _DEFAULT_TIMEZONE = tzlocal.get_localzone().zone -except ImportError: - _DEFAULT_TIMEZONE = 'Europe/Rome' +except NameError: + _DEFAULT_TIMEZONE = "Europe/Rome" from pelican import __version__ -locale.setlocale(locale.LC_ALL, '') +locale.setlocale(locale.LC_ALL, "") try: _DEFAULT_LANGUAGE = locale.getlocale()[0] except ValueError: # Don't fail on macosx: "unknown locale: UTF-8" _DEFAULT_LANGUAGE = None if _DEFAULT_LANGUAGE is None: - _DEFAULT_LANGUAGE = 'en' + _DEFAULT_LANGUAGE = "en" else: - _DEFAULT_LANGUAGE = _DEFAULT_LANGUAGE.split('_')[0] + _DEFAULT_LANGUAGE = _DEFAULT_LANGUAGE.split("_")[0] -_TEMPLATES_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), - "templates") +_TEMPLATES_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), "templates") _jinja_env = Environment( loader=FileSystemLoader(_TEMPLATES_DIR), trim_blocks=True, ) -_GITHUB_PAGES_BRANCHES = { - 'personal': 'main', - 'project': 'gh-pages' -} +_GITHUB_PAGES_BRANCHES = {"personal": "main", "project": "gh-pages"} CONF = { - 'pelican': 'pelican', - 'pelicanopts': '', - 'basedir': os.curdir, - 'ftp_host': 'localhost', - 'ftp_user': 'anonymous', - 'ftp_target_dir': '/', - 'ssh_host': 'localhost', - 'ssh_port': 22, - 'ssh_user': 'root', - 'ssh_target_dir': '/var/www', - 's3_bucket': 'my_s3_bucket', - 'cloudfiles_username': 'my_rackspace_username', - 'cloudfiles_api_key': 'my_rackspace_api_key', - 'cloudfiles_container': 'my_cloudfiles_container', - 'dropbox_dir': '~/Dropbox/Public/', - 'github_pages_branch': _GITHUB_PAGES_BRANCHES['project'], - 'default_pagination': 10, - 'siteurl': '', - 'lang': _DEFAULT_LANGUAGE, - 'timezone': _DEFAULT_TIMEZONE + "pelican": "pelican", + "pelicanopts": "", + "basedir": os.curdir, + "ftp_host": "localhost", + "ftp_user": "anonymous", + "ftp_target_dir": "/", + "ssh_host": "localhost", + "ssh_port": 22, + "ssh_user": "root", + "ssh_target_dir": "/var/www", + "s3_bucket": "my_s3_bucket", + "cloudfiles_username": "my_rackspace_username", + "cloudfiles_api_key": "my_rackspace_api_key", + "cloudfiles_container": "my_cloudfiles_container", + "dropbox_dir": "~/Dropbox/Public/", + "github_pages_branch": _GITHUB_PAGES_BRANCHES["project"], + "default_pagination": 10, + "siteurl": "", + "lang": _DEFAULT_LANGUAGE, + "timezone": _DEFAULT_TIMEZONE, } # url for list of valid timezones -_TZ_URL = 'https://en.wikipedia.org/wiki/List_of_tz_database_time_zones' +_TZ_URL = "https://en.wikipedia.org/wiki/List_of_tz_database_time_zones" # Create a 'marked' default path, to determine if someone has supplied @@ -84,12 +87,12 @@ class _DEFAULT_PATH_TYPE(str): def ask(question, answer=str, default=None, length=None): if answer == str: - r = '' + r = "" while True: if default: - r = input('> {} [{}] '.format(question, default)) + r = input("> {} [{}] ".format(question, default)) else: - r = input('> {} '.format(question)) + r = input("> {} ".format(question)) r = r.strip() @@ -98,10 +101,10 @@ def ask(question, answer=str, default=None, length=None): r = default break else: - print('You must enter something') + print("You must enter something") else: if length and len(r) != length: - print('Entry must be {} characters long'.format(length)) + print("Entry must be {} characters long".format(length)) else: break @@ -111,18 +114,18 @@ def ask(question, answer=str, default=None, length=None): r = None while True: if default is True: - r = input('> {} (Y/n) '.format(question)) + r = input("> {} (Y/n) ".format(question)) elif default is False: - r = input('> {} (y/N) '.format(question)) + r = input("> {} (y/N) ".format(question)) else: - r = input('> {} (y/n) '.format(question)) + r = input("> {} (y/n) ".format(question)) r = r.strip().lower() - if r in ('y', 'yes'): + if r in ("y", "yes"): r = True break - elif r in ('n', 'no'): + elif r in ("n", "no"): r = False break elif not r: @@ -135,9 +138,9 @@ def ask(question, answer=str, default=None, length=None): r = None while True: if default: - r = input('> {} [{}] '.format(question, default)) + r = input("> {} [{}] ".format(question, default)) else: - r = input('> {} '.format(question)) + r = input("> {} ".format(question)) r = r.strip() @@ -149,186 +152,255 @@ def ask(question, answer=str, default=None, length=None): r = int(r) break except ValueError: - print('You must enter an integer') + print("You must enter an integer") return r else: - raise NotImplementedError( - 'Argument `answer` must be str, bool, or integer') + raise NotImplementedError("Argument `answer` must be str, bool, or integer") def ask_timezone(question, default, tzurl): """Prompt for time zone and validate input""" - lower_tz = [tz.lower() for tz in pytz.all_timezones] + try: + tz_dict = {tz.lower(): tz for tz in zoneinfo.available_timezones()} + except NameError: + tz_dict = {tz.lower(): tz for tz in backports.zoneinfo.available_timezones()} + while True: r = ask(question, str, default) - r = r.strip().replace(' ', '_').lower() - if r in lower_tz: - r = pytz.all_timezones[lower_tz.index(r)] + r = r.strip().replace(" ", "_").lower() + if r in tz_dict.keys(): + r = tz_dict[r] break else: - print('Please enter a valid time zone:\n' - ' (check [{}])'.format(tzurl)) + print("Please enter a valid time zone:\n" " (check [{}])".format(tzurl)) return r def render_jinja_template(tmpl_name: str, tmpl_vars: Mapping, target_path: str): try: - with open(os.path.join(CONF['basedir'], target_path), - 'w', encoding='utf-8') as fd: + with open( + os.path.join(CONF["basedir"], target_path), "w", encoding="utf-8" + ) as fd: _template = _jinja_env.get_template(tmpl_name) fd.write(_template.render(**tmpl_vars)) except OSError as e: - print('Error: {}'.format(e)) + print("Error: {}".format(e)) def main(): parser = argparse.ArgumentParser( description="A kickstarter for Pelican", - formatter_class=argparse.ArgumentDefaultsHelpFormatter) - parser.add_argument('-p', '--path', default=_DEFAULT_PATH, - help="The path to generate the blog into") - parser.add_argument('-t', '--title', metavar="title", - help='Set the title of the website') - parser.add_argument('-a', '--author', metavar="author", - help='Set the author name of the website') - parser.add_argument('-l', '--lang', metavar="lang", - help='Set the default web site language') + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) + parser.add_argument( + "-p", "--path", default=_DEFAULT_PATH, help="The path to generate the blog into" + ) + parser.add_argument( + "-t", "--title", metavar="title", help="Set the title of the website" + ) + parser.add_argument( + "-a", "--author", metavar="author", help="Set the author name of the website" + ) + parser.add_argument( + "-l", "--lang", metavar="lang", help="Set the default web site language" + ) args = parser.parse_args() - print('''Welcome to pelican-quickstart v{v}. + print( + """Welcome to pelican-quickstart v{v}. This script will help you create a new Pelican-based website. Please answer the following questions so this script can generate the files needed by Pelican. - '''.format(v=__version__)) + """.format( + v=__version__ + ) + ) - project = os.path.join( - os.environ.get('VIRTUAL_ENV', os.curdir), '.project') - no_path_was_specified = hasattr(args.path, 'is_default_path') + project = os.path.join(os.environ.get("VIRTUAL_ENV", os.curdir), ".project") + no_path_was_specified = hasattr(args.path, "is_default_path") if os.path.isfile(project) and no_path_was_specified: - CONF['basedir'] = open(project).read().rstrip("\n") - print('Using project associated with current virtual environment. ' - 'Will save to:\n%s\n' % CONF['basedir']) + CONF["basedir"] = open(project).read().rstrip("\n") + print( + "Using project associated with current virtual environment. " + "Will save to:\n%s\n" % CONF["basedir"] + ) else: - CONF['basedir'] = os.path.abspath(os.path.expanduser( - ask('Where do you want to create your new web site?', - answer=str, default=args.path))) - - CONF['sitename'] = ask('What will be the title of this web site?', - answer=str, default=args.title) - CONF['author'] = ask('Who will be the author of this web site?', - answer=str, default=args.author) - CONF['lang'] = ask('What will be the default language of this web site?', - str, args.lang or CONF['lang'], 2) - - if ask('Do you want to specify a URL prefix? e.g., https://example.com ', - answer=bool, default=True): - CONF['siteurl'] = ask('What is your URL prefix? (see ' - 'above example; no trailing slash)', - str, CONF['siteurl']) - - CONF['with_pagination'] = ask('Do you want to enable article pagination?', - bool, bool(CONF['default_pagination'])) - - if CONF['with_pagination']: - CONF['default_pagination'] = ask('How many articles per page ' - 'do you want?', - int, CONF['default_pagination']) + CONF["basedir"] = os.path.abspath( + os.path.expanduser( + ask( + "Where do you want to create your new web site?", + answer=str, + default=args.path, + ) + ) + ) + + CONF["sitename"] = ask( + "What will be the title of this web site?", answer=str, default=args.title + ) + CONF["author"] = ask( + "Who will be the author of this web site?", answer=str, default=args.author + ) + CONF["lang"] = ask( + "What will be the default language of this web site?", + str, + args.lang or CONF["lang"], + 2, + ) + + if ask( + "Do you want to specify a URL prefix? e.g., https://example.com ", + answer=bool, + default=True, + ): + CONF["siteurl"] = ask( + "What is your URL prefix? (see " "above example; no trailing slash)", + str, + CONF["siteurl"], + ) + + CONF["with_pagination"] = ask( + "Do you want to enable article pagination?", + bool, + bool(CONF["default_pagination"]), + ) + + if CONF["with_pagination"]: + CONF["default_pagination"] = ask( + "How many articles per page " "do you want?", + int, + CONF["default_pagination"], + ) else: - CONF['default_pagination'] = False + CONF["default_pagination"] = False - CONF['timezone'] = ask_timezone('What is your time zone?', - CONF['timezone'], _TZ_URL) + CONF["timezone"] = ask_timezone( + "What is your time zone?", CONF["timezone"], _TZ_URL + ) - automation = ask('Do you want to generate a tasks.py/Makefile ' - 'to automate generation and publishing?', bool, True) + automation = ask( + "Do you want to generate a tasks.py/Makefile " + "to automate generation and publishing?", + bool, + True, + ) if automation: - if ask('Do you want to upload your website using FTP?', - answer=bool, default=False): - CONF['ftp'] = True, - CONF['ftp_host'] = ask('What is the hostname of your FTP server?', - str, CONF['ftp_host']) - CONF['ftp_user'] = ask('What is your username on that server?', - str, CONF['ftp_user']) - CONF['ftp_target_dir'] = ask('Where do you want to put your ' - 'web site on that server?', - str, CONF['ftp_target_dir']) - if ask('Do you want to upload your website using SSH?', - answer=bool, default=False): - CONF['ssh'] = True, - CONF['ssh_host'] = ask('What is the hostname of your SSH server?', - str, CONF['ssh_host']) - CONF['ssh_port'] = ask('What is the port of your SSH server?', - int, CONF['ssh_port']) - CONF['ssh_user'] = ask('What is your username on that server?', - str, CONF['ssh_user']) - CONF['ssh_target_dir'] = ask('Where do you want to put your ' - 'web site on that server?', - str, CONF['ssh_target_dir']) - - if ask('Do you want to upload your website using Dropbox?', - answer=bool, default=False): - CONF['dropbox'] = True, - CONF['dropbox_dir'] = ask('Where is your Dropbox directory?', - str, CONF['dropbox_dir']) - - if ask('Do you want to upload your website using S3?', - answer=bool, default=False): - CONF['s3'] = True, - CONF['s3_bucket'] = ask('What is the name of your S3 bucket?', - str, CONF['s3_bucket']) - - if ask('Do you want to upload your website using ' - 'Rackspace Cloud Files?', answer=bool, default=False): - CONF['cloudfiles'] = True, - CONF['cloudfiles_username'] = ask('What is your Rackspace ' - 'Cloud username?', str, - CONF['cloudfiles_username']) - CONF['cloudfiles_api_key'] = ask('What is your Rackspace ' - 'Cloud API key?', str, - CONF['cloudfiles_api_key']) - CONF['cloudfiles_container'] = ask('What is the name of your ' - 'Cloud Files container?', - str, - CONF['cloudfiles_container']) - - if ask('Do you want to upload your website using GitHub Pages?', - answer=bool, default=False): - CONF['github'] = True, - if ask('Is this your personal page (username.github.io)?', - answer=bool, default=False): - CONF['github_pages_branch'] = \ - _GITHUB_PAGES_BRANCHES['personal'] + if ask( + "Do you want to upload your website using FTP?", answer=bool, default=False + ): + CONF["ftp"] = (True,) + CONF["ftp_host"] = ask( + "What is the hostname of your FTP server?", str, CONF["ftp_host"] + ) + CONF["ftp_user"] = ask( + "What is your username on that server?", str, CONF["ftp_user"] + ) + CONF["ftp_target_dir"] = ask( + "Where do you want to put your " "web site on that server?", + str, + CONF["ftp_target_dir"], + ) + if ask( + "Do you want to upload your website using SSH?", answer=bool, default=False + ): + CONF["ssh"] = (True,) + CONF["ssh_host"] = ask( + "What is the hostname of your SSH server?", str, CONF["ssh_host"] + ) + CONF["ssh_port"] = ask( + "What is the port of your SSH server?", int, CONF["ssh_port"] + ) + CONF["ssh_user"] = ask( + "What is your username on that server?", str, CONF["ssh_user"] + ) + CONF["ssh_target_dir"] = ask( + "Where do you want to put your " "web site on that server?", + str, + CONF["ssh_target_dir"], + ) + + if ask( + "Do you want to upload your website using Dropbox?", + answer=bool, + default=False, + ): + CONF["dropbox"] = (True,) + CONF["dropbox_dir"] = ask( + "Where is your Dropbox directory?", str, CONF["dropbox_dir"] + ) + + if ask( + "Do you want to upload your website using S3?", answer=bool, default=False + ): + CONF["s3"] = (True,) + CONF["s3_bucket"] = ask( + "What is the name of your S3 bucket?", str, CONF["s3_bucket"] + ) + + if ask( + "Do you want to upload your website using " "Rackspace Cloud Files?", + answer=bool, + default=False, + ): + CONF["cloudfiles"] = (True,) + CONF["cloudfiles_username"] = ask( + "What is your Rackspace " "Cloud username?", + str, + CONF["cloudfiles_username"], + ) + CONF["cloudfiles_api_key"] = ask( + "What is your Rackspace " "Cloud API key?", + str, + CONF["cloudfiles_api_key"], + ) + CONF["cloudfiles_container"] = ask( + "What is the name of your " "Cloud Files container?", + str, + CONF["cloudfiles_container"], + ) + + if ask( + "Do you want to upload your website using GitHub Pages?", + answer=bool, + default=False, + ): + CONF["github"] = (True,) + if ask( + "Is this your personal page (username.github.io)?", + answer=bool, + default=False, + ): + CONF["github_pages_branch"] = _GITHUB_PAGES_BRANCHES["personal"] else: - CONF['github_pages_branch'] = \ - _GITHUB_PAGES_BRANCHES['project'] + CONF["github_pages_branch"] = _GITHUB_PAGES_BRANCHES["project"] try: - os.makedirs(os.path.join(CONF['basedir'], 'content')) + os.makedirs(os.path.join(CONF["basedir"], "content")) except OSError as e: - print('Error: {}'.format(e)) + print("Error: {}".format(e)) try: - os.makedirs(os.path.join(CONF['basedir'], 'output')) + os.makedirs(os.path.join(CONF["basedir"], "output")) except OSError as e: - print('Error: {}'.format(e)) + print("Error: {}".format(e)) conf_python = dict() for key, value in CONF.items(): conf_python[key] = repr(value) - render_jinja_template('pelicanconf.py.jinja2', conf_python, 'pelicanconf.py') + render_jinja_template("pelicanconf.py.jinja2", conf_python, "pelicanconf.py") - render_jinja_template('publishconf.py.jinja2', CONF, 'publishconf.py') + render_jinja_template("publishconf.py.jinja2", CONF, "publishconf.py") if automation: - render_jinja_template('tasks.py.jinja2', CONF, 'tasks.py') - render_jinja_template('Makefile.jinja2', CONF, 'Makefile') + render_jinja_template("tasks.py.jinja2", CONF, "tasks.py") + render_jinja_template("Makefile.jinja2", CONF, "Makefile") - print('Done. Your new project is available at %s' % CONF['basedir']) + print("Done. Your new project is available at %s" % CONF["basedir"]) if __name__ == "__main__": diff --git a/pelican/utils.py b/pelican/utils.py index de6ef9bfe..6476cad5d 100644 --- a/pelican/utils.py +++ b/pelican/utils.py @@ -18,47 +18,49 @@ import dateutil.parser -from markupsafe import Markup +try: + import zoneinfo +except ModuleNotFoundError: + import backports.zoneinfo -import pytz +from markupsafe import Markup logger = logging.getLogger(__name__) def sanitised_join(base_directory, *parts): - joined = posixize_path( - os.path.abspath(os.path.join(base_directory, *parts))) + joined = posixize_path(os.path.abspath(os.path.join(base_directory, *parts))) base = posixize_path(os.path.abspath(base_directory)) if not joined.startswith(base): raise RuntimeError( - "Attempted to break out of output directory to {}".format( - joined - ) + "Attempted to break out of output directory to {}".format(joined) ) return joined def strftime(date, date_format): - ''' + """ Enhanced replacement for built-in strftime with zero stripping This works by 'grabbing' possible format strings (those starting with %), formatting them with the date, stripping any leading zeros if - prefix is used and replacing formatted output back. - ''' + """ + def strip_zeros(x): - return x.lstrip('0') or '0' + return x.lstrip("0") or "0" + # includes ISO date parameters added by Python 3.6 - c89_directives = 'aAbBcdfGHIjmMpSUuVwWxXyYzZ%' + c89_directives = "aAbBcdfGHIjmMpSUuVwWxXyYzZ%" # grab candidate format options - format_options = '%[-]?.' + format_options = "%[-]?." candidates = re.findall(format_options, date_format) # replace candidates with placeholders for later % formatting - template = re.sub(format_options, '%s', date_format) + template = re.sub(format_options, "%s", date_format) formatted_candidates = [] for candidate in candidates: @@ -67,7 +69,7 @@ def strip_zeros(x): # check for '-' prefix if len(candidate) == 3: # '-' prefix - candidate = '%{}'.format(candidate[-1]) + candidate = "%{}".format(candidate[-1]) conversion = strip_zeros else: conversion = None @@ -90,10 +92,10 @@ def strip_zeros(x): class SafeDatetime(datetime.datetime): - '''Subclass of datetime that works with utf-8 format strings on PY2''' + """Subclass of datetime that works with utf-8 format strings on PY2""" def strftime(self, fmt, safe=True): - '''Uses our custom strftime if supposed to be *safe*''' + """Uses our custom strftime if supposed to be *safe*""" if safe: return strftime(self, fmt) else: @@ -101,11 +103,11 @@ def strftime(self, fmt, safe=True): class DateFormatter: - '''A date formatter object used as a jinja filter + """A date formatter object used as a jinja filter Uses the `strftime` implementation and makes sure jinja uses the locale defined in LOCALE setting - ''' + """ def __init__(self): self.locale = locale.setlocale(locale.LC_TIME) @@ -154,7 +156,7 @@ def __repr__(self): return self.func.__doc__ def __get__(self, obj, objtype): - '''Support instance methods.''' + """Support instance methods.""" fn = partial(self.__call__, obj) fn.cache = self.cache return fn @@ -176,17 +178,16 @@ def __init__(self): Note that the decorator needs a dummy method to attach to, but the content of the dummy method is ignored. """ + def _warn(): - version = '.'.join(str(x) for x in since) - message = ['{} has been deprecated since {}'.format(old, version)] + version = ".".join(str(x) for x in since) + message = ["{} has been deprecated since {}".format(old, version)] if remove: - version = '.'.join(str(x) for x in remove) - message.append( - ' and will be removed by version {}'.format(version)) - message.append('. Use {} instead.'.format(new)) - logger.warning(''.join(message)) - logger.debug(''.join(str(x) for x - in traceback.format_stack())) + version = ".".join(str(x) for x in remove) + message.append(" and will be removed by version {}".format(version)) + message.append(". Use {} instead.".format(new)) + logger.warning("".join(message)) + logger.debug("".join(str(x) for x in traceback.format_stack())) def fget(self): _warn() @@ -207,21 +208,20 @@ def get_date(string): If no format matches the given date, raise a ValueError. """ - string = re.sub(' +', ' ', string) - default = SafeDatetime.now().replace(hour=0, minute=0, - second=0, microsecond=0) + string = re.sub(" +", " ", string) + default = SafeDatetime.now().replace(hour=0, minute=0, second=0, microsecond=0) try: return dateutil.parser.parse(string, default=default) except (TypeError, ValueError): - raise ValueError('{!r} is not a valid date'.format(string)) + raise ValueError("{!r} is not a valid date".format(string)) @contextmanager -def pelican_open(filename, mode='r', strip_crs=(sys.platform == 'win32')): +def pelican_open(filename, mode="r", strip_crs=(sys.platform == "win32")): """Open a file and return its content""" # utf-8-sig will clear any BOM if present - with open(filename, mode, encoding='utf-8-sig') as infile: + with open(filename, mode, encoding="utf-8-sig") as infile: content = infile.read() yield content @@ -243,7 +243,7 @@ def slugify(value, regex_subs=(), preserve_case=False, use_unicode=False): def normalize_unicode(text): # normalize text by compatibility composition # see: https://en.wikipedia.org/wiki/Unicode_equivalence - return unicodedata.normalize('NFKC', text) + return unicodedata.normalize("NFKC", text) # strip tags from value value = Markup(value).striptags() @@ -258,10 +258,8 @@ def normalize_unicode(text): # perform regex substitutions for src, dst in regex_subs: value = re.sub( - normalize_unicode(src), - normalize_unicode(dst), - value, - flags=re.IGNORECASE) + normalize_unicode(src), normalize_unicode(dst), value, flags=re.IGNORECASE + ) if not preserve_case: value = value.lower() @@ -282,8 +280,7 @@ def copy(source, destination, ignores=None): """ def walk_error(err): - logger.warning("While copying %s: %s: %s", - source_, err.filename, err.strerror) + logger.warning("While copying %s: %s: %s", source_, err.filename, err.strerror) source_ = os.path.abspath(os.path.expanduser(source)) destination_ = os.path.abspath(os.path.expanduser(destination)) @@ -291,39 +288,40 @@ def walk_error(err): if ignores is None: ignores = [] - if any(fnmatch.fnmatch(os.path.basename(source), ignore) - for ignore in ignores): - logger.info('Not copying %s due to ignores', source_) + if any(fnmatch.fnmatch(os.path.basename(source), ignore) for ignore in ignores): + logger.info("Not copying %s due to ignores", source_) return if os.path.isfile(source_): dst_dir = os.path.dirname(destination_) if not os.path.exists(dst_dir): - logger.info('Creating directory %s', dst_dir) + logger.info("Creating directory %s", dst_dir) os.makedirs(dst_dir) - logger.info('Copying %s to %s', source_, destination_) + logger.info("Copying %s to %s", source_, destination_) copy_file_metadata(source_, destination_) elif os.path.isdir(source_): if not os.path.exists(destination_): - logger.info('Creating directory %s', destination_) + logger.info("Creating directory %s", destination_) os.makedirs(destination_) if not os.path.isdir(destination_): - logger.warning('Cannot copy %s (a directory) to %s (a file)', - source_, destination_) + logger.warning( + "Cannot copy %s (a directory) to %s (a file)", source_, destination_ + ) return for src_dir, subdirs, others in os.walk(source_, followlinks=True): - dst_dir = os.path.join(destination_, - os.path.relpath(src_dir, source_)) + dst_dir = os.path.join(destination_, os.path.relpath(src_dir, source_)) - subdirs[:] = (s for s in subdirs if not any(fnmatch.fnmatch(s, i) - for i in ignores)) - others[:] = (o for o in others if not any(fnmatch.fnmatch(o, i) - for i in ignores)) + subdirs[:] = ( + s for s in subdirs if not any(fnmatch.fnmatch(s, i) for i in ignores) + ) + others[:] = ( + o for o in others if not any(fnmatch.fnmatch(o, i) for i in ignores) + ) if not os.path.isdir(dst_dir): - logger.info('Creating directory %s', dst_dir) + logger.info("Creating directory %s", dst_dir) # Parent directories are known to exist, so 'mkdir' suffices. os.mkdir(dst_dir) @@ -331,24 +329,27 @@ def walk_error(err): src_path = os.path.join(src_dir, o) dst_path = os.path.join(dst_dir, o) if os.path.isfile(src_path): - logger.info('Copying %s to %s', src_path, dst_path) + logger.info("Copying %s to %s", src_path, dst_path) copy_file_metadata(src_path, dst_path) else: - logger.warning('Skipped copy %s (not a file or ' - 'directory) to %s', - src_path, dst_path) + logger.warning( + "Skipped copy %s (not a file or " "directory) to %s", + src_path, + dst_path, + ) def copy_file_metadata(source, destination): - '''Copy a file and its metadata (perm bits, access times, ...)''' + """Copy a file and its metadata (perm bits, access times, ...)""" # This function is a workaround for Android python copystat # bug ([issue28141]) https://bugs.python.org/issue28141 try: shutil.copy2(source, destination) except OSError as e: - logger.warning("A problem occurred copying file %s to %s; %s", - source, destination, e) + logger.warning( + "A problem occurred copying file %s to %s; %s", source, destination, e + ) def clean_output_dir(path, retention): @@ -369,15 +370,15 @@ def clean_output_dir(path, retention): for filename in os.listdir(path): file = os.path.join(path, filename) if any(filename == retain for retain in retention): - logger.debug("Skipping deletion; %s is on retention list: %s", - filename, file) + logger.debug( + "Skipping deletion; %s is on retention list: %s", filename, file + ) elif os.path.isdir(file): try: shutil.rmtree(file) logger.debug("Deleted directory %s", file) except Exception as e: - logger.error("Unable to delete directory %s; %s", - file, e) + logger.error("Unable to delete directory %s; %s", file, e) elif os.path.isfile(file) or os.path.islink(file): try: os.remove(file) @@ -409,29 +410,31 @@ def posixize_path(rel_path): """Use '/' as path separator, so that source references, like '{static}/foo/bar.jpg' or 'extras/favicon.ico', will work on Windows as well as on Mac and Linux.""" - return rel_path.replace(os.sep, '/') + return rel_path.replace(os.sep, "/") class _HTMLWordTruncator(HTMLParser): - - _word_regex = re.compile(r"{DBC}|(\w[\w'-]*)".format( - # DBC means CJK-like characters. An character can stand for a word. - DBC=("([\u4E00-\u9FFF])|" # CJK Unified Ideographs - "([\u3400-\u4DBF])|" # CJK Unified Ideographs Extension A - "([\uF900-\uFAFF])|" # CJK Compatibility Ideographs - "([\U00020000-\U0002A6DF])|" # CJK Unified Ideographs Extension B - "([\U0002F800-\U0002FA1F])|" # CJK Compatibility Ideographs Supplement - "([\u3040-\u30FF])|" # Hiragana and Katakana - "([\u1100-\u11FF])|" # Hangul Jamo - "([\uAC00-\uD7FF])|" # Hangul Compatibility Jamo - "([\u3130-\u318F])" # Hangul Syllables - )), re.UNICODE) - _word_prefix_regex = re.compile(r'\w', re.U) - _singlets = ('br', 'col', 'link', 'base', 'img', 'param', 'area', - 'hr', 'input') + _word_regex = re.compile( + r"{DBC}|(\w[\w'-]*)".format( + # DBC means CJK-like characters. An character can stand for a word. + DBC=( + "([\u4E00-\u9FFF])|" # CJK Unified Ideographs + "([\u3400-\u4DBF])|" # CJK Unified Ideographs Extension A + "([\uF900-\uFAFF])|" # CJK Compatibility Ideographs + "([\U00020000-\U0002A6DF])|" # CJK Unified Ideographs Extension B + "([\U0002F800-\U0002FA1F])|" # CJK Compatibility Ideographs Supplement + "([\u3040-\u30FF])|" # Hiragana and Katakana + "([\u1100-\u11FF])|" # Hangul Jamo + "([\uAC00-\uD7FF])|" # Hangul Compatibility Jamo + "([\u3130-\u318F])" # Hangul Syllables + ) + ), + re.UNICODE, + ) + _word_prefix_regex = re.compile(r"\w", re.U) + _singlets = ("br", "col", "link", "base", "img", "param", "area", "hr", "input") class TruncationCompleted(Exception): - def __init__(self, truncate_at): super().__init__(truncate_at) self.truncate_at = truncate_at @@ -457,7 +460,7 @@ def getoffset(self): line_start = 0 lineno, line_offset = self.getpos() for i in range(lineno - 1): - line_start = self.rawdata.index('\n', line_start) + 1 + line_start = self.rawdata.index("\n", line_start) + 1 return line_start + line_offset def add_word(self, word_end): @@ -484,7 +487,7 @@ def handle_endtag(self, tag): else: # SGML: An end tag closes, back to the matching start tag, # all unclosed intervening start tags with omitted end tags - del self.open_tags[:i + 1] + del self.open_tags[: i + 1] def handle_data(self, data): word_end = 0 @@ -533,7 +536,7 @@ def _handle_ref(self, name, char): ref_end = offset + len(name) + 1 try: - if self.rawdata[ref_end] == ';': + if self.rawdata[ref_end] == ";": ref_end += 1 except IndexError: # We are at the end of the string and there's no ';' @@ -558,7 +561,7 @@ def handle_entityref(self, name): codepoint = entities.name2codepoint[name] char = chr(codepoint) except KeyError: - char = '' + char = "" self._handle_ref(name, char) def handle_charref(self, name): @@ -569,17 +572,17 @@ def handle_charref(self, name): `#x2014`) """ try: - if name.startswith('x'): + if name.startswith("x"): codepoint = int(name[1:], 16) else: codepoint = int(name) char = chr(codepoint) except (ValueError, OverflowError): - char = '' - self._handle_ref('#' + name, char) + char = "" + self._handle_ref("#" + name, char) -def truncate_html_words(s, num, end_text='…'): +def truncate_html_words(s, num, end_text="…"): """Truncates HTML to a certain number of words. (not counting tags and comments). Closes opened tags if they were correctly @@ -590,23 +593,23 @@ def truncate_html_words(s, num, end_text='…'): """ length = int(num) if length <= 0: - return '' + return "" truncator = _HTMLWordTruncator(length) truncator.feed(s) if truncator.truncate_at is None: return s - out = s[:truncator.truncate_at] + out = s[: truncator.truncate_at] if end_text: - out += ' ' + end_text + out += " " + end_text # Close any tags still open for tag in truncator.open_tags: - out += '' % tag + out += "" % tag # Return string return out def process_translations(content_list, translation_id=None): - """ Finds translations and returns them. + """Finds translations and returns them. For each content_list item, populates the 'translations' attribute, and returns a tuple with two lists (index, translations). Index list includes @@ -634,19 +637,23 @@ def process_translations(content_list, translation_id=None): try: content_list.sort(key=attrgetter(*translation_id)) except TypeError: - raise TypeError('Cannot unpack {}, \'translation_id\' must be falsy, a' - ' string or a collection of strings' - .format(translation_id)) + raise TypeError( + "Cannot unpack {}, 'translation_id' must be falsy, a" + " string or a collection of strings".format(translation_id) + ) except AttributeError: - raise AttributeError('Cannot use {} as \'translation_id\', there ' - 'appear to be items without these metadata ' - 'attributes'.format(translation_id)) + raise AttributeError( + "Cannot use {} as 'translation_id', there " + "appear to be items without these metadata " + "attributes".format(translation_id) + ) for id_vals, items in groupby(content_list, attrgetter(*translation_id)): # prepare warning string id_vals = (id_vals,) if len(translation_id) == 1 else id_vals - with_str = 'with' + ', '.join([' {} "{{}}"'] * len(translation_id))\ - .format(*translation_id).format(*id_vals) + with_str = "with" + ", ".join([' {} "{{}}"'] * len(translation_id)).format( + *translation_id + ).format(*id_vals) items = list(items) original_items = get_original_items(items, with_str) @@ -664,24 +671,24 @@ def _warn_source_paths(msg, items, *extra): args = [len(items)] args.extend(extra) args.extend(x.source_path for x in items) - logger.warning('{}: {}'.format(msg, '\n%s' * len(items)), *args) + logger.warning("{}: {}".format(msg, "\n%s" * len(items)), *args) # warn if several items have the same lang - for lang, lang_items in groupby(items, attrgetter('lang')): + for lang, lang_items in groupby(items, attrgetter("lang")): lang_items = list(lang_items) if len(lang_items) > 1: - _warn_source_paths('There are %s items "%s" with lang %s', - lang_items, with_str, lang) + _warn_source_paths( + 'There are %s items "%s" with lang %s', lang_items, with_str, lang + ) # items with `translation` metadata will be used as translations... candidate_items = [ - i for i in items - if i.metadata.get('translation', 'false').lower() == 'false'] + i for i in items if i.metadata.get("translation", "false").lower() == "false" + ] # ...unless all items with that slug are translations if not candidate_items: - _warn_source_paths('All items ("%s") "%s" are translations', - items, with_str) + _warn_source_paths('All items ("%s") "%s" are translations', items, with_str) candidate_items = items # find items with default language @@ -693,13 +700,14 @@ def _warn_source_paths(msg, items, *extra): # warn if there are several original items if len(original_items) > 1: - _warn_source_paths('There are %s original (not translated) items %s', - original_items, with_str) + _warn_source_paths( + "There are %s original (not translated) items %s", original_items, with_str + ) return original_items -def order_content(content_list, order_by='slug'): - """ Sorts content. +def order_content(content_list, order_by="slug"): + """Sorts content. order_by can be a string of an attribute or sorting function. If order_by is defined, content will be ordered by that attribute or sorting function. @@ -715,22 +723,22 @@ def order_content(content_list, order_by='slug'): try: content_list.sort(key=order_by) except Exception: - logger.error('Error sorting with function %s', order_by) + logger.error("Error sorting with function %s", order_by) elif isinstance(order_by, str): - if order_by.startswith('reversed-'): + if order_by.startswith("reversed-"): order_reversed = True - order_by = order_by.replace('reversed-', '', 1) + order_by = order_by.replace("reversed-", "", 1) else: order_reversed = False - if order_by == 'basename': + if order_by == "basename": content_list.sort( - key=lambda x: os.path.basename(x.source_path or ''), - reverse=order_reversed) + key=lambda x: os.path.basename(x.source_path or ""), + reverse=order_reversed, + ) else: try: - content_list.sort(key=attrgetter(order_by), - reverse=order_reversed) + content_list.sort(key=attrgetter(order_by), reverse=order_reversed) except AttributeError: for content in content_list: try: @@ -738,26 +746,29 @@ def order_content(content_list, order_by='slug'): except AttributeError: logger.warning( 'There is no "%s" attribute in "%s". ' - 'Defaulting to slug order.', + "Defaulting to slug order.", order_by, content.get_relative_source_path(), extra={ - 'limit_msg': ('More files are missing ' - 'the needed attribute.') - }) + "limit_msg": ( + "More files are missing " + "the needed attribute." + ) + }, + ) else: logger.warning( - 'Invalid *_ORDER_BY setting (%s). ' - 'Valid options are strings and functions.', order_by) + "Invalid *_ORDER_BY setting (%s). " + "Valid options are strings and functions.", + order_by, + ) return content_list class FileSystemWatcher: def __init__(self, settings_file, reader_class, settings=None): - self.watchers = { - 'settings': FileSystemWatcher.file_watcher(settings_file) - } + self.watchers = {"settings": FileSystemWatcher.file_watcher(settings_file)} self.settings = None self.reader_class = reader_class @@ -771,9 +782,9 @@ def __init__(self, settings_file, reader_class, settings=None): def update_watchers(self, settings): new_extensions = set(self.reader_class(settings).extensions) - new_content_path = settings.get('PATH', '') - new_theme_path = settings.get('THEME', '') - new_ignore_files = set(settings.get('IGNORE_FILES', [])) + new_content_path = settings.get("PATH", "") + new_theme_path = settings.get("THEME", "") + new_ignore_files = set(settings.get("IGNORE_FILES", [])) extensions_changed = new_extensions != self._extensions content_changed = new_content_path != self._content_path @@ -782,31 +793,25 @@ def update_watchers(self, settings): # Refresh content watcher if related settings changed if extensions_changed or content_changed or ignore_changed: - self.add_watcher('content', - new_content_path, - new_extensions, - new_ignore_files) + self.add_watcher( + "content", new_content_path, new_extensions, new_ignore_files + ) # Refresh theme watcher if related settings changed if theme_changed or ignore_changed: - self.add_watcher('theme', - new_theme_path, - [''], - new_ignore_files) + self.add_watcher("theme", new_theme_path, [""], new_ignore_files) # Watch STATIC_PATHS - old_static_watchers = set(key - for key in self.watchers - if key.startswith('[static]')) + old_static_watchers = set( + key for key in self.watchers if key.startswith("[static]") + ) - for path in settings.get('STATIC_PATHS', []): - key = '[static]{}'.format(path) + for path in settings.get("STATIC_PATHS", []): + key = "[static]{}".format(path) if ignore_changed or (key not in self.watchers): self.add_watcher( - key, - os.path.join(new_content_path, path), - [''], - new_ignore_files) + key, os.path.join(new_content_path, path), [""], new_ignore_files + ) if key in old_static_watchers: old_static_watchers.remove(key) @@ -822,34 +827,35 @@ def update_watchers(self, settings): self._ignore_files = new_ignore_files def check(self): - '''return a key:watcher_status dict for all watchers''' + """return a key:watcher_status dict for all watchers""" result = {key: next(watcher) for key, watcher in self.watchers.items()} # Various warnings - if result.get('content') is None: + if result.get("content") is None: reader_descs = sorted( { - ' | %s (%s)' % (type(r).__name__, ', '.join(r.file_extensions)) + " | %s (%s)" % (type(r).__name__, ", ".join(r.file_extensions)) for r in self.reader_class(self.settings).readers.values() if r.enabled } ) logger.warning( - 'No valid files found in content for the active readers:\n' - + '\n'.join(reader_descs)) + "No valid files found in content for the active readers:\n" + + "\n".join(reader_descs) + ) - if result.get('theme') is None: - logger.warning('Empty theme folder. Using `basic` theme.') + if result.get("theme") is None: + logger.warning("Empty theme folder. Using `basic` theme.") return result - def add_watcher(self, key, path, extensions=[''], ignores=[]): + def add_watcher(self, key, path, extensions=[""], ignores=[]): watcher = self.get_watcher(path, extensions, ignores) if watcher is not None: self.watchers[key] = watcher - def get_watcher(self, path, extensions=[''], ignores=[]): - '''return a watcher depending on path type (file or folder)''' + def get_watcher(self, path, extensions=[""], ignores=[]): + """return a watcher depending on path type (file or folder)""" if not os.path.exists(path): logger.warning("Watched path does not exist: %s", path) return None @@ -861,27 +867,25 @@ def get_watcher(self, path, extensions=[''], ignores=[]): @staticmethod def folder_watcher(path, extensions, ignores=[]): - '''Generator for monitoring a folder for modifications. + """Generator for monitoring a folder for modifications. Returns a boolean indicating if files are changed since last check. - Returns None if there are no matching files in the folder''' + Returns None if there are no matching files in the folder""" def file_times(path): - '''Return `mtime` for each file in path''' + """Return `mtime` for each file in path""" for root, dirs, files in os.walk(path, followlinks=True): dirs[:] = [x for x in dirs if not x.startswith(os.curdir)] for f in files: valid_extension = f.endswith(tuple(extensions)) - file_ignored = any( - fnmatch.fnmatch(f, ignore) for ignore in ignores - ) + file_ignored = any(fnmatch.fnmatch(f, ignore) for ignore in ignores) if valid_extension and not file_ignored: try: yield os.stat(os.path.join(root, f)).st_mtime except OSError as e: - logger.warning('Caught Exception: %s', e) + logger.warning("Caught Exception: %s", e) LAST_MTIME = 0 while True: @@ -897,14 +901,14 @@ def file_times(path): @staticmethod def file_watcher(path): - '''Generator for monitoring a file for modifications''' + """Generator for monitoring a file for modifications""" LAST_MTIME = 0 while True: if path: try: mtime = os.stat(path).st_mtime except OSError as e: - logger.warning('Caught Exception: %s', e) + logger.warning("Caught Exception: %s", e) continue if mtime > LAST_MTIME: @@ -919,10 +923,14 @@ def file_watcher(path): def set_date_tzinfo(d, tz_name=None): """Set the timezone for dates that don't have tzinfo""" if tz_name and not d.tzinfo: - tz = pytz.timezone(tz_name) - d = tz.localize(d) - return SafeDatetime(d.year, d.month, d.day, d.hour, d.minute, d.second, - d.microsecond, d.tzinfo) + try: + timezone = zoneinfo.ZoneInfo(tz_name) + except NameError: + timezone = backports.zoneinfo.ZoneInfo(tz_name) + d = d.replace(tzinfo=timezone) + return SafeDatetime( + d.year, d.month, d.day, d.hour, d.minute, d.second, d.microsecond, d.tzinfo + ) return d @@ -940,7 +948,7 @@ def split_all(path): ['a', 'b', 'c'] """ components = [] - path = path.lstrip('/') + path = path.lstrip("/") while path: head, tail = os.path.split(path) if tail: @@ -953,25 +961,25 @@ def split_all(path): def is_selected_for_writing(settings, path): - '''Check whether path is selected for writing + """Check whether path is selected for writing according to the WRITE_SELECTED list If WRITE_SELECTED is an empty list (default), any path is selected for writing. - ''' - if settings['WRITE_SELECTED']: - return path in settings['WRITE_SELECTED'] + """ + if settings["WRITE_SELECTED"]: + return path in settings["WRITE_SELECTED"] else: return True def path_to_file_url(path): - '''Convert file-system path to file:// URL''' + """Convert file-system path to file:// URL""" return urllib.parse.urljoin("file://", urllib.request.pathname2url(path)) def maybe_pluralize(count, singular, plural): - ''' + """ Returns a formatted string containing count and plural if count is not 1 Returns count and singular if count is 1 @@ -979,8 +987,8 @@ def maybe_pluralize(count, singular, plural): maybe_pluralize(1, 'Article', 'Articles') -> '1 Article' maybe_pluralize(2, 'Article', 'Articles') -> '2 Articles' - ''' + """ selection = plural if count == 1: selection = singular - return '{} {}'.format(count, selection) + return "{} {}".format(count, selection) diff --git a/pyproject.toml b/pyproject.toml index eb3a8f526..d16ce2dcd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,10 +37,10 @@ feedgenerator = ">=1.9" jinja2 = ">=2.7" pygments = ">=2.6" python-dateutil = ">=2.8" -pytz = ">=2020.1" rich = ">=10.1" unidecode = ">=1.1" markdown = {version = ">=3.1", optional = true} +backports-zoneinfo = {version = "^0.2.1", python = "3.9"} [tool.poetry.dev-dependencies] BeautifulSoup4 = "^4.9"