diff --git a/browse/controllers/archive_page/__init__.py b/browse/controllers/archive_page/__init__.py index 3b9d7324..c1733389 100644 --- a/browse/controllers/archive_page/__init__.py +++ b/browse/controllers/archive_page/__init__.py @@ -1,6 +1,6 @@ """Archive landing page.""" -import datetime +from datetime import datetime from typing import Any, Dict, List, Tuple, Optional from http import HTTPStatus as status @@ -48,15 +48,14 @@ def get_archive(archive_id: Optional[str]) -> Tuple[Dict[str, Any], int, Dict[st archive = subsuming_category.get_archive() years = years_operating(archive) - data["years"] = years + data["years"] = [datetime.now().year, datetime.now().year-1] #only last 90 days allowed anyways data["months"] = MONTHS data["days"] = DAYS data["archive"] = archive data["list_form"] = ByMonthForm(archive, years) data["stats_by_year"] = stats_by_year(archive, years) data["category_list"] = category_list(archive) - - data["catchup_to"] = datetime.date.today() - datetime.timedelta(days=7) + data["current_month"] = datetime.now().strftime('%m') data["template"] = "archive/single_archive.html" return data, status.OK, response_headers diff --git a/browse/controllers/catchup_page.py b/browse/controllers/catchup_page.py new file mode 100644 index 00000000..70fbb85b --- /dev/null +++ b/browse/controllers/catchup_page.py @@ -0,0 +1,256 @@ +"""handles requests to the catchup page. +Allows users to access something equivalent to the /new page for up to 90 days back +""" +import re +from typing import Tuple, Union, Dict, Any, List +from datetime import date, datetime, timedelta + +from http import HTTPStatus +from flask import request, redirect, url_for +from werkzeug.exceptions import BadRequest + +from arxiv.document.metadata import DocMetadata +from arxiv.integration.fastly.headers import add_surrogate_key +from arxiv.taxonomy.category import Group, Archive, Category +from arxiv.taxonomy.definitions import CATEGORIES, ARCHIVES, GROUPS, ARCHIVES_ACTIVE + +from browse.controllers.archive_page.by_month_form import MONTHS +from browse.controllers.list_page import latexml_links_for_articles, dl_for_articles, authors_for_articles, sub_sections_for_types, Response +from browse.services.database.catchup import get_catchup_data, CATCHUP_LIMIT, get_next_announce_day + +def get_catchup_page(subject_str:str, date:str)-> Response: + """get the catchup page for a given set of request parameters + see process_catchup_params for details on parameters + """ + subject, start_day, include_abs, page=_process_catchup_params(subject_str, date) + #check for redirects for noncanon subjects + if subject.id != subject.canonical_id: + return redirect( + url_for('catchup', + subject=subject.canonical_id, + date=start_day, + page=page, + abs=include_abs), + HTTPStatus.MOVED_PERMANENTLY) #type: ignore + + headers: Dict[str,str]={} + headers=add_surrogate_key(headers,["catchup",f"list-{start_day.year:04d}-{start_day.month:02d}-{subject.id}"]) + #get data + listing=get_catchup_data(subject, start_day, include_abs, page) + next_announce_day=get_next_announce_day(start_day) + + #format data + response_data: Dict[str, Any] = {} + headers.update({'Surrogate-Control': f'max-age={listing.expires}'}) + count= listing.new_count+listing.cross_count+listing.rep_count + response_data['announced'] = listing.announced + skip=(page-1)*CATCHUP_LIMIT + response_data.update(catchup_index_for_types(listing.new_count, listing.cross_count, listing.rep_count, subject, start_day, include_abs, page)) + response_data.update(sub_sections_for_types(listing, skip, CATCHUP_LIMIT)) + + idx = 0 + for item in listing.listings: + idx = idx + 1 + setattr(item, 'list_index', idx + skip) + + response_data['listings'] = listing.listings + response_data['author_links'] = authors_for_articles(listing.listings) + response_data['downloads'] = dl_for_articles(listing.listings) + response_data['latexml'] = latexml_links_for_articles(listing.listings) + + response_data.update({ + 'subject':subject, + 'date': start_day, + 'next_day':next_announce_day, + 'page':page, + 'include_abs': include_abs, + 'count': count, + 'list_type':"new" if include_abs else "catchup", #how the list macro checks to display abstract + 'paging': catchup_paging(subject, start_day, include_abs, page, count) + }) + + def author_query(article: DocMetadata, query: str)->str: + try: + if article.primary_archive: + archive_id = article.primary_archive.id + elif article.primary_category: + archive_id = article.primary_category.in_archive + else: + archive_id='' + return str(url_for('search_archive', + searchtype='author', + archive=archive_id, + query=query)) + except (AttributeError, KeyError): + return str(url_for('search_archive', + searchtype='author', + archive=archive_id, + query=query)) + + response_data['url_for_author_search'] = author_query + + return response_data, 200, headers + +def get_catchup_form() -> Response: + headers: Dict[str,str]={} + headers=add_surrogate_key(headers,["catchup"]) + + #check for form/parameter requests + subject = request.args.get('subject') + date = request.args.get('date') + include_abs = request.args.get('include_abs') + if subject and date: + if include_abs: + new_address= url_for('.catchup', subject=subject, date=date, abs=include_abs) + else: + new_address=url_for('.catchup', subject=subject, date=date) + headers.update({'Location':new_address}) + headers.update({'Surrogate-Control': f'max-age=2600000'}) #one month, url construction should never change + headers=add_surrogate_key(headers,["catchup-redirect"]) + return {}, 301, headers + + #otherwise create catchup form + response_data: Dict[str, Any]= {} + response_data['years']= [datetime.now().year, datetime.now().year-1] #only last 90 days allowed anyways + response_data['months']= MONTHS[1:] + response_data['current_month']=datetime.now().strftime('%m') + response_data['days']= [str(day).zfill(2) for day in range(1, 32)] + response_data['groups']= GROUPS + + headers=add_surrogate_key(headers,["catchup-form"]) + headers.update({'Surrogate-Control': f'max-age=604800'}) #one week, form never changes except for autoselecting currently month + return response_data, 200, headers + + +def _process_catchup_params(subject_str:str, date_str:str)->Tuple[Union[Group, Archive, Category], date, bool, int]: + """processes the request parameters to the catchup page + raises an error or returns usable values + + Returns: + subject: as a Group, Archive, or Category. Still needs to be checked for canonicalness + start_day: date (date to catchup on) + abs: bool (include abstracts or not ) + page: int (which page of results, default is 1) + """ + + #check for valid arguments + ALLOWED_PARAMS={"abs", "page"} + unexpected_params = request.args.keys() - ALLOWED_PARAMS + if unexpected_params: + raise BadRequest(f"Unexpected parameters. Only accepted parameters are: 'page', and 'abs'") + + #subject validation + subject: Union[Group, Archive, Category] + if subject_str == "grp_physics": + subject=GROUPS["grp_physics"] + elif subject_str in ARCHIVES: + subject= ARCHIVES[subject_str] + elif subject_str in CATEGORIES: + subject= CATEGORIES[subject_str] + else: + raise BadRequest("Invalid subject. Subject must be an archive, category or 'grp_physics'") + + #date validation + if not re.match(r"^\d{4}-\d{2}-\d{2}$", date_str): #enforce two digit days and months + raise BadRequest(f"Invalid date format. Use format: YYYY-MM-DD") + try: + start_day= datetime.strptime(date_str, "%Y-%m-%d").date() + except ValueError: + raise BadRequest(f"Invalid date format. Use format: YYYY-MM-DD") + #only allow dates within the last 90 days (91 just in case time zone differences) + today=datetime.now().date() + earliest_allowed=today - timedelta(days=91) + if start_day < earliest_allowed: + #TODO link to earliest allowed date + raise BadRequest(f"Invalid date: {start_day}. Catchup only allowed for past 90 days") + elif start_day > today: + raise BadRequest(f"Invalid date: {start_day}. Can't request date later than today") + + #include abstract or not + abs_str=request.args.get("abs","False") + if abs_str == "True": + include_abs=True + elif abs_str == "False": + include_abs=False + else: + raise BadRequest(f"Invalid abs value. Use ?abs=True to include abstracts or ?abs=False to not") + + #select page number (each page has 2000 items) + page_str = request.args.get("page", "1") #page defaults to 1 + if page_str.isdigit(): + page=int(page_str) + else: + raise BadRequest(f"Invalid page value. Page value should be a positive integer like ?page=3") + if page<1: + raise BadRequest(f"Invalid page value. Page value should be a positive integer like ?page=3") + + return subject, start_day, include_abs, page + +def catchup_paging(subject: Union[Group, Archive, Category], day:date, include_abs:bool, page: int, count:int)-> List[Tuple[str,str]]: + '''creates a dictionary of page links for the case that there is more than one page of data''' + if CATCHUP_LIMIT >= count: #only one page + return [] + + total_pages=count//CATCHUP_LIMIT+1 + url_base=url_for('.catchup', subject=subject.id, date=day.strftime('%Y-%m-%d'), abs=include_abs) + page_links=[] + + if total_pages <10: #realistically there should be at most 2-3 pages per day + for i in range(1,total_pages+1): + if i == page: + page_links.append((str(i),'no-link')) + else: + page_links.append((str(i),url_base+f'&page={i}')) + + else: #shouldnt happen but its handled + if page !=1: + page_links.append(('1',url_base+f'&page=1')) + if page >2: + page_links.append(('...','no-link')) + page_links.append((str(page),'no-link')) + if page Dict[str, Any]: + """Creates index for types for catchup papers. + page count and index both start at 1 + """ + ift = [] + + if new_count > 0: + if page != 1: + ift.append(('New submissions', + url_for('.catchup', subject=subject.id, date=day.strftime('%Y-%m-%d'), abs=include_abs, page=1), + 1)) + else: + ift.append(('New submissions', '', 1)) + + if cross_count > 0: + cross_start = new_count + 1 + cross_start_page=(cross_start-1)//CATCHUP_LIMIT +1 #item 2000 is on page 1, 2001 is on page 2 + cross_index=cross_start-(cross_start_page-1)*CATCHUP_LIMIT + + if page==cross_start_page: + ift.append(('Cross-lists', '', cross_index)) + else: + ift.append(('Cross-lists', + url_for('.catchup', subject=subject.id, date=day.strftime('%Y-%m-%d'), abs=include_abs, page=cross_start_page), + cross_index)) + + if rep_count > 0: + rep_start = new_count + cross_count+ 1 + rep_start_page=(rep_start-1)//CATCHUP_LIMIT +1 #item 2000 is on page 1, 2001 is on page 2 + rep_index=rep_start-(rep_start_page-1)*CATCHUP_LIMIT + + if page==rep_start_page: + ift.append(('Replacements', '', rep_index)) + else: + ift.append(('Replacements', + url_for('.catchup', subject=subject.id, date=day.strftime('%Y-%m-%d'), abs=include_abs, page=rep_start_page), + rep_index)) + + return {'index_for_types': ift} diff --git a/browse/controllers/list_page/__init__.py b/browse/controllers/list_page/__init__.py index 9972aaf4..6ae02888 100644 --- a/browse/controllers/list_page/__init__.py +++ b/browse/controllers/list_page/__init__.py @@ -549,7 +549,7 @@ def sub_sections_for_types( continued=skipn > 0, last=skipn >= new_count - shown, visible=len(news)>0, - heading=f'New submissions for {date} ' + heading=f'New submissions ' ) sec_cross=ListingSection( @@ -559,7 +559,7 @@ def sub_sections_for_types( continued=skipn + 1 > cross_start, last=skipn >= rep_start - shown, visible=len(crosses)>0, - heading=f'Cross submissions for {date} ' + heading=f'Cross submissions ' ) sec_rep=ListingSection( @@ -569,7 +569,7 @@ def sub_sections_for_types( continued=skipn + 1 > rep_start, last=last_shown >= new_count + cross_count + rep_count, visible=len(reps)>0, - heading=f'Replacement submissions for {date} ' + heading=f'Replacement submissions ' ) secs=[sec_new, sec_cross, sec_rep] @@ -582,7 +582,7 @@ def sub_sections_for_types( showing = showing + 'last ' if not sec.last and not sec.continued: showing = showing + 'first ' - sec.heading += f'({showing}{len(sec.items)} of {sec.total} entries )' + sec.heading += f'({showing}{len(sec.items)} of {sec.total} entries)' return {'sub_sections_for_types': secs} diff --git a/browse/routes/ui.py b/browse/routes/ui.py index fef3ee7f..04ff28aa 100644 --- a/browse/routes/ui.py +++ b/browse/routes/ui.py @@ -30,7 +30,8 @@ list_page, prevnext, stats_page, - tb_page + tb_page, + catchup_page ) from browse.controllers.openurl_cookie import make_openurl_cookie, get_openurl_page from browse.controllers.cookies import get_cookies_page, cookies_to_set @@ -129,6 +130,21 @@ def category_taxonomy() -> Any: None, ) +@blueprint.route("catchup", methods=["GET"], endpoint="catchup_form") +def catchup_form() -> Response: + response, code, headers = catchup_page.get_catchup_form() + if code == status.OK: + return render_template("catchup_form.html", **response), code, headers # type: ignore + return response, code, headers # type: ignore + +@blueprint.route("catchup//", methods=["GET"]) +def catchup(subject:str, date:str) -> Response: + response, code, headers = catchup_page.get_catchup_page(subject, date) + headers=add_surrogate_key(headers,["catchup"]) + if code == status.OK: + return render_template("catchup.html", **response), code, headers # type: ignore + return response, code, headers # type: ignore + @blueprint.route("institutional_banner", methods=["GET"]) def institutional_banner() -> Any: try: diff --git a/browse/routes/unimplemented.py b/browse/routes/unimplemented.py index 48ca7a71..6df427db 100644 --- a/browse/routes/unimplemented.py +++ b/browse/routes/unimplemented.py @@ -18,9 +18,6 @@ # these commenetd out blueprints are ones that the arxiv-browse or # arxiv-search systems should sooner or later should probably implement. -# -# @blueprint.route("/pdf/", defaults={"path":""}) -# @blueprint.route("/pdf/") # @blueprint.route("/docmeta/", defaults={"path":""}) # @blueprint.route("/docmeta/") # @blueprint.route("/docmeta_bulk/", defaults={"path":""}) @@ -31,18 +28,8 @@ # @blueprint.route("/tar/") # @blueprint.route("/abstar/", defaults={"path":""}) # @blueprint.route("/abstar/") -# @blueprint.route("/e-print/", defaults={"path":""}) -# @blueprint.route("/e-print/") -# @blueprint.route("/src/", defaults={"path":""}) -# @blueprint.route("/src/") -# @blueprint.route("/list/", defaults={"path":""}) -# @blueprint.route("/list/") # @blueprint.route("/view/", defaults={"path":""}) # @blueprint.route("/view/") -# @blueprint.route("/catchup/", defaults={"path":""}) -# @blueprint.route("/catchup/") -# @blueprint.route("/year/", defaults={"path":""}) -# @blueprint.route("/year/") # @blueprint.route("/cits/", defaults={"path":""}) # @blueprint.route("/cits/") # @blueprint.route("/refs/", defaults={"path":""}) @@ -51,18 +38,12 @@ # @blueprint.route("/ps/") # @blueprint.route("/psfigs/", defaults={"path":""}) # @blueprint.route("/psfigs/") -# @blueprint.route("/format/", defaults={"path":""}) -# @blueprint.route("/format/") # @blueprint.route("/dvi/", defaults={"path":""}) # @blueprint.route("/dvi/") -# @blueprint.route("/pdf/", defaults={"path":""}) -# @blueprint.route("/pdf/") # @blueprint.route("/openurl-cookie/", defaults={"path":""}) # @blueprint.route("/openurl-cookie/") # @blueprint.route("/openurl-resolver/", defaults={"path":""}) # @blueprint.route("/openurl-resolver/") -# @blueprint.route("/html/", defaults={"path":""}) -# @blueprint.route("/html/") # @blueprint.route("/ftp/") diff --git a/browse/services/database/catchup.py b/browse/services/database/catchup.py new file mode 100644 index 00000000..e2180194 --- /dev/null +++ b/browse/services/database/catchup.py @@ -0,0 +1,189 @@ +from typing import Union, Optional +from datetime import date + +from sqlalchemy import or_, and_, case +from sqlalchemy.orm import aliased, load_only +from sqlalchemy.sql import func + +from arxiv.db import Session +from arxiv.db.models import Metadata, NextMail, t_arXiv_in_category +from arxiv.taxonomy.category import Group, Archive, Category + +from browse.services.database.listings import _metadata_to_listing_item, process_requested_subject +from browse.services.listing import ListingNew, gen_expires + +CATCHUP_LIMIT=2000 + +def get_catchup_data(subject: Union[Group, Archive, Category], day:date, include_abs:bool, page_num:int)-> ListingNew: + """ + parameters should already be verified (only canonical subjects, date acceptable) + """ + offset=(page_num-1)*CATCHUP_LIMIT + + mail_id=date_to_mail_id(day) + #get document ids + doc_ids=( + Session.query( + NextMail.document_id, + NextMail.version, + NextMail.type + ) + .filter(NextMail.mail_id==mail_id) + .filter(NextMail.type!="jref") + .filter( + or_( + NextMail.type != 'rep', + NextMail.version <= 5 + ) + ) + .subquery() + ) + + #filter by subject + aic = aliased(t_arXiv_in_category) + archives, categories=process_requested_subject(subject) + cat_conditions = [and_(aic.c.archive == arch_part, aic.c.subject_class == subj_part) for arch_part, subj_part in categories] + + all_items=( + Session.query( + doc_ids.c.document_id, + doc_ids.c.type, + func.max(aic.c.is_primary).label('is_primary') + ) + .join(aic, aic.c.document_id == doc_ids.c.document_id) + .where( + or_( + aic.c.archive.in_(archives), + or_(*cat_conditions) + ) + ) + .group_by(aic.c.document_id) + .subquery() + ) + + #categorize by type of listing + listing_type = case(* + [ + (and_(all_items.c.type == 'new', all_items.c.is_primary == 1), 'new'), + (and_(all_items.c.type == 'cross', all_items.c.is_primary == 1), 'no-match'), #removes intra archive crosses + (or_(all_items.c.type == 'new', all_items.c.type == 'cross'), 'cross'), + (and_(or_(all_items.c.type == 'rep', all_items.c.type == 'wdr'), all_items.c.is_primary == 1), 'rep'), + (or_(all_items.c.type == 'rep', all_items.c.type == 'wdr'), 'repcross') + ], + else_="no_match" + ).label('listing_type') + + case_order = case(* + [ + (listing_type == 'new', 0), + (listing_type == 'cross', 1), + (listing_type == 'rep', 2), + (listing_type == 'repcross', 3), + ], + else_=4 + ).label('case_order') + + valid_types=["new", "cross", 'rep','repcross'] + + #counts + counts = ( + Session.query( + listing_type, + func.count().label('type_count') + ) + .filter(listing_type.label('case_order').in_(valid_types)) + .group_by(listing_type) + .order_by(case_order) + .all() + ) + + new_count=0 + cross_count=0 + rep_count=0 + for name, number in counts: + if name =="new": + new_count+=number + elif name=="cross": + cross_count+=number + else: #rep and repcross + rep_count+=number + + #data + meta = aliased(Metadata) + load_fields = [ + meta.document_id, + meta.paper_id, + meta.updated, + meta.source_flags, + meta.title, + meta.authors, + meta.abs_categories, + meta.comments, + meta.journal_ref, + meta.version, + meta.modtime, + ] + if include_abs: + load_fields.append(meta.abstract) + + #sort, limit and fetch + results=( + Session.query( + listing_type, + meta + ) + .join(meta, meta.document_id == all_items.c.document_id) + .filter(listing_type.label('case_order').in_(valid_types)) + .filter(meta.is_current ==1) + .order_by(case_order, meta.paper_id) + .offset(offset) + .limit(CATCHUP_LIMIT) + .options(load_only(*load_fields, raiseload=True)) + .all() + ) + + #process similar to listings + items=[] + for row in results: + listing_case, metadata = row + if listing_case=="repcross": + listing_case="rep" + item= _metadata_to_listing_item(metadata, listing_case) + items.append(item) + + return ListingNew(listings=items, + new_count=new_count, + cross_count=cross_count, + rep_count=rep_count, + announced=day, + expires=gen_expires()) + +def get_next_announce_day(day: date)->Optional[date]: + """returns the next day with announcements after the parameter day + returns None if the input data was the most recent mailing date and there are no mailings past then + """ + mail_id=date_to_mail_id(day) + next_day = ( + Session.query(NextMail.mail_id) + .filter(NextMail.mail_id > mail_id) + .filter(NextMail.is_written ==1) + .order_by(NextMail.mail_id.asc()) + .first() + ) + if not next_day: + return None + + return mail_id_to_date(next_day[0]) + +def date_to_mail_id(day:date)->str: + """converts a date to the mail_id it would have""" + return f"{(day.year-2000):02d}{day.month:02d}{day.day:02d}" + +def mail_id_to_date(mail_id:str)->date: + "converts an arxiv mailid into a date" + year=int(mail_id[0:2])+1900 + if year <1990: + year+=100 + month=int(mail_id[2:4]) + day=int(mail_id[4:6]) + return date(year=year, month=month, day=day) \ No newline at end of file diff --git a/browse/services/database/listings.py b/browse/services/database/listings.py index d9851dda..6ee38abc 100644 --- a/browse/services/database/listings.py +++ b/browse/services/database/listings.py @@ -1,9 +1,10 @@ from datetime import datetime from dateutil.tz import gettz -from typing import List, Optional, Tuple +from typing import List, Optional, Tuple, Set, Union from sqlalchemy import case, distinct, or_, and_, desc -from sqlalchemy.sql import func, select +from sqlalchemy.exc import InvalidRequestError +from sqlalchemy.sql import func from sqlalchemy.engine import Row from sqlalchemy.orm import aliased, load_only @@ -16,15 +17,16 @@ ListingNew, AnnounceTypes ) + from arxiv.db import Session -from arxiv.db.models import Metadata, DocumentCategory, Document, Updates, t_arXiv_in_category +from arxiv.db.models import Metadata, Document, Updates, t_arXiv_in_category from arxiv.document.metadata import DocMetadata, AuthorList -from arxiv.taxonomy.definitions import CATEGORIES, ARCHIVES, ARCHIVES_SUBSUMED +from arxiv.taxonomy.category import Group, Archive, Category +from arxiv.taxonomy.definitions import CATEGORIES, ARCHIVES from arxiv.document.version import VersionEntry, SourceFlag from arxiv.base.globals import get_application_config from arxiv.base import logging -from logging import Logger from werkzeug.exceptions import BadRequest logger = logging.getLogger(__name__) @@ -427,12 +429,17 @@ def _metadata_to_listing_item(meta: Metadata, type: AnnounceTypes) -> ListingIte primary_cat=CATEGORIES["bad-arch.bad-cat"] secondary_cats=[] + try: #incase abstract wasnt loaded + abstract=getattr(meta, 'abstract','') + except InvalidRequestError: + abstract='' + doc = DocMetadata( arxiv_id=meta.paper_id, arxiv_id_v=f"{meta.paper_id}v{meta.version}", title= getattr(meta, 'title',''), authors=AuthorList(getattr(meta, 'authors',"")), - abstract= getattr(meta, 'abstract','') , + abstract= abstract, categories= getattr(meta, 'abs_categories',""), primary_category=primary_cat, secondary_categories=secondary_cats, @@ -488,35 +495,59 @@ def _entries_into_monthly_listing_items( return new_listings, cross_listings + +def process_requested_subject(subject: Union[Group, Archive, Category])-> Tuple[Set[str], Set[Tuple[str,str]]]: + """ + set of archives to search if appliable, + set of tuples are the categories to check for in addition to the archive broken into archive and category parts + only categories not contained by the set of archives will be returned seperately to work with the archive in category table + """ + archs=set() + cats=set() + + #utility functions + def process_cat_name(name: str) -> None: + #splits category name into parts and adds it + if "." in name: + arch_part, cat_part = name.split(".") + if arch_part not in archs: + cats.add((arch_part, cat_part)) + elif name not in archs: + archs.add(name) + + #handle category request + if isinstance(subject, Category): + process_cat_name(subject.id) + if subject.alt_name: + process_cat_name(subject.alt_name) + + elif isinstance(subject, Archive): + archs.add(subject.id) + for category in subject.get_categories(True): + process_cat_name(category.alt_name) if category.alt_name else None + + elif isinstance(subject, Group): + for arch in subject.get_archives(True): + archs.add(arch.id) + for arch in subject.get_archives(True): #twice to avoid adding cateogires covered by archives + for category in arch.get_categories(True): + process_cat_name(category.alt_name) if category.alt_name else None + + return archs, cats + + def _request_categories(archive_or_cat:str) -> Tuple[List[str],List[Tuple[str,str]]]: """ list of archives to search if appliable, list of tuples are the categories to check for (possibly in addition to the archive) broken into archvie and category parts if a category is received, return the category and possible alternate names if an archive is received return the archive name and a list of all categories that should be included but arent nominally part of the archive """ - arch=[] - cats=[] - - def process_alt_name(alt_name: str) -> None: - if "." in alt_name: - arch_part, cat_part = alt_name.split(".") - cats.append((arch_part, cat_part)) - else: - arch.append(alt_name) - - if archive_or_cat in ARCHIVES: #get all categories for archive - archive=ARCHIVES[archive_or_cat] - arch.append(archive_or_cat) - for category in archive.get_categories(True): - process_alt_name(category.alt_name) if category.alt_name else None - - else: #otherwise its just a category requested - category=CATEGORIES[archive_or_cat] - process_alt_name(archive_or_cat) - if category.alt_name: - process_alt_name(category.alt_name) if category.alt_name else None - - return arch, cats + if archive_or_cat in ARCHIVES: + arch, cats=process_requested_subject(ARCHIVES[archive_or_cat]) + elif archive_or_cat in CATEGORIES: + arch, cats=process_requested_subject(CATEGORIES[archive_or_cat]) + + return list(arch), list(cats) def _all_possible_categories(archive_or_cat:str) -> List[str]: """returns a list of all categories in an archive, or all possible alternate names for categories diff --git a/browse/templates/archive/single_archive.html b/browse/templates/archive/single_archive.html index 573f5adf..51cb2c15 100644 --- a/browse/templates/archive/single_archive.html +++ b/browse/templates/archive/single_archive.html @@ -48,39 +48,53 @@

{{archive.full_name}} (since {{archive.start_d
  • Catch-up: - {# hard coded for legacy /catchup for now #} -
    - + + + + + {% set categories = archive.get_categories() %} + {% if categories|length > 1 %} + + +
    + {% endif %} + Changes since: - {% for day in days %} - + {% endfor %} - + {% for val, name in months %} + + {% endfor %} - + {% for year in years %} + + {% endfor %} - , view results + + , view results + abstracts +
    • +
  • Search within the {{archive.id}} archive
  • Article statistics by year:
    @@ -106,4 +120,26 @@

    Categories within {{archive.full_name}}

    {% endfor %} {% endif %} + + {%- endblock %} + + diff --git a/browse/templates/catchup.html b/browse/templates/catchup.html new file mode 100644 index 00000000..0109a49f --- /dev/null +++ b/browse/templates/catchup.html @@ -0,0 +1,104 @@ +{%- extends "base.html" -%} +{% import "list/list_macros.html" as list_macros with context %} + +{#Catchup is a varaiation of the new page, but for days in the past and has set pagination#} + +{% block head %} + {{super()-}} + + +{% endblock head %} + +{% block header_h1 %} +
    + arxiv logo > + catchup +
    +{% endblock %} + +{%- block content %} +
    +
    +

    Catchup results for {{subject.full_name}} on {{date.strftime("%a, %d %b %Y")}}

    + + {% block list_index %} +
      + {% for ityp in index_for_types %} +
    • {{ityp[0]}}
      • + {% endfor %} +
    + + {% endblock %} + + {% block pre_items %} +
    + {% if next_day is not none %} + Continue to the next day +

    + {% endif %} + + Total of {{count}} entries for {{date.strftime("%a, %d %b %Y")}} + {{catchup_pages()}} + +
    + {% endblock %} + + {% block items %} + + {% for sec in sub_sections_for_types %} + {%if sec.visible%} + {{ list_macros.item_list(sec.items, sec.heading) }} + {% endif %} + {% endfor %} + + {% if not (sub_sections_for_types[0].visible or sub_sections_for_types[1].visible or sub_sections_for_types[2].visible) %} +
    + {% if page > 1 %} +
    + No further updates for {{date.strftime("%a, %d %b %Y")}} +
    + Go to page 1 +
    + {% else %} +
    +

    No updates for {{date.strftime("%a, %d %b %Y")}}. There was either no announcement for this day, or no updates within the requested subject.

    +
    + {% endif %} +
    + {% endif %} + + {% endblock %} + + {% block post_items %} +
    + Total of {{count}} entries for {{date.strftime("%a, %d %b %Y")}} + {{catchup_pages()}} + + {% if next_day is not none %} +

    + Continue to the next day + {% endif %} +
    + {% endblock %} + +
    + +{%- endblock %} + + +{%- macro catchup_pages() -%} + {% if paging | length > 0 %} + View page: + {%- for number, link in paging -%} + + {% if link == 'no-link' %} + {{number}} + {% else %} + {{number}} + {% endif %} + + {% endfor %} + {% endif %} +{%- endmacro -%} + + diff --git a/browse/templates/catchup_form.html b/browse/templates/catchup_form.html new file mode 100644 index 00000000..877f39bf --- /dev/null +++ b/browse/templates/catchup_form.html @@ -0,0 +1,184 @@ +{%- extends "base.html" -%} + +{% block title %}Catchup{% endblock %} + +{% block header_h1 %} +
    + arxiv logo + > + catchup +
    +{% endblock %} + +{%- block content %} +

    Catchup on:

    + +
    +
    +

    Subject:

    + +
    + + +
    + +
    + + + + + {% for group in groups.values() %} + {% if group.is_active and not group.is_test %} + + + + {% endif %} + {% endfor %} +
    + +
    + + + + + {% for group in groups.values() %} + {% if group.is_active and not group.is_test %} + {% for archive in group.get_archives() %} + + + + {% endfor %} + {% endif %} + {% endfor %} +
    +
    + + +
    +

    Changes Since:

    + + + + + + +
    + +
    +
    + + + +
    +
    + + +
    + + + +{%- endblock %} diff --git a/browse/templates/home/home.html b/browse/templates/home/home.html index 3079c364..6bf3a568 100644 --- a/browse/templates/home/home.html +++ b/browse/templates/home/home.html @@ -36,9 +36,14 @@ - +