diff --git a/components/renku_data_services/migrations/versions/f254f678b82b_add_documentation_to_projects.py b/components/renku_data_services/migrations/versions/f254f678b82b_add_documentation_to_projects.py new file mode 100644 index 00000000..452db0d5 --- /dev/null +++ b/components/renku_data_services/migrations/versions/f254f678b82b_add_documentation_to_projects.py @@ -0,0 +1,28 @@ +"""add documentation to projects + +Revision ID: f254f678b82b +Revises: 726d5d0e1f28 +Create Date: 2024-10-09 12:07:08.959418 + +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "f254f678b82b" +down_revision = "726d5d0e1f28" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column("projects", sa.Column("documentation", sa.String(), nullable=True), schema="projects") + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("projects", "documentation", schema="projects") + # ### end Alembic commands ### diff --git a/components/renku_data_services/project/api.spec.yaml b/components/renku_data_services/project/api.spec.yaml index dfc385a3..9b4b007a 100644 --- a/components/renku_data_services/project/api.spec.yaml +++ b/components/renku_data_services/project/api.spec.yaml @@ -80,6 +80,11 @@ paths: required: true schema: $ref: "#/components/schemas/Ulid" + - in: query + name: with_documentation + required: false + schema: + $ref: "#/components/schemas/WithDocumentation" responses: "200": description: The project @@ -337,6 +342,8 @@ components: $ref: "#/components/schemas/Description" keywords: $ref: "#/components/schemas/KeywordsList" + documentation: + $ref: "#/components/schemas/ProjectDocumentation" required: - "name" - "namespace" @@ -357,12 +364,17 @@ components: $ref: "#/components/schemas/Description" keywords: $ref: "#/components/schemas/KeywordsList" + documentation: + $ref: "#/components/schemas/ProjectDocumentation" Ulid: description: ULID identifier type: string minLength: 26 maxLength: 26 pattern: "^[0-7][0-9A-HJKMNP-TV-Z]{25}$" # This is case-insensitive + WithDocumentation: + description: Projects with or without possibly extensive documentation? + type: boolean ProjectName: description: Renku project name type: string @@ -396,6 +408,12 @@ components: description: A description for the resource type: string maxLength: 500 + Keyword: + description: A single keyword + type: string + minLength: 1 + maxLength: 99 + pattern: '^[A-Za-z0-9\s\-_.]*$' KeywordsList: description: Project keywords type: array @@ -405,12 +423,12 @@ components: example: - "project" - "keywords" - Keyword: - description: A single keyword + ProjectDocumentation: + description: Renku project documentation type: string minLength: 1 - maxLength: 99 - pattern: '^[A-Za-z0-9\s\-_.]*$' + maxLength: 5000 + example: "My Renku Project Documentation :)" RepositoriesList: description: A list of repositories type: array diff --git a/components/renku_data_services/project/apispec.py b/components/renku_data_services/project/apispec.py index af8b24ed..7642b2f2 100644 --- a/components/renku_data_services/project/apispec.py +++ b/components/renku_data_services/project/apispec.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: api.spec.yaml -# timestamp: 2024-09-30T13:50:44+00:00 +# timestamp: 2024-10-10T07:28:43+00:00 from __future__ import annotations @@ -55,6 +55,12 @@ class ErrorResponse(BaseAPISpec): error: Error +class ProjectsProjectIdGetParametersQuery(BaseAPISpec): + with_documentation: Optional[bool] = Field( + None, description="Projects with or without possibly extensive documentation?" + ) + + class ProjectMemberPatchRequest(BaseAPISpec): model_config = ConfigDict( extra="forbid", @@ -232,6 +238,13 @@ class ProjectPost(BaseAPISpec): example=["project", "keywords"], min_length=0, ) + documentation: Optional[str] = Field( + None, + description="Renku project documentation", + example="My Renku Project Documentation :)", + max_length=5000, + min_length=1, + ) class ProjectPatch(BaseAPISpec): @@ -272,6 +285,13 @@ class ProjectPatch(BaseAPISpec): example=["project", "keywords"], min_length=0, ) + documentation: Optional[str] = Field( + None, + description="Renku project documentation", + example="My Renku Project Documentation :)", + max_length=5000, + min_length=1, + ) class ProjectMemberListPatchRequest(RootModel[List[ProjectMemberPatchRequest]]): diff --git a/components/renku_data_services/project/blueprints.py b/components/renku_data_services/project/blueprints.py index 1ce5a72a..bb2c8ae4 100644 --- a/components/renku_data_services/project/blueprints.py +++ b/components/renku_data_services/project/blueprints.py @@ -36,6 +36,25 @@ class ProjectsBP(CustomBlueprint): user_repo: UserRepo authenticator: base_models.Authenticator + def _project2dict(self, project: project_models.Project, with_documentation: bool = False) -> dict[str, Any]: + result = dict( + id=str(project.id), + name=project.name, + namespace=project.namespace.slug, + slug=project.slug, + creation_date=project.creation_date.isoformat(), + created_by=project.created_by, + updated_at=project.updated_at.isoformat() if project.updated_at else None, + repositories=project.repositories, + visibility=project.visibility.value, + description=project.description, + etag=project.etag, + keywords=project.keywords or [], + ) + if with_documentation: + result = dict(result, documentation=project.documentation) + return result + def get_all(self) -> BlueprintFactoryResponse: """List all projects.""" @@ -48,23 +67,7 @@ async def _get_all( projects, total_num = await self.project_repo.get_projects( user=user, pagination=pagination, namespace=query.namespace, direct_member=query.direct_member ) - return [ - dict( - id=str(p.id), - name=p.name, - namespace=p.namespace.slug, - slug=p.slug, - creation_date=p.creation_date.isoformat(), - created_by=p.created_by, - updated_at=p.updated_at.isoformat() if p.updated_at else None, - repositories=p.repositories, - visibility=p.visibility.value, - description=p.description, - etag=p.etag, - keywords=p.keywords or [], - ) - for p in projects - ], total_num + return [self._project2dict(project) for project in projects], total_num return "/projects", ["GET"], _get_all @@ -85,24 +88,10 @@ async def _post(_: Request, user: base_models.APIUser, body: apispec.ProjectPost created_by=user.id, # type: ignore[arg-type] visibility=Visibility(body.visibility.value), keywords=keywords, + documentation=body.documentation, ) - result = await self.project_repo.insert_project(user, project) - return json( - dict( - id=str(result.id), - name=result.name, - namespace=result.namespace.slug, - slug=result.slug, - creation_date=result.creation_date.isoformat(), - created_by=result.created_by, - repositories=result.repositories, - visibility=result.visibility.value, - description=result.description, - etag=result.etag, - keywords=result.keywords or [], - ), - 201, - ) + project = await self.project_repo.insert_project(user, project) + return json(self._project2dict(project), 201) return "/projects", ["POST"], _post @@ -112,29 +101,17 @@ def get_one(self) -> BlueprintFactoryResponse: @authenticate(self.authenticator) @validate_path_project_id async def _get_one(request: Request, user: base_models.APIUser, project_id: str) -> JSONResponse | HTTPResponse: - project = await self.project_repo.get_project(user=user, project_id=ULID.from_str(project_id)) + with_documentation = bool(request.args.get("with_documentation", False)) + project = await self.project_repo.get_project( + user=user, project_id=ULID.from_str(project_id), with_documentation=with_documentation + ) etag = request.headers.get("If-None-Match") if project.etag is not None and project.etag == etag: return HTTPResponse(status=304) headers = {"ETag": project.etag} if project.etag is not None else None - return json( - dict( - id=str(project.id), - name=project.name, - namespace=project.namespace.slug, - slug=project.slug, - creation_date=project.creation_date.isoformat(), - created_by=project.created_by, - repositories=project.repositories, - visibility=project.visibility.value, - description=project.description, - etag=project.etag, - keywords=project.keywords or [], - ), - headers=headers, - ) + return json(self._project2dict(project, with_documentation=with_documentation), headers=headers) return "/projects/", ["GET"], _get_one @@ -152,22 +129,7 @@ async def _get_one_by_namespace_slug( return HTTPResponse(status=304) headers = {"ETag": project.etag} if project.etag is not None else None - return json( - dict( - id=str(project.id), - name=project.name, - namespace=project.namespace.slug, - slug=project.slug, - creation_date=project.creation_date.isoformat(), - created_by=project.created_by, - repositories=project.repositories, - visibility=project.visibility.value, - description=project.description, - etag=project.etag, - keywords=project.keywords or [], - ), - headers=headers, - ) + return json(self._project2dict(project), headers=headers) return "/projects//", ["GET"], _get_one_by_namespace_slug @@ -206,22 +168,7 @@ async def _patch( ) updated_project = project_update.new - return json( - dict( - id=str(updated_project.id), - name=updated_project.name, - namespace=updated_project.namespace.slug, - slug=updated_project.slug, - creation_date=updated_project.creation_date.isoformat(), - created_by=updated_project.created_by, - repositories=updated_project.repositories, - visibility=updated_project.visibility.value, - description=updated_project.description, - etag=updated_project.etag, - keywords=updated_project.keywords or [], - ), - 200, - ) + return json(self._project2dict(updated_project), 200) return "/projects/", ["PATCH"], _patch diff --git a/components/renku_data_services/project/db.py b/components/renku_data_services/project/db.py index ae5f427d..e31466cc 100644 --- a/components/renku_data_services/project/db.py +++ b/components/renku_data_services/project/db.py @@ -9,6 +9,7 @@ from sqlalchemy import Select, delete, func, select, update from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import undefer from sqlalchemy.sql.functions import coalesce from ulid import ULID @@ -92,7 +93,9 @@ async def get_all_projects(self, requested_by: base_models.APIUser) -> AsyncGene async for project in projects: yield project.dump() - async def get_project(self, user: base_models.APIUser, project_id: ULID) -> models.Project: + async def get_project( + self, user: base_models.APIUser, project_id: ULID, with_documentation: bool = False + ) -> models.Project: """Get one project from the database.""" authorized = await self.authz.has_permission(user, ResourceType.project, project_id, Scope.READ) if not authorized: @@ -102,13 +105,15 @@ async def get_project(self, user: base_models.APIUser, project_id: ULID) -> mode async with self.session_maker() as session: stmt = select(schemas.ProjectORM).where(schemas.ProjectORM.id == project_id) + if with_documentation: + stmt = stmt.options(undefer(schemas.ProjectORM.documentation)) result = await session.execute(stmt) project_orm = result.scalars().first() if project_orm is None: raise errors.MissingResourceError(message=f"Project with id '{project_id}' does not exist.") - return project_orm.dump() + return project_orm.dump(with_documentation=with_documentation) async def get_project_by_namespace_slug( self, user: base_models.APIUser, namespace: str, slug: str @@ -188,6 +193,7 @@ async def insert_project( repositories=repositories, creation_date=datetime.now(UTC).replace(microsecond=0), keywords=project.keywords, + documentation=project.documentation, ) project_slug = schemas.ProjectSlug(slug, project_id=project_orm.id, namespace_id=ns.id) diff --git a/components/renku_data_services/project/models.py b/components/renku_data_services/project/models.py index 13c1bb9c..c85d92fd 100644 --- a/components/renku_data_services/project/models.py +++ b/components/renku_data_services/project/models.py @@ -26,6 +26,7 @@ class BaseProject: repositories: list[Repository] = field(default_factory=list) description: Optional[str] = None keywords: Optional[list[str]] = None + documentation: Optional[str] = None @property def etag(self) -> str | None: diff --git a/components/renku_data_services/project/orm.py b/components/renku_data_services/project/orm.py index b50232fe..9e935b6e 100644 --- a/components/renku_data_services/project/orm.py +++ b/components/renku_data_services/project/orm.py @@ -34,6 +34,7 @@ class ProjectORM(BaseORM): created_by_id: Mapped[str] = mapped_column("created_by_id", String()) description: Mapped[str | None] = mapped_column("description", String(500)) keywords: Mapped[Optional[list[str]]] = mapped_column("keywords", ARRAY(String(99)), nullable=True) + documentation: Mapped[str | None] = mapped_column("documentation", String(), nullable=True, deferred=True) # NOTE: The project slugs table has a foreign key from the projects table, but there is a stored procedure # triggered by the deletion of slugs to remove the project used by the slug. See migration 89aa4573cfa9. slug: Mapped["ProjectSlug"] = relationship(lazy="joined", init=False, repr=False, viewonly=True) @@ -51,7 +52,7 @@ class ProjectORM(BaseORM): "updated_at", DateTime(timezone=True), default=None, server_default=func.now(), onupdate=func.now() ) - def dump(self) -> models.Project: + def dump(self, with_documentation: bool = False) -> models.Project: """Create a project model from the ProjectORM.""" return models.Project( id=self.id, @@ -67,6 +68,7 @@ def dump(self) -> models.Project: repositories=[models.Repository(r.url) for r in self.repositories], description=self.description, keywords=self.keywords, + documentation=self.documentation if with_documentation else None, ) diff --git a/test/bases/renku_data_services/data_api/test_projects.py b/test/bases/renku_data_services/data_api/test_projects.py index f1e229ad..9953388e 100644 --- a/test/bases/renku_data_services/data_api/test_projects.py +++ b/test/bases/renku_data_services/data_api/test_projects.py @@ -37,6 +37,7 @@ async def test_project_creation(sanic_client, user_headers, regular_user, app_co "repositories": ["http://renkulab.io/repository-1", "http://renkulab.io/repository-2"], "namespace": f"{regular_user.first_name}.{regular_user.last_name}", "keywords": ["keyword 1", "keyword.2", "keyword-3", "KEYWORD_4"], + "documentation": "$\\sqrt(2)$", } _, response = await sanic_client.post("/api/data/projects", headers=user_headers, json=payload) @@ -46,13 +47,15 @@ async def test_project_creation(sanic_client, user_headers, regular_user, app_co assert project["name"] == "Renku Native Project" assert project["slug"] == "project-slug" assert project["description"] == "First Renku native project" - assert set(project["keywords"]) == {"keyword 1", "keyword.2", "keyword-3", "KEYWORD_4"} assert project["visibility"] == "public" - assert project["created_by"] == "user" assert {r for r in project["repositories"]} == { "http://renkulab.io/repository-1", "http://renkulab.io/repository-2", } + assert set(project["keywords"]) == {"keyword 1", "keyword.2", "keyword-3", "KEYWORD_4"} + assert "documentation" not in project + assert project["created_by"] == "user" + project_id = project["id"] events = await app_config.event_repo._get_pending_events() @@ -78,13 +81,22 @@ async def test_project_creation(sanic_client, user_headers, regular_user, app_co assert project["name"] == "Renku Native Project" assert project["slug"] == "project-slug" assert project["description"] == "First Renku native project" - assert set(project["keywords"]) == {"keyword 1", "keyword.2", "keyword-3", "KEYWORD_4"} assert project["visibility"] == "public" - assert project["created_by"] == "user" assert {r for r in project["repositories"]} == { "http://renkulab.io/repository-1", "http://renkulab.io/repository-2", } + assert set(project["keywords"]) == {"keyword 1", "keyword.2", "keyword-3", "KEYWORD_4"} + assert "documentation" not in project + assert project["created_by"] == "user" + + _, response = await sanic_client.get( + f"/api/data/projects/{project_id}", params={"with_documentation": True}, headers=user_headers + ) + + assert response.status_code == 200, response.text + project = response.json + assert project["documentation"] == "$\\sqrt(2)$" # same as above, but using namespace/slug to retreive the pr _, response = await sanic_client.get( @@ -314,6 +326,7 @@ async def test_patch_project(create_project, get_project, sanic_client, user_hea "keywords": ["keyword 1", "keyword 2"], "visibility": "public", "repositories": ["http://renkulab.io/repository-1", "http://renkulab.io/repository-2"], + "documentation": "$\\infty$", } project_id = project["id"] _, response = await sanic_client.patch(f"/api/data/projects/{project_id}", headers=headers, json=patch) @@ -343,6 +356,15 @@ async def test_patch_project(create_project, get_project, sanic_client, user_hea "http://renkulab.io/repository-1", "http://renkulab.io/repository-2", } + assert "documentation" not in project + + _, response = await sanic_client.get( + f"/api/data/projects/{project_id}", params={"with_documentation": True}, headers=user_headers + ) + + assert response.status_code == 200, response.text + project = response.json + assert project["documentation"] == "$\\infty$" @pytest.mark.asyncio