diff --git a/.github/workflows/black.yml b/.github/workflows/black.yml index 8b48ddf..052e2ec 100644 --- a/.github/workflows/black.yml +++ b/.github/workflows/black.yml @@ -6,6 +6,6 @@ jobs: lint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 - uses: psf/black@stable diff --git a/docs/changelog.md b/docs/changelog.md index 76ac388..9fee62b 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,10 +2,18 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format. + +## [0.12.0] -- 2025-03-27 +- New database model for schemas +- Added schema version and schema record +- Added tags to schemas + + ## [0.11.1] -- 2024-09-04 - Added archive table of namespaces - Added sort by stars + ## [0.11.0] -- 2024-07-24 - Added validation schemas diff --git a/pepdbagent/__init__.py b/pepdbagent/__init__.py index cced953..a8b67f2 100644 --- a/pepdbagent/__init__.py +++ b/pepdbagent/__init__.py @@ -1,4 +1,4 @@ -""" Package-level data """ +"""Package-level data""" import coloredlogs import logmuse diff --git a/pepdbagent/_version.py b/pepdbagent/_version.py index fee46bd..ea370a8 100644 --- a/pepdbagent/_version.py +++ b/pepdbagent/_version.py @@ -1 +1 @@ -__version__ = "0.11.1" +__version__ = "0.12.0" diff --git a/pepdbagent/const.py b/pepdbagent/const.py index aae67f6..cf0577b 100644 --- a/pepdbagent/const.py +++ b/pepdbagent/const.py @@ -22,3 +22,6 @@ PEPHUB_SAMPLE_ID_KEY = "ph_id" MAX_HISTORY_SAMPLES_NUMBER = 2000 + +DEFAULT_TAG_VERSION = "1.0.0" +LATEST_SCHEMA_VERSION = "latest" diff --git a/pepdbagent/db_utils.py b/pepdbagent/db_utils.py index 45d7391..d048dca 100644 --- a/pepdbagent/db_utils.py +++ b/pepdbagent/db_utils.py @@ -86,12 +86,11 @@ class Projects(Base): last_update_date: Mapped[Optional[datetime.datetime]] = mapped_column( default=deliver_update_date, # onupdate=deliver_update_date, # This field should not be updated, while we are adding project to favorites ) - pep_schema: Mapped[Optional[str]] schema_id: Mapped[Optional[int]] = mapped_column( - ForeignKey("schemas.id", ondelete="SET NULL"), nullable=True + ForeignKey("schema_versions.id", ondelete="SET NULL"), nullable=True ) - schema_mapping: Mapped["Schemas"] = relationship("Schemas", lazy="joined") + schema_mapping: Mapped["SchemaVersions"] = relationship("SchemaVersions", lazy="joined") pop: Mapped[Optional[bool]] = mapped_column(default=False) samples_mapping: Mapped[List["Samples"]] = relationship( @@ -200,10 +199,14 @@ class User(Base): order_by="Stars.star_date.desc()", ) number_of_projects: Mapped[int] = mapped_column(default=0) + number_of_schemas: Mapped[int] = mapped_column(default=0) projects_mapping: Mapped[List["Projects"]] = relationship( "Projects", back_populates="namespace_mapping" ) + schemas_mapping: Mapped[List["SchemaRecords"]] = relationship( + "SchemaRecords", back_populates="user_mapping" + ) class Stars(Base): @@ -302,65 +305,68 @@ class HistorySamples(Base): ) -class Schemas(Base): - - __tablename__ = "schemas" +class SchemaRecords(Base): + __tablename__ = "schema_records" - id: Mapped[int] = mapped_column(primary_key=True, index=True) + id: Mapped[int] = mapped_column(primary_key=True) namespace: Mapped[str] = mapped_column(ForeignKey("users.namespace", ondelete="CASCADE")) - name: Mapped[str] = mapped_column(nullable=False, index=True) - description: Mapped[Optional[str]] = mapped_column(nullable=True, index=True) - schema_json: Mapped[dict] = mapped_column(JSON, server_default=FetchedValue()) + name: Mapped[str] = mapped_column(nullable=False) + maintainers: Mapped[str] = mapped_column(nullable=True) + lifecycle_stage: Mapped[str] = mapped_column(nullable=True) + description: Mapped[Optional[str]] = mapped_column(nullable=True) private: Mapped[bool] = mapped_column(default=False) - submission_date: Mapped[datetime.datetime] = mapped_column(default=deliver_update_date) last_update_date: Mapped[Optional[datetime.datetime]] = mapped_column( default=deliver_update_date, onupdate=deliver_update_date ) - projects_mappings: Mapped[List["Projects"]] = relationship( - "Projects", back_populates="schema_mapping" - ) - group_relation_mapping: Mapped[List["SchemaGroupRelations"]] = relationship( - "SchemaGroupRelations", back_populates="schema_mapping" - ) - __table_args__ = (UniqueConstraint("namespace", "name"),) + versions_mapping: Mapped[List["SchemaVersions"]] = relationship( + "SchemaVersions", + back_populates="schema_mapping", + cascade="all, delete-orphan", + order_by="SchemaVersions.version.desc()", + ) + user_mapping: Mapped["User"] = relationship("User", back_populates="schemas_mapping") -class SchemaGroups(Base): - __tablename__ = "schema_groups" +class SchemaVersions(Base): + __tablename__ = "schema_versions" - id: Mapped[int] = mapped_column(primary_key=True, index=True) - namespace: Mapped[str] = mapped_column( - ForeignKey("users.namespace", ondelete="CASCADE"), index=True + id: Mapped[int] = mapped_column(primary_key=True) + schema_id: Mapped[int] = mapped_column(ForeignKey("schema_records.id", ondelete="CASCADE")) + version: Mapped[str] = mapped_column(nullable=False) + schema_value: Mapped[dict] = mapped_column(JSON, server_default=FetchedValue()) + release_date: Mapped[datetime.datetime] = mapped_column(default=deliver_update_date) + last_update_date: Mapped[Optional[datetime.datetime]] = mapped_column( + default=deliver_update_date, onupdate=deliver_update_date ) - name: Mapped[str] = mapped_column(nullable=False, index=True) - description: Mapped[Optional[str]] = mapped_column(nullable=True) + contributors: Mapped[Optional[str]] = mapped_column(nullable=True) + release_notes: Mapped[Optional[str]] = mapped_column(nullable=True) - schema_relation_mapping: Mapped[List["SchemaGroupRelations"]] = relationship( - "SchemaGroupRelations", back_populates="group_mapping" - ) + __table_args__ = (UniqueConstraint("schema_id", "version"),) - __table_args__ = (UniqueConstraint("namespace", "name"),) + schema_mapping: Mapped["SchemaRecords"] = relationship( + "SchemaRecords", back_populates="versions_mapping" + ) + tags_mapping: Mapped[List["SchemaTags"]] = relationship( + "SchemaTags", back_populates="schema_mapping", lazy="joined", cascade="all, delete-orphan" + ) -class SchemaGroupRelations(Base): - __tablename__ = "schema_group_relations" +class SchemaTags(Base): + __tablename__ = "schema_tags" - schema_id: Mapped[int] = mapped_column( - ForeignKey("schemas.id", ondelete="CASCADE"), index=True, primary_key=True - ) - group_id: Mapped[int] = mapped_column( - ForeignKey("schema_groups.id", ondelete="CASCADE"), index=True, primary_key=True + id: Mapped[int] = mapped_column(primary_key=True) + tag_name: Mapped[str] = mapped_column(nullable=False) + tag_value: Mapped[str] = mapped_column(nullable=True) + schema_version_id: Mapped[int] = mapped_column( + ForeignKey("schema_versions.id", ondelete="CASCADE") ) - schema_mapping: Mapped["Schemas"] = relationship( - "Schemas", back_populates="group_relation_mapping" - ) - group_mapping: Mapped["SchemaGroups"] = relationship( - "SchemaGroups", back_populates="schema_relation_mapping" + schema_mapping: Mapped["SchemaVersions"] = relationship( + "SchemaVersions", back_populates="tags_mapping" ) diff --git a/pepdbagent/exceptions.py b/pepdbagent/exceptions.py index caf152b..17be697 100644 --- a/pepdbagent/exceptions.py +++ b/pepdbagent/exceptions.py @@ -1,4 +1,4 @@ -""" Custom error types """ +"""Custom error types""" class PEPDatabaseAgentError(Exception): @@ -129,21 +129,21 @@ def __init__(self, msg=""): super().__init__(f"""Schema already exists. {msg}""") -class SchemaGroupDoesNotExistError(PEPDatabaseAgentError): +class SchemaVersionDoesNotExistError(PEPDatabaseAgentError): def __init__(self, msg=""): - super().__init__(f"""Schema group does not exist. {msg}""") + super().__init__(f"""Schema version does not exist. {msg}""") -class SchemaGroupAlreadyExistsError(PEPDatabaseAgentError): +class SchemaVersionAlreadyExistsError(PEPDatabaseAgentError): def __init__(self, msg=""): - super().__init__(f"""Schema group already exists. {msg}""") + super().__init__(f"""Schema version already exists. {msg}""") -class SchemaAlreadyInGroupError(PEPDatabaseAgentError): +class SchemaTagDoesNotExistError(PEPDatabaseAgentError): def __init__(self, msg=""): - super().__init__(f"""Schema already in the group. {msg}""") + super().__init__(f"""Schema tag does not exist. {msg}""") -class SchemaIsNotInGroupError(PEPDatabaseAgentError): +class SchemaTagAlreadyExistsError(PEPDatabaseAgentError): def __init__(self, msg=""): - super().__init__(f"""Schema not found in group. {msg}""") + super().__init__(f"""Schema tag already exists. {msg}""") diff --git a/pepdbagent/models.py b/pepdbagent/models.py index 3bcd402..b37b181 100644 --- a/pepdbagent/models.py +++ b/pepdbagent/models.py @@ -52,6 +52,12 @@ def is_private_should_be_bool(cls, v): return v +class PaginationResult(BaseModel): + page: int = 0 + page_size: int = 10 + total: int + + class AnnotationList(BaseModel): """ Annotation return model. @@ -156,8 +162,10 @@ class NamespaceInfo(BaseModel): Model with information about namespace """ - namespace: str + namespace_name: str + contact_url: Optional[str] = None number_of_projects: int + number_of_schemas: int class ListOfNamespaceInfo(BaseModel): @@ -165,8 +173,7 @@ class ListOfNamespaceInfo(BaseModel): Namespace information response model """ - number_of_namespaces: int - limit: int + pagination: PaginationResult results: List[NamespaceInfo] @@ -249,17 +256,34 @@ class HistoryAnnotationModel(BaseModel): history: List[HistoryChangeModel] -class SchemaAnnotation(BaseModel): +class SchemaVersionAnnotation(BaseModel): + """ + Schema version annotation model + """ + + namespace: str + schema_name: str + version: str + contributors: Optional[str] = "" + release_notes: Optional[str] = "" + tags: Dict[str, Union[str, None]] = {} + release_date: datetime.datetime + last_update_date: datetime.datetime + + +class SchemaRecordAnnotation(BaseModel): """ Schema annotation model """ namespace: str - name: str - last_update_date: str - submission_date: str + schema_name: str description: Optional[str] = "" - popularity_number: Optional[int] = 0 + maintainers: str = "" + lifecycle_stage: str = "" + latest_released_version: str + private: bool = False + last_update_date: datetime.datetime class SchemaSearchResult(BaseModel): @@ -267,32 +291,31 @@ class SchemaSearchResult(BaseModel): Schema search result model """ - count: int - limit: int - offset: int - results: List[SchemaAnnotation] + pagination: PaginationResult + results: List[SchemaRecordAnnotation] -class SchemaGroupAnnotation(BaseModel): +class SchemaVersionSearchResult(BaseModel): """ - Schema group annotation model + Schema version search result model """ - namespace: str - name: str - description: Optional[str] = "" - schemas: List[SchemaAnnotation] + pagination: PaginationResult + results: List[SchemaVersionAnnotation] -class SchemaGroupSearchResult(BaseModel): - """ - Schema group search result model - """ +class UpdateSchemaRecordFields(BaseModel): + maintainers: Optional[str] = None + lifecycle_stage: Optional[str] = None + private: Optional[bool] = False + name: Optional[str] = None + description: Optional[str] = None - count: int - limit: int - offset: int - results: List[SchemaGroupAnnotation] + +class UpdateSchemaVersionFields(BaseModel): + contributors: Optional[str] = None + schema_value: Optional[dict] = None + release_notes: Optional[str] = None class TarNamespaceModel(BaseModel): diff --git a/pepdbagent/modules/annotation.py b/pepdbagent/modules/annotation.py index 8f755b0..6481796 100644 --- a/pepdbagent/modules/annotation.py +++ b/pepdbagent/modules/annotation.py @@ -212,7 +212,7 @@ def _get_single_annotation( last_update_date=str(query_result.last_update_date), digest=query_result.digest, pep_schema=( - f"{query_result.schema_mapping.namespace}/{query_result.schema_mapping.name}" + f"{query_result.schema_mapping.schema_mapping.namespace}/{query_result.schema_mapping.schema_mapping.name}:{query_result.schema_mapping.version}" if query_result.schema_mapping else None ), @@ -326,6 +326,7 @@ def _get_projects( statement = self._add_date_filter_if_provided( statement, filter_by, filter_start_date, filter_end_date ) + statement = statement statement = self._add_order_by_keyword(statement, by=order_by, desc=order_desc) statement = statement.limit(limit).offset(offset) if pep_type: @@ -333,7 +334,8 @@ def _get_projects( results_list = [] with Session(self._sa_engine) as session: - results = session.scalars(statement) + # Unique should be called because of the join with schema_mapping + results = session.scalars(statement).unique() for result in results: results_list.append( AnnotationModel( @@ -347,7 +349,7 @@ def _get_projects( last_update_date=str(result.last_update_date), digest=result.digest, pep_schema=( - f"{result.schema_mapping.namespace}/{result.schema_mapping.name}" + f"{result.schema_mapping.schema_mapping.namespace}/{result.schema_mapping.schema_mapping.name}:{result.schema_mapping.version}" if result.schema_mapping else None ), @@ -548,7 +550,7 @@ def get_by_rp_list( statement = select(Projects).where(or_(*or_statement_list)) anno_results = [] with Session(self._sa_engine) as session: - query_result = session.scalars(statement) + query_result = session.scalars(statement).unique() for result in query_result: project_obj = result annot = AnnotationModel( @@ -562,7 +564,7 @@ def get_by_rp_list( last_update_date=str(project_obj.last_update_date), digest=project_obj.digest, pep_schema=( - f"{project_obj.schema_mapping.namespace}/{project_obj.schema_mapping.name}" + f"{project_obj.schema_mapping.schema_mapping.namespace}/{project_obj.schema_mapping.schema_mapping.name}:{project_obj.schema_mapping.version}" if project_obj.schema_mapping else None ), diff --git a/pepdbagent/modules/namespace.py b/pepdbagent/modules/namespace.py index 332c1cb..7766fb6 100644 --- a/pepdbagent/modules/namespace.py +++ b/pepdbagent/modules/namespace.py @@ -18,6 +18,7 @@ NamespaceStats, TarNamespaceModel, TarNamespaceModelReturn, + PaginationResult, ) from pepdbagent.utils import tuple_converter @@ -174,78 +175,56 @@ def _add_condition( ) return statement - # old function, that counts namespace info based on Projects table - # def info(self, limit: int = DEFAULT_LIMIT_INFO) -> ListOfNamespaceInfo: - # """ - # Get list of top n namespaces in the database - # - # :param limit: limit of results (top namespace ) - # :return: number_of_namespaces: int - # limit: int - # results: { namespace: str - # number_of_projects: int - # } - # """ - # total_number_of_namespaces = self._count_namespace() - # - # statement = ( - # select( - # func.count(Projects.namespace).label("number_of_projects"), - # Projects.namespace, - # ) - # .select_from(Projects) - # .where(Projects.private.is_(False)) - # .limit(limit) - # .order_by(text("number_of_projects desc")) - # .group_by(Projects.namespace) - # ) - # - # with Session(self._sa_engine) as session: - # query_results = session.execute(statement).all() - # - # list_of_results = [] - # for result in query_results: - # list_of_results.append( - # NamespaceInfo( - # namespace=result.namespace, - # number_of_projects=result.number_of_projects, - # ) - # ) - # return ListOfNamespaceInfo( - # number_of_namespaces=total_number_of_namespaces, - # limit=limit, - # results=list_of_results, - # ) - - def info(self, limit: int = DEFAULT_LIMIT_INFO) -> ListOfNamespaceInfo: + def info( + self, + page: int = 0, + page_size: int = DEFAULT_LIMIT_INFO, + order_by: str = "number_of_projects", + ) -> ListOfNamespaceInfo: """ Get list of top n namespaces in the database ! Warning: this function counts number of all projects in namespaces. ! it does not filter private projects (It was done for efficiency reasons) - :param limit: limit of results (top namespace ) + :param page: page number + :param page_size: number of namespaces to show + :param order_by: order by field. Options: number_of_projects, number_of_schemas [Default: number_of_projects] + :return: number_of_namespaces: int limit: int results: { namespace: str number_of_projects: int + number_of_schemas: int } """ + + statement = select(User) + + if order_by == "number_of_projects": + statement = statement.order_by(User.number_of_projects.desc()) + elif order_by == "number_of_schemas": + statement = statement.order_by(User.number_of_schemas.desc()) + with Session(self._sa_engine) as session: - results = session.scalars( - select(User).limit(limit).order_by(User.number_of_projects.desc()) - ) + results = session.scalars(statement.limit(page_size).offset(page_size * page)) + total_number_of_namespaces = session.execute(select(func.count(User.id))).one()[0] list_of_results = [] for result in results: list_of_results.append( NamespaceInfo( - namespace=result.namespace, + namespace_name=result.namespace, + contact_url=f"https://github.com/{result.namespace}", number_of_projects=result.number_of_projects, + number_of_schemas=result.number_of_schemas, ) ) return ListOfNamespaceInfo( - number_of_namespaces=len(list_of_results), - limit=limit, + pagination=PaginationResult( + page=page, + page_size=page_size, + total=total_number_of_namespaces, + ), results=list_of_results, ) diff --git a/pepdbagent/modules/project.py b/pepdbagent/modules/project.py index cf2cfbe..d5ee5a7 100644 --- a/pepdbagent/modules/project.py +++ b/pepdbagent/modules/project.py @@ -31,9 +31,9 @@ HistorySamples, Projects, Samples, - Schemas, + SchemaRecords, + SchemaVersions, Subsamples, - TarNamespace, UpdateTypes, User, ) @@ -47,8 +47,6 @@ SchemaDoesNotExistError, ) from pepdbagent.models import ( - TarNamespaceModel, - TarNamespaceModelReturn, HistoryAnnotationModel, HistoryChangeModel, ProjectDict, @@ -368,13 +366,17 @@ def create( number_of_samples = len(proj_dict[SAMPLE_RAW_DICT_KEY]) if pep_schema: - schema_namespace, schema_name = schema_path_converter(pep_schema) + schema_namespace, schema_name, schema_version = schema_path_converter(pep_schema) with Session(self._sa_engine) as session: + schema_mapping = session.scalar( - select(Schemas).where( + select(SchemaVersions) + .join(SchemaRecords, SchemaRecords.id == SchemaVersions.schema_id) + .where( and_( - Schemas.namespace == schema_namespace, - Schemas.name == schema_name, + SchemaRecords.namespace == schema_namespace, + SchemaRecords.name == schema_name, + SchemaVersions.version == schema_version, ) ) ) @@ -693,14 +695,17 @@ def _convert_update_schema_id(session: Session, update_values: dict): return None """ if "pep_schema" in update_values: - schema_namespace, schema_name = schema_path_converter(update_values["pep_schema"]) + schema_namespace, schema_name, schema_version = schema_path_converter( + update_values["pep_schema"] + ) + where_clause = and_( + SchemaRecords.namespace == schema_namespace, + SchemaRecords.name == schema_name, + SchemaVersions.version == schema_version, + ) + schema_mapping = session.scalar( - select(Schemas).where( - and_( - Schemas.namespace == schema_namespace, - Schemas.name == schema_name, - ) - ) + select(SchemaVersions).join(SchemaRecords).where(where_clause) ) if not schema_mapping: raise SchemaDoesNotExistError( @@ -708,6 +713,7 @@ def _convert_update_schema_id(session: Session, update_values: dict): f"Project won't be updated." ) update_values["schema_id"] = schema_mapping.id + del update_values["pep_schema"] def _update_samples( self, @@ -1065,7 +1071,7 @@ def fork( fork_prj.forked_from_id = original_prj.id fork_prj.pop = original_prj.pop fork_prj.submission_date = original_prj.submission_date - fork_prj.pep_schema = original_prj.pep_schema + fork_prj.schema_id = original_prj.schema_id fork_prj.description = description or original_prj.description session.commit() diff --git a/pepdbagent/modules/schema.py b/pepdbagent/modules/schema.py index ff6f274..fe216ad 100644 --- a/pepdbagent/modules/schema.py +++ b/pepdbagent/modules/schema.py @@ -1,25 +1,29 @@ import logging -from sqlalchemy import Select, and_, delete, func, or_, select -from sqlalchemy.exc import IntegrityError +from typing import List, Optional, Union, Dict + +from sqlalchemy import Select, and_, func, or_, select from sqlalchemy.orm import Session from sqlalchemy.orm.attributes import flag_modified -from pepdbagent.const import PKG_NAME -from pepdbagent.db_utils import BaseEngine, SchemaGroupRelations, SchemaGroups, Schemas, User +from pepdbagent.const import PKG_NAME, DEFAULT_TAG_VERSION, LATEST_SCHEMA_VERSION +from pepdbagent.db_utils import BaseEngine, SchemaRecords, SchemaTags, SchemaVersions, User from pepdbagent.exceptions import ( SchemaAlreadyExistsError, - SchemaAlreadyInGroupError, + SchemaVersionDoesNotExistError, SchemaDoesNotExistError, - SchemaGroupAlreadyExistsError, - SchemaGroupDoesNotExistError, - SchemaIsNotInGroupError, + SchemaTagAlreadyExistsError, + SchemaTagDoesNotExistError, + SchemaVersionAlreadyExistsError, ) from pepdbagent.models import ( - SchemaAnnotation, - SchemaGroupAnnotation, - SchemaGroupSearchResult, + SchemaRecordAnnotation, + SchemaVersionAnnotation, + PaginationResult, + SchemaVersionSearchResult, SchemaSearchResult, + UpdateSchemaRecordFields, + UpdateSchemaVersionFields, ) _LOGGER = logging.getLogger(PKG_NAME) @@ -27,7 +31,7 @@ class PEPDatabaseSchema: """ - Class that represents Schemas in Database. + Class that represents SchemaRecords in Database. While using this class, user can create, retrieve, delete, and update schemas from database """ @@ -39,603 +43,894 @@ def __init__(self, pep_db_engine: BaseEngine): self._sa_engine = pep_db_engine.engine self._pep_db_engine = pep_db_engine - def get(self, namespace: str, name: str) -> dict: + def get(self, namespace: str, name: str, version: str) -> dict: """ Get schema from the database. :param namespace: user namespace :param name: schema name + :param version: schema version :return: schema dict """ with Session(self._sa_engine) as session: - schema_obj = session.scalar( - select(Schemas).where(and_(Schemas.namespace == namespace, Schemas.name == name)) - ) + if version == LATEST_SCHEMA_VERSION: + schema_obj = session.scalar( + select(SchemaVersions) + .join(SchemaRecords, SchemaRecords.id == SchemaVersions.schema_id) + .where( + and_( + SchemaRecords.namespace == namespace, + SchemaRecords.name == name, + ) + ) + .order_by(SchemaVersions.version.desc()) + ) + + else: + + schema_obj = session.scalar( + select(SchemaVersions) + .join(SchemaRecords, SchemaRecords.id == SchemaVersions.schema_id) + .where( + and_( + SchemaRecords.namespace == namespace, + SchemaRecords.name == name, + SchemaVersions.version == version, + ) + ) + ) if not schema_obj: - raise SchemaDoesNotExistError(f"Schema '{name}' does not exist in the database") + raise SchemaVersionDoesNotExistError( + f"Schema '{name}' does not exist in the database" + ) - return schema_obj.schema_json + return schema_obj.schema_value - def info(self, namespace: str, name: str) -> SchemaAnnotation: + def create( + self, + namespace: str, + name: str, + schema_value: dict, + version: str = DEFAULT_TAG_VERSION, + description: str = "", + lifecycle_stage: str = "", + maintainers: str = "", + contributors: str = "", + release_notes: str = "", + tags: Optional[Union[List[str], str, Dict[str, str], List[Dict[str, str]]]] = None, + private: bool = False, # TODO: for simplicity was not implemented yet + ) -> None: """ - Get schema information from the database. + Create or update schema in the database. :param namespace: user namespace :param name: schema name + :param schema_value: schema dict + :param version: schema version [Default: "1.0.0"] + :param description: schema description [Default: ""] + :param lifecycle_stage: schema lifecycle stage [Default: ""] + :param maintainers: schema maintainers [Default: ""] + :param contributors: schema contributors [Default: ""] + :param release_notes: schema release notes [Default: ""] + :param tags: schema tags [Default: None] + :param private: schema privacy [Default: False] - :return: SchemaAnnotation object: - - namespace: schema namespace - - name: schema name - - last_update_date: last update date - - submission_date: submission date - - description: schema description + :return: None """ + tags = self._unify_tags(tags) + with Session(self._sa_engine) as session: schema_obj = session.scalar( - select(Schemas).where(and_(Schemas.namespace == namespace, Schemas.name == name)) + select(SchemaRecords).where( + and_(SchemaRecords.namespace == namespace, SchemaRecords.name == name) + ) ) - if not schema_obj: - raise SchemaDoesNotExistError(f"Schema '{name}' does not exist in the database") - - return SchemaAnnotation( - namespace=schema_obj.namespace, - name=schema_obj.name, - last_update_date=str(schema_obj.last_update_date), - submission_date=str(schema_obj.submission_date), - description=schema_obj.description, - popularity_number=len(schema_obj.projects_mappings), - ) + if schema_obj: + raise SchemaAlreadyExistsError(f"Schema '{name}' already exists in the database") - def search( - self, - namespace: str = None, - search_str: str = "", - limit: int = 100, - offset: int = 0, - order_by: str = "update_date", - order_desc: bool = False, - ) -> SchemaSearchResult: - """ - Search schemas in the database. + user = session.scalar(select(User).where(User.namespace == namespace)) - :param namespace: user namespace [Default: None]. If None, search in all namespaces - :param search_str: query string. [Default: ""]. If empty, return all schemas - :param limit: limit number of schemas [Default: 100] - :param offset: offset number of schemas [Default: 0] - :param order_by: sort the result-set by the information - Options: ["name", "update_date", "submission_date"] - [Default: update_date] - :param order_desc: Sort the records in descending order. [Default: False] + if not user: + user = User(namespace=namespace) + session.add(user) + session.commit() - :return: list of schema dicts - """ + user.number_of_schemas += 1 - statement = select(Schemas) - statement = self._add_condition(statement, namespace, search_str) - statement = statement.limit(limit).offset(offset) - statement = self._add_order_by_keyword(statement, by=order_by, desc=order_desc) + schema_obj = SchemaRecords( + namespace=namespace, + name=name, + description=description, + maintainers=maintainers, + lifecycle_stage=lifecycle_stage, + private=private, + ) - return_list = [] + session.add(schema_obj) - with Session(self._sa_engine) as session: - results = session.scalars(statement) + schema_version_obj = SchemaVersions( + schema_mapping=schema_obj, + version=version, + schema_value=schema_value, + release_notes=release_notes, + contributors=contributors, + ) - for result in results: - return_list.append( - SchemaAnnotation( - namespace=result.namespace, - name=result.name, - last_update_date=str(result.last_update_date), - submission_date=str(result.submission_date), - description=result.description, - # popularity_number=sum(result.projects_mappings), + for tag_name, tag_value in tags.items(): + tag_obj = session.scalar(select(SchemaTags).where(SchemaTags.tag_name == tag_name)) + if not tag_obj: + tag_obj = SchemaTags( + tag_name=tag_name, tag_value=tag_value, schema_mapping=schema_version_obj ) - ) + session.add(tag_obj) - return SchemaSearchResult( - count=self._count_search(namespace=namespace, search_str=search_str), - limit=limit, - offset=offset, - results=return_list, - ) - - def _count_search(self, namespace: str = None, search_str: str = "") -> int: - """ - Count number of found schemas + session.add(schema_version_obj) + session.commit() - :param namespace: user namespace [Default: None]. If None, search in all namespaces - :param search_str: query string. [Default: ""]. If empty, return all schemas + return None - :return: list of schema dicts - """ - statement = select(func.count(Schemas.id)) + def add_version( + self, + namespace: str, + name: str, + version: str, + schema_value: dict, + release_notes: str = "", + contributors: str = "", + overwrite: bool = False, + tags: Optional[Union[List[str], str, Dict[str, str], List[Dict[str, str]]]] = None, + ) -> None: - statement = self._add_condition(statement, namespace, search_str) + tags = self._unify_tags(tags) with Session(self._sa_engine) as session: - result = session.execute(statement).one() - - return result[0] - - @staticmethod - def _add_order_by_keyword( - statement: Select, by: str = "update_date", desc: bool = False - ) -> Select: - """ - Add order by clause to sqlalchemy statement + schema_obj = session.scalar( + select(SchemaRecords).where( + and_(SchemaRecords.namespace == namespace, SchemaRecords.name == name) + ) + ) + if not schema_obj: + raise SchemaDoesNotExistError( + f"Schema '{name}' does not exist in the database. Unable to add version." + ) - :param statement: sqlalchemy representation of a SELECT statement. - :param by: sort the result-set by the information - Options: ["name", "update_date", "submission_date"] - [Default: "update_date"] - :param desc: Sort the records in descending order. [Default: False] - :return: sqlalchemy representation of a SELECT statement with order by keyword - """ - if by == "update_date": - order_by_obj = Schemas.last_update_date - elif by == "name": - order_by_obj = Schemas.name - elif by == "submission_date": - order_by_obj = Schemas.submission_date - else: - _LOGGER.warning( - f"order by: '{by}' statement is unavailable. Projects are sorted by 'update_date'" + version_obj = session.scalar( + select(SchemaVersions) + .join(SchemaRecords, SchemaRecords.id == SchemaVersions.schema_id) + .where( + and_( + SchemaRecords.namespace == namespace, + SchemaRecords.name == name, + SchemaVersions.version == version, + ) + ) ) - order_by_obj = Schemas.last_update_date - if desc and by == "name": - order_by_obj = order_by_obj.desc() + if version_obj: + if not overwrite: + raise SchemaVersionAlreadyExistsError( + f"Schema '{name}' with version '{version}' already exists in the database" + ) - elif by != "name" and not desc: - order_by_obj = order_by_obj.desc() + return self.update_schema_version( + namespace, + name, + version, + update_fields=UpdateSchemaVersionFields( + schema_value=schema_value, + release_notes=release_notes, + contributors=contributors, + ), + ) - return statement.order_by(order_by_obj) + schema_obj.last_update_date = func.now() - @staticmethod - def _add_condition( - statement: Select, - namespace: str = None, - search_str: str = None, - ) -> Select: - if search_str: - sql_search_str = f"%{search_str}%" - search_query = or_( - Schemas.name.ilike(sql_search_str), - Schemas.description.ilike(sql_search_str), - Schemas.namespace.ilike(sql_search_str), + schema_version_obj = SchemaVersions( + schema_id=schema_obj.id, + version=version, + schema_value=schema_value, + release_notes=release_notes, + contributors=contributors, ) - statement = statement.where(search_query) - if namespace: - statement = statement.where(Schemas.namespace == namespace) - return statement - def create( + for tag_name, tag_value in tags.items(): + tag_obj = SchemaTags( + tag_name=tag_name, tag_value=tag_value, schema_mapping=schema_version_obj + ) + session.add(tag_obj) + + session.add(schema_version_obj) + session.commit() + + return None + + def update_schema_version( self, namespace: str, name: str, - schema: dict, - description: str = "", - # private: bool = False, # TODO: for simplicity was not implemented yet - overwrite: bool = False, - update_only: bool = False, + version: str, + update_fields: Union[UpdateSchemaVersionFields, dict], ) -> None: """ - Create or update schema in the database. + Update schema version in the database. :param namespace: user namespace :param name: schema name - :param schema: schema dict - :param description: schema description [Default: ""] - :param overwrite: overwrite schema if exists [Default: False] - :param update_only: update only schema if exists [Default: False] + :param version: schema version + :param update_fields: fields to be updated. Fields are optional, and include: + - contributors: str + - schema_value: dict + - release_notes: str """ + if isinstance(update_fields, dict): + update_fields = UpdateSchemaVersionFields(**update_fields) + update_fields = update_fields.model_dump(exclude_unset=True, exclude_defaults=True) - if description: - schema["description"] = description - else: - description = schema.get("description", "") - - if self.exist(namespace, name): - if overwrite: - self.update(namespace, name, schema, description) - return None - elif update_only: - self.update(namespace, name, schema, description) - return None - else: - raise SchemaAlreadyExistsError(f"Schema '{name}' already exists in the database") - - if update_only: - raise SchemaDoesNotExistError( - f"Schema '{name}' does not exist in the database" - f"Cannot update schema that does not exist" + with Session(self._sa_engine) as session: + schema_obj = session.scalar( + select(SchemaVersions) + .join(SchemaRecords, SchemaRecords.id == SchemaVersions.schema_id) + .where( + and_( + SchemaRecords.namespace == namespace, + SchemaRecords.name == name, + SchemaVersions.version == version, + ) + ) ) - with Session(self._sa_engine) as session: - user = session.scalar(select(User).where(User.namespace == namespace)) + if not schema_obj: + raise SchemaVersionDoesNotExistError( + f"Schema '{name}' with version '{version}' does not exist in the database. Unable to update version." + ) + schema_obj.last_update_date = func.now() - if not user: - user = User(namespace=namespace) - session.add(user) - session.commit() + for field, value in update_fields.items(): + setattr(schema_obj, field, value) + if field == "schema_value": + flag_modified(schema_obj, field) - schema_obj = Schemas( - namespace=namespace, - name=name, - schema_json=schema, - description=description, - ) - session.add(schema_obj) session.commit() - def update( + def update_schema_record( self, namespace: str, name: str, - schema: dict, - description: str = "", - # private: bool = False, # TODO: for simplicity was not implemented yet + update_fields: Union[UpdateSchemaRecordFields, dict], ) -> None: """ - Update schema in the database. + Update schema record in the database. :param namespace: user namespace :param name: schema name - :param schema: schema dict - :param description: schema description [Default: ""] - - :return: None + :param update_fields: fields to be updated. Fields are optional, and include: + - maintainers: str + - lifecycle_stage: str + - private: bool + - name: str """ + if isinstance(update_fields, dict): + update_fields = UpdateSchemaRecordFields(**update_fields) + + update_fields = update_fields.model_dump(exclude_unset=True, exclude_defaults=True) + with Session(self._sa_engine) as session: schema_obj = session.scalar( - select(Schemas).where(and_(Schemas.namespace == namespace, Schemas.name == name)) + select(SchemaRecords).where( + and_(SchemaRecords.namespace == namespace, SchemaRecords.name == name) + ) ) if not schema_obj: raise SchemaDoesNotExistError(f"Schema '{name}' does not exist in the database") - schema_obj.schema_json = schema - schema_obj.description = description - flag_modified(schema_obj, "schema_json") + for field, value in update_fields.items(): + setattr(schema_obj, field, value) session.commit() - def delete(self, namespace: str, name: str) -> None: + def schema_exist(self, namespace: str, name: str) -> bool: """ - Delete schema from the database. + Check if schema exists in the database. :param namespace: user namespace :param name: schema name - :return: None + :return: True if schema exists, False otherwise """ with Session(self._sa_engine) as session: schema_obj = session.scalar( - select(Schemas).where(and_(Schemas.namespace == namespace, Schemas.name == name)) + select(SchemaRecords).where( + and_(SchemaRecords.namespace == namespace, SchemaRecords.name == name) + ) ) + return True if schema_obj else False - if not schema_obj: - raise SchemaDoesNotExistError(f"Schema '{name}' does not exist in the database") - - session.delete(schema_obj) - - session.commit() - - def exist(self, namespace: str, name: str) -> bool: + def version_exist(self, namespace: str, name: str, version: str) -> bool: """ - Check if schema exists in the database. + Check if schema version exists in the database. :param namespace: user namespace :param name: schema name + :param version: schema version - :return: True if schema exists, False otherwise + :return: True if schema version exists, False otherwise """ with Session(self._sa_engine) as session: schema_obj = session.scalar( - select(Schemas).where(and_(Schemas.namespace == namespace, Schemas.name == name)) + select(SchemaVersions) + .join(SchemaRecords, SchemaRecords.id == SchemaVersions.schema_id) + .where( + and_( + SchemaRecords.namespace == namespace, + SchemaRecords.name == name, + SchemaVersions.version == version, + ) + ) ) return True if schema_obj else False - def group_create(self, namespace: str, name: str, description: str = "") -> None: + def get_schema_info(self, namespace: str, name: str) -> SchemaRecordAnnotation: """ - Create schema group in the database. + Get schema information from the database. :param namespace: user namespace - :param name: schema group name - :param description: schema group description [Default: ""] + :param name: schema name - :return: None + :return: SchemaRecordAnnotation """ - try: - with Session(self._sa_engine) as session: - session.add( - SchemaGroups( - namespace=namespace, - name=name, - description=description, - ) + + with Session(self._sa_engine) as session: + schema_obj = session.scalar( + select(SchemaRecords).where( + and_(SchemaRecords.namespace == namespace, SchemaRecords.name == name) ) - session.commit() + ) - except IntegrityError: - raise SchemaGroupAlreadyExistsError + if not schema_obj: + raise SchemaDoesNotExistError(f"Schema '{name}' does not exist in the database") - def group_get(self, namespace: str, name: str) -> SchemaGroupAnnotation: + return SchemaRecordAnnotation( + namespace=schema_obj.namespace, + schema_name=schema_obj.name, + description=schema_obj.description, + latest_released_version=schema_obj.versions_mapping[0].version, + maintainers=schema_obj.maintainers, + private=schema_obj.private, + last_update_date=schema_obj.last_update_date, + lifecycle_stage=schema_obj.lifecycle_stage, + ) + + def get_version_info(self, namespace: str, name: str, version: str) -> SchemaVersionAnnotation: """ - Get schema group from the database. + Get schema version information from the database. :param namespace: user namespace - :param name: schema group name + :param name: schema name + :param version: schema version - :return: SchemaGroupAnnotation object: - - namespace: schema group namespace - - name: schema group name - - description: schema group description - - schemas: list of SchemaAnnotation objects + :return: SchemaVersionAnnotation """ with Session(self._sa_engine) as session: - schema_group_obj = session.scalar( - select(SchemaGroups).where( - and_(SchemaGroups.namespace == namespace, SchemaGroups.name == name) - ) - ) - if not schema_group_obj: - raise SchemaGroupDoesNotExistError( - f"Schema group '{name}' does not exist in the database" + # if user provided "latest" version + if version == LATEST_SCHEMA_VERSION: + version_obj = session.scalar( + select(SchemaVersions) + .join(SchemaRecords, SchemaRecords.id == SchemaVersions.schema_id) + .where(and_(SchemaRecords.namespace == namespace, SchemaRecords.name == name)) + .order_by(SchemaVersions.version.desc()) + .limit(1) ) - - schemas = [] - for schema_relation in schema_group_obj.schema_relation_mapping: - schema_annotation = schema_relation.schema_mapping - schemas.append( - SchemaAnnotation( - namespace=schema_annotation.namespace, - name=schema_annotation.name, - last_update_date=str(schema_annotation.last_update_date), - submission_date=str(schema_annotation.submission_date), - desciription=schema_annotation.description, + else: + version_obj = session.scalar( + select(SchemaVersions) + .join(SchemaRecords, SchemaRecords.id == SchemaVersions.schema_id) + .where( + and_( + SchemaRecords.namespace == namespace, + SchemaRecords.name == name, + SchemaVersions.version == version, + ) ) ) - return SchemaGroupAnnotation( - namespace=schema_group_obj.namespace, - name=schema_group_obj.name, - description=schema_group_obj.description, - schemas=schemas, + if not version_obj: + raise SchemaVersionDoesNotExistError( + f"Schema '{name}' with version '{version}' does not exist in the database" + ) + + return SchemaVersionAnnotation( + namespace=version_obj.schema_mapping.namespace, + schema_name=version_obj.schema_mapping.name, + version=version_obj.version, + contributors=version_obj.contributors, + release_notes=version_obj.release_notes, + tags={tag.tag_name: tag.tag_value for tag in version_obj.tags_mapping}, + release_date=version_obj.release_date, + last_update_date=version_obj.last_update_date, ) - def group_search( - self, namespace: str = None, search_str: str = "", limit: int = 100, offset: int = 0 - ) -> SchemaGroupSearchResult: + def fetch_schemas( + self, + namespace: str = None, + name: str = None, + maintainer: str = None, + lifecycle_stage: str = None, + latest_version: str = None, + page: int = 0, + page_size: int = 10, + order_by: str = "update_date", + order_desc: bool = False, + ) -> SchemaSearchResult: """ - Search schema groups in the database. + Get schemas with providing filters. + If not filters provided, return all schemas. :param namespace: user namespace [Default: None]. If None, search in all namespaces - :param search_str: query string. [Default: ""]. If empty, return all schema groups - :param limit: limit of the search - :param offset: offset of the search + :param name: schema name [Default: None] + :param maintainer: schema maintainer [Default: None] + :param lifecycle_stage: schema lifecycle stage [Default: None] + :param latest_version: schema latest version [Default: None] - :return: SchemaGroupSearchResult object: - - count: number of found schema groups - - limit: limit number of schema groups - - offset: offset number of schema groups - - results: list of SchemaGroupAnnotation objects - """ + :param page: page number [Default: 0] + :param page_size: number of schemas per page [Default: 0] + :param order_by: sort the result-set by the information + Options: ["name", "update_date"] + [Default: update_date] + :param order_desc: Sort the records in descending order. [Default: False] - statement = select(SchemaGroups) - statement = self._add_group_condition( - statement=statement, namespace=namespace, search_str=search_str + :return: { + pagination: {page: int, + page_size: int, + total: int}, + results: [SchemaRecordAnnotation] + """ + + # filters = [ + # SchemaRecords.namespace == namespace if namespace else None, + # SchemaRecords.name == name if name else None, + # SchemaRecords.maintainers == maintainer if maintainer else None, + # SchemaRecords.lifecycle_stage == lifecycle_stage if lifecycle_stage else None, + # ] + filters = [ + SchemaRecords.namespace.ilike(f"%{namespace}%") if namespace else None, + SchemaRecords.name.ilike(f"%{name}%") if name else None, + SchemaRecords.maintainers.ilike(f"%{maintainer}%") if maintainer else None, + ( + SchemaRecords.lifecycle_stage.ilike(f"%{lifecycle_stage}%") + if lifecycle_stage + else None + ), + ] + + # Remove None values before applying and_ + conditions = [f for f in filters if f is not None] + + statement = ( + select(SchemaRecords).where(and_(*conditions)) if conditions else select(SchemaRecords) + ) + statement_count = ( + select(func.count(SchemaRecords.id)).where(and_(*conditions)) + if conditions + else select(func.count(SchemaRecords.id)) ) with Session(self._sa_engine) as session: - results = session.scalars(statement) - - return_results = [] - for result in results: - return_results.append( - SchemaGroupAnnotation( + total = session.scalar(statement_count) + + statement = self._add_order_by_schemas_keyword(statement, by=order_by, desc=order_desc) + + results_objects = session.scalars(statement.limit(page_size).offset(page * page_size)) + return SchemaSearchResult( + pagination=PaginationResult( + page=page, + page_size=page_size, + total=total, + ), + results=[ + SchemaRecordAnnotation( namespace=result.namespace, - name=result.name, + schema_name=result.name, + latest_released_version=result.versions_mapping[0].version, description=result.description, - schemas=[], + maintainers=result.maintainers, + private=result.private, + last_update_date=result.last_update_date, ) - ) - - return SchemaGroupSearchResult( - count=self._group_search_count(namespace, search_str), - limit=limit, - offset=offset, - results=return_results, - ) + for result in results_objects + ], + ) - @staticmethod - def _add_group_condition( - statement: Select, + def query_schemas( + self, namespace: str = None, search_str: str = "", - ) -> Select: + page: int = 0, + page_size: int = 10, + order_by: str = "update_date", + order_desc: bool = False, + ) -> SchemaSearchResult: """ - Add query condition to statement in group search + Search schemas in the database with pagination. - :param statement: Select statement - :param namespace: Namespace of schema group [Default: None]. If none set, all search in all namespaces - :param search_str: Search string to look for schemas. Search in name and description of the group + :param namespace: user namespace [Default: None]. If None, search in all namespaces + :param search_str: query string. [Default: ""]. If empty, return all schemas + :param page: page number [Default: 0] + :param page_size: number of schemas per page [Default: 0] + :param order_by: sort the result-set by the information + Options: ["name", "update_date"] + [Default: update_date] + :param order_desc: Sort the records in descending order. [Default: False] + + :return: { + pagination: {page: int, + page_size: int, + total: int}, + results: [SchemaRecordAnnotation] """ - if search_str: - sql_search_str = f"%{search_str}%" - search_query = or_( - SchemaGroups.name.ilike(sql_search_str), - SchemaGroups.description.ilike(sql_search_str), - ) - statement = statement.where(search_query) + + search_str = search_str.lower() if search_str else "" + + where_statement = or_( + SchemaRecords.name.ilike(f"%{search_str}%"), + SchemaRecords.description.ilike(f"%{search_str}%"), + ) if namespace: - statement = statement.where(SchemaGroups.namespace == namespace) - return statement + where_statement = and_(where_statement, SchemaRecords.namespace == namespace) + + with Session(self._sa_engine) as session: + total = session.scalar(select(func.count(SchemaRecords.id)).where(where_statement)) + statement = ( + select(SchemaRecords) + .where(where_statement) + .limit(page_size) + .offset(page * page_size) + ) + statement = self._add_order_by_schemas_keyword(statement, by=order_by, desc=order_desc) + results_objects = session.scalars(statement) + + return SchemaSearchResult( + pagination=PaginationResult( + page=page, + page_size=page_size, + total=total, + ), + results=[ + SchemaRecordAnnotation( + namespace=result.namespace, + schema_name=result.name, + latest_released_version=result.versions_mapping[0].version, + description=result.description, + maintainers=result.maintainers, + private=result.private, + last_update_date=result.last_update_date, + ) + for result in results_objects + ], + ) - def _group_search_count(self, namespace: str = None, search_str: str = ""): + def query_schema_version( + self, + namespace: str, + name: str, + tag: str = None, + search_str: str = "", + page: int = 0, + page_size: int = 10, + ) -> SchemaVersionSearchResult: """ - Count number of found group of schemas + Search schema versions in the database with pagination. - :param namespace: user namespace [Default: None]. If None, search in all namespaces + :param namespace: user namespace + :param name: schema name + :param tag: tag name. [Default: None]. If None, return versions with all tags :param search_str: query string. [Default: ""]. If empty, return all schemas + :param page: result page number [Default: 10] + :param page_size: number of schemas per page [Default: 10] - :return: list of schema dicts + :return: { + pagination: {page: int, + page_size: int, + total: int}, + results: [SchemaVersionAnnotation] """ - statement = select(func.count(SchemaGroups.id)) - statement = self._add_group_condition(statement, namespace, search_str) + search_str = search_str.lower() if search_str else "" with Session(self._sa_engine) as session: - result = session.execute(statement).one() + schema_obj = session.scalar( + select(SchemaRecords).where( + and_(SchemaRecords.namespace == namespace, SchemaRecords.name == name) + ) + ) + + if not schema_obj: + raise SchemaDoesNotExistError(f"Schema '{name}' does not exist in the database") - return result[0] + where_statement = and_( + SchemaRecords.namespace == namespace, + SchemaRecords.name == name, + or_( + SchemaVersions.version.ilike(f"%{search_str}%"), + SchemaVersions.release_notes.ilike(f"%{search_str}%"), + ), + ) - def group_delete(self, namespace: str, name: str) -> None: + if tag: + where_statement = and_(where_statement, SchemaTags.tag_name == tag) + total_statement = ( + select(func.count(SchemaVersions.id)) + .join(SchemaRecords) + .join(SchemaTags) + .where(where_statement) + ) + find_statement = ( + select(SchemaVersions) + .join(SchemaRecords) + .join(SchemaTags) + .where(where_statement) + ) + + else: + total_statement = ( + select(func.count(SchemaVersions.id)) + .join(SchemaRecords) + .where(where_statement) + ) + find_statement = select(SchemaVersions).join(SchemaRecords).where(where_statement) + + total = session.scalar(total_statement) + + results_objects = session.scalars( + find_statement.order_by(SchemaVersions.version.desc()) + .limit(page_size) + .offset(page * page_size) + ).unique() + + return SchemaVersionSearchResult( + pagination=PaginationResult( + page=page, + page_size=page_size, + total=total, + ), + results=[ + SchemaVersionAnnotation( + namespace=result.schema_mapping.namespace, + schema_name=result.schema_mapping.name, + version=result.version, + contributors=result.contributors, + release_notes=result.release_notes, + tags={tag.tag_name: tag.tag_value for tag in result.tags_mapping}, + release_date=result.release_date, + last_update_date=result.last_update_date, + ) + for result in results_objects + ], + ) + + def delete_schema(self, namespace: str, name: str) -> None: """ - Delete schema group from the database. + Delete schema from the database. :param namespace: user namespace - :param name: schema group name - + :param name: schema name :return: None """ - if not self.group_exist(namespace, name): - raise SchemaGroupDoesNotExistError( - f"Schema group '{name}' does not exist in the database" + with Session(self._sa_engine) as session: + schema_obj = session.scalar( + select(SchemaRecords).where( + and_(SchemaRecords.namespace == namespace, SchemaRecords.name == name) + ) ) + if not schema_obj: + raise SchemaDoesNotExistError(f"Schema '{name}' does not exist in the database") + + statement = select(User).where(User.namespace == namespace) + user = session.scalar(statement) + if user: + user.number_of_schemas -= 1 + session.commit() + + session.delete(schema_obj) + session.commit() + + def delete_version(self, namespace: str, name: str, version: str) -> None: + """ + Delete version of the schema + + :param namespace: Namespace of the schema + :param name: Name of the schema + :param version: Version of the Schema + + :raise: SchemaVersionDoesNotExistError if version doesn't exist + :return: None + """ with Session(self._sa_engine) as session: - session.execute( - delete(SchemaGroups).where( - and_(SchemaGroups.namespace == namespace, SchemaGroups.name == name) + schema_obj = session.scalar( + select(SchemaVersions) + .join(SchemaRecords, SchemaRecords.id == SchemaVersions.schema_id) + .where( + and_( + SchemaRecords.namespace == namespace, + SchemaRecords.name == name, + SchemaVersions.version == version, + ) ) ) + if not schema_obj: + raise SchemaVersionDoesNotExistError( + f"Schema '{name}' with version '{version}' does not exist in the database. Unable to update version." + ) + session.delete(schema_obj) session.commit() - def group_add_schema( - self, namespace: str, name: str, schema_namespace: str, schema_name: str + def add_tag_to_schema( + self, + namespace: str, + name: str, + version: str, + tag: Optional[Union[List[str], str, Dict[str, str]]], ) -> None: """ - Add schema to the schema group. + Add tag to the schema - :param namespace: user namespace - :param name: schema group name - :param schema_namespace: schema namespace - :param schema_name: schema name + :param namespace: Namespace of the schema + :param name: Name of the schema + :param version: Version of the Schema + :param tag: Tag to be added. Can be a string, list of strings or dictionaries + :raise: SchemaVersionDoesNotExistError if version doesn't exist :return: None """ - try: - with Session(self._sa_engine) as session: - group_mapping = session.scalar( - select(SchemaGroups).where( - and_( - SchemaGroups.namespace == namespace, - SchemaGroups.name == name, - ) - ) - ) + tag = self._unify_tags(tag) - if not group_mapping: - raise SchemaGroupDoesNotExistError( - f"Group of Schemas with namespace='{namespace}' and name='{name}' does not exist" - ) - - schema_mapping = session.scalar( - select(Schemas).where( - and_( - Schemas.namespace == schema_namespace, - Schemas.name == schema_name, - ) + with Session(self._sa_engine) as session: + schema_obj = session.scalar( + select(SchemaVersions) + .join(SchemaRecords, SchemaRecords.id == SchemaVersions.schema_id) + .where( + and_( + SchemaRecords.namespace == namespace, + SchemaRecords.name == name, + SchemaVersions.version == version, ) ) - - if not schema_mapping: - raise SchemaDoesNotExistError( - f"Schema with namespace='{schema_namespace}' and name='{schema_name}' does not exist" + ) + if not schema_obj: + raise SchemaVersionDoesNotExistError( + f"Schema '{name}' with version '{version}' does not exist in the database. Unable to add tag." + ) + if isinstance(tag, str): + tag = [tag] + + for tag_name, tag_value in tag.items(): + tag_obj = session.scalar(select(SchemaTags).where(SchemaTags.tag_name == tag_name)) + if not tag_obj: + tag_obj = SchemaTags( + tag_name=tag_name, tag_value=tag_value, schema_mapping=schema_obj ) - - session.add( - SchemaGroupRelations( - schema_id=schema_mapping.id, - group_id=group_mapping.id, + session.add(tag_obj) + else: + raise SchemaTagAlreadyExistsError( + f"Tag '{tag_name}' already exists in the schema" ) - ) - session.commit() - except IntegrityError: - raise SchemaAlreadyInGroupError - def group_remove_schema( - self, namespace: str, name: str, schema_namespace: str, schema_name: str - ) -> None: + session.commit() + + def remove_tag_from_schema(self, namespace: str, name: str, version: str, tag: str) -> None: """ - Remove schema from the schema group. + Remove tag from the schema - :param namespace: user namespace - :param name: schema group name - :param schema_namespace: schema namespace - :param schema_name: schema name + :param namespace: Namespace of the schema + :param name: Name of the schema + :param version: Version of the Schema + :param tag: Tag to be removed + :raise: SchemaVersionDoesNotExistError if version doesn't exist :return: None """ - - try: - with Session(self._sa_engine) as session: - delete_statement = delete(SchemaGroupRelations).where( + with Session(self._sa_engine) as session: + schema_obj = session.scalar( + select(SchemaVersions) + .join(SchemaRecords, SchemaRecords.id == SchemaVersions.schema_id) + .where( and_( - SchemaGroupRelations.schema_id - == select(Schemas.id) - .where( - and_( - Schemas.namespace == schema_namespace, - Schemas.name == schema_name, - ) - ) - .scalar_subquery(), - SchemaGroupRelations.group_id - == select(SchemaGroups.id) - .where( - and_( - SchemaGroups.namespace == namespace, - SchemaGroups.name == name, - ) - ) - .scalar_subquery(), + SchemaRecords.namespace == namespace, + SchemaRecords.name == name, + SchemaVersions.version == version, ) ) + ) + if not schema_obj: + raise SchemaVersionDoesNotExistError( + f"Schema '{name}' with version '{version}' does not exist in the database. Unable to remove tag." + ) - session.execute(delete_statement) - session.commit() - except IntegrityError: - raise SchemaIsNotInGroupError("Schema not found in the group") + tag_obj = session.scalar( + select(SchemaTags).where( + SchemaTags.tag_name == tag, SchemaTags.schema_version_id == schema_obj.id + ) + ) + if not tag_obj: + raise SchemaTagDoesNotExistError(f"Tag '{tag}' does not exist in the schema") - def group_exist(self, namespace: str, name: str) -> bool: + session.delete(tag_obj) + session.commit() + + @staticmethod + def _add_order_by_schemas_keyword( + statement: Select, by: str = "update_date", desc: bool = False + ) -> Select: """ - Check if schema group exists in the database. + Add order by clause to sqlalchemy statement - :param namespace: user namespace - :param name: schema group name + :param statement: sqlalchemy representation of a SELECT statement. + :param by: sort the result-set by the information + Options: ["name", "update_date"] + [Default: "update_date"] + :param desc: Sort the records in descending order. [Default: False] + :return: sqlalchemy representation of a SELECT statement with order by keyword + """ + if by == "update_date": + order_by_obj = SchemaRecords.last_update_date + elif by == "name": + order_by_obj = SchemaRecords.name + else: + _LOGGER.warning( + f"order by: '{by}' statement is unavailable. Projects are sorted by 'update_date'" + ) + order_by_obj = SchemaRecords.last_update_date + + if desc and by == "name": + order_by_obj = order_by_obj.desc() - :return: True if schema group exists, False otherwise + elif by != "name" and not desc: + order_by_obj = order_by_obj.desc() + + return statement.order_by(order_by_obj) + + def _unify_tags( + self, tags: Optional[Union[List[str], str, Dict[str, str], List[Dict[str, str]]]] + ) -> [Dict[str, str]]: """ + Convert provided tags to one standard - with Session(self._sa_engine) as session: - schema_group_obj = session.scalar( - select(SchemaGroups).where( - and_(SchemaGroups.namespace == namespace, SchemaGroups.name == name) + :param tags: tags to be converted from types: str, dict, list of str, list of dict + + :raise: ValueError if tags are not in the correct format + :return: dictionary of tags + """ + if not tags: + tags = {} + if tags == (None,): + tags = {} + elif isinstance(tags, str): + tags = {tags: None} + elif isinstance(tags, dict): + pass + elif isinstance(tags, list): + if all(isinstance(tag, str) for tag in tags): + tags = {tag: None for tag in tags} + else: + raise ValueError( + f"tags should be a list of strings or a list of dictionaries. Tag values: {tags}" ) + else: + raise ValueError( + f"tags should be a list of strings or a list of dictionaries. Tag values: {tags}" ) - return True if schema_group_obj else False + return tags diff --git a/pepdbagent/modules/user.py b/pepdbagent/modules/user.py index 125c7c1..e6e1ce0 100644 --- a/pepdbagent/modules/user.py +++ b/pepdbagent/modules/user.py @@ -177,7 +177,7 @@ def get_favorites(self, namespace: str) -> AnnotationList: last_update_date=str(prj.last_update_date), submission_date=str(prj.submission_date), digest=prj.digest, - pep_schema=prj.pep_schema, + pep_schema=f"{prj.schema_mapping.schema_mapping.namespace}/{prj.schema_mapping.schema_mapping.name}:{prj.schema_mapping.version}", pop=prj.pop, stars_number=prj.number_of_stars, forked_from=( diff --git a/pepdbagent/utils.py b/pepdbagent/utils.py index 1bec815..fc96684 100644 --- a/pepdbagent/utils.py +++ b/pepdbagent/utils.py @@ -80,17 +80,21 @@ def registry_path_converter(registry_path: str) -> Tuple[str, str, str]: raise RegistryPathError(f"Error in: '{registry_path}'") -def schema_path_converter(schema_path: str) -> Tuple[str, str]: +def schema_path_converter(schema_path: str) -> Tuple[str, str, str]: """ Convert schema path to namespace, name :param schema_path: schema path that has structure: "namespace/name.yaml" - :return: tuple(namespace, name) + :return: tuple(namespace, name, version) """ if "/" in schema_path: - namespace, name = schema_path.split("/") - return namespace, name - raise RegistryPathError(f"Incorrect schema registry path: '{schema_path}'") + namespace, name_tag = schema_path.split("/") + if ":" in name_tag: + name, version = name_tag.split(":") + return namespace, name, version + + return namespace, name_tag, "latest" + raise RegistryPathError(f"Error in: '{schema_path}'") def tuple_converter(value: Union[tuple, list, str, None]) -> tuple: diff --git a/tests/test_schema.py b/tests/test_schema.py index 131b67f..d4b8ecb 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -2,6 +2,10 @@ from .utils import PEPDBAgentContextManager +from pepdbagent.models import UpdateSchemaVersionFields, UpdateSchemaRecordFields + +DEFAULT_SCHEMA_VERSION = "1.0.0" + @pytest.mark.skipif( not PEPDBAgentContextManager().db_setup(), @@ -17,207 +21,239 @@ class TestSchemas: ) def test_get(self, namespace, name): with PEPDBAgentContextManager(add_schemas=True) as agent: - schema = agent.schema.get(namespace=namespace, name=name) - assert agent.schema.exist(namespace=namespace, name=name) + assert agent.schema.schema_exist(namespace=namespace, name=name) + assert agent.schema.version_exist( + namespace=namespace, name=name, version=DEFAULT_SCHEMA_VERSION + ) + schema = agent.schema.get( + namespace=namespace, name=name, version=DEFAULT_SCHEMA_VERSION + ) assert schema - @pytest.mark.parametrize( - "namespace, name", - [ - ["namespace1", "2.0.0"], - ], - ) - def test_delete(self, namespace, name): + def test_update_schema(self): with PEPDBAgentContextManager(add_schemas=True) as agent: - assert agent.schema.exist(namespace=namespace, name=name) - agent.schema.delete(namespace=namespace, name=name) - assert not agent.schema.exist(namespace=namespace, name=name) + new_maintainers = "New Maintainer" + new_lifecycle_stage = "New Stage" + new_private = True + new_name = "new_schema_name" - @pytest.mark.parametrize( - "namespace, name", - [ - ["namespace1", "2.0.0"], - ], - ) - def test_update(self, namespace, name): - with PEPDBAgentContextManager(add_schemas=True) as agent: - schema = agent.schema.get(namespace=namespace, name=name) - schema["new"] = "hello" - agent.schema.update(namespace=namespace, name=name, schema=schema) - assert agent.schema.exist(namespace=namespace, name=name) - assert schema == agent.schema.get(namespace=namespace, name=name) + agent.schema.update_schema_record( + "namespace1", + "2.0.0", + UpdateSchemaRecordFields( + maintainers=new_maintainers, + lifecycle_stage=new_lifecycle_stage, + private=new_private, + name=new_name, + ), + ) + result = agent.schema.get_schema_info("namespace1", new_name) + assert result.maintainers == new_maintainers + assert result.lifecycle_stage == new_lifecycle_stage + assert result.private == new_private + assert result.schema_name == new_name - @pytest.mark.parametrize( - "namespace, name", - [ - ["namespace1", "2.0.0"], - ], - ) - def test_get_annotation(self, namespace, name): + def test_update_schema_update_date(self): with PEPDBAgentContextManager(add_schemas=True) as agent: - schema_annot = agent.schema.info(namespace=namespace, name=name) - assert schema_annot - assert schema_annot.model_fields_set == { - "namespace", - "name", - "last_update_date", - "submission_date", - "description", - "popularity_number", + version_name = "2.0.0" + version_schema = { + "type": "object", + "properties": { + "name": {"type": "string"}, + "age": {"type": "integer"}, + }, } - @pytest.mark.parametrize( - "namespace, name", - [ - ["namespace1", "2.0.0"], - ], - ) - def test_update_annotation(self, namespace, name): - with PEPDBAgentContextManager(add_schemas=True) as agent: - schema_annot = agent.schema.info(namespace=namespace, name=name) - schema = agent.schema.get(namespace=namespace, name=name) - agent.schema.update( - namespace=namespace, name=name, schema=schema, description="new desc" + first_time = agent.schema.get_schema_info("namespace1", "2.0.0").last_update_date + + agent.schema.add_version( + "namespace1", + version_name, + "pablo1", + schema_value=version_schema, + contributors="Teddy", + release_notes="Initial release", ) - assert schema_annot != agent.schema.info(namespace=namespace, name=name) + result = agent.schema.get_schema_info("namespace1", "2.0.0") + assert result.last_update_date != first_time - @pytest.mark.parametrize( - "namespace, name", - [ - ["namespace2", "bedboss"], - ], - ) - def test_annotation_popular(self, namespace, name): - with PEPDBAgentContextManager(add_data=True, add_schemas=True) as agent: - agent.project.update( - namespace="namespace1", - name="amendments1", - update_dict={"pep_schema": "namespace2/bedboss"}, + def test_add_schema_version(self): + with PEPDBAgentContextManager(add_schemas=True) as agent: + new_schema = { + "type": "object", + "properties": { + "name": {"type": "string"}, + "age": {"type": "integer"}, + }, + } + new_contributors = "New Maintainer" + new_release_notes = "New release" + agent.schema.update_schema_version( + "namespace1", + "2.0.0", + DEFAULT_SCHEMA_VERSION, + UpdateSchemaVersionFields( + schema_value=new_schema, + contributors=new_contributors, + release_notes=new_release_notes, + ), ) - schema_annot = agent.schema.info(namespace=namespace, name=name) - assert schema_annot.popularity_number == 1 + result = agent.schema.get_version_info("namespace1", "2.0.0", DEFAULT_SCHEMA_VERSION) + assert result.contributors == new_contributors + assert result.release_notes == new_release_notes + assert agent.schema.get("namespace1", "2.0.0", DEFAULT_SCHEMA_VERSION) == new_schema - def test_search(self): + def test_search_schema_version(self): with PEPDBAgentContextManager(add_schemas=True) as agent: - results = agent.schema.search(namespace="namespace2") - assert results - assert results.count == 3 - assert len(results.results) == 3 + result = agent.schema.query_schema_version("namespace1", "2.0.0") + assert result.pagination.total == 1 + assert len(result.results) == 1 - def test_search_offset(self): + def test_search_schema_version_with_tags(self): with PEPDBAgentContextManager(add_schemas=True) as agent: - results = agent.schema.search(namespace="namespace2", offset=1) - assert results - assert results.count == 3 - assert len(results.results) == 2 + schema1 = { + "type": "object", + "properties": { + "name1": {"type": "string"}, + "age1": {"type": "integer"}, + }, + } + schema2 = { + "type": "object", + "properties": { + "name2": {"type": "string"}, + "age2": {"type": "integer"}, + }, + } + + agent.schema.add_version( + "namespace1", + "2.0.0", + "bino1", + schema_value=schema1, + tags=["tag1", "bioinfo"], + release_notes="computer change", + ) + agent.schema.add_version( + "namespace1", + "2.0.0", + "bino2", + schema_value=schema2, + tags=["bioinfo"], + release_notes="language", + ) + + result = agent.schema.query_schema_version("namespace1", "2.0.0", tag="tag1") + + assert result.pagination.total == 1 + assert len(result.results) == 1 + + result = agent.schema.query_schema_version("namespace1", "2.0.0", tag="bioinfo") + + assert result.pagination.total == 2 + assert len(result.results) == 2 - def test_search_limit(self): + def test_search_schema(self): with PEPDBAgentContextManager(add_schemas=True) as agent: - results = agent.schema.search(namespace="namespace2", limit=1) - assert results - assert results.count == 3 - assert len(results.results) == 1 + result = agent.schema.query_schemas(search_str="bed") + assert result.pagination.total == 3 + assert len(result.results) == 3 - def test_search_limit_offset(self): + def test_search_schema_namespace(self): with PEPDBAgentContextManager(add_schemas=True) as agent: - results = agent.schema.search(namespace="namespace2", limit=2, offset=2) - assert results - assert results.count == 3 - assert len(results.results) == 1 + result = agent.schema.query_schemas("namespace1") + assert result.pagination.total == 2 + assert "namespace1" in [f.namespace for f in result.results] + assert "2.0.0" in [f.schema_name for f in result.results] - def test_search_query(self): + def test_search_schema_page_number(self): with PEPDBAgentContextManager(add_schemas=True) as agent: - results = agent.schema.search(namespace="namespace2", search_str="bedb") - assert results - assert results.count == 2 - assert len(results.results) == 2 + result = agent.schema.query_schemas("namespace2", page_size=2, page=1) + assert result.pagination.total == 3 + assert result.pagination.page == 1 + assert result.pagination.page_size == 2 + assert len(result.results) == 1 @pytest.mark.parametrize( "namespace, name", [ - ["namespace1", "2.0.0"], + ["namespace2", "bedmaker"], ], ) - def test_create_group(self, namespace, name): + def test_schema_delete(self, namespace, name): with PEPDBAgentContextManager(add_schemas=True) as agent: - group_name = "new_group" - agent.schema.group_create( - namespace=namespace, name=group_name, description="new group" + assert agent.schema.schema_exist(namespace=namespace, name=name) + agent.schema.delete_schema(namespace=namespace, name=name) + assert not agent.schema.version_exist( + namespace=namespace, name=name, version=DEFAULT_SCHEMA_VERSION ) - assert agent.schema.group_exist(namespace=namespace, name=group_name) + assert not agent.schema.schema_exist(namespace=namespace, name=name) @pytest.mark.parametrize( "namespace, name", [ - ["namespace1", "2.0.0"], + ["namespace2", "bedmaker"], ], ) - def test_delete_group(self, namespace, name): + def test_schema_version_delete(self, namespace, name): with PEPDBAgentContextManager(add_schemas=True) as agent: - group_name = "new_group" - agent.schema.group_create( - namespace=namespace, name=group_name, description="new group" + assert agent.schema.version_exist( + namespace=namespace, name=name, version=DEFAULT_SCHEMA_VERSION + ) + agent.schema.delete_version( + namespace=namespace, name=name, version=DEFAULT_SCHEMA_VERSION ) - assert agent.schema.group_exist(namespace=namespace, name=group_name) - agent.schema.group_delete(namespace=namespace, name=group_name) - assert not agent.schema.group_exist(namespace=namespace, name=group_name) + assert not agent.schema.version_exist( + namespace=namespace, name=name, version=DEFAULT_SCHEMA_VERSION + ) + assert agent.schema.schema_exist(namespace=namespace, name=name) - @pytest.mark.parametrize( - "namespace, name", - [ - ["namespace1", "2.0.0"], - ], - ) - def test_add_to_group(self, namespace, name): + def test_number_of_schemas_in_namespace(self): with PEPDBAgentContextManager(add_schemas=True) as agent: - group_name = "new_group" - agent.schema.group_create( - namespace=namespace, name=group_name, description="new group" + for k in agent.namespace.info().results: + if k.namespace_name == "namespace1": + assert k.number_of_schemas == 2 + if k.namespace_name == "namespace2": + assert k.number_of_schemas == 3 + + +class TestSchemaTags: + def test_insert_tags(self): + with PEPDBAgentContextManager(add_schemas=True) as agent: + new_tag1 = "new_tag" + new_tag2 = "tag2" + agent.schema.add_tag_to_schema( + "namespace1", "2.0.0", DEFAULT_SCHEMA_VERSION, tag=[new_tag1, new_tag2] ) - agent.schema.group_add_schema( - namespace=namespace, name=group_name, schema_name=name, schema_namespace=namespace + + result = agent.schema.get_version_info("namespace1", "2.0.0", DEFAULT_SCHEMA_VERSION) + + assert new_tag1 in result.tags + assert new_tag2 in result.tags + + def test_insert_one_tag(self): + with PEPDBAgentContextManager(add_schemas=True) as agent: + new_tag1 = "new_tag" + agent.schema.add_tag_to_schema( + "namespace1", "2.0.0", DEFAULT_SCHEMA_VERSION, tag=new_tag1 ) - group_annot = agent.schema.group_get(namespace=namespace, name=group_name) - assert group_annot.schemas[0].name == name + result = agent.schema.get_version_info("namespace1", "2.0.0", DEFAULT_SCHEMA_VERSION) + assert new_tag1 in result.tags @pytest.mark.parametrize( "namespace, name", [ - ["namespace1", "2.0.0"], + ["namespace2", "bedmaker"], ], ) - def test_remove_from_group(self, namespace, name): + def test_delete_tag(self, namespace, name): with PEPDBAgentContextManager(add_schemas=True) as agent: - group_name = "new_group" - agent.schema.group_create( - namespace=namespace, name=group_name, description="new group" + new_tag1 = "new_tag" + agent.schema.add_tag_to_schema(namespace, name, DEFAULT_SCHEMA_VERSION, tag=new_tag1) + result = agent.schema.get_version_info(namespace, name, DEFAULT_SCHEMA_VERSION) + assert new_tag1 in result.tags + agent.schema.remove_tag_from_schema( + namespace, name, DEFAULT_SCHEMA_VERSION, tag=new_tag1 ) - agent.schema.group_add_schema( - namespace=namespace, name=group_name, schema_name=name, schema_namespace=namespace - ) - group_annot = agent.schema.group_get(namespace=namespace, name=group_name) - assert len(group_annot.schemas) == 1 - - agent.schema.group_remove_schema( - namespace=namespace, name=group_name, schema_name=name, schema_namespace=namespace - ) - group_annot = agent.schema.group_get(namespace=namespace, name=group_name) - assert len(group_annot.schemas) == 0 - - def test_search_group(self): - with PEPDBAgentContextManager(add_schemas=True) as agent: - group_name1 = "new_group1" - group_name2 = "new2" - group_name3 = "new_group3" - agent.schema.group_create( - namespace="namespace1", name=group_name1, description="new group" - ) - agent.schema.group_create(namespace="namespace1", name=group_name2, description="new") - agent.schema.group_create( - namespace="namespace1", name=group_name3, description="new group" - ) - - results = agent.schema.group_search(search_str="new_group") - - assert results.count == 2 - assert len(results.results) == 2 + result = agent.schema.get_version_info(namespace, name, DEFAULT_SCHEMA_VERSION) + assert new_tag1 not in result.tags diff --git a/tests/test_updates.py b/tests/test_updates.py index e057024..628772e 100644 --- a/tests/test_updates.py +++ b/tests/test_updates.py @@ -99,16 +99,16 @@ def test_update_project_description(self, namespace, name, new_description): def test_update_project_schema(self, namespace, name, new_schema): with PEPDBAgentContextManager(add_data=True) as agent: prj_annot = agent.annotation.get(namespace=namespace, name=name) - assert prj_annot.results[0].pep_schema == "namespace1/2.0.0" + assert prj_annot.results[0].pep_schema == "namespace1/2.0.0:1.0.0" agent.project.update( namespace=namespace, name=name, tag="default", - update_dict={"pep_schema": "namespace2/bedboss"}, + update_dict={"pep_schema": "namespace2/bedboss:1.0.0"}, ) prj_annot = agent.annotation.get(namespace=namespace, name=name) - assert prj_annot.results[0].pep_schema == "namespace2/bedboss" + assert prj_annot.results[0].pep_schema == "namespace2/bedboss:1.0.0" @pytest.mark.parametrize( "namespace, name, new_description", @@ -155,8 +155,8 @@ def test_update_whole_project(self, namespace, name): @pytest.mark.parametrize( "namespace, name, pep_schema", [ - ["namespace1", "amendments1", "namespace2/bedmaker"], - ["namespace2", "derive", "namespace2/bedbuncher"], + ["namespace1", "amendments1", "namespace2/bedmaker:1.0.0"], + ["namespace2", "derive", "namespace2/bedbuncher:1.0.0"], ], ) def test_update_pep_schema(self, namespace, name, pep_schema): diff --git a/tests/utils.py b/tests/utils.py index 2afc623..fc9cc89 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -95,6 +95,9 @@ def __enter__(self): def __exit__(self, exc_type, exc_value, exc_traceback): self.db_engine.delete_schema() + def __del__(self): + self.db_engine.delete_schema() + def _insert_data(self): pepdb_con = PEPDatabaseAgent(dsn=self.url, echo=self._echo) for namespace, item in list_of_available_peps().items(): @@ -111,7 +114,7 @@ def _insert_data(self): is_private=private, project=prj, overwrite=True, - pep_schema="namespace1/2.0.0", + pep_schema="namespace1/2.0.0:1.0.0", # TODO: test without this line ) def _add_schemas(self): @@ -120,7 +123,16 @@ def _add_schemas(self): for name, path in item.items(): file_dict = read_yaml_file(path) - pepdb_con.schema.create(namespace=namespace, name=name[0:-5], schema=file_dict) + pepdb_con.schema.create( + namespace=namespace, + name=name[0:-5], + version="1.0.0", + schema_value=file_dict, + maintainers="Teddy", + contributors="Teddy, John", + release_notes="Initial release", + tags={"maturity_level": "trial_use"}, + ) @property def agent(self) -> PEPDatabaseAgent: