Release V0.12.0

.github/workflows/black.yml

@@ -6,6 +6,6 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
       - uses: psf/black@stable

docs/changelog.md

@@ -2,10 +2,18 @@
 
 This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format.
 
+
+## [0.12.0] -- 2025-03-27
+- New database model for schemas
+- Added schema version and schema record
+- Added tags to schemas
+
+
 ## [0.11.1] -- 2024-09-04
 - Added archive table of namespaces
 - Added sort by stars
 
+
 ## [0.11.0] -- 2024-07-24
 - Added validation schemas
 

pepdbagent/__init__.py

@@ -1,4 +1,4 @@
-""" Package-level data """
+"""Package-level data"""
 
 import coloredlogs
 import logmuse

pepdbagent/_version.py

@@ -1 +1 @@
-__version__ = "0.11.1"
+__version__ = "0.12.0"

pepdbagent/const.py

@@ -22,3 +22,6 @@
 PEPHUB_SAMPLE_ID_KEY = "ph_id"
 
 MAX_HISTORY_SAMPLES_NUMBER = 2000
+
+DEFAULT_TAG_VERSION = "1.0.0"
+LATEST_SCHEMA_VERSION = "latest"

pepdbagent/db_utils.py

@@ -86,12 +86,11 @@ class Projects(Base):
     last_update_date: Mapped[Optional[datetime.datetime]] = mapped_column(
         default=deliver_update_date,  # onupdate=deliver_update_date, # This field should not be updated, while we are adding project to favorites
     )
-    pep_schema: Mapped[Optional[str]]
 
     schema_id: Mapped[Optional[int]] = mapped_column(
-        ForeignKey("schemas.id", ondelete="SET NULL"), nullable=True
+        ForeignKey("schema_versions.id", ondelete="SET NULL"), nullable=True
     )
-    schema_mapping: Mapped["Schemas"] = relationship("Schemas", lazy="joined")
+    schema_mapping: Mapped["SchemaVersions"] = relationship("SchemaVersions", lazy="joined")
 
     pop: Mapped[Optional[bool]] = mapped_column(default=False)
     samples_mapping: Mapped[List["Samples"]] = relationship(
@@ -200,10 +199,14 @@ class User(Base):
         order_by="Stars.star_date.desc()",
     )
     number_of_projects: Mapped[int] = mapped_column(default=0)
+    number_of_schemas: Mapped[int] = mapped_column(default=0)
 
     projects_mapping: Mapped[List["Projects"]] = relationship(
         "Projects", back_populates="namespace_mapping"
     )
+    schemas_mapping: Mapped[List["SchemaRecords"]] = relationship(
+        "SchemaRecords", back_populates="user_mapping"
+    )
 
 
 class Stars(Base):
@@ -302,65 +305,68 @@ class HistorySamples(Base):
     )
 
 
-class Schemas(Base):
-
-    __tablename__ = "schemas"
+class SchemaRecords(Base):
+    __tablename__ = "schema_records"
 
-    id: Mapped[int] = mapped_column(primary_key=True, index=True)
+    id: Mapped[int] = mapped_column(primary_key=True)
     namespace: Mapped[str] = mapped_column(ForeignKey("users.namespace", ondelete="CASCADE"))
-    name: Mapped[str] = mapped_column(nullable=False, index=True)
-    description: Mapped[Optional[str]] = mapped_column(nullable=True, index=True)
-    schema_json: Mapped[dict] = mapped_column(JSON, server_default=FetchedValue())
+    name: Mapped[str] = mapped_column(nullable=False)
+    maintainers: Mapped[str] = mapped_column(nullable=True)
+    lifecycle_stage: Mapped[str] = mapped_column(nullable=True)
+    description: Mapped[Optional[str]] = mapped_column(nullable=True)
     private: Mapped[bool] = mapped_column(default=False)
-    submission_date: Mapped[datetime.datetime] = mapped_column(default=deliver_update_date)
     last_update_date: Mapped[Optional[datetime.datetime]] = mapped_column(
         default=deliver_update_date, onupdate=deliver_update_date
     )
 
-    projects_mappings: Mapped[List["Projects"]] = relationship(
-        "Projects", back_populates="schema_mapping"
-    )
-    group_relation_mapping: Mapped[List["SchemaGroupRelations"]] = relationship(
-        "SchemaGroupRelations", back_populates="schema_mapping"
-    )
-
     __table_args__ = (UniqueConstraint("namespace", "name"),)
 
+    versions_mapping: Mapped[List["SchemaVersions"]] = relationship(
+        "SchemaVersions",
+        back_populates="schema_mapping",
+        cascade="all, delete-orphan",
+        order_by="SchemaVersions.version.desc()",
+    )
+    user_mapping: Mapped["User"] = relationship("User", back_populates="schemas_mapping")
 
-class SchemaGroups(Base):
 
-    __tablename__ = "schema_groups"
+class SchemaVersions(Base):
+    __tablename__ = "schema_versions"
 
-    id: Mapped[int] = mapped_column(primary_key=True, index=True)
-    namespace: Mapped[str] = mapped_column(
-        ForeignKey("users.namespace", ondelete="CASCADE"), index=True
+    id: Mapped[int] = mapped_column(primary_key=True)
+    schema_id: Mapped[int] = mapped_column(ForeignKey("schema_records.id", ondelete="CASCADE"))
+    version: Mapped[str] = mapped_column(nullable=False)
+    schema_value: Mapped[dict] = mapped_column(JSON, server_default=FetchedValue())
+    release_date: Mapped[datetime.datetime] = mapped_column(default=deliver_update_date)
+    last_update_date: Mapped[Optional[datetime.datetime]] = mapped_column(
+        default=deliver_update_date, onupdate=deliver_update_date
     )
-    name: Mapped[str] = mapped_column(nullable=False, index=True)
-    description: Mapped[Optional[str]] = mapped_column(nullable=True)
+    contributors: Mapped[Optional[str]] = mapped_column(nullable=True)
+    release_notes: Mapped[Optional[str]] = mapped_column(nullable=True)
 
-    schema_relation_mapping: Mapped[List["SchemaGroupRelations"]] = relationship(
-        "SchemaGroupRelations", back_populates="group_mapping"
-    )
+    __table_args__ = (UniqueConstraint("schema_id", "version"),)
 
-    __table_args__ = (UniqueConstraint("namespace", "name"),)
+    schema_mapping: Mapped["SchemaRecords"] = relationship(
+        "SchemaRecords", back_populates="versions_mapping"
+    )
 
+    tags_mapping: Mapped[List["SchemaTags"]] = relationship(
+        "SchemaTags", back_populates="schema_mapping", lazy="joined", cascade="all, delete-orphan"
+    )
 
-class SchemaGroupRelations(Base):
 
-    __tablename__ = "schema_group_relations"
+class SchemaTags(Base):
+    __tablename__ = "schema_tags"
 
-    schema_id: Mapped[int] = mapped_column(
-        ForeignKey("schemas.id", ondelete="CASCADE"), index=True, primary_key=True
-    )
-    group_id: Mapped[int] = mapped_column(
-        ForeignKey("schema_groups.id", ondelete="CASCADE"), index=True, primary_key=True
+    id: Mapped[int] = mapped_column(primary_key=True)
+    tag_name: Mapped[str] = mapped_column(nullable=False)
+    tag_value: Mapped[str] = mapped_column(nullable=True)
+    schema_version_id: Mapped[int] = mapped_column(
+        ForeignKey("schema_versions.id", ondelete="CASCADE")
     )
 
-    schema_mapping: Mapped["Schemas"] = relationship(
-        "Schemas", back_populates="group_relation_mapping"
-    )
-    group_mapping: Mapped["SchemaGroups"] = relationship(
-        "SchemaGroups", back_populates="schema_relation_mapping"
+    schema_mapping: Mapped["SchemaVersions"] = relationship(
+        "SchemaVersions", back_populates="tags_mapping"
     )
 
 

pepdbagent/exceptions.py

@@ -1,4 +1,4 @@
-""" Custom error types """
+"""Custom error types"""
 
 
 class PEPDatabaseAgentError(Exception):
@@ -129,21 +129,21 @@ def __init__(self, msg=""):
         super().__init__(f"""Schema already exists. {msg}""")
 
 
-class SchemaGroupDoesNotExistError(PEPDatabaseAgentError):
+class SchemaVersionDoesNotExistError(PEPDatabaseAgentError):
     def __init__(self, msg=""):
-        super().__init__(f"""Schema group does not exist. {msg}""")
+        super().__init__(f"""Schema version does not exist. {msg}""")
 
 
-class SchemaGroupAlreadyExistsError(PEPDatabaseAgentError):
+class SchemaVersionAlreadyExistsError(PEPDatabaseAgentError):
     def __init__(self, msg=""):
-        super().__init__(f"""Schema group already exists. {msg}""")
+        super().__init__(f"""Schema version already exists. {msg}""")
 
 
-class SchemaAlreadyInGroupError(PEPDatabaseAgentError):
+class SchemaTagDoesNotExistError(PEPDatabaseAgentError):
     def __init__(self, msg=""):
-        super().__init__(f"""Schema already in the group. {msg}""")
+        super().__init__(f"""Schema tag does not exist. {msg}""")
 
 
-class SchemaIsNotInGroupError(PEPDatabaseAgentError):
+class SchemaTagAlreadyExistsError(PEPDatabaseAgentError):
     def __init__(self, msg=""):
-        super().__init__(f"""Schema not found in group. {msg}""")
+        super().__init__(f"""Schema tag already exists. {msg}""")

pepdbagent/models.py

@@ -52,6 +52,12 @@ def is_private_should_be_bool(cls, v):
             return v
 
 
+class PaginationResult(BaseModel):
+    page: int = 0
+    page_size: int = 10
+    total: int
+
+
 class AnnotationList(BaseModel):
     """
     Annotation return model.
@@ -156,17 +162,18 @@ class NamespaceInfo(BaseModel):
     Model with information about namespace
     """
 
-    namespace: str
+    namespace_name: str
+    contact_url: Optional[str] = None
     number_of_projects: int
+    number_of_schemas: int
 
 
 class ListOfNamespaceInfo(BaseModel):
     """
     Namespace information response model
     """
 
-    number_of_namespaces: int
-    limit: int
+    pagination: PaginationResult
     results: List[NamespaceInfo]
 
 
@@ -249,50 +256,66 @@ class HistoryAnnotationModel(BaseModel):
     history: List[HistoryChangeModel]
 
 
-class SchemaAnnotation(BaseModel):
+class SchemaVersionAnnotation(BaseModel):
+    """
+    Schema version annotation model
+    """
+
+    namespace: str
+    schema_name: str
+    version: str
+    contributors: Optional[str] = ""
+    release_notes: Optional[str] = ""
+    tags: Dict[str, Union[str, None]] = {}
+    release_date: datetime.datetime
+    last_update_date: datetime.datetime
+
+
+class SchemaRecordAnnotation(BaseModel):
     """
     Schema annotation model
     """
 
     namespace: str
-    name: str
-    last_update_date: str
-    submission_date: str
+    schema_name: str
     description: Optional[str] = ""
-    popularity_number: Optional[int] = 0
+    maintainers: str = ""
+    lifecycle_stage: str = ""
+    latest_released_version: str
+    private: bool = False
+    last_update_date: datetime.datetime
 
 
 class SchemaSearchResult(BaseModel):
     """
     Schema search result model
     """
 
-    count: int
-    limit: int
-    offset: int
-    results: List[SchemaAnnotation]
+    pagination: PaginationResult
+    results: List[SchemaRecordAnnotation]
 
 
-class SchemaGroupAnnotation(BaseModel):
+class SchemaVersionSearchResult(BaseModel):
     """
-    Schema group annotation model
+    Schema version search result model
     """
 
-    namespace: str
-    name: str
-    description: Optional[str] = ""
-    schemas: List[SchemaAnnotation]
+    pagination: PaginationResult
+    results: List[SchemaVersionAnnotation]
 
 
-class SchemaGroupSearchResult(BaseModel):
-    """
-    Schema group search result model
-    """
+class UpdateSchemaRecordFields(BaseModel):
+    maintainers: Optional[str] = None
+    lifecycle_stage: Optional[str] = None
+    private: Optional[bool] = False
+    name: Optional[str] = None
+    description: Optional[str] = None
 
-    count: int
-    limit: int
-    offset: int
-    results: List[SchemaGroupAnnotation]
+
+class UpdateSchemaVersionFields(BaseModel):
+    contributors: Optional[str] = None
+    schema_value: Optional[dict] = None
+    release_notes: Optional[str] = None
 
 
 class TarNamespaceModel(BaseModel):

pepdbagent/modules/annotation.py

@@ -212,7 +212,7 @@ def _get_single_annotation(
                     last_update_date=str(query_result.last_update_date),
                     digest=query_result.digest,
                     pep_schema=(
-                        f"{query_result.schema_mapping.namespace}/{query_result.schema_mapping.name}"
+                        f"{query_result.schema_mapping.schema_mapping.namespace}/{query_result.schema_mapping.schema_mapping.name}:{query_result.schema_mapping.version}"
                         if query_result.schema_mapping
                         else None
                     ),
@@ -326,14 +326,16 @@ def _get_projects(
         statement = self._add_date_filter_if_provided(
             statement, filter_by, filter_start_date, filter_end_date
         )
+        statement = statement
         statement = self._add_order_by_keyword(statement, by=order_by, desc=order_desc)
         statement = statement.limit(limit).offset(offset)
         if pep_type:
             statement = statement.where(Projects.pop.is_(pep_type == "pop"))
 
         results_list = []
         with Session(self._sa_engine) as session:
-            results = session.scalars(statement)
+            # Unique should be called because of the join with schema_mapping
+            results = session.scalars(statement).unique()
             for result in results:
                 results_list.append(
                     AnnotationModel(
@@ -347,7 +349,7 @@ def _get_projects(
                         last_update_date=str(result.last_update_date),
                         digest=result.digest,
                         pep_schema=(
-                            f"{result.schema_mapping.namespace}/{result.schema_mapping.name}"
+                            f"{result.schema_mapping.schema_mapping.namespace}/{result.schema_mapping.schema_mapping.name}:{result.schema_mapping.version}"
                             if result.schema_mapping
                             else None
                         ),
@@ -548,7 +550,7 @@ def get_by_rp_list(
             statement = select(Projects).where(or_(*or_statement_list))
             anno_results = []
             with Session(self._sa_engine) as session:
-                query_result = session.scalars(statement)
+                query_result = session.scalars(statement).unique()
                 for result in query_result:
                     project_obj = result
                     annot = AnnotationModel(
@@ -562,7 +564,7 @@ def get_by_rp_list(
                         last_update_date=str(project_obj.last_update_date),
                         digest=project_obj.digest,
                         pep_schema=(
-                            f"{project_obj.schema_mapping.namespace}/{project_obj.schema_mapping.name}"
+                            f"{project_obj.schema_mapping.schema_mapping.namespace}/{project_obj.schema_mapping.schema_mapping.name}:{project_obj.schema_mapping.version}"
                             if project_obj.schema_mapping
                             else None
                         ),