diff --git a/apps/worker/src/index.ts b/apps/worker/src/index.ts index f19c693..31c2ac5 100644 --- a/apps/worker/src/index.ts +++ b/apps/worker/src/index.ts @@ -5,4 +5,5 @@ import { Tasks } from "@playfulprogramming/common"; createWorker(Tasks.POST_IMAGES, "./tasks/post-images/processor.ts"); createWorker(Tasks.URL_METADATA, "./tasks/url-metadata/processor.ts"); createWorker(Tasks.SYNC_AUTHOR, "./tasks/sync-author/processor.ts"); +createWorker(Tasks.SYNC_COLLECTION, "./tasks/sync-collection/processor.ts"); createHealthcheck(); diff --git a/apps/worker/src/tasks/sync-collection/processor.test.ts b/apps/worker/src/tasks/sync-collection/processor.test.ts new file mode 100644 index 0000000..ce72cf6 --- /dev/null +++ b/apps/worker/src/tasks/sync-collection/processor.test.ts @@ -0,0 +1,369 @@ +import processor from "./processor.ts"; +import type { TaskInputs } from "@playfulprogramming/common"; +import type { Job } from "bullmq"; +import { collectionAuthors, collectionData, db } from "@playfulprogramming/db"; +import { s3 } from "@playfulprogramming/s3"; +import * as github from "@playfulprogramming/github-api"; +import { Readable } from "node:stream"; +import { eq } from "drizzle-orm"; + +const mockImage = `iVBORw0KGgoAAAANSUhEUgAAAPIAAADOCAYAAAAE0F9yAAAACXBIWXMAABYZAAAWGQFJGZrZAAAN+ElEQVR4Aeyd65mjOBpGeSaU3c2hOpWqzqErlq4ccKfSymFm8tjdH7t6sbHxBSGEBLqceUYFBl0+nU8HsKu7/ce///Pf/1FgwBooew380fEfBCBQPAFELj6FTAACXYfIrAIIVECgZpErSA9TgIAfAUT240QtCGRNAJGzTg/BQcCPACL7caIWBLImgMhZp2c2OE5A4I4AIt/h4AUEyiSAyGXmjaghcEcAke9w8AICZRJA5DLzVnPUzC2AACIHQKMJBHIjgMi5ZYR4IBBAAJEDoNEEArkRQOTcMkI8NRNINjdEToaWjiGwHwFE3o81I0EgGQFEToaWjiGwHwFE3o81I0EgGYEMRE42NzqGQDMEELmZVDPRmgkgcs3ZZW7NEEDkZlLNRGsmgMhJs0vnENiHACLvw5lRIJCUACInxUvnENiHACLvw5lRIJCUACInxVtz58wtJwKInFM2iAUCgQQQORAczSCQEwFEzikbxAKBQAKIHAiOZjUTKG9uiFxezogYAk8EEPkJCQcgUB4BRC4vZ0QMgScCiPyEhAMQKI+Av8jlzY2IIdAMAURuJtVMtGYCiFxzdplbMwQQuZlUM9GaCSCyskuBQOEEELnwBBI+BEQAkUWBAoHCCSBy4QkkfAiIACKLQs2FuTVBAJGbSDOTrJ0AIteeYebXBAFEbiLNTLJ2Aohce4Zrnh9zuxJA5CsKdiBQLgFELjd3RA6BKwFEvqJgBwLlEkDkcnNH5DUTWDk3RF4JjOoQyJEAIueYFWKCwEoCiLwSGNUhkCMBRM4xK8QEgZUEihJ55dyoDoFmCCByZqk2xnQqX18/u+8fH522KjqWS6iKRUXxqSg+lVziazEORM4k6xLhX//8h5X3fShfP39aoX932qp8/3jvdF71jgh5FFcxKBYVY37fxahzKkfFeASXXMZE5IMzYewdeLirWXF9QpHUe8siMUdx18SoufnUp852Aoi8nWFwD1NB1nYiodV+bbu19TWGxlrbTvUlv9prn5KWACKn5TvbuxZ4qCBjp2qvfsbXsbfqW2Ns6VftuTNvIejXFpH9OEWvpQUeo1P1k0KUGBKP81OM4z7bNAQQOQ1XZ6+SxFlh5ckUosTs09gPxfQ5wMppUX0FAUReAStW1SVJ3t6+df3p11B+fH52Kq6xJYqxH5q56qw559PXmhg1trEya0tJQwCR03Cd7dUsCCdp+9Ope3t7G8qPH1ZkFSv0bKeRT5gF6frhIrM+RrMw98jTaKo7RN453cYhiSSWuK9C0nGdf3VOx5bu8qoToygGXWRe9aUYdad+dY5jaQkgclq+UXvfSxLXRUGyuiYl0efOu/qda8NxPwKI7McpWi3XYl6SZO5OqOBcd3qdp6QhkEuviLxzJlx3VcN7yJ2zUc9wiFxQLl2/tnJdINZO0dXX0sXGOD4DePv2tjYU6nsSQGRPULGqud5D6o80Gsdd2fw2s2HsJYnrrYFid52fDZ4Tmwkg8maEcTuQzI93XgmiP1BhHHe7mFG4LjaKQbE8jmfsBUixPx6fvl76DGBal/11BFKIvC6CxmrrAyvXo6tw6K6mv+EkYc7b9+GvC+rcXNlTEmMvKIpLZRrjXGw67ro46DxlGwFE3sYvqLXvopYwPgP49ufTl+roYuPbp2+M6peSjgAip2M72/MaUWY7uZyQcCnuxupz6cnhEsLiJlWMiwM3VAGRD0p2LFHUT6opSMCtfauPlDFuja+W9oi8LpNRa/en0+JfiJgbUHfLP//6e+50lON6ctAYGiukQyQOoRbWBpHDuEVrpbuVFvyaDlVfF4E1bbbU1Vgac00fqq+5rWlD3XACiBzOLlpLLXjd+YbF//nZPd4B9VqlH/7W0a9O9aMN7tmRxvSNcaj349OzZ6rFIIDIMShG6kOyqPT2kVsy9Fbc8/bU9faYHnVVIg0X1I3iU1E859h+deftLcagjmm0iQAib8KXtvHO0gZNpoQYgyZWWCNELixhhAuBVwQQ+RUVjkGgMAKIXFjCCBcCrwgg8isqHKuNQPXzQeTqU8wEWyCAyC1kmTlWTwCRq08xE2yBACK3kGXmWDOBYW6IPGDgBwTKJoDIZeeP6CEwEEDkAQM/IFA2AUQuO39ED4GBQKUiD3PjBwSaIYDIzaSaidZMAJFrzi5za4YAIjeTaiZaMwFELi67BAyBZwKI/MyEIxAojgAiF5cyAobAMwFEfmbCEQgURwCRi0tZzQEzt1ACTYtsjAnlRrtMCCiH+kbIr6+fnUomYe0eRpMij8n//vHe6atBW14Au6+4iAMqb8qhMb87fRXtUKzQEYcopqsmRR6TP2ZJC0BX9fE12/wJSGLl7TFSHTMNPmk1J7IWwGPy9droqt7o1VzzL61I2LmYlcu5c8cdTztycyK7cGpxmAav5i4mOZ6buxjnGOteMTUnsr4MzQVXj92u85w7loAk1gXXFYW+m8p1vsZzDYr89vRth4+J5f3yI5F8Xi9K/Nnmt0A2J7KWpL6+VNu5Yni/PIfm0OO6G7sCUF5bvBuLybEiK4IDir5BUEl3Da0rv+H9sgvRrucksXLiGrRVicWkSZE1cSV96f3y0sJRP5T0BHRBXcrF0oU5fZTHjtCsyMLen07O98uGR2xhOrQY+1S09AGkJNaF+dBADx68aZHFXotA27miO4Ee6+bOczwtgSWJNXrrEotB8yL7vl9eLbPoUjYR8PntwdKFeFMABTVuXmTlSld0n/fLyCxa+xRJbOxbG9dokli5c9Vp5RwiXzKtRXHZnd3wmD2LJuoJJF6PE5EvzPSI3Z9+XV7Nb5B5nk2MMz4S6+mJO/E9bUSe8EDmCQzXbqJzvhL39rcNiUIotltEfkgdMj8A2eGlGX7F9NGZhffEuhMj8euEIPILLmtkPt9F+JdGXmD0OqQPEPUrJiT2wjVbCZFn0Ehmnw/AtAC1ELUgZ7ri8AwBMdNnDjOn7w5zJ77D8fQCkZ+Q3A7oAxUfmdVCC1ILU/sUNwFzeZQWM3fN89ne40PIc80IPwvtApEXErdWZj1qL3TZ9OmzxO+L74dHSJJYT0fja7avCSDyay53R9fIbOwHNvyDfnf4ri/0xKK3IdcDjh19sPXnX393SOyANDmFyBMYrl3JfF5Y31zVruf02KiFa+xj5PVgoztioCcVMfFBoLczPb9i8kF1rYPIVxR+O1pgWmg+tbVwdQeS0D71a6szCiwGxj6p+Myvt++HddH0qUudGwEvkW/V2RMBLTRfmVVfQrf0uG3sU4juwGsE1qO0JOZRWitmfUHk9cyGFpL5vPD8HrXVqHahQwQWF10Ue/sojcSiEVYQOYzb0EoLTwtQC3E44PljKnQNj92hAguX2OmiqH1KOAFEDmd3bamFqAV5PeC5I6FVSnzsHuVV7GseoUc046O02I3H2IYTaF7kcHT3LbUg9al2iNDqaSp0znfpUeAQeTVPld5+oNXzKC0U0QoiR0N57khC93ahbhH6UWrJc+59358aV+X8wdXH8IV3WwTWXVgXO70l2Xcm9Y+GyAlyrIUqoSWzFm/oEBJaRfKcH2E/hq8OTXXHlrQqo7gaV8XYXx2phM5DDHRx6+1dOLQP2rkJILKbz6azklmLV0Jv6ujSWDJJbBWJrXKT7ia5RFcx9tdA06JjY5m20/65r/cuhriXcLupwLq4jcfZxieAyPGZPvUoofVIKaFVnipsOGAud0ttJfi0SMppmZ5T/WnZEMJTUwR+QpL8ACInR3wbQEKrjFJrwd/Olr2nufT2swHNrbeP0NyB980nIu/L+zqahNaC7+3ij32Xvg6SeEfyKnbNoUfexLTd3SOym0/ys7pzSWrdySSFSvJBNwwgeVV6ewHqrbyKXXPY0CVNIxBA5AgQY3UhKVQkdW9FkdQqsfoP6UfSqiieczl1vRX4aHlD5lJzG0TONLsSRVKrSGwVSa0isVKFrb41Rm8vJBqzt9KqKB6VVOPS7zYCiLyN366tJbWKxJJkKr0VbiwS8LFIzLFMz41ttFU/Y+mtuBoDaXdN7ebBEHkzwmM7kHBjkYCPRWKOZXpubKPtsTNg9BgEEDkGRfqAQEQCIV0hcgg12kAgMwKInFlCCAcCIQQQOYQabSCQGQFEziwhhAOBEAKliBwyN9pAoBkCiNxMqplozQQQuebsMrdmCCByM6lmojUTQOTjs0sEENhMAJE3I6QDCBxPAJGPzwERQGAzAUTejJAOIHA8AUQ+Pgc1R8DcdiKAyDuBZhgIpCSAyCnp0jcEdiKAyDuBZhgIpCSAyCnp0nfNBLKaGyJnlQ6CgUAYAUQO40YrCGRFAJGzSgfBQCCMACKHcaMVBLIiEFnkrOZGMBBohgAiN5NqJlozAUSuObvMrRkCiNxMqplozQQQ2Tu7VIRAvgQQOd/cEBkEvAkgsjcqKkIgXwKInG9uiAwC3gQQ2RtVzRWZW+kEELn0DBI/BCwBRLYQ+B8CpRNA5NIzSPwQsAQQ2ULg/5oJtDE3RG4jz8yycgKIXHmCmV4bBBC5jTwzy8oJIHLlCWZ6NRO4zQ2RbyzYg0CxBBC52NQROARuBBD5xoI9CBRLAJGLTR2BQ+BGoD6Rb3NjDwLNEEDkZlLNRGsmgMg1Z5e5NUMAkZtJNROtmQAil5RdYoXADAFEngHDYQiURACRS8oWsUJghgAiz4DhMARKIoDIJWWr5liZ2yYCiLwJH40hkAcBRM4jD0QBgU0EEHkTPhpDIA8CiJxHHoiiZgI7zA2Rd4DMEBBITQCRUxOmfwjsQACRd4DMEBBITeD/AAAA//+PgPcJAAAABklEQVQDAEL72xBBlWtZAAAAAElFTkSuQmCC`; + +test("Creates an example collection successfully", async () => { + const insertCollectionValues = vi.fn().mockReturnValue({ + onConflictDoUpdate: vi.fn(), + }); + const insertAuthorValues = vi.fn(); + vi.mocked(db.insert).mockImplementation((table) => { + if (table === collectionData) { + return { values: insertCollectionValues } as never; + } + if (table === collectionAuthors) { + return { values: insertAuthorValues } as never; + } + throw new Error(`Unexpected table: ${table}`); + }); + + const deleteWhere = vi.fn(); + vi.mocked(db.delete).mockReturnValue({ + where: deleteWhere, + } as never); + + vi.mocked(db.select).mockReturnValue({ + from: vi.fn().mockReturnValue({ + where: vi.fn().mockResolvedValue([{ slug: "example-author" }]), + }), + } as never); + + vi.mocked(github.getContents).mockImplementation(((params: { + path: string; + }) => { + if ( + params.path === "/content/example-author/collections/example-collection/" + ) { + return Promise.resolve({ + data: { + entries: [ + { + name: "index.md", + path: "content/example-author/collections/example-collection/index.md", + }, + ], + }, + error: undefined, + response: {} as never, + }); + } + return Promise.reject(); + }) as never); + + vi.mocked(github.getContentsRaw).mockImplementation((params) => { + if ( + params.path === + "/content/example-author/collections/example-collection/index.md" + ) { + return Promise.resolve({ + data: `--- +title: "Example Collection" +description: "A test collection" +coverImg: "./cover.png" +published: "2023-01-01T00:00:00Z" +--- +`, + response: {} as never, + }); + } + return Promise.reject(); + }); + + vi.mocked(github.getContentsRawStream).mockImplementation((params) => { + if ( + params.path === + "/content/example-author/collections/example-collection/cover.png" + ) { + const buffer = Buffer.from(mockImage, "base64"); + return Promise.resolve({ + data: Readable.toWeb(Readable.from(buffer)) as never, + response: {} as never, + }); + } + return Promise.reject(); + }); + + await processor({ + data: { + author: "example-author", + collection: "example-collection", + ref: "main", + }, + } as unknown as Job); + + // The cover image was uploaded to S3 + expect(s3.upload).toBeCalledWith( + "example-bucket", + "collections/example-collection/en/cover.jpg", + undefined, + expect.anything(), + "image/jpeg", + ); + + // The collection was inserted into the database + expect(insertCollectionValues).toBeCalledWith({ + slug: "example-collection", + locale: "en", + title: "Example Collection", + description: "A test collection", + coverImage: "collections/example-collection/en/cover.jpg", + socialImage: null, + meta: { + buttons: undefined, + tags: undefined, + chapterList: undefined, + }, + }); + + // The author association was deleted and re-inserted + expect(deleteWhere).toBeCalledWith( + eq(collectionAuthors.collectionSlug, "example-collection"), + ); + expect(insertAuthorValues).toBeCalledWith([ + { + collectionSlug: "example-collection", + authorSlug: "example-author", + }, + ]); +}); + +test("Deletes a collection record if it no longer exists", async () => { + const deleteWhere = vi.fn(); + vi.mocked(db.delete).mockReturnValue({ + where: deleteWhere, + } as never); + + vi.mocked(github.getContents).mockImplementation(((params: { + path: string; + }) => { + if ( + params.path === "/content/example-author/collections/example-collection/" + ) { + return Promise.resolve({ + data: undefined, + error: {}, + response: { + status: 404, + } as never, + }); + } + return Promise.reject(); + }) as never); + + await processor({ + data: { + author: "example-author", + collection: "example-collection", + ref: "main", + }, + } as unknown as Job); + + // The collection was deleted from the database + expect(deleteWhere).toBeCalledWith( + eq(collectionData.slug, "example-collection"), + ); +}); + +test("Fails if author profile does not exist", async () => { + const insertCollectionValues = vi.fn().mockReturnValue({ + onConflictDoUpdate: vi.fn(), + }); + vi.mocked(db.insert).mockImplementation((table) => { + if (table === collectionData) { + return { values: insertCollectionValues } as never; + } + throw new Error(`Unexpected table: ${table}`); + }); + + vi.mocked(db.delete).mockReturnValue({ + where: vi.fn(), + } as never); + + // Return empty array - author does not exist + vi.mocked(db.select).mockReturnValue({ + from: vi.fn().mockReturnValue({ + where: vi.fn().mockResolvedValue([]), + }), + } as never); + + vi.mocked(github.getContents).mockImplementation(((params: { + path: string; + }) => { + if ( + params.path === "/content/example-author/collections/example-collection/" + ) { + return Promise.resolve({ + data: { + entries: [ + { + name: "index.md", + path: "content/example-author/collections/example-collection/index.md", + }, + ], + }, + error: undefined, + response: {} as never, + }); + } + return Promise.reject(); + }) as never); + + vi.mocked(github.getContentsRaw).mockImplementation((params) => { + if ( + params.path === + "/content/example-author/collections/example-collection/index.md" + ) { + return Promise.resolve({ + data: `--- +title: "Example Collection" +description: "A test collection" +coverImg: "./cover.png" +published: "2023-01-01T00:00:00Z" +--- +`, + response: {} as never, + }); + } + return Promise.reject(); + }); + + vi.mocked(github.getContentsRawStream).mockImplementation((params) => { + if ( + params.path === + "/content/example-author/collections/example-collection/cover.png" + ) { + const buffer = Buffer.from(mockImage, "base64"); + return Promise.resolve({ + data: Readable.toWeb(Readable.from(buffer)) as never, + response: {} as never, + }); + } + return Promise.reject(); + }); + + await expect( + processor({ + data: { + author: "example-author", + collection: "example-collection", + ref: "main", + }, + } as unknown as Job), + ).rejects.toThrow( + "Author profiles not found for collection example-collection: example-author", + ); +}); + +test("Handles collection with multiple authors", async () => { + const insertCollectionValues = vi.fn().mockReturnValue({ + onConflictDoUpdate: vi.fn(), + }); + const insertAuthorValues = vi.fn(); + vi.mocked(db.insert).mockImplementation((table) => { + if (table === collectionData) { + return { values: insertCollectionValues } as never; + } + if (table === collectionAuthors) { + return { values: insertAuthorValues } as never; + } + throw new Error(`Unexpected table: ${table}`); + }); + + const deleteWhere = vi.fn(); + vi.mocked(db.delete).mockReturnValue({ + where: deleteWhere, + } as never); + + // Both authors exist + vi.mocked(db.select).mockReturnValue({ + from: vi.fn().mockReturnValue({ + where: vi + .fn() + .mockResolvedValue([{ slug: "example-author" }, { slug: "co-author" }]), + }), + } as never); + + vi.mocked(github.getContents).mockImplementation(((params: { + path: string; + }) => { + if ( + params.path === "/content/example-author/collections/example-collection/" + ) { + return Promise.resolve({ + data: { + entries: [ + { + name: "index.md", + path: "content/example-author/collections/example-collection/index.md", + }, + ], + }, + error: undefined, + response: {} as never, + }); + } + return Promise.reject(); + }) as never); + + vi.mocked(github.getContentsRaw).mockImplementation((params) => { + if ( + params.path === + "/content/example-author/collections/example-collection/index.md" + ) { + return Promise.resolve({ + data: `--- +title: "Example Collection" +description: "A test collection" +authors: + - co-author +coverImg: "./cover.png" +published: "2023-01-01T00:00:00Z" +--- +`, + response: {} as never, + }); + } + return Promise.reject(); + }); + + vi.mocked(github.getContentsRawStream).mockImplementation((params) => { + if ( + params.path === + "/content/example-author/collections/example-collection/cover.png" + ) { + const buffer = Buffer.from(mockImage, "base64"); + return Promise.resolve({ + data: Readable.toWeb(Readable.from(buffer)) as never, + response: {} as never, + }); + } + return Promise.reject(); + }); + + await processor({ + data: { + author: "example-author", + collection: "example-collection", + ref: "main", + }, + } as unknown as Job); + + // Both authors should be inserted (co-author from frontmatter + example-author as the folder owner) + expect(insertAuthorValues).toBeCalledWith([ + { + collectionSlug: "example-collection", + authorSlug: "co-author", + }, + { + collectionSlug: "example-collection", + authorSlug: "example-author", + }, + ]); +}); diff --git a/apps/worker/src/tasks/sync-collection/processor.ts b/apps/worker/src/tasks/sync-collection/processor.ts new file mode 100644 index 0000000..f439161 --- /dev/null +++ b/apps/worker/src/tasks/sync-collection/processor.ts @@ -0,0 +1,234 @@ +import { Tasks, env } from "@playfulprogramming/common"; +import { + collectionAuthors, + collectionData, + db, + profiles, +} from "@playfulprogramming/db"; +import * as github from "@playfulprogramming/github-api"; +import { createProcessor } from "../../createProcessor.ts"; +import { eq, inArray } from "drizzle-orm"; +import matter from "gray-matter"; +import { CollectionMetaSchema } from "./types.ts"; +import { Value } from "@sinclair/typebox/value"; +import sharp from "sharp"; +import { Readable } from "node:stream"; +import { s3 } from "@playfulprogramming/s3"; + +function extractLocale(name: string) { + const match = name.match(/\.([a-z]+)\.md$/); + return match ? match[1] : "en"; +} + +const IMAGE_SIZE_MAX = 2048; + +async function processImg( + stream: ReadableStream, + uploadKey: string, +) { + const pipeline = sharp() + .resize({ + width: IMAGE_SIZE_MAX, + height: IMAGE_SIZE_MAX, + fit: "inside", + }) + .jpeg({ mozjpeg: true }); + + Readable.fromWeb(stream as never).pipe(pipeline); + + const bucket = await s3.createBucket(env.S3_BUCKET); + await s3.upload(bucket, uploadKey, undefined, pipeline, "image/jpeg"); +} + +export default createProcessor( + Tasks.SYNC_COLLECTION, + async (job, { signal }) => { + const authorId = job.data.author; + const collectionId = job.data.collection; + + const collectionMetaUrl = new URL( + `content/${encodeURIComponent(authorId)}/collections/${encodeURIComponent(collectionId)}/`, + "http://localhost", + ); + + const collectionMetaResponse = await github.getContents({ + ref: job.data.ref, + path: collectionMetaUrl.pathname, + repoOwner: env.GITHUB_REPO_OWNER, + repoName: env.GITHUB_REPO_NAME, + signal, + }); + + if (collectionMetaResponse.data === undefined) { + if (collectionMetaResponse.response.status == 404) { + console.log( + `Metadata for ${collectionId} (${collectionMetaUrl.pathname}) returned 404 - removing collection entry.`, + ); + await db + .delete(collectionData) + .where(eq(collectionData.slug, collectionId)); + return; + } + + throw new Error(`Unable to fetch collection data for ${collectionId}`); + } + + if ( + !collectionMetaResponse.data.entries || + !Array.isArray(collectionMetaResponse.data.entries) + ) { + throw new Error(`Unable to fetch collection data for ${collectionId}`); + } + + type Entry = (typeof collectionMetaResponse.data.entries)[number]; + + const collectionEntries = collectionMetaResponse.data.entries.reduce( + ( + prev, + // entry.name is `index.md` and path is `content/{authorId}/collections/{collectionId}/index.md` + // We may have many locales in the future, so we need to check the path as well. + entry, + ) => { + if (!(entry.name.startsWith("index") && entry.name.endsWith(".md"))) { + return prev; + } + + prev.push({ entry, locale: extractLocale(entry.name) }); + return prev; + }, + [] as Array<{ entry: Entry; locale: string }>, + ); + + // Check if coverImg or socialImg have changed since last edit, if so upload to S3 + for (const { entry, locale } of collectionEntries) { + const contentUrl = new URL(entry.path, "http://localhost"); + + const contentResponse = await github.getContentsRaw({ + ref: job.data.ref, + path: contentUrl.pathname, + repoOwner: env.GITHUB_REPO_OWNER, + repoName: env.GITHUB_REPO_NAME, + signal, + }); + + if (contentResponse.data === undefined) { + throw new Error( + `Unable to fetch collection content for ${collectionId} locale ${locale}`, + ); + } + + const { data } = matter(contentResponse.data); + const collectionParsedData = Value.Parse(CollectionMetaSchema, data); + + let coverImgKey: string | null = null; + let socialImgKey: string | null = null; + if (collectionParsedData.coverImg) { + const coverImgUrl = new URL( + collectionParsedData.coverImg, + collectionMetaUrl, + ); + const { data: coverImgStream } = await github.getContentsRawStream({ + ref: job.data.ref, + path: coverImgUrl.pathname, + repoOwner: env.GITHUB_REPO_OWNER, + repoName: env.GITHUB_REPO_NAME, + signal, + }); + + if (coverImgStream === null || typeof coverImgStream === "undefined") { + throw new Error( + `Unable to fetch cover image for ${collectionId} (${coverImgUrl.pathname})`, + ); + } + + coverImgKey = `collections/${collectionId}/${locale}/cover.jpg`; + await processImg(coverImgStream, coverImgKey); + } + + if (collectionParsedData.socialImg) { + const socialImgUrl = new URL( + collectionParsedData.socialImg, + collectionMetaUrl, + ); + const { data: socialImgStream } = await github.getContentsRawStream({ + ref: job.data.ref, + path: socialImgUrl.pathname, + repoOwner: env.GITHUB_REPO_OWNER, + repoName: env.GITHUB_REPO_NAME, + signal, + }); + + if ( + socialImgStream === null || + typeof socialImgStream === "undefined" + ) { + throw new Error( + `Unable to fetch social image for ${collectionId} (${socialImgUrl.pathname})`, + ); + } + + socialImgKey = `collections/${collectionId}/${locale}/social.jpg`; + await processImg(socialImgStream, socialImgKey); + } + + const result = { + slug: collectionId, + locale: locale, + title: collectionParsedData.title, + description: collectionParsedData.description, + coverImage: coverImgKey, + socialImage: socialImgKey, + meta: { + buttons: collectionParsedData.buttons, + tags: collectionParsedData.tags, + chapterList: collectionParsedData.chapterList, + }, + }; + + await db + .insert(collectionData) + .values(result) + .onConflictDoUpdate({ + target: [collectionData.slug, collectionData.locale], + set: result, + }); + + // Handle authors + const authorSlugs = collectionParsedData.authors + ? [...new Set([...collectionParsedData.authors, authorId])] + : [authorId]; + + // Verify all authors exist in the database + const existingAuthors = await db + .select({ slug: profiles.slug }) + .from(profiles) + .where(inArray(profiles.slug, authorSlugs)); + + const existingSlugs = new Set(existingAuthors.map((a) => a.slug)); + const missingAuthors = authorSlugs.filter( + (slug) => !existingSlugs.has(slug), + ); + + if (missingAuthors.length > 0) { + throw new Error( + `Author profiles not found for collection ${collectionId}: ${missingAuthors.join(", ")}`, + ); + } + + // Delete existing author associations for this collection + await db + .delete(collectionAuthors) + .where(eq(collectionAuthors.collectionSlug, collectionId)); + + // Insert new author associations + if (authorSlugs.length > 0) { + await db.insert(collectionAuthors).values( + authorSlugs.map((authorSlug) => ({ + collectionSlug: collectionId, + authorSlug, + })), + ); + } + } + }, +); diff --git a/apps/worker/src/tasks/sync-collection/types.ts b/apps/worker/src/tasks/sync-collection/types.ts new file mode 100644 index 0000000..de4e16c --- /dev/null +++ b/apps/worker/src/tasks/sync-collection/types.ts @@ -0,0 +1,92 @@ +import { Type } from "@sinclair/typebox"; + +const CollectionButtonSchema = Type.Object( + { + text: Type.String(), + url: Type.String(), + }, + { + additionalProperties: false, + examples: [ + { + text: "Learn More", + url: "https://example.com/learn-more", + }, + ], + }, +); + +const CollectionCurrentPost = Type.Object( + { + post: Type.String(), + }, + { + additionalProperties: false, + examples: [ + { + post: "abc123", + }, + ], + }, +); + +const CollectionFuturePost = Type.Object( + { + order: Type.Number(), + title: Type.String(), + description: Type.String({ default: "" }), + }, + { + additionalProperties: false, + examples: [ + { + order: 1, + title: "Chapter One", + description: "An introduction to chapter one.", + }, + ], + }, +); + +export const CollectionMetaSchema = Type.Object( + { + title: Type.String(), + description: Type.String({ default: "" }), + authors: Type.Optional(Type.Array(Type.String())), + coverImg: Type.String(), + socialImg: Type.Optional(Type.String()), + type: Type.Optional(Type.Literal("book")), + pageLayout: Type.Optional(Type.Literal("none")), + customChaptersText: Type.Optional(Type.String()), + tags: Type.Optional(Type.Array(Type.String())), + published: Type.String(), + noindex: Type.Optional(Type.Boolean({ default: false })), + version: Type.Optional(Type.String()), + upToDateSlug: Type.Optional(Type.String()), + buttons: Type.Optional(Type.Array(CollectionButtonSchema)), + chapterList: Type.Optional( + Type.Array(Type.Union([CollectionCurrentPost, CollectionFuturePost])), + ), + }, + { + additionalProperties: false, + examples: [ + { + title: "My Collection", + description: "A collection of my favorite posts.", + coverImg: "./cover.jpg", + published: "2023-01-01T00:00:00Z", + tags: ["tag1", "tag2"], + buttons: [ + { + text: "Learn More", + url: "/learn-more", + }, + { + post: "abc123", + }, + ], + }, + ], + }, +); diff --git a/apps/worker/test-utils/setup.ts b/apps/worker/test-utils/setup.ts index 81a7f86..ee92d70 100644 --- a/apps/worker/test-utils/setup.ts +++ b/apps/worker/test-utils/setup.ts @@ -6,6 +6,31 @@ afterEach(() => { vi.setSystemTime(new Date("2025-05-05")); }); +vi.mock("@playfulprogramming/common", () => { + return { + Tasks: { + SYNC_AUTHOR: "sync-author", + SYNC_COLLECTION: "sync-collection", + SYNC_POST: "sync-post", + URL_METADATA: "url-metadata", + POST_IMAGES: "post-images", + }, + env: { + ENVIRONMENT: "development", + SITE_URL: "https://example.com", + S3_PUBLIC_URL: "https://s3.example.com", + S3_ENDPOINT: "https://s3.example.com", + S3_KEY_ID: "test-key-id", + S3_KEY_SECRET: "test-key-secret", + S3_BUCKET: "example-bucket", + POSTGRES_URL: "postgresql://localhost/test", + REDIS_URL: "redis://localhost:6379", + GITHUB_REPO_OWNER: "playfulprogramming", + GITHUB_REPO_NAME: "playfulprogramming", + }, + }; +}); + vi.mock("@playfulprogramming/s3", () => { return { s3: { @@ -20,9 +45,18 @@ vi.mock("@playfulprogramming/db", () => { profiles: { slug: {}, }, + collectionData: { + slug: {}, + locale: {}, + }, + collectionAuthors: { + collectionSlug: {}, + authorSlug: {}, + }, db: { insert: vi.fn(), delete: vi.fn(), + select: vi.fn(), }, }; }); diff --git a/packages/db/src/schema/collections.ts b/packages/db/src/schema/collections.ts index ab6ecd1..8dc84ff 100644 --- a/packages/db/src/schema/collections.ts +++ b/packages/db/src/schema/collections.ts @@ -14,6 +14,7 @@ export const collections = pgTable("collections", { slug: text("slug").primaryKey(), }); +// TODO: This is missing a ton of fields that are in the Collections task type, what do? export const collectionData = pgTable( "collection_data", { @@ -25,6 +26,8 @@ export const collectionData = pgTable( description: text("description").notNull().default(""), publishedAt: timestamp("published_at", { withTimezone: true }), meta: jsonb("meta").notNull(), + coverImage: text("cover_image"), + socialImage: text("social_image"), }, (table) => [primaryKey({ columns: [table.slug, table.locale] })], ); @@ -46,6 +49,7 @@ export const collectionAuthors = pgTable( ], ); +// TODO: Do we need this in the DB or can it just be in the meta JSON? export const collectionChapters = pgTable("collection_chapters", { id: integer("id").primaryKey().generatedAlwaysAsIdentity(), locale: text("locale").notNull(), diff --git a/packages/github-api/src/getContents.ts b/packages/github-api/src/getContents.ts index 9c4e8d9..5e3fcea 100644 --- a/packages/github-api/src/getContents.ts +++ b/packages/github-api/src/getContents.ts @@ -9,7 +9,7 @@ export interface GetContentsParams { } export async function getContents(params: GetContentsParams) { - const response = await clientWithType("application/vnd.github.object").GET( + return await clientWithType("application/vnd.github.object").GET( "/repos/{owner}/{repo}/contents/{path}", { params: { @@ -28,14 +28,4 @@ export async function getContents(params: GetContentsParams) { signal: params.signal, }, ); - - const data = response.data; - - if (typeof data === "undefined" || response.error) { - throw new Error( - `GitHub API (getContents) returned ${response.response.status} ${response.error}`, - ); - } - - return data; }