Species identification UI

apps/api/src/endpoints/plantPhoto/get.ts → apps/api/src/endpoints/media/plantPhotos/get.ts

@@ -4,8 +4,8 @@ import { Readable } from 'stream'
 
 import { get as getBlob } from '@vercel/blob'
 
-import { config } from '../../config'
-import { Plant } from '../../models'
+import { config } from '../../../config'
+import { Plant } from '../../../models'
 
 /**
  * Stream a private plant photo blob. Requires exactly one of:
@@ -61,7 +61,7 @@ export async function getPlantPhoto(req: Request, res: Response): Promise<void>
 
       return
     }
-    const url = (plant as { photoUrl?: string | null }).photoUrl
+    const url = plant.photoUrl
     if (!url) {
       res.status(404).json({ message: 'Plant has no photo' })
 

apps/api/src/endpoints/trpc/me/delete.ts

@@ -53,7 +53,7 @@ export const deleteMe = authProcedure
 
     // Delete all plant photos from blob storage
     for (const plant of plants) {
-      const photoUrl = (plant as { photoUrl?: string | null }).photoUrl
+      const photoUrl = plant.photoUrl
       if (photoUrl) {
         await blobService.deletePlantPhotoFromBlob(photoUrl)
       }

apps/api/src/endpoints/trpc/plants/chat.ts

@@ -103,7 +103,7 @@ export const chat = authProcedure
       model: 'gpt-4o-mini',
       messages: [
         { role: 'system', content: systemParts.join('\n') },
-        ...input.messages.map(m => ({ role: m.role as 'user' | 'assistant', content: m.content })),
+        ...input.messages.map(m => ({ role: m.role, content: m.content })),
       ],
     })
 

apps/api/src/index.ts

@@ -9,11 +9,11 @@ import { mongo } from './db'
 
 import { index } from './endpoints/index'
 import { getHealth } from './endpoints/health/get'
-import { getPlantPhoto } from './endpoints/plantPhoto/get'
 
 import { debugEndpoints } from './middlewares/debugEndpoints'
 
 import { cronRouter } from './routers/cron'
+import { mediaRouter } from './routers/media'
 import { trpcRouter } from './routers/trpc'
 
 import { isTest } from './utils/isTest'
@@ -42,8 +42,8 @@ app.use('/health', getHealth)
 // Cronjob endpoints
 app.use('/cron', cronRouter)
 
-// Plant photo proxy (private blob access; requires Authorization: Bearer <jwt>; use ?plantId=... or ?url=...)
-app.get('/api/plant-photo', getPlantPhoto)
+// Media endpoints
+app.use('/media', mediaRouter)
 
 // tRPC endpoints
 app.use(

apps/api/src/models/UserFertilizerPreferences.ts

@@ -27,6 +27,7 @@ export const userFertilizerPreferencesSchema = new mongoose.Schema<IUserFertiliz
     ref: 'User',
     required: true,
     unique: true,
+    index: true,
   },
   favoriteFertilizerTypes: {
     type: [String],
@@ -43,7 +44,4 @@ export const userFertilizerPreferencesSchema = new mongoose.Schema<IUserFertiliz
   timestamps: true,
 })
 
-// One preferences document per user
-userFertilizerPreferencesSchema.index({ user: 1 }, { unique: true })
-
 export const UserFertilizerPreferences = mongoose.model<IUserFertilizerPreferences>('UserFertilizerPreferences', userFertilizerPreferencesSchema)

apps/api/src/routers/media/index.ts

@@ -0,0 +1,10 @@
+import { Router } from 'express'
+
+import { getPlantPhoto } from '../../endpoints/media/plantPhotos/get'
+
+const mediaRouter = Router()
+
+mediaRouter
+  .get('/plant-photos', getPlantPhoto)
+
+export { mediaRouter }

apps/api/src/services/plantIdentification/index.ts

@@ -1,33 +1,53 @@
 import * as plantNetProvider from './providers/plantNet'
 
 export type PlantIdentificationResult = {
-  confidence: number,
+  id: string | null, // Generated ID that we derive from other fields
+  source: 'plantNet',
   commonNames: string[],
-  scientificName: string,
-  scientificNameAuthorship: string,
+  confidence: number,
   genus: string,
   genusAuthorship: string,
   family: string,
   familyAuthorship: string,
+  images: PlantIdentificationImage[],
+  scientificName: string,
+  scientificNameAuthorship: string,
+}
+
+export type PlantIdentificationImage = {
+  organ?: string,
+  url: string,
 }
 
-const MIN_CONFIDENCE = 0.1
+const MIN_CONFIDENCE = 0.0 // Don't filter any results out - we will show multiple to the user to choose from
 
 export const identifyPlantByImages = async (images: Buffer[]): Promise<PlantIdentificationResult[]> => {
   const plantNetResult = await plantNetProvider.identifyPlantByImages(images)
 
   const plantIdentificationResult = plantNetResult
     .filter(result => result.score > MIN_CONFIDENCE)
-    .map(result => ({
-      confidence: result.score,
-      commonNames: result.species.commonNames,
-      scientificName: result.species.scientificNameWithoutAuthor,
-      scientificNameAuthorship: result.species.scientificNameAuthorship,
-      genus: result.species.genus.scientificNameWithoutAuthor,
-      genusAuthorship: result.species.genus.scientificNameAuthorship,
-      family: result.species.family.scientificNameWithoutAuthor,
-      familyAuthorship: result.species.family.scientificNameAuthorship,
-    }))
+    .map(result => {
+      const images: PlantIdentificationImage[] = (result.images || [])
+        .map(img => ({
+          url: img.url?.m || img.url?.s || img.url?.o,
+          ...(img.organ && { organ: img.organ }),
+        }))
+        .filter((item): item is PlantIdentificationImage => !!item.url)
+
+      return {
+        id: result.powo?.id || null,
+        source: result.source,
+        commonNames: result.species.commonNames,
+        confidence: result.score,
+        genus: result.species.genus.scientificNameWithoutAuthor,
+        genusAuthorship: result.species.genus.scientificNameAuthorship,
+        family: result.species.family.scientificNameWithoutAuthor,
+        familyAuthorship: result.species.family.scientificNameAuthorship,
+        images,
+        scientificName: result.species.scientificNameWithoutAuthor,
+        scientificNameAuthorship: result.species.scientificNameAuthorship,
+      }
+    })
 
   return plantIdentificationResult
 }

apps/api/src/services/plantIdentification/providers/plantNet.ts

@@ -8,36 +8,49 @@ import * as debugService from '../../debug'
 const debugPlantIdentification = debugService.init('app:plantIdentification')
 
 const plantNetIdentificationResultSchema = z.object({
+  gbif: z.object({ id: z.string() }).optional(), // Global Biodiversity Information Facility id (see powo below)
+  iucn: z.object({ id: z.string(), category: z.string().optional() }).optional(), // International Union for Conservation of Nature id (see powo below)
+  images: z.array(
+    z.object({
+      organ: z.string().optional(),
+      url: z.object({
+        o: z.string(), // original, largest
+        m: z.string().optional(), // medium
+        s: z.string().optional(), // small
+      }),
+    })
+  ).optional(),
+  powo: z.object({ id: z.string() }).optional(), // Plants of the World Online id - this is the preferred and primary source
   score: z.number().min(0).max(1),
   species: z.object({
-    scientificName: z.string(),
-    scientificNameWithoutAuthor: z.string(),
-    scientificNameAuthorship: z.string(),
-    genus: z.object({
-      scientificNameWithoutAuthor: z.string(),
-      scientificNameAuthorship: z.string(),
-    }),
+    commonNames: z.array(z.string()),
     family: z.object({
+      scientificNameAuthorship: z.string(),
       scientificNameWithoutAuthor: z.string(),
+    }),
+    genus: z.object({
       scientificNameAuthorship: z.string(),
+      scientificNameWithoutAuthor: z.string(),
     }),
-    commonNames: z.array(z.string()),
+    scientificName: z.string(),
+    scientificNameAuthorship: z.string(),
+    scientificNameWithoutAuthor: z.string(),
   }),
 })
 
 const plantNetResponseSchema = z.object({
+  language: z.string(),
+  preferedReferential: z.string(),
   query: z.object({
     project: z.string(),
     images: z.array(z.string()),
     organs: z.array(z.string()),
   }),
-  language: z.string(),
-  preferedReferential: z.string(),
-  results: z.array(plantNetIdentificationResultSchema),
   remainingIdentificationRequests: z.number(),
+  results: z.array(plantNetIdentificationResultSchema),
 })
 
-export type PlantNetIdentificationResult = z.infer<typeof plantNetIdentificationResultSchema>
+export type PlantNetIdentificationResult = z.infer<typeof plantNetIdentificationResultSchema> & { source: 'plantNet' }
 
 export type PlantNetResponse = z.infer<typeof plantNetResponseSchema>
 
@@ -64,7 +77,7 @@ export const identifyPlantByImages = async (images: Buffer[]): Promise<PlantNetI
     formData.append('images', new Blob([new Uint8Array(image)]))
   }
 
-  const url = getUrl('/identify/all')
+  const url = getUrl('/identify/all', { 'include-related-images': 'true' })
   const response = await fetch(url, {
     method: 'POST',
     body: formData,
@@ -73,7 +86,9 @@ export const identifyPlantByImages = async (images: Buffer[]): Promise<PlantNetI
   debugPlantIdentification(`PlantNet API response status: ${response.status} ${response.statusText}`)
 
   if (!response.ok) {
-    throw new Error(`PlantNet API error: ${response.status} ${response.statusText}`)
+    debugPlantIdentification(`PlantNet API response not OK: ${response.status} ${response.statusText}`)
+
+    return []
   }
 
   const json = await response.json()
@@ -87,10 +102,10 @@ export const identifyPlantByImages = async (images: Buffer[]): Promise<PlantNetI
   } = plantNetResponseSchema.safeParse(json)
 
   if (!success || !data?.results) {
-    console.error(`PlantNet API error: ${error?.message || 'No results returned.'}`)
+    debugPlantIdentification(`PlantNet API not successful: ${error?.message || 'No results returned.'}`)
   }
 
-  const results = data?.results
+  const results = data?.results.map(result => ({ ...result, source: 'plantNet' as const }))
 
   return results || []
 }

apps/mobile/src/app/(tabs)/fertilizers.tsx

@@ -323,7 +323,6 @@ export function FertilizersScreen() {
           buttons={
             <IconFilter
               onPress={() => setFilterModalVisible(true)}
-              color={palette.brandPrimary}
               isPulsating={!!searchQuery.trim()}
               testID='fertilizers-filter-button'
             />

apps/mobile/src/app/(tabs)/index.tsx

@@ -377,7 +377,6 @@ function ToDoScreen() {
           buttons={
             <IconFilter
               onPress={() => setFilterModalVisible(true)}
-              color={palette.brandPrimary}
               isPulsating={!!searchQuery.trim()}
               testID='todolist-filter-button'
             />

apps/mobile/src/app/(tabs)/plants.tsx

@@ -23,14 +23,12 @@ import { trpc } from '../../trpc'
 import { getLifecycleLabelWithIcon } from '../../utils/lifecycle'
 import { getPlantPhotoImageSource } from '../../utils/plantPhoto'
 
-import { config } from '../../config'
 import { palette, styles } from '../../styles'
 
 export function PlantsScreen() {
   const router = useRouter()
   const { alert } = useAlert()
   const { token } = useAuth()
-  const apiBaseUrl = config.api.baseUrl
 
   const { expandPlantId } = useLocalSearchParams<{ expandPlantId?: string }>()
   const scrollViewRef = React.useRef<ScrollView>(null)
@@ -157,7 +155,6 @@ export function PlantsScreen() {
           buttons={
             <IconFilter
               onPress={() => setFilterModalVisible(true)}
-              color={palette.brandPrimary}
               isPulsating={!!searchQuery.trim()}
               testID='plants-filter-button'
             />
@@ -220,9 +217,9 @@ export function PlantsScreen() {
                   >
                     <View style={[styles.listItem, localStyles.plantListItem]}>
                       <View style={localStyles.plantListImageContainer}>
-                        {getPlantPhotoImageSource({ plant }, { apiBaseUrl, token }) ? (
+                        {getPlantPhotoImageSource({ plant }, { token }) ? (
                           <RNImage
-                            source={getPlantPhotoImageSource({ plant }, { apiBaseUrl, token })!}
+                            source={getPlantPhotoImageSource({ plant }, { token })!}
                             style={localStyles.plantListImage}
                             resizeMode="cover"
                           />

apps/mobile/src/app/chores/[id].tsx

@@ -262,7 +262,7 @@ export default function ChoreDetailScreen() {
 
       const title = (fertilizers && fertilizers.length > 0
         ? fertilizers.map(f => {
-            const fertObj = f.fertilizer as any
+            const fertObj = f.fertilizer
             const name = fertObj && typeof fertObj === 'object' && fertObj.name ? fertObj.name : ''
 
             return `${name}${f.amount ? ` (${f.amount})` : ''}`