Create expense from receipt (#69)

* Create expense from receipt

* Add modal

* Update README

src/app/groups/[groupId]/expenses/create-from-receipt-button-actions.ts

@@ -0,0 +1,48 @@
+'use server'
+import { getCategories } from '@/lib/api'
+import { env } from '@/lib/env'
+import OpenAI from 'openai'
+
+const openai = new OpenAI({ apiKey: env.OPENAI_API_KEY })
+
+export async function extractExpenseInformationFromImage(imageUrl: string) {
+  'use server'
+  const categories = await getCategories()
+
+  const body = {
+    model: 'gpt-4-vision-preview',
+    messages: [
+      {
+        role: 'user',
+        content: [
+          {
+            type: 'text',
+            text: `
+              This image contains a receipt.
+              Read the total amount and store it as a non-formatted number without any other text or currency.
+              Then guess the category for this receipt amoung the following categories and store its ID: ${categories.map(
+                ({ id, grouping, name }) => `"${grouping}/${name}" (ID: ${id})`,
+              )}.
+              Guess the expense’s date and store it as yyyy-mm-dd.
+              Guess a title for the expense.
+              Return the amount, the category, the date and the title with just a comma between them, without anything else.`,
+          },
+        ],
+      },
+      {
+        role: 'user',
+        content: [{ type: 'image_url', image_url: { url: imageUrl } }],
+      },
+    ],
+  }
+  const completion = await openai.chat.completions.create(body as any)
+
+  const [amountString, categoryId, date, title] = completion.choices
+    .at(0)
+    ?.message.content?.split(',') ?? [null, null, null, null]
+  return { amount: Number(amountString), categoryId, date, title }
+}
+
+export type ReceiptExtractedInfo = Awaited<
+  ReturnType<typeof extractExpenseInformationFromImage>
+>

src/app/groups/[groupId]/expenses/create-from-receipt-button.tsx

@@ -0,0 +1,278 @@
+'use client'
+
+import { CategoryIcon } from '@/app/groups/[groupId]/expenses/category-icon'
+import {
+  ReceiptExtractedInfo,
+  extractExpenseInformationFromImage,
+} from '@/app/groups/[groupId]/expenses/create-from-receipt-button-actions'
+import { Badge } from '@/components/ui/badge'
+import { Button } from '@/components/ui/button'
+import {
+  Dialog,
+  DialogContent,
+  DialogDescription,
+  DialogHeader,
+  DialogTitle,
+  DialogTrigger,
+} from '@/components/ui/dialog'
+import {
+  Drawer,
+  DrawerContent,
+  DrawerDescription,
+  DrawerHeader,
+  DrawerTitle,
+  DrawerTrigger,
+} from '@/components/ui/drawer'
+import { ToastAction } from '@/components/ui/toast'
+import { useToast } from '@/components/ui/use-toast'
+import { useMediaQuery } from '@/lib/hooks'
+import { formatExpenseDate } from '@/lib/utils'
+import { Category } from '@prisma/client'
+import { ChevronRight, Loader2, Receipt } from 'lucide-react'
+import { getImageData, useS3Upload } from 'next-s3-upload'
+import Image from 'next/image'
+import { useRouter } from 'next/navigation'
+import { PropsWithChildren, ReactNode, useState } from 'react'
+
+type Props = {
+  groupId: string
+  groupCurrency: string
+  categories: Category[]
+}
+
+export function CreateFromReceiptButton({
+  groupId,
+  groupCurrency,
+  categories,
+}: Props) {
+  const [pending, setPending] = useState(false)
+  const { uploadToS3, FileInput, openFileDialog } = useS3Upload()
+  const { toast } = useToast()
+  const router = useRouter()
+  const [receiptInfo, setReceiptInfo] = useState<
+    | null
+    | (ReceiptExtractedInfo & { url: string; width?: number; height?: number })
+  >(null)
+  const isDesktop = useMediaQuery('(min-width: 640px)')
+
+  const handleFileChange = async (file: File) => {
+    const upload = async () => {
+      try {
+        setPending(true)
+        console.log('Uploading image…')
+        let { url } = await uploadToS3(file)
+        console.log('Extracting information from receipt…')
+        const { amount, categoryId, date, title } =
+          await extractExpenseInformationFromImage(url)
+        const { width, height } = await getImageData(file)
+        setReceiptInfo({ amount, categoryId, date, title, url, width, height })
+      } catch (err) {
+        console.error(err)
+        toast({
+          title: 'Error while uploading document',
+          description:
+            'Something wrong happened when uploading the document. Please retry later or select a different file.',
+          variant: 'destructive',
+          action: (
+            <ToastAction altText="Retry" onClick={() => upload()}>
+              Retry
+            </ToastAction>
+          ),
+        })
+      } finally {
+        setPending(false)
+      }
+    }
+    upload()
+  }
+
+  const receiptInfoCategory =
+    (receiptInfo?.categoryId &&
+      categories.find((c) => String(c.id) === receiptInfo.categoryId)) ||
+    null
+
+  const DialogOrDrawer = isDesktop
+    ? CreateFromReceiptDialog
+    : CreateFromReceiptDrawer
+
+  return (
+    <DialogOrDrawer
+      trigger={
+        <Button
+          size="icon"
+          variant="secondary"
+          title="Create expense from receipt"
+        >
+          <Receipt className="w-4 h-4" />
+        </Button>
+      }
+      title={
+        <>
+          <span>Create from receipt</span>
+          <Badge className="bg-pink-700 hover:bg-pink-600 dark:bg-pink-500 dark:hover:bg-pink-600">
+            Beta
+          </Badge>
+        </>
+      }
+      description={<>Extract the expense information from a receipt photo.</>}
+    >
+      <div className="prose prose-sm dark:prose-invert">
+        <p>
+          Upload the photo of a receipt, and we’ll scan it to extract the
+          expense information if we can.
+        </p>
+        <div>
+          <FileInput
+            onChange={handleFileChange}
+            accept="image/jpeg,image/png"
+          />
+          <div className="grid gap-x-4 gap-y-2 grid-cols-3">
+            <Button
+              variant="secondary"
+              className="row-span-3 w-full h-full relative"
+              title="Create expense from receipt"
+              onClick={openFileDialog}
+              disabled={pending}
+            >
+              {pending ? (
+                <Loader2 className="w-8 h-8 animate-spin" />
+              ) : receiptInfo ? (
+                <div className="absolute top-2 left-2 bottom-2 right-2">
+                  <Image
+                    src={receiptInfo.url}
+                    width={receiptInfo.width}
+                    height={receiptInfo.height}
+                    className="w-full h-full m-0 object-contain drop-shadow-lg"
+                    alt="Scanned receipt"
+                  />
+                </div>
+              ) : (
+                <span className="text-xs sm:text-sm text-muted-foreground">
+                  Select image…
+                </span>
+              )}
+            </Button>
+            <div className="col-span-2">
+              <strong>Title:</strong>
+              <div>{receiptInfo?.title ?? '…'}</div>
+            </div>
+            <div className="col-span-2">
+              <strong>Category:</strong>
+              <div>
+                {receiptInfoCategory ? (
+                  <div className="flex items-center">
+                    <CategoryIcon
+                      category={receiptInfoCategory}
+                      className="inline w-4 h-4 mr-2"
+                    />
+                    <span className="mr-1">{receiptInfoCategory.grouping}</span>
+                    <ChevronRight className="inline w-3 h-3 mr-1" />
+                    <span>{receiptInfoCategory.name}</span>
+                  </div>
+                ) : (
+                  '' || '…'
+                )}
+              </div>
+            </div>
+            <div>
+              <strong>Amount:</strong>
+              <div>
+                {receiptInfo?.amount ? (
+                  <>
+                    {groupCurrency} {receiptInfo.amount.toFixed(2)}
+                  </>
+                ) : (
+                  '…'
+                )}
+              </div>
+            </div>
+            <div>
+              <strong>Date:</strong>
+              <div>
+                {receiptInfo?.date
+                  ? formatExpenseDate(
+                      new Date(`${receiptInfo?.date}T12:00:00.000Z`),
+                    )
+                  : '…'}
+              </div>
+            </div>
+          </div>
+        </div>
+        <p>You’ll be able to edit the expense information after creating it.</p>
+        <div className="text-center">
+          <Button
+            disabled={pending || !receiptInfo}
+            onClick={() => {
+              if (!receiptInfo) return
+              router.push(
+                `/groups/${groupId}/expenses/create?amount=${
+                  receiptInfo.amount
+                }&categoryId=${receiptInfo.categoryId}&date=${
+                  receiptInfo.date
+                }&title=${encodeURIComponent(
+                  receiptInfo.title ?? '',
+                )}&imageUrl=${encodeURIComponent(receiptInfo.url)}&imageWidth=${
+                  receiptInfo.width
+                }&imageHeight=${receiptInfo.height}`,
+              )
+            }}
+          >
+            Create expense
+          </Button>
+        </div>
+      </div>
+    </DialogOrDrawer>
+  )
+}
+
+function CreateFromReceiptDialog({
+  trigger,
+  title,
+  description,
+  children,
+}: PropsWithChildren<{
+  trigger: ReactNode
+  title: ReactNode
+  description: ReactNode
+}>) {
+  return (
+    <Dialog>
+      <DialogTrigger asChild>{trigger}</DialogTrigger>
+      <DialogContent>
+        <DialogHeader>
+          <DialogTitle className="flex items-center gap-2">{title}</DialogTitle>
+          <DialogDescription className="text-left">
+            {description}
+          </DialogDescription>
+        </DialogHeader>
+        {children}
+      </DialogContent>
+    </Dialog>
+  )
+}
+
+function CreateFromReceiptDrawer({
+  trigger,
+  title,
+  description,
+  children,
+}: PropsWithChildren<{
+  trigger: ReactNode
+  title: ReactNode
+  description: ReactNode
+}>) {
+  return (
+    <Drawer>
+      <DrawerTrigger asChild>{trigger}</DrawerTrigger>
+      <DrawerContent>
+        <DrawerHeader>
+          <DrawerTitle className="flex items-center gap-2">{title}</DrawerTitle>
+          <DrawerDescription className="text-left">
+            {description}
+          </DrawerDescription>
+        </DrawerHeader>
+        <div className="px-4 pb-4">{children}</div>
+      </DrawerContent>
+    </Drawer>
+  )
+}

src/app/groups/[groupId]/expenses/expense-list.tsx

@@ -3,7 +3,7 @@ import { CategoryIcon } from '@/app/groups/[groupId]/expenses/category-icon'
 import { Button } from '@/components/ui/button'
 import { SearchBar } from '@/components/ui/search-bar'
 import { getGroupExpenses } from '@/lib/api'
-import { cn } from '@/lib/utils'
+import { cn, formatExpenseDate } from '@/lib/utils'
 import { Expense, Participant } from '@prisma/client'
 import dayjs, { type Dayjs } from 'dayjs'
 import { ChevronRight } from 'lucide-react'
@@ -159,7 +159,7 @@ export function ExpenseList({
                     {currency} {(expense.amount / 100).toFixed(2)}
                   </div>
                   <div className="text-xs text-muted-foreground">
-                    {formatDate(expense.expenseDate)}
+                    {formatExpenseDate(expense.expenseDate)}
                   </div>
                 </div>
                 <Button
@@ -189,10 +189,3 @@ export function ExpenseList({
     </p>
   )
 }
-
-function formatDate(date: Date) {
-  return date.toLocaleDateString('en-US', {
-    dateStyle: 'medium',
-    timeZone: 'UTC',
-  })
-}

src/app/groups/[groupId]/expenses/page.tsx

@@ -1,5 +1,6 @@
 import { cached } from '@/app/cached-functions'
 import { ActiveUserModal } from '@/app/groups/[groupId]/expenses/active-user-modal'
+import { CreateFromReceiptButton } from '@/app/groups/[groupId]/expenses/create-from-receipt-button'
 import { ExpenseList } from '@/app/groups/[groupId]/expenses/expense-list'
 import { Button } from '@/components/ui/button'
 import {
@@ -10,7 +11,8 @@ import {
   CardTitle,
 } from '@/components/ui/card'
 import { Skeleton } from '@/components/ui/skeleton'
-import { getGroupExpenses } from '@/lib/api'
+import { getCategories, getGroupExpenses } from '@/lib/api'
+import { env } from '@/lib/env'
 import { Download, Plus } from 'lucide-react'
 import { Metadata } from 'next'
 import Link from 'next/link'
@@ -31,6 +33,8 @@ export default async function GroupExpensesPage({
   const group = await cached.getGroup(groupId)
   if (!group) notFound()
 
+  const categories = await getCategories()
+
   return (
     <>
       <Card className="mb-4 rounded-none -mx-4 border-x-0 sm:border-x sm:rounded-lg sm:mx-0">
@@ -51,6 +55,13 @@ export default async function GroupExpensesPage({
                 <Download className="w-4 h-4" />
               </Link>
             </Button>
+            {env.NEXT_PUBLIC_ENABLE_RECEIPT_EXTRACT && (
+              <CreateFromReceiptButton
+                groupId={groupId}
+                groupCurrency={group.currency}
+                categories={categories}
+              />
+            )}
             <Button asChild size="icon">
               <Link href={`/groups/${groupId}/expenses/create`}>
                 <Plus className="w-4 h-4" />

src/components/expense-form.tsx

@@ -34,7 +34,7 @@ import {
   SelectTrigger,
   SelectValue,
 } from '@/components/ui/select'
-import { getCategories, getExpense, getGroup } from '@/lib/api'
+import { getCategories, getExpense, getGroup, randomId } from '@/lib/api'
 import { ExpenseFormValues, expenseFormSchema } from '@/lib/schemas'
 import { cn } from '@/lib/utils'
 import { zodResolver } from '@hookform/resolvers/zod'
@@ -105,10 +105,14 @@ export function ExpenseForm({
           documents: [],
         }
       : {
-          title: '',
-          expenseDate: new Date(),
-          amount: 0,
-          category: 0, // category with Id 0 is General
+          title: searchParams.get('title') ?? '',
+          expenseDate: searchParams.get('date')
+            ? new Date(searchParams.get('date') as string)
+            : new Date(),
+          amount: (searchParams.get('amount') || 0) as unknown as number, // hack,
+          category: searchParams.get('categoryId')
+            ? Number(searchParams.get('categoryId'))
+            : 0, // category with Id 0 is General
           // paid for all, split evenly
           paidFor: group.participants.map(({ id }) => ({
             participant: id,
@@ -117,7 +121,16 @@ export function ExpenseForm({
           paidBy: getSelectedPayer(),
           isReimbursement: false,
           splitMode: 'EVENLY',
-          documents: [],
+          documents: searchParams.get('imageUrl')
+            ? [
+                {
+                  id: randomId(),
+                  url: searchParams.get('imageUrl') as string,
+                  width: Number(searchParams.get('imageWidth')),
+                  height: Number(searchParams.get('imageHeight')),
+                },
+              ]
+            : [],
         },
   })
 

src/lib/env.ts

@@ -17,6 +17,8 @@ const envSchema = z
     S3_UPLOAD_SECRET: z.string().optional(),
     S3_UPLOAD_BUCKET: z.string().optional(),
     S3_UPLOAD_REGION: z.string().optional(),
+    NEXT_PUBLIC_ENABLE_RECEIPT_EXTRACT: z.coerce.boolean().default(false),
+    OPENAI_API_KEY: z.string().optional(),
   })
   .superRefine((env, ctx) => {
     if (
@@ -32,6 +34,13 @@ const envSchema = z
           'If NEXT_PUBLIC_ENABLE_EXPENSE_DOCUMENTS is specified, then S3_* must be specified too',
       })
     }
+    if (env.NEXT_PUBLIC_ENABLE_RECEIPT_EXTRACT && !env.OPENAI_API_KEY) {
+      ctx.addIssue({
+        code: ZodIssueCode.custom,
+        message:
+          'If NEXT_PUBLIC_ENABLE_RECEIPT_EXTRACT is specified, then OPENAI_API_KEY must be specified too',
+      })
+    }
   })
 
 export const env = envSchema.parse(process.env)

src/lib/utils.ts

@@ -8,3 +8,10 @@ export function cn(...inputs: ClassValue[]) {
 export function delay(ms: number) {
   return new Promise((resolve) => setTimeout(resolve, ms))
 }
+
+export function formatExpenseDate(date: Date) {
+  return date.toLocaleDateString('en-US', {
+    dateStyle: 'medium',
+    timeZone: 'UTC',
+  })
+}