upload the invoice as a PDF and autofill the form via OCR extraction

This commit is contained in:
Yashas 2026-07-29 12:19:21 +05:30
parent cc997838bb
commit db8233fa45
3 changed files with 488 additions and 19 deletions

View File

@ -13,12 +13,9 @@ function headers(): Record<string, string> {
return h;
}
async function request<T>(method: string, path: string, body?: unknown): Promise<T> {
const res = await fetch(`${baseUrl()}${path}`, {
method,
headers: headers(),
body: body ? JSON.stringify(body) : undefined,
});
// Shared response handling — 401 bounces to login, error bodies are unwrapped
// from { error }. Used by both the JSON and multipart request paths.
async function parseResponse<T>(res: Response): Promise<T> {
if (res.status === 401) {
localStorage.removeItem(TOKEN_KEY);
localStorage.removeItem("p2p_user");
@ -34,6 +31,37 @@ async function request<T>(method: string, path: string, body?: unknown): Promise
return res.json() as Promise<T>;
}
async function request<T>(method: string, path: string, body?: unknown): Promise<T> {
const res = await fetch(`${baseUrl()}${path}`, {
method,
headers: headers(),
body: body ? JSON.stringify(body) : undefined,
});
return parseResponse<T>(res);
}
// Multipart POST for the field-scoped core-service endpoints (/upload,
// /ocr-extract). Content-Type is deliberately omitted so the browser sets the
// multipart boundary itself.
async function postFile<T>(path: string, file: File, ctx: FieldFileContext): Promise<T> {
const form = new FormData();
form.append("file", file);
form.append("workflow_uuid", ctx.workflowUuid);
form.append("activity_id", ctx.activityId);
form.append("field_id", ctx.fieldId);
if (ctx.columnId) form.append("column_id", ctx.columnId);
if (ctx.versionUuid) form.append("version_uuid", ctx.versionUuid);
if (ctx.instanceId) form.append("instance_id", String(ctx.instanceId));
const token = localStorage.getItem(TOKEN_KEY);
const res = await fetch(`${baseUrl()}${path}`, {
method: "POST",
headers: token ? { Authorization: `Bearer ${token}` } : {},
body: form,
});
return parseResponse<T>(res);
}
// ---------------------------------------------------------------------------
// Header
// ---------------------------------------------------------------------------
@ -248,6 +276,33 @@ export function getDetailViewLegacy(dvUid: string, instanceId: string): Promise<
// Form Screens
// ---------------------------------------------------------------------------
// One key the vision model must return, as configured in studio's OCR modal.
export interface OcrExtractionField {
key: string;
label: string;
data_type?: string;
description?: string;
}
// extraction_key → the form field that receives the extracted value.
export interface OcrFieldMapping {
extraction_key: string;
target_field: string;
}
// Studio's ocr_config block. `prompt` / `extraction_fields` are resolved
// server-side by /ocr-extract and are present here only for picker UX; the
// client genuinely needs `field_mappings` to know where to write the results.
export interface OcrConfig {
prompt?: string;
template?: string;
allowed_types?: string[] | string;
max_size_mb?: number | string;
placeholder?: string;
extraction_fields?: OcrExtractionField[];
field_mappings?: OcrFieldMapping[];
}
export interface FormScreenField {
id: string;
uid: string;
@ -259,6 +314,15 @@ export interface FormScreenField {
properties?: {
options?: Array<{ label: string; value: string }>;
country_code?: string;
placeholder?: string;
multiple?: boolean;
ocr_config?: OcrConfig;
// Some studio configs put the OCR keys flat on properties instead of
// nested under ocr_config — mirrored from FFOcrField's resolveOcrConfig.
allowed_types?: string[] | string;
max_size_mb?: number | string;
extraction_fields?: OcrExtractionField[];
field_mappings?: OcrFieldMapping[];
};
}
@ -270,6 +334,10 @@ export interface FormScreenResponse {
fields: FormScreenField[];
grid_config: Array<{ i: string; x: number; y: number; w: number; h: number }>;
layout: any[];
// Identifiers the field-scoped core endpoints (/upload, /ocr-extract) need
// so they can resolve the field's config + RBAC server-side.
workflow_uuid?: string;
version_uuid?: string;
}
export function getFormScreen(
@ -287,6 +355,53 @@ export function getFormScreen(
});
}
// ---------------------------------------------------------------------------
// Field files & OCR (core-service)
// ---------------------------------------------------------------------------
// Identifiers sent alongside every field-scoped file upload. The backend uses
// these to resolve the field's deployed allowed_types / max_size_mb / ocr_config
// and to run activity RBAC — anything the client sends about that config is
// ignored, so this trio is mandatory.
export interface FieldFileContext {
workflowUuid: string;
activityId: string;
fieldId: string;
columnId?: string;
versionUuid?: string;
instanceId?: string;
}
// What /upload returns and what gets stored in instance data. `uuid` is the
// stable app-scoped handle; blob_path is kept for the legacy serve fallback.
export interface FileMeta {
uuid: string;
blob_path: string;
original_name: string;
mime_type: string;
size_bytes: number;
}
export interface OcrExtractResponse {
// Keyed by the studio-configured extraction_fields[].key. A key whose value
// wasn't found in the document comes back as null.
extracted: Record<string, unknown>;
raw?: string;
// Set when the model's reply wasn't parseable JSON — `raw` still holds the text.
parse_error?: string;
}
// Stores a form-field file and returns the metadata to submit in its place.
export function uploadFieldFile(file: File, ctx: FieldFileContext): Promise<FileMeta> {
return postFile<FileMeta>(`/app/${APP_ID}/upload`, file, ctx);
}
// Runs vision OCR over an uploaded document. The prompt and extraction schema
// live in the deployed ocr_config, so only the file + identifiers go over the wire.
export function extractOcr(file: File, ctx: FieldFileContext): Promise<OcrExtractResponse> {
return postFile<OcrExtractResponse>(`/app/${APP_ID}/ocr-extract`, file, ctx);
}
// ---------------------------------------------------------------------------
// Workflow Actions
// ---------------------------------------------------------------------------

View File

@ -1,12 +1,24 @@
import { useEffect, useState, useRef } from "react";
import { X, ArrowRight, AlertCircle, CheckCircle2, ChevronDown } from "lucide-react";
import { getFormScreen, startWorkflow, performActivity } from "../api/viewService";
import { getFormScreen, startWorkflow, performActivity, uploadFieldFile } from "../api/viewService";
import type { FormScreenField, FieldFileContext } from "../api/viewService";
import { WORKFLOW_ID } from "../config";
import { Skeleton } from "./Skeleton";
import OcrUploadField, { resolveOcrConfig } from "./OcrUploadField";
interface FormField { id: string; uid: string; name: string; type: string; data_type: string; mandatory: boolean; value?: any; properties?: { options?: Array<{ label: string; value: string }>; country_code?: string }; }
type FormField = FormScreenField;
interface GridItem { i: string; x: number; y: number; w: number; h: number; }
// Field data types whose value is a file, not a scalar. They skip the
// string-based validation and number coercion paths, and their File is uploaded
// via /upload before submit so instance data references the blob by metadata.
const MEDIA_TYPES = new Set(["ocr", "file", "image"]);
const isEmpty = (v: any) =>
v == null ||
(typeof v === "string" && v.trim() === "") ||
(Array.isArray(v) && v.length === 0);
interface Props {
activityId: string;
instanceId?: string;
@ -19,7 +31,13 @@ export default function FormModal({ activityId, instanceId, title, onClose, onSu
const [fields, setFields] = useState<FormField[]>([]);
const [gridConfig, setGridConfig] = useState<GridItem[]>([]);
const [formTitle, setFormTitle] = useState(title || "");
const [formData, setFormData] = useState<Record<string, string>>({});
// Values are strings for scalar fields and a File for media fields.
const [formData, setFormData] = useState<Record<string, any>>({});
// Identifiers /upload + /ocr-extract need to resolve a field's deployed
// config and RBAC server-side. The form screen echoes them; WORKFLOW_ID is
// the fallback for older view-service builds that don't.
const [wfUuid, setWfUuid] = useState<string>(WORKFLOW_ID);
const [versionUuid, setVersionUuid] = useState<string | undefined>(undefined);
const [loading, setLoading] = useState(true);
const [submitting, setSubmitting] = useState(false);
const [error, setError] = useState<string | null>(null);
@ -38,8 +56,14 @@ export default function FormModal({ activityId, instanceId, title, onClose, onSu
setFields(f);
setGridConfig(res.grid_config ?? []);
setFormTitle(res.activity_name || title || "Form");
const pre: Record<string, string> = {};
f.forEach((field) => { if (field.value != null) pre[field.id] = String(field.value); });
if (res.workflow_uuid) setWfUuid(res.workflow_uuid);
setVersionUuid(res.version_uuid || undefined);
const pre: Record<string, any> = {};
f.forEach((field) => {
// Media prefills are already-uploaded metadata arrays, not strings.
if (field.value == null) return;
pre[field.id] = MEDIA_TYPES.has(field.data_type) ? field.value : String(field.value);
});
setFormData(pre);
})
.catch((err: any) => setError(err.message))
@ -51,11 +75,44 @@ export default function FormModal({ activityId, instanceId, title, onClose, onSu
setTimeout(callback, 200);
};
// Base context for the field-scoped file endpoints; each call adds its fieldId.
const fileCtx: Omit<FieldFileContext, "fieldId"> = {
workflowUuid: wfUuid,
activityId,
versionUuid,
instanceId,
};
// Write an OCR result into the fields ocr_config.field_mappings points at.
// Returns the number of fields that actually received a value, which the OCR
// field reports back to the user ("Filled 9 fields…").
const applyExtraction = (ocrField: FormField, extracted: Record<string, unknown>): number => {
const mappings = resolveOcrConfig(ocrField).field_mappings ?? [];
const next: Record<string, any> = {};
for (const m of mappings) {
if (!m?.extraction_key || !m?.target_field) continue;
const raw = extracted[m.extraction_key];
if (raw == null || raw === "") continue; // key absent, or the model found nothing
const target = fields.find((f) => f.id === m.target_field);
const coerced = coerceExtracted(raw, target?.data_type);
if (coerced === "") continue;
next[m.target_field] = coerced;
}
const filled = Object.keys(next);
if (filled.length > 0) {
setFormData((p) => ({ ...p, ...next }));
// Mark them touched so a value the model got wrong still shows validation.
setTouched((t) => { const s = new Set(t); filled.forEach((k) => s.add(k)); return s; });
setError(null);
}
return filled.length;
};
const handleSubmit = async (e: React.FormEvent) => {
e.preventDefault();
setTouched(new Set(fields.map((f) => f.id)));
const missing = fields.filter((f) => f.mandatory && !formData[f.id]?.trim());
const missing = fields.filter((f) => f.mandatory && isEmpty(formData[f.id]));
if (missing.length > 0) {
setError(`Please fill in: ${missing.map((f) => f.name).join(", ")}`);
return;
@ -64,7 +121,22 @@ export default function FormModal({ activityId, instanceId, title, onClose, onSu
setSubmitting(true); setError(null);
try {
const data: Record<string, unknown> = {};
for (const f of fields) { const v = formData[f.id] ?? ""; data[f.id] = f.data_type === "number" ? (v ? Number(v) : 0) : v; }
for (const f of fields) {
const v = formData[f.id];
if (MEDIA_TYPES.has(f.data_type)) {
// Freshly picked Files are uploaded now and submitted as metadata;
// an array is already-uploaded metadata from a prefill. Instance data
// holds an array either way, matching the platform's file fields.
if (v instanceof File) {
data[f.id] = [await uploadFieldFile(v, { ...fileCtx, fieldId: f.id })];
} else if (Array.isArray(v)) {
data[f.id] = v;
}
continue;
}
const s = v ?? "";
data[f.id] = f.data_type === "number" ? (s !== "" ? Number(s) : 0) : String(s);
}
instanceId ? await performActivity(WORKFLOW_ID, instanceId, activityId, data) : await startWorkflow(WORKFLOW_ID, activityId, data);
setSuccess(true);
setTimeout(() => animateClose(onSuccess), 800);
@ -81,7 +153,7 @@ export default function FormModal({ activityId, instanceId, title, onClose, onSu
const rows: FormField[][] = []; let cy = -1;
for (const f of sorted) { const g = gridConfig.find((gi) => gi.i === f.uid); const y = g?.y ?? rows.length; if (y !== cy) { rows.push([]); cy = y; } rows[rows.length - 1].push(f); }
const isMissing = (f: FormField) => f.mandatory && touched.has(f.id) && !formData[f.id]?.trim();
const isMissing = (f: FormField) => f.mandatory && touched.has(f.id) && isEmpty(formData[f.id]);
return (
<div className="fixed inset-0 z-50 flex items-center justify-center" onClick={() => animateClose(onClose)}>
@ -141,11 +213,26 @@ export default function FormModal({ activityId, instanceId, title, onClose, onSu
{f.name}
{f.mandatory && <span className="text-red-400 ml-0.5">*</span>}
</label>
{renderInput(f, formData[f.id] ?? "", (v) => {
setFormData((p) => ({ ...p, [f.id]: v }));
setTouched((t) => new Set(t).add(f.id));
if (error) setError(null);
}, isMissing(f))}
{MEDIA_TYPES.has(f.data_type) ? (
<OcrUploadField
field={f}
ctx={fileCtx}
file={formData[f.id] instanceof File ? formData[f.id] : null}
onFileChange={(file) => {
setFormData((p) => ({ ...p, [f.id]: file }));
setTouched((t) => new Set(t).add(f.id));
if (error) setError(null);
}}
onExtracted={(extracted) => applyExtraction(f, extracted)}
hasError={isMissing(f)}
/>
) : (
renderInput(f, formData[f.id] ?? "", (v) => {
setFormData((p) => ({ ...p, [f.id]: v }));
setTouched((t) => new Set(t).add(f.id));
if (error) setError(null);
}, isMissing(f))
)}
{isMissing(f) && (
<p className="text-[11px] text-red-500 mt-1 flex items-center gap-1">
<AlertCircle size={11} /> Required
@ -201,6 +288,32 @@ export default function FormModal({ activityId, instanceId, title, onClose, onSu
);
}
// ---------------------------------------------------------------------------
// OCR value coercion
// ---------------------------------------------------------------------------
// The vision model returns JSON scalars, but every input here is string-valued.
// The OCR prompt asks for plain numbers and YYYY-MM-DD dates; this is the
// belt-and-braces pass for when it doesn't comply — a currency symbol or
// thousands separator on a number, or a date in some other notation. Returns ""
// when the value can't be used, which the caller treats as "not filled".
function coerceExtracted(raw: unknown, dataType?: string): string {
if (typeof raw === "boolean") return raw ? "true" : "false";
const s = String(raw).trim();
if (s === "") return "";
if (dataType === "number") {
const n = Number(s.replace(/[^0-9.\-]/g, ""));
return Number.isFinite(n) ? String(n) : "";
}
if (dataType === "date") {
const iso = s.match(/^\d{4}-\d{2}-\d{2}/);
if (iso) return iso[0];
const d = new Date(s);
return Number.isNaN(d.getTime()) ? "" : d.toISOString().slice(0, 10);
}
return s;
}
// ---------------------------------------------------------------------------
// Input renderer
// ---------------------------------------------------------------------------

View File

@ -0,0 +1,241 @@
// OcrUploadField — document dropzone for the file-valued form field types.
//
// Handles `ocr`, `file` and `image`. The Extract action only appears for `ocr`
// fields that actually carry an extraction schema and field mappings; for a
// plain file/image field this is just a picker.
//
// The user drops a PDF (or image), we POST it to the core-service's
// /ocr-extract, and hand the extracted key→value map back to the parent, which
// walks ocr_config.field_mappings and writes each value into its target form
// field. The file itself stays in the parent's form state as a raw File and is
// uploaded via /upload at submit time.
//
// The prompt and extraction schema are NOT sent from here — the backend
// resolves them from the deployed ocr_config using the identifier trio
// (workflow_uuid / activity_id / field_id) plus activity RBAC. What we read
// out of the local config is only picker UX (allowed_types / max_size_mb /
// placeholder); mirrors renderer-desktop's FFOcrField.
import { useRef, useState } from "react";
import { FileText, UploadCloud, X, ScanText, CheckCircle2, AlertCircle } from "lucide-react";
import { extractOcr } from "../api/viewService";
import type { FieldFileContext, FormScreenField, OcrConfig } from "../api/viewService";
// Studio writes the OCR keys either nested under properties.ocr_config or flat
// on properties. The nested shape wins when it actually carries a schema.
export function resolveOcrConfig(field: FormScreenField): OcrConfig {
const nested = field.properties?.ocr_config;
if (nested && Array.isArray(nested.extraction_fields)) return nested;
return (field.properties ?? {}) as OcrConfig;
}
// allowed_types arrives as a CSV string or an array, holding any of ".pdf",
// "pdf", "application/pdf" or "image/*". Normalise to what <input accept> wants.
function toAcceptAttr(raw: OcrConfig["allowed_types"]): string {
const entries = Array.isArray(raw)
? raw
: typeof raw === "string"
? raw.split(",")
: [];
const cleaned = entries
.map((e) => e.trim().toLowerCase())
.filter(Boolean)
.map((e) => (e.includes("/") || e.startsWith(".") ? e : `.${e}`));
return cleaned.join(",");
}
function formatBytes(n: number): string {
if (n < 1024) return `${n} B`;
if (n < 1024 * 1024) return `${(n / 1024).toFixed(0)} KB`;
return `${(n / (1024 * 1024)).toFixed(1)} MB`;
}
type ExtractState =
| { status: "idle" }
| { status: "loading" }
| { status: "done"; applied: number }
| { status: "error"; message: string };
interface Props {
field: FormScreenField;
/** Identifiers the backend needs to resolve this field's ocr_config + RBAC. */
ctx: Omit<FieldFileContext, "fieldId">;
file: File | null;
onFileChange: (file: File | null) => void;
/** Receives the raw extraction map; the parent applies field_mappings. */
onExtracted: (extracted: Record<string, unknown>) => number;
disabled?: boolean;
hasError?: boolean;
}
export default function OcrUploadField({
field, ctx, file, onFileChange, onExtracted, disabled, hasError,
}: Props) {
const inputRef = useRef<HTMLInputElement>(null);
const [dragging, setDragging] = useState(false);
const [state, setState] = useState<ExtractState>({ status: "idle" });
const config = resolveOcrConfig(field);
const accept = toAcceptAttr(config.allowed_types);
const maxSizeMb = Number(config.max_size_mb) > 0 ? Number(config.max_size_mb) : null;
const mappings = Array.isArray(config.field_mappings) ? config.field_mappings : [];
// Without extraction_fields the backend has nothing to ask for, and without
// mappings there is nowhere to put the answer — either way, no Extract button.
const canExtract =
(config.extraction_fields?.length ?? 0) > 0 && mappings.length > 0 && !!ctx.workflowUuid;
const acceptHint = accept
? accept.replace(/\./g, "").replace(/,/g, ", ").toUpperCase()
: "PDF or image";
const pick = (next: File | null) => {
setState({ status: "idle" });
if (next && maxSizeMb && next.size > maxSizeMb * 1024 * 1024) {
onFileChange(null);
setState({ status: "error", message: `File exceeds the ${maxSizeMb} MB limit` });
return;
}
onFileChange(next);
};
const runExtract = async () => {
if (!file || !canExtract) return;
setState({ status: "loading" });
try {
const res = await extractOcr(file, { ...ctx, fieldId: field.id });
const applied = onExtracted(res?.extracted ?? {});
if (applied === 0) {
setState({
status: "error",
message: res?.parse_error
? `Could not read the document: ${res.parse_error}`
: "Nothing could be read from this document — please fill the fields manually.",
});
return;
}
setState({ status: "done", applied });
} catch (err: any) {
setState({ status: "error", message: err?.message || "Extraction failed" });
}
};
const clear = () => {
pick(null);
if (inputRef.current) inputRef.current.value = "";
};
return (
<div>
<input
ref={inputRef}
type="file"
accept={accept || undefined}
className="hidden"
onChange={(e) => pick(e.target.files?.[0] ?? null)}
/>
{!file ? (
/* ── Empty dropzone ─────────────────────────────────────────────── */
<button
type="button"
disabled={disabled}
onClick={() => inputRef.current?.click()}
onDragOver={(e) => { e.preventDefault(); setDragging(true); }}
onDragLeave={() => setDragging(false)}
onDrop={(e) => {
e.preventDefault();
setDragging(false);
if (!disabled) pick(e.dataTransfer.files?.[0] ?? null);
}}
className={`w-full rounded-xl border border-dashed px-5 py-7 flex flex-col items-center gap-2 transition-colors disabled:opacity-50 ${
dragging
? "border-violet-400 bg-violet-50/70 dark:border-violet-500 dark:bg-violet-950/20"
: hasError
? "border-red-300 bg-red-50/40 dark:border-red-700 dark:bg-red-950/20"
: "border-slate-200 bg-slate-50/50 hover:border-violet-300 hover:bg-violet-50/40 dark:border-zinc-700 dark:bg-zinc-800/40 dark:hover:border-violet-600 dark:hover:bg-violet-950/10"
}`}
>
<div className="w-10 h-10 rounded-full bg-white dark:bg-zinc-800 shadow-sm flex items-center justify-center">
<UploadCloud size={18} className="text-violet-500" />
</div>
<div className="text-[13px] font-medium text-slate-700 dark:text-zinc-200">
{config.placeholder || field.properties?.placeholder || `Drop ${field.name} here`}
</div>
<div className="text-[11px] text-slate-400 dark:text-zinc-500">
or click to browse · {acceptHint}
{maxSizeMb ? ` up to ${maxSizeMb} MB` : ""}
</div>
</button>
) : (
/* ── Picked file + extract action ────────────────────────────────── */
<div className="rounded-xl border border-slate-200 dark:border-zinc-700 bg-white dark:bg-zinc-800/60 px-4 py-3">
<div className="flex items-center gap-3">
<div className="w-9 h-9 shrink-0 rounded-lg bg-violet-50 dark:bg-violet-950/40 flex items-center justify-center">
<FileText size={16} className="text-violet-500" />
</div>
<div className="min-w-0 flex-1">
<div className="text-[13px] font-medium text-slate-800 dark:text-zinc-100 truncate">
{file.name}
</div>
<div className="text-[11px] text-slate-400 dark:text-zinc-500">
{formatBytes(file.size)}
</div>
</div>
{canExtract && (
<button
type="button"
disabled={disabled || state.status === "loading"}
onClick={runExtract}
className="h-8 px-3 shrink-0 text-[12px] font-semibold text-white rounded-lg flex items-center gap-1.5 disabled:opacity-60 transition-all hover:scale-[1.02] active:scale-[0.98]"
style={{ background: "linear-gradient(135deg, #7c3aed, #a855f7)" }}
>
{state.status === "loading" ? (
<>
<div className="w-3 h-3 border-2 border-white/30 border-t-white rounded-full animate-spin" />
Reading
</>
) : (
<>
<ScanText size={13} />
{state.status === "done" ? "Re-read" : "Read invoice"}
</>
)}
</button>
)}
<button
type="button"
disabled={disabled || state.status === "loading"}
onClick={clear}
className="w-7 h-7 shrink-0 flex items-center justify-center text-slate-400 dark:text-zinc-500 hover:text-slate-600 dark:hover:text-zinc-200 hover:bg-slate-100 dark:hover:bg-zinc-700 rounded-lg transition-colors disabled:opacity-40"
title="Remove"
>
<X size={14} />
</button>
</div>
{state.status === "done" && (
<div className="mt-2.5 flex items-center gap-1.5 text-[11.5px] text-emerald-600 dark:text-emerald-400">
<CheckCircle2 size={12} />
Filled {state.applied} field{state.applied === 1 ? "" : "s"} from the document review before submitting.
</div>
)}
{state.status === "error" && (
<div className="mt-2.5 flex items-start gap-1.5 text-[11.5px] text-red-600 dark:text-red-400">
<AlertCircle size={12} className="shrink-0 mt-0.5" />
{state.message}
</div>
)}
</div>
)}
{state.status === "error" && !file && (
<div className="mt-2 flex items-start gap-1.5 text-[11.5px] text-red-600 dark:text-red-400">
<AlertCircle size={12} className="shrink-0 mt-0.5" />
{state.message}
</div>
)}
</div>
);
}