adding some counter measures to reduce the pressure on the server

scripts/test.js

@@ -1,23 +0,0 @@
-const { promises: fs } = require("node:fs")
-
-const main = async () => {
-  console.log('generating shot..')
-  const response = await fetch("http://localhost:3000/api/shot", {
-    method: "POST",
-    headers: {
-      "Accept": "application/json",
-      "Content-Type": "application/json"
-    },
-    body: JSON.stringify({
-      token: process.env.VC_SECRET_ACCESS_TOKEN,
-      shotPrompt: "video of a dancing cat"
-    })
-  });
-
-  console.log('response:', response)
-  const buffer = await response.buffer()
-
-  fs.writeFile(`./test-juju.mp4`, buffer)
-}
-
-main()

src/app/engine/render.ts

@@ -51,14 +51,16 @@ export async function newRender({
         // negativePrompt, unused for now
         nbFrames: 1,
         nbSteps: 25, // 20 = fast, 30 = better, 50 = best
-        actionnables: [],
-        segmentation: "disabled", // one day we will remove this param, to make it automatic
+        actionnables: [], // ["text block"],
+        segmentation: "disabled", // "firstframe", // one day we will remove this param, to make it automatic
         width,
         height,
 
         // no need to upscale right now as we generate tiny panels
         // maybe later we can provide an "export" button to PDF
-        upscalingFactor: 2,
+        // unfortunately there are too many requests for upscaling,
+        // the server is always down
+        upscalingFactor: 1, // 2,
 
         // analyzing doesn't work yet, it seems..
         analyze: false, // analyze: true,

src/app/interface/bottom-bar/index.tsx

@@ -28,7 +28,9 @@ export function BottomBar() {
           onClick={handlePrint}
           disabled={!prompt?.length}
         >{
-          remainingImages ? `Print (${allStatus.length - remainingImages}/4 in HD ⌛)` : `Print (in HD)`
+          remainingImages ? `${allStatus.length - remainingImages}/4 panels ⌛` : `Print as PDF`
+          // remainingImages ? `Print (${allStatus.length - remainingImages}/4 in HD ⌛)` : `Print (in HD)`
+        
         }</Button>
       </div>
     </div>

src/app/interface/page/index.tsx

@@ -0,0 +1,50 @@
+import { allLayouts } from "@/app/layouts"
+import { useStore } from "@/app/store"
+import { cn } from "@/lib/utils"
+import { useEffect, useState } from "react"
+
+
+export function Page({ page }: { page: number }) {
+  const zoomLevel = useStore(state => state.zoomLevel)
+  const layouts = useStore(state => state.layouts)
+  const prompt = useStore(state => state.prompt)
+
+  const LayoutElement = (allLayouts as any)[layouts[page]]
+
+  /*
+  const [canLoad, setCanLoad] = useState(false)
+  useEffect(() => {
+    if (prompt?.length) {
+      setCanLoad(false)
+      setTimeout(() => {
+        setCanLoad(true)
+      }, page * 4000)
+    }
+  }, [prompt])
+  */
+
+  return (
+    <div
+      className={cn(
+        `w-full`,
+        // we are trying to reach a "book" look
+        // we are using aspect-[297/210] because it matches A4 (297mm x 210mm)
+        // `aspect-[210/297]`,
+        `aspect-[250/297]`,
+
+        `transition-all duration-100 ease-in-out`,
+        `border border-stone-200`,
+        `shadow-2xl`,
+        `print:shadow-none`,
+        `print:border-0`,
+        `print:width-screen`
+      )}
+      style={{
+        padding: `${Math.round((zoomLevel / 100) * 16)}px`
+        // marginLeft: `${zoomLevel > 100 ? `100`}`
+      }}
+      >
+      <LayoutElement />
+    </div>
+  )
+}

src/app/interface/panel/index.tsx

@@ -12,6 +12,7 @@ import { cn } from "@/lib/utils"
 import { getInitialRenderedScene } from "@/lib/getInitialRenderedScene"
 import { Progress } from "@/app/interface/progress"
 import { see } from "@/app/engine/caption"
+import { writeIntoBubble } from "@/lib/writeIntoBubble"
 // import { Bubble } from "./bubble"
 
 export function Panel({
@@ -49,23 +50,25 @@ export function Panel({
   // since this run in its own loop, we need to use references everywhere
   // but perhaps this could be refactored
   useEffect(() => {
-    startTransition(async () => {
-      // console.log("Panel prompt: "+ prompt)
-      if (!prompt?.length) { return }
-
-      console.log("Loading panel..")
+    // console.log("Panel prompt: "+ prompt)
+    if (!prompt?.length) { return }
 
-      // console.log("calling:\nconst newRendered = await newRender({ prompt, preset, width, height })")
-      console.log({
-        panel, prompt, width, height
-      })
+    // important: update the status, and clear the scene
+    setGeneratingImages(panel, true)
+    setRendered(getInitialRenderedScene())
 
-      console.log("")
-      // important: update the status, and clear the scene
-      setGeneratingImages(panel, true)
-      setRendered(getInitialRenderedScene())
+    setTimeout(() => {
+      startTransition(async () => {
 
-      const newRendered = await newRender({ prompt, width, height })
+      console.log(`Loading panel ${panel}..`)
+    
+      let newRendered = await newRender({ prompt, width, height })
+      try {
+        newRendered = await newRender({ prompt, width, height })
+      } catch (err) {
+        console.log("Failed to load the panel! Don't worry, we are retrying..")
+        newRendered = await newRender({ prompt, width, height })
+      }
 
       if (newRendered) {
         // console.log("newRendered:", newRendered)
@@ -86,6 +89,7 @@ export function Panel({
         return
       }
     })
+  }, 1000 * panel)
   }, [prompt, width, height])
 
 
@@ -94,7 +98,7 @@ export function Panel({
       clearTimeout(timeoutRef.current)
 
       if (!renderedRef.current?.renderId || renderedRef.current?.status !== "pending") {
-        timeoutRef.current = setTimeout(checkStatus, 1000)
+        timeoutRef.current = setTimeout(checkStatus, 1500)
         return
       }
       try {
@@ -112,20 +116,20 @@ export function Panel({
 
         if (newRendered.status === "pending") {
           // console.log("job not finished")
-          timeoutRef.current = setTimeout(checkStatus, 1000)
+          timeoutRef.current = setTimeout(checkStatus, 1500)
         } else {
           console.log("panel finished!")
           setGeneratingImages(panel, false)
         }
       } catch (err) {
         console.error(err)
-        timeoutRef.current = setTimeout(checkStatus, 1000)
+        timeoutRef.current = setTimeout(checkStatus, 1500)
       }
     })
   }
  
   useEffect(() => {
-    console.log("starting timeout")
+    // console.log("starting timeout")
     clearTimeout(timeoutRef.current)
     
     // normally it should reply in < 1sec, but we could also use an interval
@@ -176,6 +180,22 @@ export function Panel({
     `print:border-[1.5px] print:shadow-none`,
   )
 
+  const [newMask, setNewMask] = useState("")
+
+  useEffect(() => {
+    const transformMask = async () => {
+      if (rendered.maskUrl) {
+        const imgSrc = await writeIntoBubble(
+          rendered.maskUrl,
+          "LOREM IPSUM! Dolor sit amet.."
+        )
+        setNewMask(imgSrc)
+      }
+    }
+    transformMask()
+  }, [rendered.maskUrl])
+
+
   if (prompt && !rendered.assetUrl) {
     return (
       <div className={cn(
@@ -194,11 +214,13 @@ export function Panel({
       { "grayscale": preset.color === "grayscale" },
       className
     )}>
-      {rendered.assetUrl && <img
-        src={rendered.assetUrl}
-        className="w-full h-full object-cover"
-        alt={rendered.alt}
-      />}
+        {rendered.assetUrl && <img
+          src={rendered.assetUrl}
+          className=" w-full h-full object-cover"
+          alt={rendered.alt}
+        />}
+
+
 
       {/*<Bubble className="absolute top-4 left-4">
         Hello, world!

src/app/interface/progress/index.tsx

@@ -24,7 +24,7 @@ export function Progress({
 
     // normally it takes 45, and we will try to go below,
     // but to be safe let's set the counter a 1 min
-    const nbSeconds = 32 // 1 min
+    const nbSeconds = 60 // 1 min
     const amountInPercent =  100 / (nbUpdatesPerSec * nbSeconds) // 0.333
 
     progressRef.current = Math.min(100, progressRef.current + amountInPercent)

src/app/interface/zoom/index.tsx

@@ -15,7 +15,7 @@ export function Zoom() {
       `animation-all duration-300 ease-in-out`,
       isGeneratingStory ? `scale-0 opacity-0` : ``,
     )}>
-      <div className="font-mono text-xs pb-1 text-stone-700">
+      <div className="font-mono text-xs pb-1 text-stone-700 bg-stone-50 rounded-full">
         Zoom
       </div>
       <div className="w-2">

src/app/layouts/index.tsx

@@ -17,14 +17,14 @@ export function Layout1() {
       <div className="bg-zinc-100 row-span-2">
         <Panel
           panel={1}
-          width={1024}
+          width={768}
           height={1024}
         />
       </div>
       <div className="bg-gray-100 row-span-2 col-span-1">
         <Panel
           panel={2}
-          width={1024}
+          width={768}
           height={1024}
         />
       </div>
@@ -154,20 +154,20 @@ export function Layout5() {
         <Panel
           panel={0}
           width={768}
-          height={768}
+          height={1024}
         />
       </div>
       <div className="bg-zinc-100 col-span-1 row-span-1">
         <Panel
           panel={1}
           width={768}
-          height={768}
+          height={1024}
         />
       </div>
       <div className="bg-stone-100 row-span-2 col-span-1">
         <Panel
           panel={2}
-          width={768}
+          width={512}
           height={1024}
         />
       </div>
@@ -175,7 +175,7 @@ export function Layout5() {
         <Panel
           panel={3}
           width={1024}
-          height={768}
+          height={1024}
         />
       </div>
     </Grid>
@@ -196,14 +196,14 @@ export function Layout6() {
         <Panel
           panel={1}
           width={768}
-          height={768}
+          height={1024}
         />
       </div>
       <div className="bg-stone-100 row-span-1 col-span-1">
         <Panel
           panel={2}
           width={768}
-          height={768}
+          height={1024}
         />
       </div>
       <div className="bg-slate-100 row-span-1 col-span-2">
@@ -218,11 +218,18 @@ export function Layout6() {
 }
 
 // export const layouts = { Layout1, Layout2, Layout3, Layout4, Layout5, Layout6 }
-export const layouts = { Layout1, Layout5, Layout6  }
+export const allLayouts = { 
+  Layout1,
+  Layout5,
+  Layout6 
+}
 
-export type LayoutName = keyof typeof layouts 
+export type LayoutName = keyof typeof allLayouts 
 
 export function getRandomLayoutName(): LayoutName {
-  return pick(Object.keys(layouts) as LayoutName[]) as LayoutName
+  return pick(Object.keys(allLayouts) as LayoutName[]) as LayoutName
 }
 
+export function getRandomLayoutNames(): LayoutName[] {
+  return Object.keys(allLayouts).sort(() => Math.random() - 0.5) as LayoutName[]
+}

src/app/main.tsx

@@ -1,18 +1,19 @@
 "use client"
 
-import { useEffect, useRef, useTransition } from "react"
+import { useEffect, useRef, useState, useTransition } from "react"
 import { useSearchParams } from "next/navigation"
 
 import { PresetName, defaultPreset, getPreset } from "@/app/engine/presets"
 
 import { cn } from "@/lib/utils"
 import { TopMenu } from "./interface/top-menu"
-import { FontName, defaultFont, fontList, fonts } from "@/lib/fonts"
-import { getRandomLayoutName, layouts } from "./layouts"
+import { FontName, defaultFont, fonts } from "@/lib/fonts"
+import { getRandomLayoutName } from "./layouts"
 import { useStore } from "./store"
 import { Zoom } from "./interface/zoom"
 import { getStory } from "./queries/getStory"
 import { BottomBar } from "./interface/bottom-bar"
+import { Page } from "./interface/page"
 
 export default function Main() {
   const [_isPending, startTransition] = useTransition()
@@ -34,8 +35,7 @@ export default function Main() {
   const prompt = useStore(state => state.prompt)
   const setPrompt = useStore(state => state.setPrompt)
 
-  const layout = useStore(state => state.layout)
-  const setLayout = useStore(state => state.setLayout)
+  const setLayouts = useStore(state => state.setLayouts)
 
   const setPanels = useStore(state => state.setPanels)
 
@@ -44,6 +44,8 @@ export default function Main() {
   const setPage = useStore(state => state.setPage)
   const pageRef = useRef<HTMLDivElement>(null)
 
+  const [waitABitMore, setWaitABitMore] = useState(false)
+
   useEffect(() => {
     const element = pageRef.current
     if (!element) { return }
@@ -68,23 +70,28 @@ export default function Main() {
     if (!prompt) { return }
 
     startTransition(async () => {
-
+      setWaitABitMore(false)
       setGeneratingStory(true)
 
-      const newLayout = getRandomLayoutName()
-      console.log("using layout " + newLayout)
-      setLayout(newLayout)
+      const newLayouts = [
+        getRandomLayoutName(),
+        getRandomLayoutName(),
+      ]
+
+      console.log("using layouts " + newLayouts)
+      setLayouts(newLayouts)
 
       try {
+
         const llmResponse = await getStory({ preset, prompt })
         console.log("response:", llmResponse)
 
-        // TODO call the LLM here!
         const panelPromptPrefix = preset.imagePrompt(prompt).join(", ")
         console.log("panel prompt prefix:", panelPromptPrefix)
     
         const nbPanels = 4
         const newPanels: string[] = []
+        setWaitABitMore(true)
 
         for (let p = 0; p < nbPanels; p++) {
           const newPanel = [panelPromptPrefix, llmResponse[p] || ""]
@@ -95,13 +102,14 @@ export default function Main() {
       } catch (err) {
         console.error(err)
       } finally {
-        setGeneratingStory(false)
+        setTimeout(() => {
+          setGeneratingStory(false)
+          setWaitABitMore(false)
+        }, 9000)
       }
     })
   }, [prompt, preset?.label]) // important: we need to react to preset changes too
 
-  const LayoutElement = (layouts as any)[layout]
-
   return (
     <div>
       <TopMenu />
@@ -112,35 +120,27 @@ export default function Main() {
         `print:pt-0 print:px-0 print:pl-0 print:pr-0`,
         fonts.actionman.className
       )}>
-        <div className={cn(
-          `flex flex-col w-full`,
-          zoomLevel > 105 ? `items-start` : `items-center`
-        )}>
+        <div
+          ref={pageRef}
+          className={cn(
+            `flex flex-col w-full`,
+            zoomLevel > 105 ? `items-start` : `items-center`
+          )}>
           <div
-            ref={pageRef}
             className={cn(
               `comic-page`,
-              `flex flex-col items-center justify-start`,
-
-              // we are trying to reach a "book" look
-              // we are using aspect-[297/210] because it matches A4 (297mm x 210mm)
-              // `aspect-[210/297]`,
-              `aspect-[250/297]`,
-
-              `transition-all duration-100 ease-in-out`,
-              `border border-stone-200`,
-              `shadow-2xl`,
-              `print:shadow-none`,
-              `print:border-0`,
-              `print:width-screen`
+              `flex flex-col md:flex-row md:space-x-16 md:items-center md:justify-start`,
             )}
             style={{
-              width: `${zoomLevel}%`,
-              padding: `${Math.round((zoomLevel / 100) * 16)}px`
-              // marginLeft: `${zoomLevel > 100 ? `100`}`
-            }}
-            >
-            <LayoutElement />
+              width: `${zoomLevel}%`
+            }}>
+            <Page page={0} />
+
+            {/*
+            // we could support multiple pages here,
+            // but let's disable it for now
+            <Page page={1} />
+            */}
           </div>
         </div>
       </div>
@@ -157,11 +157,12 @@ export default function Main() {
         fonts.actionman.className
       )}>
         <div className={cn(
-          `text-center text-lg text-stone-600 w-[70%]`,
+          `text-center text-xl text-stone-600 w-[70%]`,
           isGeneratingStory ? ``: `scale-0 opacity-0`,
           `transition-all duration-300 ease-in-out`,
         )}>
-          Generating your story.. (hold tight)
+          {waitABitMore ? `Story is ready, but server is a bit busy!`: 'Generating a new story..'}<br/>
+          {waitABitMore ? `Please hold tight..` : ''}
         </div>
       </div>
     </div>

src/app/queries/getStory.ts

@@ -19,10 +19,10 @@ export const getStory = async ({
       role: "system",
       content: [
         `You are a comic book author specialized in ${preset.llmPrompt}`,
-        `Please generate detailed drawing instructions for the 4 panels of a new silent comic book page.`,
+        `Please write detailed drawing instructions for the 5 panels of a new silent comic book page.`,
         `Give your response as a JSON array like this: \`Array<{ panel: number; caption: string}>\`.`,
         // `Give your response as Markdown bullet points.`,
-        `Be brief in your caption don't add your own comments. Be straight to the point, and never reply things like "Sure, I can.." etc.`
+        `Be brief in your 5 captions, don't add your own comments. Be straight to the point, and never reply things like "Sure, I can.." etc.`
       ].filter(item => item).join("\n")
     },
     {

src/app/store/index.ts

@@ -4,16 +4,17 @@ import { create } from "zustand"
 
 import { FontName } from "@/lib/fonts"
 import { Preset, getPreset } from "@/app/engine/presets"
-import { LayoutName, getRandomLayoutName } from "../layouts"
+import { LayoutName, getRandomLayoutNames } from "../layouts"
 import html2canvas from "html2canvas"
 
 export const useStore = create<{
   prompt: string
   font: FontName
   preset: Preset
+  nbFrames: number
   panels: string[]
   captions: Record<string, string>
-  layout: LayoutName
+  layouts: LayoutName[]
   zoomLevel: number
   page: HTMLDivElement
   isGeneratingStory: boolean
@@ -24,7 +25,7 @@ export const useStore = create<{
   setFont: (font: FontName) => void
   setPreset: (preset: Preset) => void
   setPanels: (panels: string[]) => void
-  setLayout: (layout: LayoutName) => void
+  setLayouts: (layouts: LayoutName[]) => void
   setCaption: (panelId: number, caption: string) => void
   setZoomLevel: (zoomLevel: number) => void
   setPage: (page: HTMLDivElement) => void
@@ -36,9 +37,10 @@ export const useStore = create<{
   prompt: "",
   font: "actionman",
   preset: getPreset("japanese_manga"),
+  nbFrames: 1,
   panels: [],
   captions: {},
-  layout: getRandomLayoutName(),
+  layouts: getRandomLayoutNames(),
   zoomLevel: 60,
   page: undefined as unknown as HTMLDivElement,
   isGeneratingStory: false,
@@ -50,7 +52,7 @@ export const useStore = create<{
     if (prompt === existingPrompt) { return }
     set({
       prompt,
-      layout: getRandomLayoutName(),
+      layouts: getRandomLayoutNames(),
       panels: [],
       captions: {},
     })
@@ -60,7 +62,7 @@ export const useStore = create<{
     if (font === existingFont) { return }
     set({
       font,
-      layout: getRandomLayoutName(),
+      layouts: getRandomLayoutNames(),
       panels: [],
       captions: {}
     })
@@ -70,7 +72,17 @@ export const useStore = create<{
     if (preset.label === existingPreset.label) { return }
     set({
       preset,
-      layout: getRandomLayoutName(),
+      layouts: getRandomLayoutNames(),
+      panels: [],
+      captions: {}
+    })
+  },
+  setNbFrames: (nbFrames: number) => {
+    const existingNbFrames = get().nbFrames
+    if (nbFrames === existingNbFrames) { return }
+    set({
+      nbFrames,
+      layouts: getRandomLayoutNames(),
       panels: [],
       captions: {}
     })
@@ -84,7 +96,7 @@ export const useStore = create<{
       }
     })
   },
-  setLayout: (layout: LayoutName) => set({ layout }),
+  setLayouts: (layouts: LayoutName[]) => set({ layouts }),
   setZoomLevel: (zoomLevel: number) =>  set({ zoomLevel }),
   setPage: (page: HTMLDivElement) => {
     if (!page) { return }

src/lib/cropImage.ts

@@ -0,0 +1,53 @@
+async function cropImage(inputImage: string): Promise<{ croppedImage: string; x: number; y: number; width: number; height: number }> {
+  return new Promise((resolve, reject) => {
+      const img = new Image();
+      img.src = inputImage;
+      img.onload = () => {
+          const canvas = document.createElement('canvas');
+          const context = canvas.getContext('2d');
+          if (!context) {
+            reject("Context is null");
+            return;
+          }
+          canvas.width = img.width;
+          canvas.height = img.height;
+          context.drawImage(img, 0, 0, img.width, img.height);
+          const imageData = context.getImageData(0, 0, img.width, img.height);
+          const data = imageData.data;
+          let minX = img.width, minY = img.height, maxX = 0, maxY = 0;
+
+          for (let y = 0; y < img.height; y++) {
+              for (let x = 0; x < img.width; x++) {
+                  const i = (y * 4) * img.width + x * 4;
+                  const avg = (data[i] + data[i + 1] + data[i + 2]) / 3;
+                  if (avg < 255) {
+                      minX = Math.min(minX, x);
+                      minY = Math.min(minY, y);
+                      maxX = Math.max(maxX, x);
+                      maxY = Math.max(maxY, y);
+                  }
+              }
+          }
+
+          const width = maxX - minX;
+          const height = maxY - minY;
+          const croppedCanvas = document.createElement('canvas');
+          croppedCanvas.width = width;
+          croppedCanvas.height = height;
+          const croppedCtx = croppedCanvas.getContext('2d');
+          if (!croppedCtx) {
+            reject("croppedCtx is null");
+            return;
+          }
+          croppedCtx.drawImage(canvas, minX, minY, width, height, 0, 0, width, height);
+          resolve({
+              croppedImage: croppedCanvas.toDataURL(),
+              x: minX,
+              y: minY,
+              width,
+              height
+          });
+      };
+      img.onerror = reject;
+  });
+}

src/lib/loadImage.ts

@@ -0,0 +1,14 @@
+export async function loadImage(image: string): Promise<HTMLImageElement> {
+  const img = new Image();
+  img.src = image;
+
+  const imgOnLoad = () => {
+    return new Promise<HTMLImageElement>((resolve, reject) => {
+      img.onload = () => { resolve(img) };
+      img.onerror = (err) => { reject(err) };
+    })
+  };
+
+  const loadImg = await imgOnLoad();
+  return loadImg
+}

src/lib/replaceNonWhiteWithTransparent.ts

@@ -0,0 +1,46 @@
+export function replaceNonWhiteWithTransparent(imageBase64: string): Promise<string> {
+    return new Promise((resolve, reject) => {
+        const img = new Image();
+        img.onload = () => {
+            const canvas = document.createElement('canvas');
+            const ctx = canvas.getContext('2d');
+            if (!ctx) {
+                reject('Unable to get canvas context');
+                return;
+            }
+
+            const ratio = window.devicePixelRatio || 1;
+            canvas.width = img.width * ratio;
+            canvas.height = img.height * ratio;
+            ctx.scale(ratio, ratio);
+
+            ctx.drawImage(img, 0, 0);
+
+            const imageData = ctx.getImageData(0, 0, img.width, img.height);
+            const data = imageData.data;
+            console.log("ok")
+
+            for (let i = 0; i < data.length; i += 4) {
+                if (data[i] === 255 && data[i + 1] === 255 && data[i + 2] === 255) {
+                    // Change white (also shades of grays) pixels to black
+                    data[i] = 0;
+                    data[i + 1] = 0;
+                    data[i + 2] = 0;
+                } else {
+                    // Change all other pixels to transparent
+                    data[i + 3] = 0;
+                }
+            }
+
+            ctx.putImageData(imageData, 0, 0);
+
+            resolve(canvas.toDataURL());
+        };
+
+        img.onerror = (err) => {
+            reject(err);
+        };
+
+        img.src = imageBase64;
+    });
+}

src/lib/replaceWhiteWithTransparent.ts

@@ -0,0 +1,37 @@
+export function replaceWhiteWithTransparent(imageBase64: string): Promise<string> {
+  return new Promise((resolve, reject) => {
+      const img = new Image();
+      img.onload = () => {
+          const canvas = document.createElement('canvas');
+          canvas.width = img.width;
+          canvas.height = img.height;
+
+          const ctx = canvas.getContext('2d');
+          if (!ctx) {
+              reject('Unable to get canvas 2D context');
+              return;
+          }
+
+          ctx.drawImage(img, 0, 0);
+
+          const imageData = ctx.getImageData(0, 0, canvas.width, canvas.height);
+          const data = imageData.data;
+
+          for (let i = 0; i < data.length; i += 4) {
+              if (data[i] === 255 && data[i + 1] === 255 && data[i + 2] === 255) {
+                  data[i + 3] = 0;
+              }
+          }
+
+          ctx.putImageData(imageData, 0, 0);
+
+          resolve(canvas.toDataURL());
+      };
+
+      img.onerror = (err) => {
+          reject(err);
+      };
+
+      img.src = imageBase64;
+  });
+}

src/lib/writeIntoBubble.ts

@@ -0,0 +1,115 @@
+/*
+I have a PNG image which contains a colored shape (roughly in the shape of a speech bubble), surrounded by white
+
+Please write a TypeScript function (it should work in the browser) to:
+
+1. replace all the white pixels with a transparent PNG pixel
+2.  replace all the colored pixels with a white pixel
+3. write some input text into the colored shape
+4. Make sure line returns are handled
+5. It should have some padding (eg. 20px)
+6. use Comic Sans MS
+
+You can use the canvas for your operation. The signature should be something like:
+
+- Please adjust the font size, based on the available number of pixels inside the bubble, taking some margin into account. 
+- The text should not be below 8px
+- If there is not enough room to display it without going outside the shape, then crop the text.
+- in other words, NEVER write outside the shape!
+
+The function should be something like:
+
+writeIntoBubble(image: string, text: string): Promise<string>
+*/
+
+export async function writeIntoBubble(image: string, text: string): Promise<string> {
+  const padding = 20;  // Pixels
+  return new Promise((resolve, reject) => {
+    const img = new Image();
+    img.onload = () => {
+      const physicalWidth = img.width;
+      const physicalHeight = img.height;
+      const canvas = document.createElement('canvas');
+      const ctx = canvas.getContext('2d');
+      if (!ctx) {
+        reject('Unable to get canvas context');
+        return;
+      }
+      canvas.width = physicalWidth;
+      canvas.height = physicalHeight;
+      ctx.drawImage(img, 0, 0, physicalWidth, physicalHeight);
+
+      const imageData = ctx.getImageData(0, 0, physicalWidth, physicalHeight);
+      const data = imageData.data;
+
+      let minX = physicalWidth, minY = physicalHeight, maxX = 0, maxY = 0;
+
+      for (let y = 0; y < physicalHeight; y++) {
+        for (let x = 0; x < physicalWidth; x++) {
+          const i = (y * physicalWidth + x) * 4;
+          if (data[i] !== 255 || data[i + 1] !== 255 || data[i + 2] !== 255) {
+            minX = Math.min(minX, x);
+            minY = Math.min(minY, y);
+            maxX = Math.max(maxX, x);
+            maxY = Math.max(maxY, y);
+            data[i] = data[i + 1] = data[i + 2] = 255;
+            data[i + 3] = 255;
+          } else {
+            data[i + 3] = 0;
+          }
+        }
+      }
+
+      ctx.putImageData(imageData, 0, 0);
+
+      ctx.save();
+      ctx.setTransform(1, 0, 0, 1, 0, 0);  // Reset transforms to handle padding correctly
+
+      const textX = minX + padding;
+      const textY = minY  + padding;
+      const textWidth = (maxX - minX) - 2 * padding;
+      const textHeight = (maxY - minY) - 2 * padding;
+
+      ctx.restore();
+
+      ctx.rect(textX, textY, textWidth, textHeight);
+      ctx.clip();  // Clip outside of the region
+
+      let fontSize = 20;  // Start with a large size
+      let lines = [];
+      do {
+        ctx.font = `${fontSize}px Comic Sans MS`;
+        lines = wrapText(ctx, text, textWidth);
+        fontSize -= 2;  // Reduce size and try again if text doesn't fit
+      } while(lines.length > textHeight / fontSize && fontSize > 8);
+      ctx.font = `${fontSize}px Comic Sans MS`;
+
+      lines.forEach((line, i) => ctx.fillText(line, textX, textY + padding + i * fontSize));
+
+      resolve(canvas.toDataURL());
+    };
+    img.onerror = reject;
+    img.src = image;
+  });
+}
+
+// Function to wrap text into lines that fit inside a specified width
+function wrapText(context: CanvasRenderingContext2D, text: string, maxWidth: number): string[] {
+  const words = text.split(' ');
+  const lines = [];
+  let line = '';
+
+  for (let n = 0; n < words.length; n++) {
+    const testLine = line + words[n] + ' ';
+    const metrics = context.measureText(testLine);
+    const testWidth = metrics.width;
+    if (testWidth > maxWidth && n > 0) {
+      lines.push(line);
+      line = words[n] + ' ';
+    } else {
+      line = testLine;
+    }
+  }
+  lines.push(line);
+  return lines;
+}

src/lib/writeIntoBubbles.ts

@@ -0,0 +1,65 @@
+import { loadImage } from "./loadImage"
+import { writeIntoBubble } from "./writeIntoBubble"
+
+export async function writeIntoBubbles(image: string, texts: string[]): Promise<string> {
+  const loadImg = await loadImage(image);
+  
+  const canvas = document.createElement('canvas');
+  const context = canvas.getContext('2d');
+  
+  canvas.width = loadImg.width;
+  canvas.height = loadImg.height;
+  context?.drawImage(loadImg, 0, 0, loadImg.width, loadImg.height);
+  
+  const untouchedImageData = context?.getImageData(0, 0, loadImg.width, loadImg.height);
+  if (!untouchedImageData) {
+    throw new Error("untouchedImageData is invalid")
+  }
+  const colorSet = new Set<string>(); // This is the unique color container
+  
+  for(let i = 0; i < untouchedImageData?.data.length; i += 4){
+    const r = untouchedImageData?.data[i];
+    const g = untouchedImageData?.data[i+1];
+    const b = untouchedImageData?.data[i+2];
+    const colorString = `rgb(${r},${g},${b})`;
+    
+    if(!colorSet.has(colorString)){
+      colorSet.add(colorString);
+      var newCanvas = document.createElement('canvas');
+      newCanvas.width = loadImg.width;
+      newCanvas.height = loadImg.height;
+      
+      var newContext = newCanvas.getContext('2d');
+      newContext?.drawImage(loadImg, 0, 0, loadImg.width, loadImg.height);
+      var newImageData = newContext?.getImageData(0, 0, loadImg.width, loadImg.height);
+      if (!newImageData) {
+        throw new Error("newImageData is invalid")
+      }
+
+      for(let j = 0; j < newImageData?.data.length; j += 4){
+        const _r = newImageData?.data[j];
+        const _g = newImageData?.data[j+1];
+        const _b = newImageData?.data[j+2];
+        const _colorString = `rgb(${_r},${_g},${_b})`;
+        
+        if(_colorString !== colorString){
+          newImageData?.data.set([0,0,0,0], j);
+        }
+      }
+
+      newContext?.putImageData(newImageData as ImageData, 0, 0);
+
+      let imageBase64 = newCanvas.toDataURL();
+
+      if(texts.length > 0){
+        let text = texts.shift() as string;
+        if (imageBase64 != '') {
+          const processedBase64 = await writeIntoBubble(imageBase64, text);
+          const newImg = await loadImage(processedBase64);
+          context?.drawImage(newImg, 0, 0, loadImg.width, loadImg.height);
+        }
+      }
+    }
+  }
+  return canvas.toDataURL();
+}