up

src/app/globals.css

@@ -30,4 +30,10 @@ body {
 /* this is the trick to bypass the style={{}} attribute when printing */
 @media print {
   .comic-page[style] { width: 100vw !important; }
-}
+}
+
+
+.render-to-image .comic-panel {
+  height: auto !important;
+  /* max-width: fit-content !important; */
+}

src/app/interface/panel/index.tsx

@@ -35,7 +35,11 @@ export function Panel({
   const panels = useStore(state => state.panels)
   const prompt = panels[panel] || ""
 
+  const captions = useStore(state => state.captions)
+  const caption = captions[panel] || ""
+
   const zoomLevel = useStore(state => state.zoomLevel)
+  const showCaptions = useStore(state => state.showCaptions)
 
   // const setCaption = useStore(state => state.setCaption)
   // const captions = useStore(state => state.captions)
@@ -179,6 +183,7 @@ export function Panel({
   */
 
   const frameClassName = cn(
+    //`flex`,
     `w-full h-full`,
     `border-stone-800`,
     `transition-all duration-200 ease-in-out`,
@@ -214,7 +219,6 @@ export function Panel({
   }, [rendered.assetUrl, ref.current])
   */
 
-
   if (prompt && !rendered.assetUrl) {
     return (
       <div className={cn(
@@ -227,6 +231,7 @@ export function Panel({
     )
   }
 
+
   return (
     <div className={cn(
       frameClassName,
@@ -240,8 +245,33 @@ export function Panel({
           width={width}
           height={height}
           alt={rendered.alt}
-          className="w-full object-cover md:h-full md:max-w-fit print:w-full print:object-cover"
+          className={cn(
+            `comic-panel w-full h-full object-cover max-w-max`,
+            // showCaptions ? `-mt-11` : ''
+            )}
         />}
+        {/*
+              <div className={cn(
+        `flex`,
+        `bg-stone-50`,
+        `border-stone-800`,
+        `transition-all duration-200 ease-in-out`,
+        zoomLevel > 140 ? `border-b-[2px] md:border-b-[4px]` :
+        zoomLevel > 120 ? `border-b-[1.5px] md:border-b-[3px]` :
+        zoomLevel > 90 ? `border-b-[1px] md:border-b-[2px]` :
+        zoomLevel > 40 ? `border-b-[0.5px] md:border-b-[1px]` :
+        `border-transparent md:border-b-[0.5px]`,
+        `print:border-b-[1.5px]`,
+        showCaptions ? `` : `hidden`,
+        `truncate`,
+        `h-11`,
+        `p-3`
+      )}
+      style={{
+        fontSize: zoomLevel * 0.2
+      }}
+      >{caption}</div>
+    */}
     </div>
   )
 }

src/app/interface/top-menu/index.tsx

@@ -25,6 +25,7 @@ import layoutPreview1 from "../../../../public/layouts/layout1.jpg"
 import layoutPreview2 from "../../../../public/layouts/layout2.jpg"
 import layoutPreview3 from "../../../../public/layouts/layout3.jpg"
 import { StaticImageData } from "next/image"
+import { Switch } from "@/components/ui/switch"
 
 const layoutIcons: Partial<Record<LayoutName, StaticImageData>> = {
   Layout0: layoutPreview0,
@@ -41,6 +42,9 @@ export function TopMenu() {
   const layout = useStore(state => state.layout)
   const setLayout = useStore(state => state.setLayout)
 
+  const setShowCaptions = useStore(state => state.setShowCaptions)
+  const showCaptions = useStore(state => state.showCaptions)
+
   const generate = useStore(state => state.generate)
 
   const isGeneratingStory = useStore(state => state.isGeneratingStory)
@@ -200,6 +204,15 @@ export function TopMenu() {
         >
           Generate
         </Button>
+        {/*
+        <Switch
+          checked={showCaptions}
+          onCheckedChange={setShowCaptions}
+        />
+        <Label>
+          Caption
+        </Label>
+        */}
       </div>
       {/*
         Let's add this feature later, because right now people

src/app/interface/zoom/index.tsx

@@ -15,7 +15,7 @@ export function Zoom() {
       `animation-all duration-300 ease-in-out`,
       isGeneratingStory ? `scale-0 opacity-0` : ``,
     )}>
-      <div className="font-mono text-xs pb-1 text-stone-700 bg-stone-50 rounded-full">
+      <div className="font-mono text-xs pb-1 text-stone-700 bg-stone-50 p-1 rounded-sm">
         Zoom
       </div>
       <div className="w-2">

src/app/main.tsx

@@ -25,6 +25,7 @@ export default function Main() {
   const setLayouts = useStore(state => state.setLayouts)
 
   const setPanels = useStore(state => state.setPanels)
+  const setCaptions = useStore(state => state.setCaptions)
 
   const zoomLevel = useStore(state => state.zoomLevel)
 
@@ -49,13 +50,16 @@ export default function Main() {
     
         const nbPanels = 4
         const newPanels: string[] = []
+        const newCaptions: string[] = []
         setWaitABitMore(true)
 
         for (let p = 0; p < nbPanels; p++) {
-          const newPanel = [panelPromptPrefix, llmResponse[p] || ""]
+          newCaptions.push(llmResponse[p]?.caption || "...")
+          const newPanel = [panelPromptPrefix, llmResponse[p]?.instructions || ""]
           newPanels.push(newPanel.map(chunk => chunk).join(", "))
         }
         console.log("newPanels:", newPanels)
+        setCaptions(newCaptions)
         setPanels(newPanels)
       } catch (err) {
         console.error(err)

src/app/queries/getStory.ts

@@ -5,6 +5,8 @@ import { dirtyCaptionCleaner } from "@/lib/dirtyCaptionCleaner"
 
 import { predict } from "./predict"
 import { Preset } from "../engine/presets"
+import { LLMResponse } from "@/types"
+import { cleanJson } from "@/lib/cleanJson"
 
 export const getStory = async ({
   preset,
@@ -12,17 +14,17 @@ export const getStory = async ({
 }: {
   preset: Preset;
   prompt: string;
-}): Promise<string[]> => {
+}): Promise<LLMResponse> => {
 
   const query = createLlamaPrompt([
     {
       role: "system",
       content: [
         `You are a comic book author specialized in ${preset.llmPrompt}`,
-        `Please write detailed drawing instructions for the 5 panels of a new silent comic book page.`,
-        `Give your response as a JSON array like this: \`Array<{ panel: number; caption: string}>\`.`,
+        `Please write detailed drawing instructions and a one-sentence short caption for the 4 panels of a new silent comic book page.`,
+        `Give your response as a JSON array like this: \`Array<{ panel: number; instructions: string; caption: string}>\`.`,
         // `Give your response as Markdown bullet points.`,
-        `Be brief in your 5 captions, don't add your own comments. Be straight to the point, and never reply things like "Sure, I can.." etc.`
+        `Be brief in your 4 instructions and captions, don't add your own comments. Be straight to the point, and never reply things like "Sure, I can.." etc.`
       ].filter(item => item).join("\n")
     },
     {
@@ -53,44 +55,28 @@ export const getStory = async ({
   }
 
   console.log("Raw response from LLM:", result)
-  const tmp = dirtyLLMResponseCleaner(result)
+  const tmp = cleanJson(result)
   
-  let captions: string[] = []
+  let llmResponse: LLMResponse = []
 
   try {
-    captions = dirtyLLMJsonParser(tmp)
+    llmResponse = dirtyLLMJsonParser(tmp)
   } catch (err) {
     console.log(`failed to read LLM response: ${err}`)
 
-    // it is possible that the LLM has generated multiple JSON files like this:
-    
-    /*
-    [ {
-      "panel": 1,
-      "caption": "A samurai stands at the edge of a bustling street in San Francisco, looking out of place among the hippies and beatniks."
-      } ]
-
-      [ {
-      "panel": 2,
-      "caption": "The samurai spots a group of young people playing music on the sidewalk. He approaches them, intrigued."
-      } ]
-    */
-    try {
-      // in that case, we can try to repair it like so:
-      let strategy2 = `[${tmp.split("[").pop() || ""}`
-      strategy2.replaceAll("[", ",")
-
-      captions = dirtyLLMJsonParser(strategy2)
-    } catch (err2) {
       // in case of failure here, it might be because the LLM hallucinated a completely different response,
       // such as markdown. There is no real solution.. but we can try a fallback:
 
-      captions = (
-        tmp.split("*")
-        .map(item => item.replaceAll("[", "[").replaceAll("]", "]").trim())
-      )
-    }
+    llmResponse = (
+      tmp.split("*")
+      .map(item => item.trim())
+      .map((cap, i) => ({
+        panel: i,
+        caption: cap,
+        instructions: cap,
+      }))
+    )
   }
 
-  return captions.map(caption => dirtyCaptionCleaner(caption))
+  return llmResponse.map(res => dirtyCaptionCleaner(res))
 }

src/app/queries/predict.ts

@@ -17,7 +17,7 @@ export async function predict(inputs: string) {
         do_sample: true,
 
         // hard limit for max_new_tokens is 1512
-        max_new_tokens: 300, // 1150,
+        max_new_tokens: 330, // 1150,
         return_full_text: false,
       }
     })) {

src/app/store/index.ts

@@ -13,7 +13,8 @@ export const useStore = create<{
   preset: Preset
   nbFrames: number
   panels: string[]
-  captions: Record<string, string>
+  captions: string[]
+  showCaptions: boolean
   layout: LayoutName
   layouts: LayoutName[]
   zoomLevel: number
@@ -26,9 +27,10 @@ export const useStore = create<{
   setFont: (font: FontName) => void
   setPreset: (preset: Preset) => void
   setPanels: (panels: string[]) => void
+  setShowCaptions: (showCaptions: boolean) => void
   setLayout: (layout: LayoutName) => void
   setLayouts: (layouts: LayoutName[]) => void
-  setCaption: (panelId: number, caption: string) => void
+  setCaptions: (captions: string[]) => void
   setZoomLevel: (zoomLevel: number) => void
   setPage: (page: HTMLDivElement) => void
   setGeneratingStory: (isGeneratingStory: boolean) => void
@@ -40,11 +42,12 @@ export const useStore = create<{
 }>((set, get) => ({
   prompt: "",
   font: "actionman",
-  preset: getPreset("japanese_manga"),
+  preset: getRandomPreset(),
   nbFrames: 1,
   panels: [],
-  captions: {},
-  layout: "Layout1",
+  captions: [],
+  showCaptions: false,
+  layout: "random",
   layouts: getRandomLayoutNames(),
   zoomLevel: 60,
   page: undefined as unknown as HTMLDivElement,
@@ -81,12 +84,14 @@ export const useStore = create<{
     })
   },
   setPanels: (panels: string[]) => set({ panels }),
-  setCaption: (panelId: number, caption: string) => {
+  setCaptions: (captions: string[]) => {
     set({
-      captions: {
-        ...get().captions,
-        [panelId]: caption
-      }
+      captions,
+    })
+  },
+  setShowCaptions: (showCaptions: boolean) => {
+    set({
+      showCaptions,
     })
   },
   setLayout: (layoutName: LayoutName) => {
@@ -125,6 +130,7 @@ export const useStore = create<{
     const { page } = get()
     if (!page) { return "" }
     
+    
     const canvas = await html2canvas(page)
     console.log("canvas:", canvas)
 
@@ -154,7 +160,7 @@ export const useStore = create<{
     set({
       prompt,
       panels: [],
-      captions: {},
+      captions: [],
       preset: presetName === "random"
         ? getRandomPreset()
         : getPreset(presetName),

src/lib/cleanJson.ts

@@ -0,0 +1,16 @@
+import { dirtyLLMResponseCleaner } from "./dirtyLLMResponseCleaner"
+
+export function cleanJson(input: string) {
+  
+  let tmp = dirtyLLMResponseCleaner(input)
+  
+  // we only keep what's after the first [
+  tmp = `[${tmp.split("[").pop() || ""}`
+
+  // and before the first ]
+  tmp = `${tmp.split("]").shift() || ""}]`
+  
+  tmp = dirtyLLMResponseCleaner(tmp)
+  
+  return tmp 
+}

src/lib/dirtyCaptionCleaner.ts

@@ -1,3 +1,28 @@
-export function dirtyCaptionCleaner(input: string) {
-  return input.split(":").pop()?.trim() || ""
+export function dirtyCaptionCleaner({
+  panel,
+  instructions,
+  caption
+}: {
+  panel: number;
+  instructions: string;
+  caption: string
+}) {
+  return {
+    panel,
+    instructions: (
+      // need to remove from LLM garbage here, too
+      (instructions.split(":").pop() || "")
+      .replaceAll("Show a", "")
+      .replaceAll("Show the", "")
+      .replaceAll("Opens with a", "")
+      .replaceAll("Opens with the", "")
+      .replaceAll("Opens with", "")
+      .replaceAll("Cut to a", "")
+      .replaceAll("Cut to the", "")
+      .replaceAll("Cut to", "")
+      .replaceAll("End with a", "")
+      .replaceAll("End with", "").trim() || ""
+    ),
+    caption: caption.split(":").pop()?.trim() || "",
+  }
 }

src/lib/dirtyLLMJsonParser.ts

@@ -1,15 +1,28 @@
 import { LLMResponse } from "@/types"
+import { cleanJson } from "./cleanJson"
 
-export function dirtyLLMJsonParser(input: string): string[] {
-  // we only keep what's after the first [
-  let jsonOrNot = `[${input.split("[").pop() || ""}`
+export function dirtyLLMJsonParser(input: string): LLMResponse {
 
-  // and before the first ]
-  jsonOrNot = `${jsonOrNot.split("]").shift() || ""}]`
+  if (input.includes("```")) {
+    input = input.split("```")[0]
+  }
+  // we only keep what's after the first [
+  let jsonOrNot = cleanJson(input)
 
   const jsonData = JSON.parse(jsonOrNot) as LLMResponse
 
-  const captions = jsonData.map(item => item.caption.trim())
+  const results = jsonData.map((item, i) => {
+    let panel = i
+    let caption = item.caption ? item.caption.trim() : ''
+    let instructions = item.instructions ? item.instructions.trim() : ''
+    if (!instructions && caption) {
+      instructions = caption
+    } 
+    if (!caption && instructions) {
+      caption = instructions
+    }
+    return { panel, caption, instructions }
+  })
 
-  return captions
+  return results
 }

src/lib/dirtyLLMResponseCleaner.ts

@@ -1,5 +1,5 @@
 export function dirtyLLMResponseCleaner(input: string) {
-  return (
+  let str = (
     `${input || ""}`
     // a summary of all the weird hallucinations I saw it make..
     .replaceAll(`"]`, `"}]`)
@@ -10,6 +10,8 @@ export function dirtyLLMResponseCleaner(input: string) {
     .replaceAll(`"\n  ]`, `"}]`)
     .replaceAll("}}", "}")
     .replaceAll("]]", "]")
+    .replaceAll("[[", "[")
+    .replaceAll("{{", "{")
     .replaceAll(",,", ",")
     .replaceAll("[0]", "")
     .replaceAll("[1]", "")
@@ -22,4 +24,23 @@ export function dirtyLLMResponseCleaner(input: string) {
     .replaceAll("[panel 3]", "")
     .replaceAll("[panel 4]", "")
   )
+
+  // repair missing end of JSON array
+  if (str.at(-1) === '}') {
+    str = str + "]"
+  }
+
+  if (str.at(-1) === '"') {
+    str = str + "}]"
+  }
+
+  if (str[0] === '{') {
+    str = "[" + str
+  }
+  
+  if (str[0] === '"') {
+    str = "[{" + str
+  }
+
+  return str
 }

src/types.ts

@@ -79,4 +79,4 @@ export interface ImageAnalysisResponse {
   error?: string
 }
 
-export type LLMResponse = Array<{panel: number; caption: string }>
+export type LLMResponse = Array<{panel: number; instructions: string; caption: string }>