Spaces:

ibm-granite
/

granite-docling-258M-WebGPU

Running

App Files Files Community

ibibrahim

Xenova HF Staff commited on Oct 6

Commit

fbdabab

verified ·

1 Parent(s): 29d1c0c

Add WebGPU demo (#1)

Browse files

- Delete style.css (585793139918ab742943858996497d82dc451ec6)
- Delete index.js (598879cddb02ac1212a818ff2451c29e18b5ddce)
- Upload 6 files (55165a50b7a06a4230b28b7ccaace09a80c563c7)

Co-authored-by: Joshua <[email protected]>

Files changed (9) hide show

.gitattributes +1 -0
assets/chart.png +0 -0
assets/code.jpg +0 -0
assets/document.png +3 -0
assets/table.jpg +0 -0
index.html +528 -26
index.js +0 -76
parser.js +442 -0
style.css +0 -76

.gitattributes CHANGED Viewed

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text

 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+assets/document.png filter=lfs diff=lfs merge=lfs -text

assets/chart.png ADDED Viewed

assets/code.jpg ADDED Viewed

assets/document.png ADDED Viewed

Git LFS Details

SHA256: b7b3b26faea8a471b50682058ff45330ebe480c248d5171185c5d15df1537015
Pointer size: 131 Bytes
Size of remote file: 435 kB

assets/table.jpg ADDED Viewed

index.html CHANGED Viewed

@@ -1,29 +1,531 @@
-<!DOCTYPE html>
 <html lang="en">
-<head>
     <meta charset="UTF-8" />
-    <link rel="stylesheet" href="style.css" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
-    <title>Transformers.js - Object Detection</title>
-</head>
-<body>
-    <h1>Object Detection w/ 🤗 Transformers.js</h1>
-    <label id="container" for="upload">
-        <svg width="25" height="25" viewBox="0 0 25 25" fill="none" xmlns="http://www.w3.org/2000/svg">
-            <path fill="#000"
-                d="M3.5 24.3a3 3 0 0 1-1.9-.8c-.5-.5-.8-1.2-.8-1.9V2.9c0-.7.3-1.3.8-1.9.6-.5 1.2-.7 2-.7h18.6c.7 0 1.3.2 1.9.7.5.6.7 1.2.7 2v18.6c0 .7-.2 1.4-.7 1.9a3 3 0 0 1-2 .8H3.6Zm0-2.7h18.7V2.9H3.5v18.7Zm2.7-2.7h13.3c.3 0 .5 0 .6-.3v-.7l-3.7-5a.6.6 0 0 0-.6-.2c-.2 0-.4 0-.5.3l-3.5 4.6-2.4-3.3a.6.6 0 0 0-.6-.3c-.2 0-.4.1-.5.3l-2.7 3.6c-.1.2-.2.4 0 .7.1.2.3.3.6.3Z">
-            </path>
-        </svg>
-        Click to upload image
-        <label id="example">(or try example)</label>
-    </label>
-    <label id="status">Loading model...</label>
-    <input id="upload" type="file" accept="image/*" />
-    <script src="index.js" type="module"></script>
-</body>
-</html>

+<!doctype html>
 <html lang="en">
+  <head>
     <meta charset="UTF-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Granite Docling Image Converter</title>
+    <script src="https://cdn.tailwindcss.com"></script>
+    <link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&display=swap" rel="stylesheet" />
+    <style>
+      body {
+        font-family: "Inter", sans-serif;
+      }
+      .loader {
+        border-top-color: #3498db;
+        animation: spin 1s linear infinite;
+      }
+      .loader-large {
+        border: 8px solid #e5e7eb;
+        border-top: 8px solid #3498db;
+        animation: spin 1s linear infinite;
+      }
+      .loader-small {
+        border: 4px solid #e5e7eb;
+        border-top: 4px solid #3498db;
+        animation: spin 1s linear infinite;
+      }
+      @keyframes spin {
+        0% {
+          transform: rotate(0deg);
+        }
+        100% {
+          transform: rotate(360deg);
+        }
+      }
+      /* Custom toggle switch */
+      .toggle-checkbox:checked {
+        right: 0;
+        border-color: #4f46e5;
+      }
+      .toggle-checkbox:checked + .toggle-label {
+        background-color: #4f46e5;
+      }
+      .overlay {
+        background-color: rgba(255, 255, 0, 0.3);
+        border: 2px solid #fbbf24;
+        transition: background-color 0.2s;
+      }
+      .overlay:hover {
+        background-color: rgba(255, 255, 0, 0.7);
+      }
+    </style>
+  </head>
+  <body class="bg-gray-100 text-gray-800 antialiased">
+    <div id="model-loader-overlay" class="fixed inset-0 bg-black bg-opacity-60 flex flex-col items-center justify-center z-50">
+      <div class="loader-large ease-linear rounded-full h-24 w-24 mb-4"></div>
+      <h2 class="text-center text-white text-xl font-semibold">Loading Model...</h2>
+      <p class="text-center text-white text-md mt-2">This may take a moment. The model is being downloaded to your browser.</p>
+      <progress id="model-progress" value="0" max="100" class="w-64 mt-4 bg-gray-200 rounded-full h-2"></progress>
+      <p id="progress-text" class="text-center text-white text-sm mt-2">0%</p>
+    </div>
+    <main class="container mx-auto p-4 md:p-8">
+      <header class="text-center mb-8">
+        <h1 class="text-4xl font-bold text-gray-900">Granite Docling WebGPU</h1>
+        <p class="text-lg text-gray-600 mt-2">Convert document images to HTML using 🤗 Transformers.js!</p>
+      </header>
+      <div class="grid grid-cols-1 lg:grid-cols-2 gap-8">
+        <!-- Left Panel: Image Input -->
+        <div class="bg-white p-6 rounded-lg shadow-md">
+          <h2 class="text-2xl font-semibold mb-4">1. Select an Image</h2>
+          <div
+            id="image-drop-area"
+            class="border-2 border-dashed border-gray-300 rounded-lg p-8 text-center cursor-pointer transition-colors duration-200 hover:border-indigo-500 hover:bg-indigo-50"
+          >
+            <div id="image-placeholder">
+              <svg class="mx-auto h-12 w-12 text-gray-400" stroke="currentColor" fill="none" viewBox="0 0 48 48" aria-hidden="true">
+                <path
+                  d="M28 8H12a4 4 0 00-4 4v20m32-12v8m0 0v8a4 4 0 01-4 4H12a4 4 0 01-4-4v-4m32-4l-3.172-3.172a4 4 0 00-5.656 0L28 28M8 32l9.172-9.172a4 4 0 015.656 0L28 28m0 0l4 4m4-24h8m-4-4v8"
+                  stroke-width="2"
+                  stroke-linecap="round"
+                  stroke-linejoin="round"
+                />
+              </svg>
+              <p class="mt-2 text-sm text-gray-600">
+                <span class="font-semibold text-indigo-600">Drag and drop</span>
+                or click to select a file
+              </p>
+              <p class="text-xs text-gray-500">PNG, JPG, WEBP</p>
+              <input type="file" id="file-input" class="hidden" accept="image/*" />
+            </div>
+            <div id="image-preview-container" class="hidden relative">
+              <img id="image-preview" src="" alt="Selected image" class="mx-auto rounded-md shadow-sm" />
+              <button
+                id="remove-image-btn"
+                class="absolute top-2 right-2 z-10 bg-red-500 text-white rounded-full p-2 hover:bg-red-600 transition-colors focus:outline-none focus:ring-2 focus:ring-offset-2 focus:ring-red-500"
+              >
+                <svg xmlns="http://www.w3.org/2000/svg" class="h-5 w-5" viewBox="0 0 20 20" fill="currentColor">
+                  <path
+                    fill-rule="evenodd"
+                    d="M4.293 4.293a1 1 0 011.414 0L10 8.586l4.293-4.293a1 1 0 111.414 1.414L11.414 10l4.293 4.293a1 1 0 01-1.414 1.414L10 11.414l-4.293 4.293a1 1 0 01-1.414-1.414L8.586 10 4.293 5.707a1 1 0 010-1.414z"
+                    clip-rule="evenodd"
+                  />
+                </svg>
+              </button>
+            </div>
+          </div>
+          <div class="mt-4 flex">
+            <input
+              type="text"
+              id="prompt-input"
+              class="flex-1 px-3 py-2 border border-gray-300 rounded-md shadow-sm focus:outline-none focus:ring-indigo-500 focus:border-indigo-500 sm:text-sm"
+              value="Convert this page to docling."
+            />
+            <button
+              id="generate-btn"
+              class="ml-2 px-4 py-2 bg-indigo-600 text-white rounded-md hover:bg-indigo-700 disabled:bg-gray-400 disabled:cursor-not-allowed"
+            >
+              Generate
+            </button>
+          </div>
+          <h3 class="text-lg font-semibold mt-6 mb-3" id="examples-title">Or try an example:</h3>
+          <div class="flex space-x-4 overflow-x-auto" id="examples-container">
+            <img
+              src="./assets/document.png"
+              class="example-image h-36 w-auto border-2 border-gray-200 rounded-md cursor-pointer hover:border-indigo-500 transition-colors"
+              alt="Example document"
+              data-prompt="Convert this page to docling."
+              title="Document parsing"
+            />
+            <img
+              src="./assets/chart.png"
+              class="example-image h-36 w-auto border-2 border-gray-200 rounded-md cursor-pointer hover:border-indigo-500 transition-colors"
+              alt="Example chart"
+              data-prompt="Convert chart to OTSL."
+              title="Chart parsing"
+            />
+            <img
+              src="./assets/table.jpg"
+              class="example-image h-36 w-auto border-2 border-gray-200 rounded-md cursor-pointer hover:border-indigo-500 transition-colors"
+              alt="Example table"
+              data-prompt="Convert this table to OTSL."
+              title="Table parsing"
+            />
+            <img
+              src="./assets/code.jpg"
+              class="example-image h-36 w-auto border-2 border-gray-200 rounded-md cursor-pointer hover:border-indigo-500 transition-colors"
+              alt="Example code"
+              data-prompt="Convert code to text."
+              title="Code parsing"
+            />
+          </div>
+        </div>
+        <!-- Right Panel: Output -->
+        <div class="bg-white p-6 rounded-lg shadow-md flex flex-col">
+          <div class="flex justify-between items-center mb-4">
+            <h2 class="text-2xl font-semibold">2. View Result</h2>
+            <div id="processing-indicator" class="flex items-center space-x-2 text-gray-500 hidden">
+              <div class="loader-small ease-linear rounded-full h-6 w-6"></div>
+              <p class="text-sm">Processing image...</p>
+            </div>
+            <div class="flex items-center space-x-2">
+              <span class="text-sm font-medium">Docling</span>
+              <div class="relative inline-block w-10 mr-2 align-middle select-none transition duration-200 ease-in">
+                <input
+                  type="checkbox"
+                  name="toggle"
+                  id="view-toggle"
+                  class="toggle-checkbox absolute block w-6 h-6 rounded-full bg-white border-4 appearance-none cursor-pointer"
+                  checked
+                />
+                <label for="view-toggle" class="toggle-label block overflow-hidden h-6 rounded-full bg-gray-300 cursor-pointer"></label>
+              </div>
+              <span class="text-sm font-medium text-indigo-600">HTML</span>
+            </div>
+          </div>
+          <div id="output-container" class="flex-1 border border-gray-200 rounded-lg overflow-hidden bg-gray-50">
+            <div id="welcome-message" class="h-full flex items-center justify-center text-center text-gray-500">
+              <p>Select an image to see the result here.</p>
+            </div>
+            <!-- Docling Output -->
+            <div id="docling-view" class="h-full p-4 hidden">
+              <pre class="h-full whitespace-pre-wrap text-sm overflow-auto"><code id="docling-output"></code></pre>
+            </div>
+            <!-- HTML Output -->
+            <div id="html-view" class="h-full w-full">
+              <iframe id="html-iframe" sandbox="allow-scripts" class="w-full h-full border-0"></iframe>
+            </div>
+          </div>
+        </div>
+      </div>
+    </main>
+    <!-- Hidden canvas for image processing -->
+    <canvas id="hidden-canvas" class="hidden"></canvas>
+    <script type="module">
+      import { AutoProcessor, AutoModelForVision2Seq, RawImage, TextStreamer, load_image } from "https://cdn.jsdelivr.net/npm/@huggingface/[email protected]";
+      import { doclingToHtml } from "./parser.js";
+      const modelLoaderOverlay = document.getElementById("model-loader-overlay");
+      const imageDropArea = document.getElementById("image-drop-area");
+      const imagePlaceholder = document.getElementById("image-placeholder");
+      const imagePreviewContainer = document.getElementById("image-preview-container");
+      const imagePreview = document.getElementById("image-preview");
+      const removeImageBtn = document.getElementById("remove-image-btn");
+      const fileInput = document.getElementById("file-input");
+      const exampleImages = document.querySelectorAll(".example-image");
+      const examplesContainer = document.getElementById("examples-container");
+      const examplesTitle = document.getElementById("examples-title");
+      const processingIndicator = document.getElementById("processing-indicator");
+      const welcomeMessage = document.getElementById("welcome-message");
+      const doclingView = document.getElementById("docling-view");
+      const htmlView = document.getElementById("html-view");
+      const doclingOutput = document.getElementById("docling-output");
+      const htmlIframe = document.getElementById("html-iframe");
+      const viewToggle = document.getElementById("view-toggle");
+      const hiddenCanvas = document.getElementById("hidden-canvas");
+      const promptInput = document.getElementById("prompt-input");
+      const generateBtn = document.getElementById("generate-btn");
+      let model, processor;
+      let currentImageWidth, currentImageHeight;
+      let currentImage = null;
+      /**
+       * Loads and initializes the model and processor.
+       */
+      async function initializeModel() {
+        try {
+          const model_id = "onnx-community/granite-docling-258M-ONNX";
+          processor = await AutoProcessor.from_pretrained(model_id);
+          const progress = {};
+          model = await AutoModelForVision2Seq.from_pretrained(model_id, {
+            dtype: {
+              embed_tokens: "fp16", // fp32 (231 MB) | fp16 (116 MB)
+              vision_encoder: "fp32", // fp32 (374 MB)
+              decoder_model_merged: "fp32", // fp32 (658 MB) | q4 (105 MB), q4 sometimes into repetition issues
+            },
+            device: "webgpu",
+            progress_callback: (data) => {
+              if (data.status === "progress" && data.file?.endsWith?.("onnx_data")) {
+                progress[data.file] = data;
+                const progressPercent = Math.round(data.progress);
+                if (Object.keys(progress).length !== 3) return;
+                let sum = 0;
+                let total = 0;
+                for (const [key, val] of Object.entries(progress)) {
+                  sum += val.loaded;
+                  total += val.total;
+                }
+                const overallPercent = Math.round((sum / total) * 100);
+                document.getElementById("model-progress").value = overallPercent;
+                document.getElementById("progress-text").textContent = overallPercent + "%";
+              }
+            },
+          });
+          modelLoaderOverlay.style.display = "none";
+          console.log("Model loaded successfully.");
+        } catch (error) {
+          console.error("Failed to load model:", error);
+          modelLoaderOverlay.innerHTML = `
+                    <h2 class="text-center text-red-500 text-xl font-semibold">Failed to Load Model</h2>
+                    <p class="text-center text-white text-md mt-2">Please refresh the page to try again. Check the console for errors.</p>
+                `;
+        }
+      }
+      /**
+       * Processes an image and generates Docling text.
+       * @param {ImageBitmap|HTMLImageElement} imageObject An image object to process.
+       */
+      async function processImage(imageObject) {
+        if (!model || !processor) {
+          alert("Model is not loaded yet. Please wait.");
+          return;
+        }
+        // Reset UI
+        setUiState("processing");
+        clearOverlays();
+        let fullText = "";
+        doclingOutput.textContent = "";
+        htmlIframe.srcdoc = "";
+        try {
+          // 1. Draw image to canvas and get RawImage
+          const ctx = hiddenCanvas.getContext("2d");
+          hiddenCanvas.width = imageObject.width;
+          hiddenCanvas.height = imageObject.height;
+          ctx.drawImage(imageObject, 0, 0);
+          const image = RawImage.fromCanvas(hiddenCanvas);
+          // 2. Create input messages
+          const messages = [
+            {
+              role: "user",
+              content: [{ type: "image" }, { type: "text", text: promptInput.value }],
+            },
+          ];
+          // 3. Prepare inputs for the model
+          const text = processor.apply_chat_template(messages, {
+            add_generation_prompt: true,
+          });
+          const inputs = await processor(text, [image], {
+            do_image_splitting: true,
+          });
+          // 5. Generate output
+          await model.generate({
+            ...inputs,
+            max_new_tokens: 4096,
+            streamer: new TextStreamer(processor.tokenizer, {
+              skip_prompt: true,
+              skip_special_tokens: false,
+              callback_function: (streamedText) => {
+                fullText += streamedText;
+                doclingOutput.textContent += streamedText;
+              },
+            }),
+          });
+          // Strip <|end_of_text|> from the end
+          fullText = fullText.replace(/<\|end_of_text\|>$/, "");
+          doclingOutput.textContent = fullText;
+          // Parse loc tags and create overlays
+          const locRegex = /<loc_(\d+)>/g;
+          const locMatches = [];
+          let match;
+          while ((match = locRegex.exec(fullText)) !== null) {
+            locMatches.push(parseInt(match[1]));
+          }
+          const overlays = [];
+          for (let i = 0; i < locMatches.length; i += 4) {
+            if (i + 3 < locMatches.length) {
+              overlays.push(locMatches.slice(i, i + 4));
+            }
+          }
+          const imgRect = imagePreview.getBoundingClientRect();
+          const containerRect = imagePreviewContainer.getBoundingClientRect();
+          const imageOffsetLeft = imgRect.left - containerRect.left;
+          const imageOffsetTop = imgRect.top - containerRect.top;
+          const scaleX = imgRect.width / currentImageWidth;
+          const scaleY = imgRect.height / currentImageHeight;
+          overlays.forEach(([leftLoc, topLoc, rightLoc, bottomLoc]) => {
+            const left = imageOffsetLeft + (leftLoc / 500) * currentImageWidth * scaleX;
+            const top = imageOffsetTop + (topLoc / 500) * currentImageHeight * scaleY;
+            const width = ((rightLoc - leftLoc) / 500) * currentImageWidth * scaleX;
+            const height = ((bottomLoc - topLoc) / 500) * currentImageHeight * scaleY;
+            const overlay = document.createElement("div");
+            overlay.className = "overlay";
+            overlay.style.position = "absolute";
+            overlay.style.left = left + "px";
+            overlay.style.top = top + "px";
+            overlay.style.width = width + "px";
+            overlay.style.height = height + "px";
+            imagePreviewContainer.appendChild(overlay);
+          });
+          // After generation, create the HTML iframe
+          htmlIframe.srcdoc = doclingToHtml(fullText);
+        } catch (error) {
+          console.error("Error during image processing:", error);
+          doclingOutput.textContent = `An error occurred: ${error.message}`;
+        } finally {
+          setUiState("result");
+        }
+      }
+      /**
+       * Handles the selection of an image file.
+       * @param {File|string} source The image file or URL.
+       */
+      function handleImageSelection(source) {
+        const reader = new FileReader();
+        const img = new Image();
+        img.onload = () => {
+          currentImageWidth = img.naturalWidth;
+          currentImageHeight = img.naturalHeight;
+          currentImage = img;
+          imagePreview.src = img.src;
+          imagePlaceholder.classList.add("hidden");
+          imagePreviewContainer.classList.remove("hidden");
+          examplesContainer.classList.add("hidden");
+          examplesTitle.classList.add("hidden");
+          processImage(img);
+        };
+        img.onerror = () => {
+          alert("Failed to load image.");
+        };
+        if (typeof source === "string") {
+          // It's a URL
+          // To avoid CORS issues with canvas, we can try to fetch it first
+          fetch(source)
+            .then((res) => res.blob())
+            .then((blob) => {
+              img.src = URL.createObjectURL(blob);
+            })
+            .catch((e) => {
+              console.error("CORS issue likely. Trying proxy or direct load.", e);
+              // Fallback to direct load which might taint the canvas
+              img.crossOrigin = "anonymous";
+              img.src = source;
+            });
+        } else {
+          // It's a File object
+          reader.onload = (e) => {
+            img.src = e.target.result;
+          };
+          reader.readAsDataURL(source);
+        }
+      }
+      /**
+       * Manages the visibility of UI components based on the app state.
+       * @param {'initial'|'processing'|'result'} state The current state.
+       */
+      function setUiState(state) {
+        welcomeMessage.style.display = "none";
+        processingIndicator.classList.add("hidden");
+        doclingView.classList.add("hidden");
+        htmlView.classList.add("hidden");
+        if (state === "initial") {
+          welcomeMessage.style.display = "flex";
+          generateBtn.disabled = true;
+        } else if (state === "processing") {
+          viewToggle.checked = false;
+          processingIndicator.classList.remove("hidden");
+          doclingView.classList.remove("hidden");
+          generateBtn.disabled = true;
+        } else if (state === "result") {
+          viewToggle.checked = true;
+          htmlView.classList.remove("hidden");
+          generateBtn.disabled = false;
+        }
+      }
+      /**
+       * Clears all overlay divs from the image preview container.
+       */
+      function clearOverlays() {
+        document.querySelectorAll(".overlay").forEach((el) => el.remove());
+      }
+      // Drag and Drop
+      imageDropArea.addEventListener("click", () => fileInput.click());
+      imageDropArea.addEventListener("dragover", (e) => {
+        e.preventDefault();
+        imageDropArea.classList.add("border-indigo-500", "bg-indigo-50");
+      });
+      imageDropArea.addEventListener("dragleave", () => {
+        imageDropArea.classList.remove("border-indigo-500", "bg-indigo-50");
+      });
+      imageDropArea.addEventListener("drop", (e) => {
+        e.preventDefault();
+        imageDropArea.classList.remove("border-indigo-500", "bg-indigo-50");
+        const files = e.dataTransfer.files;
+        if (files.length > 0 && files[0].type.startsWith("image/")) {
+          handleImageSelection(files[0]);
+        }
+      });
+      // File input
+      fileInput.addEventListener("change", (e) => {
+        const files = e.target.files;
+        if (files.length > 0) {
+          handleImageSelection(files[0]);
+        }
+      });
+      // Example images
+      exampleImages.forEach((img) => {
+        img.addEventListener("click", () => {
+          promptInput.value = img.dataset.prompt;
+          handleImageSelection(img.src);
+        });
+      });
+      // Remove image
+      removeImageBtn.addEventListener("click", (e) => {
+        e.stopPropagation();
+        currentImage = null;
+        imagePreview.src = "";
+        fileInput.value = ""; // Reset file input
+        imagePlaceholder.classList.remove("hidden");
+        imagePreviewContainer.classList.add("hidden");
+        examplesContainer.classList.remove("hidden");
+        examplesTitle.classList.remove("hidden");
+        setUiState("initial");
+        doclingOutput.textContent = "";
+        htmlIframe.srcdoc = "";
+        clearOverlays();
+      });
+      // View toggle
+      viewToggle.addEventListener("change", () => {
+        const isHtmlView = viewToggle.checked;
+        htmlView.classList.toggle("hidden", !isHtmlView);
+        doclingView.classList.toggle("hidden", isHtmlView);
+      });
+      // Generate button
+      generateBtn.addEventListener("click", () => {
+        if (currentImage) {
+          processImage(currentImage);
+        }
+      });
+      document.addEventListener("DOMContentLoaded", () => {
+        setUiState("initial"); // Set initial view correctly
+        initializeModel();
+      });
+    </script>
+  </body>
+</html>

index.js DELETED Viewed

@@ -1,76 +0,0 @@
-import { pipeline } from 'https://cdn.jsdelivr.net/npm/@huggingface/[email protected]';
-// Reference the elements that we will need
-const status = document.getElementById('status');
-const fileUpload = document.getElementById('upload');
-const imageContainer = document.getElementById('container');
-const example = document.getElementById('example');
-const EXAMPLE_URL = 'https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/city-streets.jpg';
-// Create a new object detection pipeline
-status.textContent = 'Loading model...';
-const detector = await pipeline('object-detection', 'Xenova/detr-resnet-50');
-status.textContent = 'Ready';
-example.addEventListener('click', (e) => {
-    e.preventDefault();
-    detect(EXAMPLE_URL);
-});
-fileUpload.addEventListener('change', function (e) {
-    const file = e.target.files[0];
-    if (!file) {
-        return;
-    }
-    const reader = new FileReader();
-    // Set up a callback when the file is loaded
-    reader.onload = e2 => detect(e2.target.result);
-    reader.readAsDataURL(file);
-});
-// Detect objects in the image
-async function detect(img) {
-    imageContainer.innerHTML = '';
-    imageContainer.style.backgroundImage = `url(${img})`;
-    status.textContent = 'Analysing...';
-    const output = await detector(img, {
-        threshold: 0.5,
-        percentage: true,
-    });
-    status.textContent = '';
-    output.forEach(renderBox);
-}
-// Render a bounding box and label on the image
-function renderBox({ box, label }) {
-    const { xmax, xmin, ymax, ymin } = box;
-    // Generate a random color for the box
-    const color = '#' + Math.floor(Math.random() * 0xFFFFFF).toString(16).padStart(6, 0);
-    // Draw the box
-    const boxElement = document.createElement('div');
-    boxElement.className = 'bounding-box';
-    Object.assign(boxElement.style, {
-        borderColor: color,
-        left: 100 * xmin + '%',
-        top: 100 * ymin + '%',
-        width: 100 * (xmax - xmin) + '%',
-        height: 100 * (ymax - ymin) + '%',
-    })
-    // Draw label
-    const labelElement = document.createElement('span');
-    labelElement.textContent = label;
-    labelElement.className = 'bounding-box-label';
-    labelElement.style.backgroundColor = color;
-    boxElement.appendChild(labelElement);
-    imageContainer.appendChild(boxElement);
-}

parser.js ADDED Viewed

	@@ -0,0 +1,442 @@

+export class DoclingConverter {
+  constructor() {
+    this.simpleTagMap = {
+      doctag: "div",
+      document: "div",
+      ordered_list: "ol",
+      unordered_list: "ul",
+      list_item: "li",
+      caption: "figcaption",
+      footnote: "sup",
+      formula: "div",
+      page_footer: "footer",
+      page_header: "header",
+      picture: "figure",
+      chart: "figure",
+      table: "table",
+      otsl: "table",
+      text: "p",
+      paragraph: "p",
+      title: "h1",
+      document_index: "div",
+      form: "form",
+      key_value_region: "dl",
+      reference: "a",
+      smiles: "span",
+    };
+    this.selfClosingTagMap = {
+      checkbox_selected: '<input type="checkbox" checked disabled>',
+      checkbox_unselected: '<input type="checkbox" disabled>',
+      page_break: '<hr class="page-break">',
+    };
+    this.TABLE_TAG_CONFIG = {
+      "<ched>": { htmlTag: "th" },
+      "<rhed>": { htmlTag: "th", scope: "row" },
+      "<srow>": { htmlTag: "th", scope: "row" },
+      "<fcel>": { htmlTag: "td" },
+      "<ecel>": { htmlTag: "td" },
+      "<ucel>": { htmlTag: "td" },
+      "<lcel>": { htmlTag: "td" },
+      "<xcel>": { htmlTag: "td" },
+    };
+    this.TABLE_TAG_REGEX = new RegExp(`(${Object.keys(this.TABLE_TAG_CONFIG).join("|")})`);
+    const selfClosingNames = Object.keys(this.selfClosingTagMap).join("|");
+    this.combinedTagRegex = new RegExp(`(<([a-z_0-9]+)>(.*?)<\\/\\2>)|(<(${selfClosingNames})>)`, "s");
+  }
+  escapeHtml(text) {
+    if (!text) return "";
+    return text.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;");
+  }
+  convert(docling) {
+    let html = ` ${docling} `;
+    html = this.cleanupMetadataTokens(html);
+    html = this.processTags(html);
+    return html.trim();
+  }
+  processTags(text) {
+    let remainingText = text;
+    let result = "";
+    while (remainingText.length > 0) {
+      const match = remainingText.match(this.combinedTagRegex);
+      if (match && typeof match.index === "number") {
+        const textBefore = remainingText.substring(0, match.index);
+        result += this.escapeHtml(textBefore);
+        const fullMatch = match[0];
+        const pairedTagName = match[2];
+        const pairedContent = match[3];
+        const selfClosingTagName = match[5];
+        if (pairedTagName !== undefined) {
+          result += this.convertSingleTag(pairedTagName, pairedContent);
+        } else if (selfClosingTagName !== undefined) {
+          result += this.selfClosingTagMap[selfClosingTagName] || "";
+        }
+        remainingText = remainingText.substring(match.index + fullMatch.length);
+      } else {
+        result += this.escapeHtml(remainingText);
+        break;
+      }
+    }
+    return result;
+  }
+  convertSingleTag(tagName, content) {
+    if (tagName === "list_item") {
+      content = content.trim().replace(/^[·-]\s*/g, "");
+    }
+    switch (tagName) {
+      case "code":
+        return this.convertBlockCode(content);
+      case "otsl":
+        return this.convertTable(content);
+      case "picture":
+      case "chart":
+        return this.convertPictureOrChart(tagName, content);
+      case "inline":
+        return this.convertInlineContent(content);
+      case "section_header_level_0":
+      case "section_header_level_1":
+      case "section_header_level_2":
+      case "section_header_level_3":
+      case "section_header_level_4":
+      case "section_header_level_5":
+        const level = parseInt(tagName.at(-1), 10) + 1;
+        return `<h${level}>${this.processTags(content)}</h${level}>`;
+      default:
+        const htmlTag = this.simpleTagMap[tagName];
+        if (htmlTag) {
+          const processedContent = this.processTags(content);
+          const startTag = this.getStartTag(tagName, htmlTag);
+          return `${startTag}${processedContent}</${htmlTag}>`;
+        }
+        console.warn(`Unknown tag encountered: ${tagName}, escaping it.`);
+        return this.escapeHtml(`<${tagName}>${content}</${tagName}>`);
+    }
+  }
+  getStartTag(doclingTag, htmlTag) {
+    switch (doclingTag) {
+      case "doctag":
+      case "document":
+        return '<div class="docling-document">';
+      case "formula":
+        return '<div class="formula">';
+      case "document_index":
+        return '<div class="toc">';
+      case "smiles":
+        return '<span class="smiles">';
+      case "reference":
+        return '<a href="#">';
+      default:
+        return `<${htmlTag}>`;
+    }
+  }
+  convertInlineContent(content) {
+    const inlineTagRegex = /<(code|formula|text|smiles)>(.*?)<\/\1>/s;
+    let remainingText = content;
+    let result = "";
+    while (remainingText.length > 0) {
+      const match = remainingText.match(inlineTagRegex);
+      if (match && typeof match.index === "number") {
+        const textBefore = remainingText.substring(0, match.index);
+        result += this.escapeHtml(textBefore);
+        const [fullMatch, tagName, innerContent] = match;
+        switch (tagName) {
+          case "code":
+            const langRegex = /<_(.*?)_>/;
+            const langMatch = innerContent.match(langRegex);
+            if (langMatch && langMatch[1]) {
+              const language = this.sanitizeLanguageName(langMatch[1]);
+              const codeContent = innerContent.replace(langRegex, "").trim();
+              const escapedCode = this.escapeHtml(codeContent);
+              const langClass = language !== "unknown" ? ` class="language-${language}"` : "";
+              result += `<code${langClass}>${escapedCode}</code>`;
+            } else {
+              result += `<code>${this.escapeHtml(innerContent)}</code>`;
+            }
+            break;
+          case "formula":
+            result += `<span class="formula">${this.escapeHtml(innerContent)}</span>`;
+            break;
+          case "smiles":
+            result += `<span class="smiles">${this.escapeHtml(innerContent)}</span>`;
+            break;
+          case "text":
+            result += this.escapeHtml(innerContent);
+            break;
+        }
+        remainingText = remainingText.substring(match.index + fullMatch.length);
+      } else {
+        result += this.escapeHtml(remainingText);
+        break;
+      }
+    }
+    return result;
+  }
+  convertBlockCode(content) {
+    const langRegex = /<_(.*?)_>/;
+    const langMatch = content.match(langRegex);
+    let language = "unknown";
+    let codeContent = content;
+    if (langMatch && langMatch[1]) {
+      language = this.sanitizeLanguageName(langMatch[1]);
+      codeContent = content.replace(langRegex, "").trim();
+    }
+    const escapedCode = this.escapeHtml(codeContent);
+    const langClass = language !== "unknown" ? ` class="language-${language}"` : "";
+    return `<pre><code${langClass}>${escapedCode}</code></pre>`;
+  }
+  convertTable(content) {
+    const rows = content
+      .trim()
+      .split(/<nl>/)
+      .filter((row) => row.length > 0);
+    const cellGrid = [];
+    rows.forEach((rowStr, rowIndex) => {
+      var _a;
+      const parts = rowStr.split(this.TABLE_TAG_REGEX);
+      const currentRow = [];
+      let gridColIndex = 0;
+      for (let i = 1; i < parts.length; i += 2) {
+        const tag = parts[i];
+        const cellContent = parts[i + 1] || "";
+        switch (tag) {
+          case "<lcel>":
+            if (currentRow.length > 0) {
+              currentRow[currentRow.length - 1].colspan++;
+            }
+            break;
+          case "<ucel>":
+            if (rowIndex > 0 && ((_a = cellGrid[rowIndex - 1]) === null || _a === void 0 ? void 0 : _a[gridColIndex])) {
+              cellGrid[rowIndex - 1][gridColIndex].rowspan++;
+            }
+            gridColIndex++;
+            break;
+          case "<xcel>":
+            if (currentRow.length > 0) {
+              currentRow[currentRow.length - 1].colspan++;
+            }
+            break;
+          default:
+            if (this.TABLE_TAG_CONFIG[tag]) {
+              currentRow.push({
+                content: cellContent,
+                tag,
+                colspan: 1,
+                rowspan: 1,
+              });
+              gridColIndex++;
+            }
+            break;
+        }
+      }
+      cellGrid.push(currentRow);
+    });
+    const htmlRows = cellGrid
+      .map((row) => {
+        const cellsHtml = row
+          .map((cell) => {
+            const config = this.TABLE_TAG_CONFIG[cell.tag];
+            if (!config) return "";
+            const attrs = [];
+            if (cell.colspan > 1) attrs.push(`colspan="${cell.colspan}"`);
+            if (cell.rowspan > 1) attrs.push(`rowspan="${cell.rowspan}"`);
+            if (config.scope) attrs.push(`scope="${config.scope}"`);
+            const processedContent = this.processTags(cell.content);
+            const attrString = attrs.length > 0 ? ` ${attrs.join(" ")}` : "";
+            return `<${config.htmlTag}${attrString}>${processedContent}</${config.htmlTag}>`;
+          })
+          .join("");
+        return `<tr>${cellsHtml}</tr>`;
+      })
+      .join("");
+    return `<table><tbody>${htmlRows}</tbody></table>`;
+  }
+  convertPictureOrChart(tag, content) {
+    if (/<(fcel|ched|rhed)>/.test(content)) {
+      const cleanedContent = content.replace(/<[a-z_]+>/g, (match) => {
+        if (match.startsWith("<fcel") || match.startsWith("<ched") || match.startsWith("<rhed") || match.startsWith("<nl")) {
+          return match;
+        }
+        return "";
+      });
+      return this.convertTable(cleanedContent);
+    }
+    let captionHtml = "";
+    const captionRegex = /<caption>(.*?)<\/caption>/s;
+    const captionMatch = content.match(captionRegex);
+    if (captionMatch && captionMatch[1]) {
+      const captionContent = this.processTags(captionMatch[1]);
+      captionHtml = `<figcaption>${captionContent}</figcaption>`;
+    }
+    const contentWithoutCaption = content.replace(captionRegex, "");
+    const classificationRegex = /<([a-z_]+)>/;
+    const classMatch = contentWithoutCaption.match(classificationRegex);
+    let altText = tag;
+    if (classMatch) {
+      altText = classMatch[1].replace(/_/g, " ");
+    }
+    const imgHtml = `<img alt="${this.escapeHtml(altText)}" src="">`;
+    const figureTag = this.simpleTagMap[tag] || "figure";
+    return `<${figureTag}>${imgHtml}${captionHtml}</${figureTag}>`;
+  }
+  sanitizeLanguageName(lang) {
+    const lowerLang = lang.toLowerCase();
+    const aliasMap = {
+      "c#": "csharp",
+      "c++": "cpp",
+      objectivec: "objective-c",
+      visualbasic: "vb",
+      javascript: "js",
+      typescript: "ts",
+      python: "py",
+      ruby: "rb",
+      dockerfile: "docker",
+    };
+    return aliasMap[lowerLang] || lowerLang.replace(/[\s#+]/g, "-");
+  }
+  cleanupMetadataTokens(docling) {
+    return docling.replace(/<loc_[0-9]+>/g, "");
+  }
+}
+export function doclingToHtml(docling) {
+  const converter = new DoclingConverter();
+  const body = converter.convert(docling);
+  return `<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset="UTF-8">
+    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/[email protected]/dist/katex.min.css" integrity="sha384-//SZkxyB7axjCAopkAL1E1rve+ZSPKapD89Lo/lLhcsXR+zOYl5z6zJZEFXil+q0" crossorigin="anonymous">
+    <style>
+        html {
+            background-color: #f5f5f5;
+            font-family: Arial, sans-serif;
+            line-height: 1.6;
+        }
+        header, footer {
+            text-align: center;
+            margin-bottom: 1rem;
+            font-size: 1em;
+        }
+        body {
+            max-width: 800px;
+            margin: 0 auto;
+            padding: 2rem;
+            background-color: white;
+            box-shadow: 0 0 10px rgba(0,0,0,0.1);
+        }
+        h1, h2, h3, h4, h5, h6 {
+            color: #333;
+            margin-top: 1.5em;
+            margin-bottom: 0.5em;
+        }
+        h1 {
+            font-size: 2em;
+            border-bottom: 1px solid #eee;
+            padding-bottom: 0.3em;
+        }
+        table {
+            border-collapse: collapse;
+            margin: 1em 0;
+            width: 100%;
+        }
+        th, td {
+            border: 1px solid #ddd;
+            padding: 8px;
+            text-align: left;
+        }
+        th {
+            background-color: #f2f2f2;
+            font-weight: bold;
+        }
+        figure {
+            margin: 1.5em 0;
+            text-align: center;
+        }
+        figcaption {
+            color: #666;
+            font-style: italic;
+            margin-top: 0.5em;
+        }
+        img {
+            max-width: 100%;
+            height: auto;
+        }
+        pre {
+            background-color: #f6f8fa;
+            border-radius: 3px;
+            padding: 1em;
+            overflow: auto;
+        }
+        code {
+            font-family: monospace;
+            background-color: #f6f8fa;
+            padding: 0.2em 0.4em;
+            border-radius: 3px;
+        }
+        pre code {
+            background-color: transparent;
+            padding: 0;
+        }
+        .formula {
+            text-align: center;
+            padding: 0.5em;
+            margin: 1em 0;
+        }
+        .formula:not(:has(.katex)) {
+            color: transparent;
+        }
+        .page-break {
+            page-break-after: always;
+            border-top: 1px dashed #ccc;
+            margin: 2em 0;
+        }
+        .key-value-region {
+            background-color: #f9f9f9;
+            padding: 1em;
+            border-radius: 4px;
+            margin: 1em 0;
+        }
+        .key-value-region dt {
+            font-weight: bold;
+        }
+        .key-value-region dd {
+            margin-left: 1em;
+            margin-bottom: 0.5em;
+        }
+        .form-container {
+            border: 1px solid #ddd;
+            padding: 1em;
+            border-radius: 4px;
+            margin: 1em 0;
+        }
+        .form-item {
+            margin-bottom: 0.5em;
+        }
+    </style>
+    </head>
+<body>
+${body}
+<script type="module">
+import katex from 'https://cdn.jsdelivr.net/npm/[email protected]/dist/katex.mjs';
+import renderMathInElement from "https://cdn.jsdelivr.net/npm/[email protected]/dist/contrib/auto-render.mjs";
+const mathElements = document.querySelectorAll('.formula');
+for (let element of mathElements) {
+    katex.render(element.textContent, element, {
+        throwOnError: false,
+    });
+}
+renderMathInElement(document.body, {
+    delimiters: [
+        {left: "$$", right: "$$", display: true},
+        {left: "\\\\[", right: "\\\\]", display: true},
+        {left: "$", right: "$", display: false},
+        {left: "\\\\(", right: "\\\\)", display: false}
+    ],
+    throwOnError : false,
+});
+  </script>
+</body>
+</html>`;
+}

style.css DELETED Viewed

@@ -1,76 +0,0 @@
-* {
-    box-sizing: border-box;
-    padding: 0;
-    margin: 0;
-    font-family: sans-serif;
-}
-html,
-body {
-    height: 100%;
-}
-body {
-    padding: 32px;
-}
-body,
-#container {
-    display: flex;
-    flex-direction: column;
-    justify-content: center;
-    align-items: center;
-}
-#container {
-    position: relative;
-    gap: 0.4rem;
-    width: 640px;
-    height: 640px;
-    max-width: 100%;
-    max-height: 100%;
-    border: 2px dashed #D1D5DB;
-    border-radius: 0.75rem;
-    overflow: hidden;
-    cursor: pointer;
-    margin: 1rem;
-    background-size: 100% 100%;
-    background-position: center;
-    background-repeat: no-repeat;
-    font-size: 18px;
-}
-#upload {
-    display: none;
-}
-svg {
-    pointer-events: none;
-}
-#example {
-    font-size: 14px;
-    text-decoration: underline;
-    cursor: pointer;
-}
-#example:hover {
-    color: #2563EB;
-}
-.bounding-box {
-    position: absolute;
-    box-sizing: border-box;
-    border: solid 2px;
-}
-.bounding-box-label {
-    color: white;
-    position: absolute;
-    font-size: 12px;
-    margin: -16px 0 0 -2px;
-    padding: 1px;
-}