upload demo notebook

Gliese-OCR-7B-Post2.0-final(4bit)/Gliese_OCR_7B_Post2_0_final.ipynb

@@ -0,0 +1,299 @@
+{
+  "nbformat": 4,
+  "nbformat_minor": 0,
+  "metadata": {
+    "colab": {
+      "provenance": [],
+      "gpuType": "T4"
+    },
+    "kernelspec": {
+      "name": "python3",
+      "display_name": "Python 3"
+    },
+    "language_info": {
+      "name": "python"
+    },
+    "accelerator": "GPU"
+  },
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "source": [
+        "# **Gliese-OCR-7B-Post2.0-final**"
+      ],
+      "metadata": {
+        "id": "zeCQura_Ri5N"
+      }
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "\n",
+        "The [Gliese-OCR-7B-Post2.0-final](https://huggingface.co/prithivMLmods/Gliese-OCR-7B-Post2.0-final) model is a refined and optimized version of Gliese-OCR-7B-Post1.0, built upon the Qwen2.5-VL architecture. It represents the final iteration in the Gliese-OCR series, offering enhanced efficiency, precision, and visualization capabilities for document OCR, visual analysis, and information extraction.\n",
+        "\n",
+        "Fine-tuned with extended document visualization data and OCR-focused objectives, this model delivers superior accuracy across a wide range of document types, including scanned PDFs, handwritten pages, structured forms, and analytical reports."
+      ],
+      "metadata": {
+        "id": "pTrk5nv-HAMV"
+      }
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "oXHcxUMZGah0"
+      },
+      "outputs": [],
+      "source": [
+        "%%capture\n",
+        "!pip install git+https://github.com/huggingface/accelerate.git \\\n",
+        "             git+https://github.com/huggingface/peft.git \\\n",
+        "             transformers-stream-generator huggingface_hub albumentations \\\n",
+        "             pyvips-binary qwen-vl-utils sentencepiece opencv-python docling-core \\\n",
+        "             python-docx torchvision safetensors matplotlib num2words \\\n",
+        "\n",
+        "!pip install transformers requests pymupdf hf_xet spaces pyvips pillow gradio \\\n",
+        "             einops torch fpdf timm av decord bitsandbytes\n",
+        "#Hold tight, this will take around 1-2 minutes."
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "### **Load Gliese-OCR-7B-Post2.0 with 4-bit quantization**"
+      ],
+      "metadata": {
+        "id": "5vGwuV-4HaJv"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "import os\n",
+        "import sys\n",
+        "import random\n",
+        "import uuid\n",
+        "import json\n",
+        "import time\n",
+        "from threading import Thread\n",
+        "from typing import Iterable\n",
+        "from huggingface_hub import snapshot_download\n",
+        "\n",
+        "import gradio as gr\n",
+        "import spaces\n",
+        "import torch\n",
+        "import numpy as np\n",
+        "from PIL import Image\n",
+        "import cv2\n",
+        "\n",
+        "from transformers import (\n",
+        "    Qwen2_5_VLForConditionalGeneration,\n",
+        "    AutoProcessor,\n",
+        "    TextIteratorStreamer,\n",
+        "    BitsAndBytesConfig,\n",
+        ")\n",
+        "\n",
+        "from transformers.image_utils import load_image\n",
+        "from gradio.themes import Soft\n",
+        "from gradio.themes.utils import colors, fonts, sizes\n",
+        "\n",
+        "# --- Theme and CSS Setup ---\n",
+        "colors.steel_blue = colors.Color(\n",
+        "    name=\"steel_blue\",\n",
+        "    c50=\"#EBF3F8\",\n",
+        "    c100=\"#D3E5F0\",\n",
+        "    c200=\"#A8CCE1\",\n",
+        "    c300=\"#7DB3D2\",\n",
+        "    c400=\"#529AC3\",\n",
+        "    c500=\"#4682B4\",\n",
+        "    c600=\"#3E72A0\",\n",
+        "    c700=\"#36638C\",\n",
+        "    c800=\"#2E5378\",\n",
+        "    c900=\"#264364\",\n",
+        "    c950=\"#1E3450\",\n",
+        ")\n",
+        "\n",
+        "class SteelBlueTheme(Soft):\n",
+        "    def __init__(\n",
+        "        self,\n",
+        "        *,\n",
+        "        primary_hue: colors.Color | str = colors.gray,\n",
+        "        secondary_hue: colors.Color | str = colors.steel_blue,\n",
+        "        neutral_hue: colors.Color | str = colors.slate,\n",
+        "        text_size: sizes.Size | str = sizes.text_lg,\n",
+        "        font: fonts.Font | str | Iterable[fonts.Font | str] = (\n",
+        "            fonts.GoogleFont(\"Outfit\"), \"Arial\", \"sans-serif\",\n",
+        "        ),\n",
+        "        font_mono: fonts.Font | str | Iterable[fonts.Font | str] = (\n",
+        "            fonts.GoogleFont(\"IBM Plex Mono\"), \"ui-monospace\", \"monospace\",\n",
+        "        ),\n",
+        "    ):\n",
+        "        super().__init__(\n",
+        "            primary_hue=primary_hue,\n",
+        "            secondary_hue=secondary_hue,\n",
+        "            neutral_hue=neutral_hue,\n",
+        "            text_size=text_size,\n",
+        "            font=font,\n",
+        "            font_mono=font_mono,\n",
+        "        )\n",
+        "        super().set(\n",
+        "            background_fill_primary=\"*primary_50\",\n",
+        "            background_fill_primary_dark=\"*primary_900\",\n",
+        "            body_background_fill=\"linear-gradient(135deg, *primary_200, *primary_100)\",\n",
+        "            body_background_fill_dark=\"linear-gradient(135deg, *primary_900, *primary_800)\",\n",
+        "            button_primary_text_color=\"white\",\n",
+        "            button_primary_text_color_hover=\"white\",\n",
+        "            button_primary_background_fill=\"linear-gradient(90deg, *secondary_500, *secondary_600)\",\n",
+        "            button_primary_background_fill_hover=\"linear-gradient(90deg, *secondary_600, *secondary_700)\",\n",
+        "            button_primary_background_fill_dark=\"linear-gradient(90deg, *secondary_600, *secondary_800)\",\n",
+        "            button_primary_background_fill_hover_dark=\"linear-gradient(90deg, *secondary_500, *secondary_500)\",\n",
+        "            button_secondary_text_color=\"black\",\n",
+        "            button_secondary_text_color_hover=\"white\",\n",
+        "            button_secondary_background_fill=\"linear-gradient(90deg, *primary_300, *primary_300)\",\n",
+        "            button_secondary_background_fill_hover=\"linear-gradient(90deg, *primary_400, *primary_400)\",\n",
+        "            button_secondary_background_fill_dark=\"linear-gradient(90deg, *primary_500, *primary_600)\",\n",
+        "            button_secondary_background_fill_hover_dark=\"linear-gradient(90deg, *primary_500, *primary_500)\",\n",
+        "            slider_color=\"*secondary_500\",\n",
+        "            slider_color_dark=\"*secondary_600\",\n",
+        "            block_title_text_weight=\"600\",\n",
+        "            block_border_width=\"3px\",\n",
+        "            block_shadow=\"*shadow_drop_lg\",\n",
+        "            button_primary_shadow=\"*shadow_drop_lg\",\n",
+        "            button_large_padding=\"11px\",\n",
+        "            color_accent_soft=\"*primary_100\",\n",
+        "            block_label_background_fill=\"*primary_200\",\n",
+        "        )\n",
+        "\n",
+        "steel_blue_theme = SteelBlueTheme()\n",
+        "\n",
+        "css = \"\"\"\n",
+        "#main-title h1 {\n",
+        "    font-size: 2.3em !important;\n",
+        "}\n",
+        "#output-title h2 {\n",
+        "    font-size: 2.1em !important;\n",
+        "}\n",
+        "\"\"\"\n",
+        "\n",
+        "# --- Model Configuration ---\n",
+        "MAX_MAX_NEW_TOKENS = 4096\n",
+        "DEFAULT_MAX_NEW_TOKENS = 1024\n",
+        "MAX_INPUT_TOKEN_LENGTH = int(os.getenv(\"MAX_INPUT_TOKEN_LENGTH\", \"4096\"))\n",
+        "\n",
+        "print(\"CUDA_VISIBLE_DEVICES=\", os.environ.get(\"CUDA_VISIBLE_DEVICES\"))\n",
+        "print(\"torch.__version__ =\", torch.__version__)\n",
+        "print(\"torch.version.cuda =\", torch.version.cuda)\n",
+        "print(\"cuda available:\", torch.cuda.is_available())\n",
+        "print(\"cuda device count:\", torch.cuda.device_count())\n",
+        "if torch.cuda.is_available():\n",
+        "    print(\"current device:\", torch.cuda.current_device())\n",
+        "    print(\"device name:\", torch.cuda.get_device_name(torch.cuda.current_device()))\n",
+        "\n",
+        "# Define 4-bit quantization configuration\n",
+        "# This config will load the model in 4-bit to save VRAM.\n",
+        "quantization_config = BitsAndBytesConfig(\n",
+        "    load_in_4bit=True,\n",
+        "    bnb_4bit_compute_dtype=torch.float16,\n",
+        "    bnb_4bit_quant_type=\"nf4\",\n",
+        "    bnb_4bit_use_double_quant=True,\n",
+        ")\n",
+        "\n",
+        "# Load Gliese-OCR-7B-Post2.0 with 4-bit quantization\n",
+        "MODEL_ID_X = \"prithivMLmods/Gliese-OCR-7B-Post2.0-final\"\n",
+        "print(f\"Loading {MODEL_ID_X}🤗. This will use 4-bit quantization to save VRAM.\")\n",
+        "\n",
+        "processor = AutoProcessor.from_pretrained(MODEL_ID_X, trust_remote_code=True)\n",
+        "model = Qwen2_5_VLForConditionalGeneration.from_pretrained(\n",
+        "    MODEL_ID_X,\n",
+        "    trust_remote_code=True,\n",
+        "    quantization_config=quantization_config,\n",
+        "    device_map=\"auto\"\n",
+        ").eval()\n",
+        "\n",
+        "\n",
+        "@spaces.GPU\n",
+        "def generate_image(text: str, image: Image.Image,\n",
+        "                   max_new_tokens: int, temperature: float, top_p: float,\n",
+        "                   top_k: int, repetition_penalty: float):\n",
+        "    \"\"\"\n",
+        "    Generates responses using the Nanonets model for image input.\n",
+        "    Yields raw text and Markdown-formatted text.\n",
+        "    \"\"\"\n",
+        "    if image is None:\n",
+        "        yield \"Please upload an image.\", \"Please upload an image.\"\n",
+        "        return\n",
+        "\n",
+        "    messages = [{\n",
+        "        \"role\": \"user\",\n",
+        "        \"content\": [\n",
+        "            {\"type\": \"image\"},\n",
+        "            {\"type\": \"text\", \"text\": text},\n",
+        "        ]\n",
+        "    }]\n",
+        "    prompt_full = processor.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)\n",
+        "\n",
+        "    inputs = processor(\n",
+        "        text=[prompt_full],\n",
+        "        images=[image],\n",
+        "        return_tensors=\"pt\",\n",
+        "        padding=True).to(model.device)\n",
+        "\n",
+        "    streamer = TextIteratorStreamer(processor, skip_prompt=True, skip_special_tokens=True)\n",
+        "    generation_kwargs = {\n",
+        "        **inputs,\n",
+        "        \"streamer\": streamer,\n",
+        "        \"max_new_tokens\": max_new_tokens,\n",
+        "        \"do_sample\": True,\n",
+        "        \"temperature\": temperature,\n",
+        "        \"top_p\": top_p,\n",
+        "        \"top_k\": top_k,\n",
+        "        \"repetition_penalty\": repetition_penalty,\n",
+        "    }\n",
+        "    thread = Thread(target=model.generate, kwargs=generation_kwargs)\n",
+        "    thread.start()\n",
+        "    buffer = \"\"\n",
+        "    for new_text in streamer:\n",
+        "        buffer += new_text\n",
+        "        buffer = buffer.replace(\"<|im_end|>\", \"\")\n",
+        "        time.sleep(0.01)\n",
+        "        yield buffer, buffer\n",
+        "\n",
+        "with gr.Blocks(css=css, theme=steel_blue_theme) as demo:\n",
+        "    gr.Markdown(\"# **Gliese-OCR-7B-Post2.0 (4-bit)**\", elem_id=\"main-title\")\n",
+        "    with gr.Row():\n",
+        "        with gr.Column(scale=2):\n",
+        "            image_query = gr.Textbox(label=\"Query Input\", placeholder=\"Enter your query here...\")\n",
+        "            image_upload = gr.Image(type=\"pil\", label=\"Upload Image\", height=290)\n",
+        "\n",
+        "            image_submit = gr.Button(\"Submit\", variant=\"primary\")\n",
+        "\n",
+        "            with gr.Accordion(\"Advanced options\", open=False):\n",
+        "                max_new_tokens = gr.Slider(label=\"Max new tokens\", minimum=1, maximum=MAX_MAX_NEW_TOKENS, step=1, value=DEFAULT_MAX_NEW_TOKENS)\n",
+        "                temperature = gr.Slider(label=\"Temperature\", minimum=0.1, maximum=4.0, step=0.1, value=0.7)\n",
+        "                top_p = gr.Slider(label=\"Top-p (nucleus sampling)\", minimum=0.05, maximum=1.0, step=0.05, value=0.9)\n",
+        "                top_k = gr.Slider(label=\"Top-k\", minimum=1, maximum=1000, step=1, value=50)\n",
+        "                repetition_penalty = gr.Slider(label=\"Repetition penalty\", minimum=1.0, maximum=2.0, step=0.05, value=1.1)\n",
+        "\n",
+        "        with gr.Column(scale=3):\n",
+        "                gr.Markdown(\"## Output\", elem_id=\"output-title\")\n",
+        "                output = gr.Textbox(label=\"Raw Output Stream\", interactive=False, lines=11, show_copy_button=True)\n",
+        "                with gr.Accordion(\"(Result.md)\", open=False):\n",
+        "                    markdown_output = gr.Markdown(label=\"(Result.Md)\")\n",
+        "\n",
+        "    image_submit.click(\n",
+        "        fn=generate_image,\n",
+        "        inputs=[image_query, image_upload, max_new_tokens, temperature, top_p, top_k, repetition_penalty],\n",
+        "        outputs=[output, markdown_output]\n",
+        "    )\n",
+        "\n",
+        "if __name__ == \"__main__\":\n",
+        "    demo.queue(max_size=50).launch(show_error=True)"
+      ],
+      "metadata": {
+        "id": "cbREUHUJGrzT"
+      },
+      "execution_count": null,
+      "outputs": []
+    }
+  ]
+}