velocity

.gitattributes

@@ -36,3 +36,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.catboost filter=lfs diff=lfs merge=lfs -text
 *.png filter=lfs diff=lfs merge=lfs -text
 *.jpeg filter=lfs diff=lfs merge=lfs -text
+assets/example_video.mpg filter=lfs diff=lfs merge=lfs -text

app.py

@@ -1,5 +1,5 @@
 import gradio as gr
-from gradio_demo import main_page, nailfold_features_page, scaling_page, vdo_page
+from gradio_demo import main_page, nailfold_features_page, scaling_page, vdo_page, velocity_page
 
 with gr.Blocks() as demo:
     with gr.Tab("📐 Pixel-to-mm Scaling"):
@@ -11,6 +11,9 @@ with gr.Blocks() as demo:
     with gr.Tab("🎞️ Video to Nailfold Features"):
         vdo_page.vdo_demo.render()
 
+    with gr.Tab("🚀 Velocity Estimation [Experimental]"):
+        velocity_page.velocity_demo.render()
+        
     # with gr.Tab("🚀 Full Pipeline"):
     #     main_page.demo.render()
 

assets/example_video.mpg

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8450b6843120ece55161f0cc488f3683e4994cc5afce0b225f409c2210db72bf
+size 13416448

gradio_demo/scaling_page.py

@@ -26,7 +26,12 @@ with gr.Blocks(title="Pixel-to-mm Scaling") as scaling_demo:
 
     image_input = gr.ImageEditor(label="Upload and Crop Image", type="pil")
     measure_btn = gr.Button(" Measure Width")
-
+    # gr.Examples(
+    #             examples=[["assets/example_reticule.png"]],
+    #             inputs=[image_input],
+    #             label="Try with Example",
+    #         )
+    
     output_img = gr.Image(label="Cropped Result")
     result_text = gr.Textbox(label="Width in Pixels")
 

gradio_demo/utils.py

@@ -1,19 +1,24 @@
 import os
-import cv2
 import math
 import random
+import re
 import shutil
 import tempfile
 import textwrap
+from collections import Counter
+from pathlib import Path
 import numpy as np
 import pandas as pd
 from PIL import Image
 import torch
-from collections import Counter
+import cv2
+from ultralytics import YOLO
+from supervised.automl import AutoML 
+from skimage.morphology import skeletonize
 from matplotlib import pyplot as plt
 from matplotlib.backends.backend_pdf import PdfPages
-from ultralytics import YOLO
-from supervised.automl import AutoML
+import gradio as gr
+
 
 # Declare global variables
 midframe_path = ""
@@ -461,3 +466,412 @@ def full_pipeline(video_file, name, sex, age, weight, heightt, SBP, DBP, PR, FBS
     )
 
 
+def extract_middle_frame(video_path, output_name):
+    """
+    Extracts the middle frame from a video file and saves it as an image.
+    """
+    cap = cv2.VideoCapture(video_path)
+    total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
+    if total_frames <= 0:
+        cap.release()
+    middle_frame_number = total_frames // 2
+    cap.set(cv2.CAP_PROP_POS_FRAMES, middle_frame_number)
+    ret, frame = cap.read()
+    success = cv2.imwrite(output_name, frame, [cv2.IMWRITE_PNG_COMPRESSION, 0])
+    cap.release()
+
+
+def boxCenter(coords):
+    """
+    Calculate the center of a bounding box given its coordinates.
+    """
+    [left, top, right, bottom] = coords
+    return [int(left+right)/2, int(top+bottom)/2]
+
+
+def closestBox(boxes, coords):
+    """
+    Find the closest bounding box to a given coordinate.
+    """
+    distance = []
+    center = boxCenter(coords)
+    for box in boxes:
+        coord_pred = [int(box['x1']), int(box['y1']), int(box['x2']), int(box['y2'])]
+        boxCent = boxCenter(coord_pred)
+        distance.append(math.dist(boxCent,center))
+    return boxes[distance.index(min(distance))], distance.index(min(distance))
+
+
+def adjustBoxSize(coords, box_width, box_height):
+    """
+    Adjust the bounding box size based on the given width and height.
+    """
+    [centerX, centerY] = boxCenter(coords)
+    return [centerX-box_width/2, centerY-box_height/2, centerX+box_width/2, centerY+box_height/2]
+
+
+def adjustBoundaries(coords, screen):
+    """
+    Adjust the bounding box coordinates to fit within the screen dimensions.
+    """
+    [left, top, right, bottom] = coords
+    [width, height]=screen
+    if left<0:
+        right=right-left
+        left=0
+    if top<0:
+        bottom=bottom-top
+        top=0
+    if right>width:
+        left=left-(right-width)
+        right=width
+    if bottom>height:
+        top=top-(bottom-height)
+        bottom=height
+    return [round(left), round(top), round(right), round(bottom)]
+
+
+def get_coordinate(input_img, num_label, model, results):
+    """
+    Get the coordinates of the bounding box for a specific label in an image.
+    """
+    result = model.predict(input_img, verbose=False, save=False, conf=0.5, iou=0.3)
+    for result in results:
+        df_result = result.to_df()
+
+    x1 = df_result['box'][num_label]['x1']
+    y1 = df_result['box'][num_label]['y1']
+    x2 = df_result['box'][num_label]['x2']
+    y2 = df_result['box'][num_label]['y2']
+
+    cropCoords = [x1, y1, x2, y2]
+    return cropCoords
+
+
+def process_video_temp(input_path, output_folder=None, slow_factor=2.0):
+    """
+    Process a video file to create:
+    1. Slow-motion version
+    2. Grayscale version
+    3. Grayscale + slow-motion version
+
+    All outputs are saved as temporary files with readable names.
+
+    Args:
+        input_path (str): path to the input video
+        output_folder (str, optional): folder to save temp files. Defaults to system temp.
+        slow_factor (float or str, optional): factor to slow down video. Defaults to 2.0
+
+    Returns:
+        tuple: paths to (slow_video, gray_video, gray_slow_video)
+    """
+    if output_folder is None:
+        output_folder = tempfile.gettempdir()
+
+    slow_factor = float(slow_factor)  # ensure numeric
+    cap = cv2.VideoCapture(input_path)
+    if not cap.isOpened():
+        raise ValueError(f"Cannot open video: {input_path}")
+
+    fps = float(cap.get(cv2.CAP_PROP_FPS))
+    width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
+    height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
+    name_file = os.path.splitext(os.path.basename(input_path))[0]
+
+    new_fps = fps / slow_factor
+
+    # Generate readable temp file paths
+    output_path_slow = os.path.join(output_folder, f"{name_file}_slow.mp4")
+    output_path_gray = os.path.join(output_folder, f"{name_file}_gray.mp4")
+    output_path_both = os.path.join(output_folder, f"{name_file}_gray_slow.mp4")
+
+    fourcc = cv2.VideoWriter_fourcc(*'mp4v')  
+
+    out_slow = cv2.VideoWriter(output_path_slow, fourcc, new_fps, (width, height))
+    out_gray = cv2.VideoWriter(output_path_gray, fourcc, fps, (width, height), isColor=False)
+    out_both = cv2.VideoWriter(output_path_both, fourcc, new_fps, (width, height), isColor=False)
+
+    while True:
+        ret, frame = cap.read()
+        if not ret:
+            break
+
+        # Slow motion
+        out_slow.write(frame)
+
+        # Grayscale
+        gray_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
+        out_gray.write(gray_frame)
+
+        # Grayscale + slow
+        out_both.write(gray_frame)
+
+    cap.release()
+    out_slow.release()
+    out_gray.release()
+    out_both.release()
+
+    return output_path_slow, output_path_gray, output_path_both
+
+def process_frame_split_horizontal_lines(
+        frame, line_y_ratio=0.8, 
+        left_line_ratio=(0.1, 0.4), 
+        right_line_ratio=(0.6, 0.9), 
+        line_color=(0, 255, 0), 
+        line_thickness=2):
+    """
+    show the frame with split horizontal lines
+    """
+    height, width = frame.shape[:2]
+    line_y = int(height * line_y_ratio)
+    left_start_x = int(width * left_line_ratio[0])
+    left_end_x = int(width * left_line_ratio[1])
+    right_start_x = int(width * right_line_ratio[0])
+    right_end_x = int(width * right_line_ratio[1])
+    cv2.line(frame, (left_start_x, line_y), (left_end_x, line_y), line_color, line_thickness)
+    cv2.line(frame, (right_start_x, line_y), (right_end_x, line_y), line_color, line_thickness)
+    return frame
+
+
+def get_average_intensity_on_line(frame, line_y, x_start, x_end):
+    """
+    Calculates the average intensity along a horizontal line segment in a grayscale frame.
+    """
+    h, w = frame.shape[:2]
+    line_pixels = frame[line_y, x_start:x_end+1]
+    return np.mean(line_pixels)
+
+
+def extract_video_data_temp(video_file):
+    """
+    Analyze a video file to extract the middle frame, detect objects, and crop object videos.
+    Temporary files have readable names based on the original video name.
+    
+    Returns:
+        {
+            "middle_frame": temp_middle_frame_path,
+            "cropped_videos": [list of temp video paths],
+            "temp_files": all_temp_files
+        }
+    """
+    temp_files = []
+    base_name = os.path.splitext(os.path.basename(video_file))[0]
+    temp_dir = tempfile.gettempdir()
+
+    # Middle frame with readable name
+    temp_middle_frame_name = os.path.join(temp_dir, f"{base_name}_middle.png")
+    extract_middle_frame(video_file, temp_middle_frame_name)
+    temp_files.append(temp_middle_frame_name)
+
+    input_img = cv2.imread(temp_middle_frame_name)
+    results = obj_det_model.predict(input_img, conf=0.5, iou=0.3)
+
+    temp_videos = []
+
+    for i in range(len(results)):
+        cropCoords = get_coordinate(input_img, i, obj_det_model, results)
+
+        vidCapture = cv2.VideoCapture(video_file)
+        fps = vidCapture.get(cv2.CAP_PROP_FPS)
+        width = int(vidCapture.get(cv2.CAP_PROP_FRAME_WIDTH))
+        height = int(vidCapture.get(cv2.CAP_PROP_FRAME_HEIGHT))
+        skip_frames = int(1.5 * fps)
+
+        # Skip initial frames
+        for _ in range(skip_frames):
+            ret, frame = vidCapture.read()
+
+        box_left, box_top, box_right, box_bottom = cropCoords
+        box_left = max(box_left, 0)
+        box_top = max(box_top, 0)
+        box_right = min(box_right, width)
+        box_bottom = min(box_bottom, height)
+
+        box_width = box_right - box_left
+        box_height = box_bottom - box_top
+
+        # Temp video with readable name
+        temp_video_name = os.path.join(temp_dir, f"{base_name}_{i}_w{int(box_width)}_h{int(box_height)}.mp4")
+        temp_videos.append(temp_video_name)
+
+        outputWriter = cv2.VideoWriter(
+            temp_video_name,
+            cv2.VideoWriter_fourcc(*'mp4v'),
+            fps,
+            (int(box_width), int(box_height))
+        )
+
+        lastBoxCoords = [box_left, box_top, box_right, box_bottom]
+
+        while True:
+            ret, im = vidCapture.read()
+            if not ret:
+                break
+
+            results_frame = obj_det_model.predict(im)
+            for result in results_frame:
+                df_result = result.to_df()
+
+            boxes = df_result['box']
+            box, distance = closestBox(boxes, lastBoxCoords)
+
+            if distance < 4:
+                newCoords = adjustBoxSize([box['x1'], box['y1'], box['x2'], box['y2']], box_width, box_height)
+                newCoords = adjustBoundaries(newCoords, [width, height])
+                box_left, box_top, box_right, box_bottom = newCoords
+
+            imCropped = im[int(box_top):int(box_bottom), int(box_left): int(box_right)]
+            outputWriter.write(imCropped)
+
+        vidCapture.release()
+        outputWriter.release()
+
+    all_temp_files = temp_files + temp_videos
+    return {"middle_frame": temp_middle_frame_name, "cropped_videos": temp_videos, "temp_files": all_temp_files}
+
+
+def analyze_video_intensity(input_video_path):
+
+    base_name = os.path.splitext(os.path.basename(input_video_path))[0]
+    working_path = f"velocity_data/{base_name}"
+    os.makedirs(working_path, exist_ok=True)
+    output_slow, output_gray, output_both = process_video_temp(input_video_path, working_path)
+
+    cap = cv2.VideoCapture(output_both)
+
+    left_intensities = []
+    right_intensities = []
+    frame_numbers = []
+    frame_count = 0
+
+    line_y_ratio = 0.9
+    left_line_ratio = (0.1, 0.4)
+    right_line_ratio = (0.6, 0.9)
+
+    while True:
+        ret, frame = cap.read() 
+        if not ret:
+            break
+
+        current_frame_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
+        height, width = current_frame_gray.shape[:2]
+        line_y_pixel = int(height * line_y_ratio)
+        left_start_pixel = int(width * left_line_ratio[0])
+        left_end_pixel = int(width * left_line_ratio[1])
+        right_start_pixel = int(width * right_line_ratio[0])
+        right_end_pixel = int(width * right_line_ratio[1])
+
+        avg_left_intensity = get_average_intensity_on_line(current_frame_gray, line_y_pixel, left_start_pixel, left_end_pixel)
+        left_intensities.append(avg_left_intensity)
+
+        avg_right_intensity = get_average_intensity_on_line(current_frame_gray, line_y_pixel, right_start_pixel, right_end_pixel)
+        right_intensities.append(avg_right_intensity)
+
+        frame_numbers.append(frame_count)
+        frame_count += 1
+
+        processed_frame = process_frame_split_horizontal_lines(frame) 
+
+    cap.release()
+    cv2.destroyAllWindows()
+
+    time = np.array(frame_numbers)
+    intensity_difference = abs(np.array(right_intensities) - np.array(left_intensities))
+
+    df = pd.DataFrame(columns=["intensity_dif", "time"])
+    df["intensity_dif"] = intensity_difference
+    df["time"] = time
+
+    dif_time = []
+    for inten, time in zip(df["intensity_dif"], df["time"]):
+        if inten > 10.5:
+            round_time = round(float(time), 4)
+            dif_time.append(round_time/2)
+            if round_time/2 > 4:
+                break
+
+    differences = []
+    for i in range(1, len(dif_time)):
+        difference = dif_time[i] - dif_time[i-1]
+        differences.append(difference)
+
+    avg_time = np.mean(differences)
+    return avg_time
+
+
+def estimate_nailfold_distance_temp(input_video_path):
+    """
+    Extract velocity data from a video file:
+    - Detects capillary masks using YOLO segmentation
+    - Calculates skeleton length
+    - Saves mask as a temporary PNG for visualization
+    """
+
+    # Handle Gradio dict
+    if isinstance(input_video_path, dict):
+        input_video_path = input_video_path.get("name")
+
+    if not input_video_path or not os.path.exists(input_video_path):
+        raise FileNotFoundError(f"Video file not found: {input_video_path}")
+
+    base_name = os.path.splitext(os.path.basename(input_video_path))[0]
+    temp_dir = tempfile.mkdtemp()
+
+    # --- Extract middle frame robustly ---
+    middle_frame_path = os.path.join(temp_dir, f"{base_name}_middle.png")
+
+    cap = cv2.VideoCapture(input_video_path)
+    if not cap.isOpened():
+        raise FileNotFoundError(f"Cannot open video file: {input_video_path}")
+
+    frame_count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
+    if frame_count <= 0:
+        raise ValueError(f"No frames found in video: {input_video_path}")
+
+    middle_index = frame_count // 2
+    cap.set(cv2.CAP_PROP_POS_FRAMES, middle_index)
+    ret, frame = cap.read()
+    cap.release()
+
+    if not ret or frame is None:
+        raise FileNotFoundError(f"Failed to read middle frame from video: {input_video_path}")
+
+    cv2.imwrite(middle_frame_path, frame)
+
+    # --- Detect width/height ---
+    match = re.search(r'w(\d+)_h(\d+)', base_name)
+    if match:
+        w, h = int(match.group(1)), int(match.group(2))
+    else:
+        # fallback: use video resolution
+        h = int(frame.shape[0])
+        w = int(frame.shape[1])
+
+    # --- YOLO segmentation ---
+    model = YOLO("models/segment_yolo11.pt")
+    input_img = cv2.imread(middle_frame_path)
+
+    results = model.predict(
+        input_img, save=False, show_labels=False,
+        show_conf=False, show_boxes=False
+    )
+
+    mask_image_path = None
+    for result in results:
+        if result.masks is not None:
+            for mask in result.masks.data:
+                mask_np = (mask.cpu().numpy() * 255).astype(np.uint8)
+                mask_resized = cv2.resize(mask_np, (w, h))
+
+                mask_image_path = os.path.join(temp_dir, f"{base_name}_mask.png")
+                cv2.imwrite(mask_image_path, mask_resized)
+
+                skeleton = skeletonize(mask_resized > 0, method='lee')
+                unique_elements, counts = np.unique(skeleton, return_counts=True)
+                value_ele = dict(zip(unique_elements, counts)).get(True, 0)
+
+                skeleton_length_mm = value_ele / 587.0
+                return skeleton_length_mm, mask_image_path
+
+    raise ValueError("No mask detected in the middle frame.")
+

gradio_demo/vdo_page.py

@@ -62,6 +62,11 @@ with gr.Blocks(title="🎞️ Video to Nailfold Features") as vdo_demo:
             video_input = gr.Video(label="Upload Video File")
             mm_per_pixel = gr.Number(label="1 mm equals (pixels)", value=587, step=1)
             extract_btn = gr.Button("Extract & Analyze")
+            gr.Examples(
+                examples=[["assets/example_video.mpg", 587]],
+                inputs=[video_input, mm_per_pixel],
+                label="Try with Example",
+            )
 
         with gr.Column():
             midframe_output = gr.Image(label="Extracted Midframe Visualization")

gradio_demo/velocity_page.py

@@ -0,0 +1,137 @@
+import gradio as gr
+from gradio_demo.utils import (
+    extract_video_data_temp, 
+    analyze_video_intensity, 
+    estimate_nailfold_distance_temp,
+)
+
+# ------------------------------
+# Function to extract nailfold frames
+# ------------------------------
+def extract_nailfold_frame(input_video):
+    """
+    Extract middle frame and cropped videos from the uploaded nailfold video.
+    Returns a list of temporary file paths (middle frame + cropped videos).
+    """
+    if input_video is None:
+        return []
+    
+    results = extract_video_data_temp(input_video)
+    
+    return [results["middle_frame"]] + results["cropped_videos"]
+
+# ------------------------------
+# Function to predict velocity
+# ------------------------------
+def pred_velocity(input_video):
+    """
+    Predict the velocity of objects in a video using extracted path.
+    Returns Gradio-friendly outputs:
+    - Estimated Time
+    - Estimated Distance
+    - Estimated Velocity
+    - Segmentation Mask Image
+    """
+    if input_video is None:
+        return "No video uploaded", "No video uploaded", "No video uploaded", None
+
+    estimate_time = analyze_video_intensity(input_video)
+    estimate_distance, segmentation_path = estimate_nailfold_distance_temp(input_video)
+    predicted_velocity = estimate_distance / estimate_time if estimate_time > 0 else 0.0
+
+    # Return strings and mask path for Gradio
+    return (
+        f"{estimate_time:.2f} s",
+        f"{estimate_distance:.2f} mm",
+        f"{predicted_velocity:.2f} mm/s",
+        segmentation_path
+    )
+
+# ------------------------------
+# Build Gradio Interface
+# ------------------------------
+with gr.Blocks(
+    title="Nailfold Velocity Estimator",
+    theme=gr.themes.Default(primary_hue=gr.themes.colors.red,
+                            secondary_hue=gr.themes.colors.pink)
+) as velocity_demo:
+
+    # Header and instructions
+    gr.Markdown("# 🎥 Nailfold Velocity Estimator")
+    gr.Markdown("""
+    This page allows you to process a nailfold capillaroscopy video to estimate flow velocity.
+    
+    ⚠️ **CAUTION**: This tool is experimental and may not work with all videos.  
+    ⚠️ The values are based on video analysis and may vary based on video quality and content.  
+    ⚠️ Each parameter is set based on experimental data and may not be accurate for all cases.  
+    ⚠️ The model is trained on specific datasets and may not generalize well to other types of data.
+    
+    ## Left Side: Extract Nailfold Frames
+    Upload a nailfold capillaroscopy video. The object detection model will extract the frame of each nailfold and return cropped areas.
+
+    **Steps**:
+    1. Upload a nailfold video (.mp4 or .avi).
+    2. Click **Extract Nailfold Frame**.
+    3. Download the extracted frame and cropped videos.
+
+    ## Right Side: Predict Velocity
+    After frame extraction, upload the same video to predict velocity.
+
+    **Steps**:
+    1. Upload the video used for frame extraction.
+    2. Click **Predict Velocity**.
+    3. View estimated time, distance, velocity, and segmentation mask.
+    """)
+
+    with gr.Row():
+        # ------------------------------
+        # Left Column: Frame Extraction
+        # ------------------------------
+        with gr.Column():
+            video_input1 = gr.Video(label="Upload Video for Frame Extraction")
+            path_output = gr.File(
+                label="Download Extracted Frame & Cropped Videos",
+                file_types=[".png", ".mp4"], 
+                type="filepath"  
+            )
+            extract_button = gr.Button("Extract Nailfold Frame")
+            
+            extract_button.click(
+                fn=extract_nailfold_frame,
+                inputs=video_input1,
+                outputs=path_output
+            )
+            
+            gr.Markdown("### Example Video for Frame Extraction")
+            gr.Examples(
+                examples=[["assets/test_velo.mpg"]],
+                inputs=[video_input1],
+                label="Try with Example",
+            )
+
+        # ------------------------------
+        # Right Column: Velocity Prediction
+        # ------------------------------
+        with gr.Column():
+            video_input2 = gr.Video(label="Upload Video for Velocity Prediction")
+            time_output = gr.Textbox(label="Estimated Time", interactive=False)
+            distance_output = gr.Textbox(label="Estimated Distance", interactive=False)
+            velocity_output = gr.Textbox(label="Estimated Velocity", interactive=False)
+            segment_output = gr.Image(label="Segmentation Mask")
+            
+            velocity_button = gr.Button("Predict Velocity")
+            velocity_button.click(
+                fn=pred_velocity,
+                inputs=video_input2,
+                outputs=[time_output, distance_output, velocity_output, segment_output]
+            )
+            
+            gr.Markdown("### Example Video for Velocity Prediction")
+            gr.Examples(
+                examples=[["assets/test_velo_0_w33_h91.mp4"]],
+                inputs=[video_input2],
+                label="Try with Example",
+            )
+
+if __name__ == "__main__":
+    velocity_demo.launch()

models/segment_yolo11.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c8357f4201e9244ec3a8a8d88fda5f2981d5851e548a044aa32497d1824df64a
+size 6059890

requirements.txt

@@ -13,4 +13,5 @@ ultralytics
 torch
 lightgbm
 catboost
-mljar-supervised
+mljar-supervised
+scikit-image