Upload folder using huggingface_hub

db.py

@@ -180,7 +180,9 @@ class SanatanDatabase:
         """
         Fetch all matching verses from the collection with optional pagination,
         sorted by _global_index ascending.
+        If page or page_size is None, return all results without pagination.
         """
+
         def normalize_for_match(s: str) -> str:
             s = unicodedata.normalize("NFD", s)
             s = "".join(ch for ch in s if not unicodedata.combining(ch))
@@ -199,6 +201,10 @@ class SanatanDatabase:
             metadata_where_clause.to_chroma_where() if metadata_where_clause else None
         )
 
+        # If the conversion returns an empty dict, treat it as None
+        if isinstance(where_clause, dict) and not where_clause:
+            where_clause = None
+            
         # First, try strict filter
         data = collection.get(include=["metadatas", "documents"], where=where_clause)
 
@@ -212,7 +218,8 @@ class SanatanDatabase:
             regex_filters = [
                 f
                 for f in metadata_where_clause.filters
-                if f.metadata_search_operator == "$eq" and isinstance(f.metadata_value, str)
+                if f.metadata_search_operator == "$eq"
+                and isinstance(f.metadata_value, str)
             ]
 
             if regex_filters:
@@ -225,7 +232,9 @@ class SanatanDatabase:
                         norm_val = normalize_for_match(field_val)
                         norm_query = normalize_for_match(f.metadata_value)
 
-                        if not re.search(re.escape(norm_query), norm_val, flags=re.IGNORECASE):
+                        if not re.search(
+                            re.escape(norm_query), norm_val, flags=re.IGNORECASE
+                        ):
                             ok = False
                             break
                     if ok:
@@ -247,19 +256,25 @@ class SanatanDatabase:
 
         ids_sorted, documents_sorted, metadatas_sorted = zip(*combined)
 
-        # Apply pagination
-        start = (page - 1) * page_size
-        end = start + page_size
-
-        paged_data = {
-            "ids": list(ids_sorted[start:end]),
-            "documents": list(documents_sorted[start:end]),
-            "metadatas": list(metadatas_sorted[start:end]),
-            "total_matches": total_matches,
-        }
-
-        return paged_data
-
+        # --- Apply pagination only if both page and page_size are not None ---
+        if page is not None and page_size is not None:
+            start = (page - 1) * page_size
+            end = start + page_size
+            paged_data = {
+                "ids": list(ids_sorted[start:end]),
+                "documents": list(documents_sorted[start:end]),
+                "metadatas": list(metadatas_sorted[start:end]),
+                "total_matches": total_matches,
+            }
+            return paged_data
+        else:
+            # Return all results
+            return {
+                "ids": list(ids_sorted),
+                "documents": list(documents_sorted),
+                "metadatas": list(metadatas_sorted),
+                "total_matches": total_matches,
+            }
 
     def search(
         self,

modules/audio/model.py

@@ -1,5 +1,14 @@
+from enum import Enum
 from pydantic import BaseModel
 
 class AudioRequest(BaseModel):
     scripture_name: str
     global_index: int
+
+class AudioType(str, Enum):
+    recitation = "recitation"
+    virutham = "virutham"
+    upanyasam = "upanyasam"
+    santhai = "santhai"
+    any = "any"
+    none = "none"

modules/audio/service.py

@@ -1,8 +1,8 @@
-from modules.audio.model import AudioRequest
-from modules.dropbox.audio import get_audio_urls
+from modules.audio.model import AudioRequest, AudioType
+from modules.dropbox.audio import get_audio_urls, get_global_indices_with_audio
 from config import SanatanConfig
 from db import SanatanDatabase
-
+from typing import List
 
 async def svc_get_audio_urls(req: AudioRequest):
     config = SanatanConfig().get_scripture_by_name(req.scripture_name)
@@ -16,3 +16,36 @@ async def svc_get_audio_urls(req: AudioRequest):
         )
         urls = {"recitation": data.get("audio", "")}
     return urls
+
+
+async def svc_get_indices_with_audio(scripture_name: str, audio_type: AudioType) -> List[int]:
+    """
+    Service function to get all global indices for a scripture
+    that have audio files of the specified type.
+    
+    Args:
+        scripture_name: Name of the scripture.
+        audio_type: AudioType enum value.
+    
+    Returns:
+        List[int]: Sorted list of global indices.
+    """
+    config = SanatanConfig().get_scripture_by_name(scripture_name)
+    audio_storage = config.get("audio_storage", "dropbox")
+
+    if audio_storage == "dropbox":
+        indices = await get_global_indices_with_audio(scripture_name, audio_type)
+    else:
+        # Fallback for database storage: iterate all documents and filter by audio_type
+        db = SanatanDatabase()
+        collection_name = config["collection_name"]
+        total = db.count(collection_name=collection_name)
+        all_docs = db.fetch_all_matches(collection_name, page_size=total)
+        indices = []
+        for doc in all_docs:
+            audio_field = doc.get("audio", "")
+            if audio_field.lower().startswith(audio_type.value):
+                indices.append(doc["_global_index"])
+        indices.sort()
+
+    return indices

modules/dropbox/audio.py

@@ -4,14 +4,29 @@ from fastapi import HTTPException
 import dropbox
 from dropbox.files import FolderMetadata, FileMetadata
 from datetime import datetime, timedelta, timezone
-from modules.audio.model import AudioRequest
+from config import SanatanConfig
+from db import SanatanDatabase
+from modules.audio.model import AudioRequest, AudioType
 import logging
 from modules.dropbox.client import dbx
+from fastapi import HTTPException
+from enum import Enum
+import dropbox
+from dropbox.files import FileMetadata
+from dropbox.files import FileMetadata
+from datetime import datetime, timezone
+from fastapi import HTTPException
+from typing import List, Set
+from datetime import datetime, timezone, timedelta
+from fastapi import HTTPException
+import dropbox
+from dropbox.files import FileMetadata
 
 logging.basicConfig()
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
 
+
 def list_dropbox_folder_hierarchy(dbx: dropbox.Dropbox, base_path: str = ""):
     """
     Recursively fetches the folder/file hierarchy from Dropbox starting at base_path.
@@ -62,15 +77,12 @@ def list_dropbox_folder_hierarchy(dbx: dropbox.Dropbox, base_path: str = ""):
 
     return hierarchy
 
+
 # cache = {(scripture_name, global_index, type): {"url": ..., "expiry": ...}}
 audio_cache: dict[tuple[str, int, str], dict] = {}
 CACHE_TTL = timedelta(hours=3, minutes=30)  # refresh before 4h expiry
 
 
-from dropbox.files import FileMetadata
-from datetime import datetime, timezone
-from fastapi import HTTPException
-
 async def get_audio_urls(req: AudioRequest):
     base_path = f"/{req.scripture_name}/audio"
     prefix = f"{req.global_index}-"
@@ -93,7 +105,8 @@ async def get_audio_urls(req: AudioRequest):
 
     # Filter files matching the prefix
     matching_files = [
-        entry for entry in entries
+        entry
+        for entry in entries
         if isinstance(entry, FileMetadata) and entry.name.startswith(prefix)
     ]
 
@@ -102,7 +115,7 @@ async def get_audio_urls(req: AudioRequest):
 
     for entry in matching_files:
         filename = entry.name
-        file_type = filename[len(prefix):].rsplit(".", 1)[0]
+        file_type = filename[len(prefix) :].rsplit(".", 1)[0]
 
         cache_key = (req.scripture_name, req.global_index, file_type)
 
@@ -134,10 +147,97 @@ async def cleanup_audio_url_cache(interval_seconds: int = 600):
             print(f"Cleaned up {len(expired_keys)} expired cache entries")
         await asyncio.sleep(interval_seconds)
 
+
+from datetime import datetime, timezone, timedelta
+
+# Simple in-memory cache
+_audio_indices_cache: dict[tuple[str, str], dict] = {}
+CACHE_TTL_2 = timedelta(minutes=10)
+
+async def get_global_indices_with_audio(scripture_name: str, audio_type: AudioType):
+    """
+    Returns a sorted list of global indices for a given scripture that have audio of the specified type.
+    Supports AudioType.any, AudioType.none, and specific types.
+    Uses in-memory caching for repeated calls.
+    """
+    now = datetime.now(timezone.utc)
+    cache_key = (scripture_name, audio_type.value)
+    
+    # Check cache
+    cached = _audio_indices_cache.get(cache_key)
+    if cached and cached["expiry"] > now:
+        return cached["indices"]
+
+    # Step 1: list all files in Dropbox folder
+    base_path = f"/{scripture_name}/audio"
+    entries = []
+
+    try:
+        result = dbx.files_list_folder(base_path)
+        entries.extend(result.entries)
+        while result.has_more:
+            result = dbx.files_list_folder_continue(result.cursor)
+            entries.extend(result.entries)
+    except dropbox.exceptions.ApiError:
+        raise HTTPException(status_code=404, detail="Audio directory not found")
+
+    # Step 2: collect all global indices with any audio
+    all_indices_with_audio = set()
+    for entry in entries:
+        if not isinstance(entry, FileMetadata) or "-" not in entry.name:
+            continue
+        global_index_str, _ = entry.name.split("-", 1)
+        try:
+            global_index = int(global_index_str)
+        except ValueError:
+            continue
+        all_indices_with_audio.add(global_index)
+
+    # Step 3: filter based on audio_type
+    if audio_type == AudioType.none:
+        db = SanatanDatabase()
+        config = SanatanConfig()
+        total_verses = db.count(
+            collection_name=config.get_collection_name(scripture_name=scripture_name)
+        )
+        indices = set(range(1, total_verses + 1)) - all_indices_with_audio
+    elif audio_type == AudioType.any:
+        indices = all_indices_with_audio
+    else:
+        indices = set()
+        for entry in entries:
+            if not isinstance(entry, FileMetadata) or "-" not in entry.name:
+                continue
+            global_index_str, rest = entry.name.split("-", 1)
+            try:
+                global_index = int(global_index_str)
+            except ValueError:
+                continue
+            file_type = rest.rsplit(".", 1)[0].strip().lower()
+            if file_type.startswith(audio_type.value):
+                indices.add(global_index)
+
+    # Cache the result
+    _audio_indices_cache[cache_key] = {
+        "indices": sorted(indices),
+        "expiry": now + CACHE_TTL_2
+    }
+
+    return sorted(indices)
+
+
+
+
 if __name__ == "__main__":
     # Create Dropbox client with your access token
     # data = list_dropbox_folder_hierarchy(dbx, "")
+    # data = asyncio.run(
+    #     get_audio_urls(AudioRequest(scripture_name="divya_prabandham", global_index=0))
+    # )
     data = asyncio.run(
-        get_audio_urls(AudioRequest(scripture_name="divya_prabandham", global_index=0))
+        get_global_indices_with_audio(
+            scripture_name="divya_prabandham", audio_type=AudioType.upanyasam
+        )
     )
-    print(json.dumps(data, indent=2))
+    # print(json.dumps(data, indent=2))
+    print(len(data))

server.py

@@ -12,8 +12,8 @@ from chat_utils import chat
 from config import SanatanConfig
 from db import SanatanDatabase
 from metadata import MetadataWhereClause
-from modules.audio.model import AudioRequest
-from modules.audio.service import svc_get_audio_urls
+from modules.audio.model import AudioRequest, AudioType
+from modules.audio.service import svc_get_audio_urls, svc_get_indices_with_audio
 from modules.config.categories import get_scripture_categories
 from modules.quiz.answer_validator import validate_answer
 from modules.quiz.models import Question
@@ -336,15 +336,17 @@ async def search_scripture_find_first_match(
         logger.error("Error while searching %s", e, exc_info=True)
         return {"error": str(e)}
 
+
 class ScriptureMultiSearchRequest(BaseModel):
     filter_obj: Optional[MetadataWhereClause] = None
     page: int = 1
     page_size: int = 20
+    has_audio: Optional[AudioType] = None  # new optional field
+
 
 @router.post("/scripture/{scripture_name}/search/all")
 async def search_scripture_find_all_matches(
-    scripture_name: str,
-    req: ScriptureMultiSearchRequest
+    scripture_name: str, req: ScriptureMultiSearchRequest
 ):
     """
     Search scripture collection and return all matching results with pagination.
@@ -352,17 +354,20 @@ async def search_scripture_find_all_matches(
     - `filter_obj`: MetadataWhereClause (filters, groups, operator)
     - `page`: 1-based page number
     - `page_size`: Number of results per page
+    - `has_audio` : optional. can take values any|none|recitation|virutham|upanyasam
     """
     filter_obj = req.filter_obj
     page = req.page
     page_size = req.page_size
+    has_audio = req.has_audio
     try:
         logger.info(
-            "search_scripture_find_all_matches: searching for %s with filters %s | page=%s, page_size=%s",
+            "search_scripture_find_all_matches: searching for %s with filters %s | page=%s, page_size=%s, has_audio=%s",
             scripture_name,
             filter_obj,
             page,
             page_size,
+            has_audio
         )
 
         db = SanatanDatabase()
@@ -373,38 +378,92 @@ async def search_scripture_find_all_matches(
         if not config:
             return {"error": f"Scripture '{scripture_name}' not found"}
 
+        # 1️⃣ Fetch all matching metadata WITHOUT pagination yet
         results = db.fetch_all_matches(
             collection_name=config["collection_name"],
             metadata_where_clause=filter_obj,
-            page=page,
-            page_size=page_size,
+            page=None,  # Fetch all to apply audio filter
+            page_size=None,
         )
 
-        # Flatten + canonicalize results
         formatted_results = []
+        all_indices = []  # Keep track of all _global_index
         for i in range(len(results["metadatas"])):
             doc_id = results["ids"][i]
             metadata_doc = results["metadatas"][i]
             metadata_doc["id"] = doc_id
 
-            document_text = results["documents"][i] if results.get("documents") else None
-
+            document_text = (
+                results["documents"][i] if results.get("documents") else None
+            )
             canonical_doc = SanatanConfig().canonicalize_document(
                 scripture_name, document_text, metadata_doc
             )
             formatted_results.append(canonical_doc)
+            all_indices.append(canonical_doc["_global_index"])
+
+        # 2️⃣ Apply has_audio filter
+        if has_audio:
+            if has_audio == AudioType.none:
+                # Fetch all indices that have any audio type
+                all_audio_indices = set()
+                for atype in [
+                    AudioType.recitation,
+                    AudioType.virutham,
+                    AudioType.upanyasam,
+                    AudioType.santhai,
+                ]:
+                    indices = await svc_get_indices_with_audio(scripture_name, atype)
+                    all_audio_indices.update(indices)
+
+                # Keep only indices that are NOT in all_audio_indices
+                formatted_results = [
+                    r
+                    for r in formatted_results
+                    if r["_global_index"] not in all_audio_indices
+                ]
+            else:
+                if has_audio == AudioType.any:
+                    # Combine indices for all audio types
+                    audio_indices = set()
+                    for atype in [
+                        AudioType.recitation,
+                        AudioType.virutham,
+                        AudioType.upanyasam,
+                        AudioType.santhai,
+                    ]:
+                        indices = await svc_get_indices_with_audio(
+                            scripture_name, atype
+                        )
+                        audio_indices.update(indices)
+                else:
+                    audio_indices = set(
+                        await svc_get_indices_with_audio(scripture_name, has_audio)
+                    )
+
+                # Keep only indices that match
+                formatted_results = [
+                    r for r in formatted_results if r["_global_index"] in audio_indices
+                ]
+
+        # 3️⃣ Apply pagination on filtered results
+        total_matches = len(formatted_results)
+        start_idx = (page - 1) * page_size
+        end_idx = start_idx + page_size
+        paginated_results = formatted_results[start_idx:end_idx]
 
         return {
-            "results": formatted_results,
-            "total_matches": results.get("total_matches", 0),
+            "results": paginated_results,
+            "total_matches": total_matches,
             "page": page,
             "page_size": page_size,
         }
-
+        
     except Exception as e:
         logger.error("Error while searching %s", e, exc_info=True)
         return {"error": str(e)}
 
+
 @router.post("/audio")
 async def generate_audio_urls(req: AudioRequest):
     logger.info("generate_audio_urls: %s", req)