push all

Dockerfile

@@ -46,7 +46,7 @@ RUN python -c "from transformers import pipeline; pipeline('text-to-speech', mod
  && python -c "from transformers import pipeline; pipeline('text-to-speech', model='facebook/mms-tts-eng')" \
  && python -c "from transformers import pipeline; pipeline('text-to-speech', model='facebook/mms-tts-yor')"
 
-# NCAIR1 models will be downloaded at runtime when HF_TOKEN is available
+# Models will be downloaded at runtime when HF_TOKEN is available
 
 # Copy project files
 COPY . .

app.py

@@ -39,7 +39,6 @@ app.add_middleware(
 
 ASK_URL = "https://remostart-milestone-one-farmlingua-ai.hf.space/ask"
 tts_ha, tts_en, tts_yo, tts_ig = None, None, None, None
-natlas_tokenizer, natlas_model = None, None
 
 asr_models = {
     "ha": {"repo": "NCAIR1/Hausa-ASR", "model": None, "proc": None},
@@ -49,7 +48,7 @@ asr_models = {
 }
 
 def load_models():
-    global tts_ha, tts_en, tts_yo, tts_ig, natlas_tokenizer, natlas_model
+    global tts_ha, tts_en, tts_yo, tts_ig
     device = 0 if torch.cuda.is_available() else -1
     hf_token = os.getenv("HF_TOKEN")
     if hf_token:
@@ -60,8 +59,7 @@ def load_models():
     else:
         logger.info("HF_TOKEN is set and ready for authenticated model access.")
     
-    logger.info("Loading N-ATLaS language identification model...")
-    _load_natlas()
+    logger.info("Using lightweight keyword-based language detection (no heavy models)")
     
     logger.info("Loading TTS models...")
     try:
@@ -185,130 +183,78 @@ def get_ai_response(text: str) -> str:
         logger.error(f"AI request error: {e}")
         return f"I'm sorry, I couldn't connect to the AI service. You said: '{text}'."
 
+# Enhanced keyword lists for language detection
 HAUSA_WORDS = [
-    "aikin","manoma","gona","amfanin","yanayi","tsaba","fasaha","bisa","noman","shuka",
-    "daji","rani","damina","amfani","bidi'a","noma","bashi","manure","tsiro","gishiri"
+    # Agricultural terms
+    "aikin", "manoma", "gona", "amfanin", "yanayi", "tsaba", "fasaha", "bisa", "noman", "shuka",
+    "daji", "rani", "damina", "amfani", "bidi'a", "noma", "bashi", "manure", "tsiro", "gishiri",
+    # Common Hausa words
+    "da", "shi", "ta", "su", "mu", "ku", "ni", "kai", "ita", "shi", "ita", "su", "mu", "ku",
+    "ina", "yana", "tana", "suna", "muna", "kuna", "na", "ka", "ta", "sa", "mu", "ku",
+    "wani", "wata", "wasu", "wadansu", "wadannan", "wannan", "wancan", "wannan",
+    "kamar", "kusa", "nisa", "gaba", "baya", "hagu", "dama", "sama", "kasa",
+    "lokaci", "wani", "wata", "wasu", "wadansu", "wadannan", "wannan", "wancan"
 ]
 
 YORUBA_WORDS = [
-    "ilé","ọmọ","òun","awọn","agbẹ","oko","ọgbà","irugbin","àkọsílẹ","omi","ojo","àgbàlá","irọlẹ"
+    # Agricultural terms
+    "ilé", "ọmọ", "òun", "awọn", "agbẹ", "oko", "ọgbà", "irugbin", "àkọsílẹ", "omi", "ojo", "àgbàlá", "irọlẹ",
+    # Common Yoruba words
+    "ni", "ti", "si", "fun", "lati", "ninu", "lori", "labe", "pelu", "ati", "tabi", "sugbon",
+    "o", "a", "e", "won", "mi", "re", "wa", "yin", "won", "mi", "re", "wa", "yin",
+    "kan", "kankan", "die", "pupo", "gbogbo", "kookan", "kookan", "gbogbo",
+    "nibi", "nibe", "nibi", "nibe", "nibi", "nibe", "nibi", "nibe",
+    "igba", "akoko", "ojo", "osu", "odun", "ise", "owo", "owo", "owo"
 ]
 
 IGBO_WORDS = [
-    "ugbo","akụkọ","mmiri","ala","ọrụ","ncheta","ọhụrụ","ugwu","nri","ahụhụ"
+    # Agricultural terms
+    "ugbo", "akụkọ", "mmiri", "ala", "ọrụ", "ncheta", "ọhụrụ", "ugwu", "nri", "ahụhụ",
+    # Common Igbo words
+    "na", "n'", "maka", "n'ihi", "n'ime", "n'elu", "n'okpuru", "na", "na", "na",
+    "m", "i", "o", "ya", "anyị", "unu", "ha", "m", "i", "o", "ya", "anyị", "unu", "ha",
+    "otu", "ọtụtụ", "ọtụtụ", "ọtụtụ", "ọtụtụ", "ọtụtụ", "ọtụtụ", "ọtụtụ",
+    "ebe", "ebe", "ebe", "ebe", "ebe", "ebe", "ebe", "ebe",
+    "oge", "oge", "oge", "oge", "oge", "oge", "oge", "oge"
 ]
 
-def _load_natlas():
-    global natlas_tokenizer, natlas_model
-    if natlas_tokenizer is not None and natlas_model is not None:
-        logger.info("N-ATLaS model already loaded")
-        return True
+def detect_language_keywords(text: str) -> str:
+    """
+    Lightweight keyword-based language detection.
+    Returns language code: 'ha' (Hausa), 'yo' (Yoruba), 'ig' (Igbo), 'en' (English)
+    """
+    text_lower = text.lower().strip()
     
-    hf_token = os.getenv("HF_TOKEN")
-    if hf_token:
-        hf_token = hf_token.strip()
+    if not text_lower:
+        return "en"  # Default to English for empty text
     
-    if not hf_token:
-        logger.error("HF_TOKEN not available for N-ATLaS model access")
-        return False
+    # Count matches for each language
+    hausa_count = sum(1 for word in HAUSA_WORDS if word in text_lower)
+    yoruba_count = sum(1 for word in YORUBA_WORDS if word in text_lower)
+    igbo_count = sum(1 for word in IGBO_WORDS if word in text_lower)
     
-    try:
-        logger.info("Loading N-ATLaS language identification model...")
-        logger.info("Downloading model files from Hugging Face...")
-        
-        natlas_tokenizer = AutoTokenizer.from_pretrained("NCAIR1/N-ATLaS", token=hf_token)
-        natlas_model = AutoModelForCausalLM.from_pretrained(
-            "NCAIR1/N-ATLaS",
-            torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32,
-            device_map="auto" if torch.cuda.is_available() else None,
-            token=hf_token,
-            trust_remote_code=True,
-            low_cpu_mem_usage=True,
-            use_cache=True
-        )
-        logger.info("Successfully loaded N-ATLaS language identification model")
-        return True
-    except Exception as e:
-        logger.exception(f"Failed to load N-ATLaS model: {e}")
-        natlas_tokenizer, natlas_model = None, None
-        return False
+    logger.info(f"Language detection scores - Hausa: {hausa_count}, Yoruba: {yoruba_count}, Igbo: {igbo_count}")
+    
+    # Return language with highest count, default to English if no matches
+    if hausa_count > yoruba_count and hausa_count > igbo_count:
+        logger.info("Keyword detection: Hausa")
+        return "ha"
+    elif yoruba_count > igbo_count:
+        logger.info("Keyword detection: Yoruba")
+        return "yo"
+    elif igbo_count > 0:
+        logger.info("Keyword detection: Igbo")
+        return "ig"
+    else:
+        logger.info("Keyword detection: English (default)")
+        return "en"
 
 def detect_language(text: str) -> str:
+    """
+    Main language detection function using lightweight keyword-based approach.
+    """
     logger.info(f"Detecting language for text: '{text[:50]}...'")
-    
-    if not _load_natlas():
-        logger.warning("N-ATLaS model not available, falling back to keyword detection")
-        text_lower = text.lower()
-        if any(word in text_lower for word in HAUSA_WORDS):
-            logger.info("Keyword detection: Hausa")
-            return "ha"
-        elif any(word in text_lower for word in YORUBA_WORDS):
-            logger.info("Keyword detection: Yoruba")
-            return "yo"
-        elif any(word in text_lower for word in IGBO_WORDS):
-            logger.info("Keyword detection: Igbo")
-            return "ig"
-        else:
-            logger.info("Keyword detection: English (default)")
-            return "en"
-    
-    try:
-        logger.info("Using N-ATLaS for language detection")
-        messages = [
-            {'role': 'system', 'content': 'You are a language identification assistant. Identify the language of the given text and respond with only the language code: "en" for English, "ha" for Hausa, "yo" for Yoruba, or "ig" for Igbo.'},
-            {'role': 'user', 'content': f'What language is this text written in? "{text}"'}
-        ]
-        
-        formatted_text = natlas_tokenizer.apply_chat_template(
-            messages,
-            add_generation_prompt=True,
-            tokenize=False
-        )
-        
-        input_tokens = natlas_tokenizer(formatted_text, return_tensors='pt', add_special_tokens=False)
-        if torch.cuda.is_available():
-            input_tokens = input_tokens.to('cuda')
-        
-        with torch.no_grad():
-            outputs = natlas_model.generate(
-                **input_tokens,
-                max_new_tokens=10,
-                use_cache=True,
-                repetition_penalty=1.1,
-                temperature=0.1,
-                do_sample=False
-            )
-        
-        response = natlas_tokenizer.batch_decode(outputs)[0]
-        response_text = response.split(messages[1]['content'])[-1].strip().lower()
-        
-        logger.info(f"N-ATLaS response: '{response_text}'")
-        
-        if 'ha' in response_text:
-            logger.info("N-ATLaS detection: Hausa")
-            return "ha"
-        elif 'yo' in response_text:
-            logger.info("N-ATLaS detection: Yoruba")
-            return "yo"
-        elif 'ig' in response_text:
-            logger.info("N-ATLaS detection: Igbo")
-            return "ig"
-        else:
-            logger.info("N-ATLaS detection: English (default)")
-            return "en"
-            
-    except Exception as e:
-        logger.exception(f"Language detection failed: {e}")
-        logger.warning("Falling back to keyword detection due to N-ATLaS error")
-        text_lower = text.lower()
-        if any(word in text_lower for word in HAUSA_WORDS):
-            return "ha"
-        elif any(word in text_lower for word in YORUBA_WORDS):
-            return "yo"
-        elif any(word in text_lower for word in IGBO_WORDS):
-            return "ig"
-        else:
-            return "en"
+    return detect_language_keywords(text)
 
 def text_to_speech_file(text: str) -> str:
     lang = detect_language(text)
@@ -365,10 +311,9 @@ async def root():
 
 @app.get("/health")
 async def health():
-    natlas_status = "loaded" if natlas_tokenizer is not None and natlas_model is not None else "not_loaded"
     return {
         "message": "Farmlingua AI Speech Interface is running!",
-        "natlas_status": natlas_status,
+        "language_detection": "keyword-based (lightweight)",
         "tts_models": {
             "hausa": tts_ha is not None,
             "english": tts_en is not None,
@@ -380,8 +325,9 @@ async def health():
 @app.get("/status")
 async def status():
     return {
-        "natlas_loaded": natlas_tokenizer is not None and natlas_model is not None,
-        "loading_message": "N-ATLaS model is loading shards, please wait..." if natlas_tokenizer is None else "N-ATLaS model is ready"
+        "language_detection": "keyword-based (lightweight)",
+        "status": "ready",
+        "message": "Using lightweight keyword-based language detection - no heavy models required"
     }
 
 @app.post("/chat")