Upload 23 files

.gitattributes

@@ -1,35 +1,35 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

Dockerfile

@@ -1,9 +1,9 @@
-FROM python:3.9-slim
-WORKDIR /webapp
-COPY . .
-RUN chmod +x /webapp/start.sh
-RUN pip install --upgrade pip
-RUN pip install --no-cache-dir -r requirements.txt
-RUN apt update && apt install -y redis-server
-EXPOSE 7860 6379
-CMD ["/webapp/start.sh"]
+FROM python:3.9-slim
+WORKDIR /webapp
+COPY . .
+RUN chmod +x /webapp/start.sh
+RUN pip install --upgrade pip
+RUN pip install --no-cache-dir -r requirements.txt
+RUN apt update && apt install -y redis-server
+EXPOSE 7860 6379
+CMD ["newrelic-admin", "run-program", "/webapp/start.sh"]

README.md

@@ -1,11 +1,11 @@
----
-title: News Aggregator
-emoji: ⚡
-colorFrom: indigo
-colorTo: blue
-sdk: docker
-pinned: false
-license: mit
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+---
+title: News Aggregator
+emoji: ⚡
+colorFrom: indigo
+colorTo: blue
+sdk: docker
+pinned: false
+license: mit
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -1,262 +1,258 @@
-import numpy as np
-import pandas as pd
-from dateutil import parser
-from quart_cors import cors
-from quart import Quart
-from quart import render_template
-from db_operations.db_operations import DBOperations
-import logging
-import traceback 
-import redis
-import uuid
-from datetime import datetime
-from functools import lru_cache
-import gc
-from word_cloud import get_frequent_words_html
-from config import NEWS_RETENTION_SECONDS, UK_EDITION_URL
-
-
-app = Quart(__name__)
-app = cors(app, allow_origin="*")
-redis_client = redis.Redis(host='localhost', port=6379, decode_responses=True)
-logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=logging.INFO)
-logging.warning(f'Is Redis available?: {redis_client.ping()}') 
-db = DBOperations()
-session_id = None
-
-
-REFRESH_FREQ = 300 # 300 secs = 5 mins
-
-def is_db_fetch_reqd():
-    try:
-        env_news_time = redis_client.get('NEWSFETCHTIME')
-        logging.warning(f'[session_id: {session_id}] fetch_time_env_var: {env_news_time}')
-        fetch_flag = 1
-        if env_news_time is None:
-            redis_client.set("NEWSFETCHTIME", str(datetime.now()))
-            fetch_flag = 1
-    
-        if env_news_time is not None:
-            fetch_time_lapse_seconds = (datetime.now() - datetime.strptime(env_news_time, '%Y-%m-%d %H:%M:%S.%f')).seconds
-            if fetch_time_lapse_seconds <= REFRESH_FREQ: 
-                fetch_flag = 0 
-            else:
-                redis_client.set("NEWSFETCHTIME", str(datetime.now()))
-                fetch_flag = 1
-    except Exception as e:
-        print(e)
-        fetch_flag = 1
-    return fetch_flag
-
-
-def correct_date(x):
-    if (not isinstance(x, str)) or (str(x).find(":") == -1):
-        logging.error(f'[session_id: {session_id}] correct_date() error: {x} is not the right date format')
-        return "2020-11-07 00:36:44+05:30"
-    return x
-
-def date_time_parser(dt):
-    """
-    Computes the minutes elapsed since published time.
-    :param dt: date
-    :return: int, minutes elapsed.
-    """
-    try:
-        return int(np.round((dt.now(dt.tz) - dt).total_seconds() / 60, 0))
-    except:
-        logging.error(f'[session_id: {session_id}] date_time_parser() error: {dt} is not the right date format')
-        return 100000
-
-
-def elapsed_time_str(mins):
-    """
-    Return the time elapsed string from minutes passed as an argument.
-    :param mins: int, minutes elapsed.
-    :return: str, time elapsed string
-    """
-    try:
-        time_str = ''
-        hours = int(mins / 60)
-        days = np.round(mins / (60 * 24), 1)
-        remaining_mins = int(mins - (hours * 60))
-        if days >= 1:
-            time_str = f'{str(days)} days ago'
-            if days == 1:
-                time_str = 'a day ago'
-        elif (days < 1) & (hours < 24) & (mins >= 60):
-            time_str = f'{str(hours)} hours and {str(remaining_mins)} mins ago'
-            if (hours == 1) & (remaining_mins > 1):
-                time_str = f'an hour and {str(remaining_mins)} mins ago'
-            if (hours == 1) & (remaining_mins == 1):
-                time_str = f'an hour and a min ago'
-            if (hours > 1) & (remaining_mins == 1):
-                time_str = f'{str(hours)} hours and a min ago'
-            if (hours > 1) & (remaining_mins == 0):
-                time_str = f'{str(hours)} hours ago'
-            if ((mins / 60) == 1) & (remaining_mins == 0):
-                time_str = 'an hour ago'
-        elif (days < 1) & (hours < 24) & (mins == 0):
-            time_str = 'Just in'
-        else:
-            time_str = f'{str(mins)} minutes ago'
-            if mins == 1:
-                time_str = 'a minute ago'
-        return time_str
-    except:
-        return "-"
-
-
-
-async def fetch_from_db(fetch_flag):
-    try:
-        logging.warning(f'[session_id: {session_id}] fetch_flag: {fetch_flag}')
-        if fetch_flag == 1:
-            final_df = await db.read_news_from_db()
-            freq_tokens = await get_frequent_words_html(final_df)
-            logging.warning(f'[session_id: {session_id}] Fetched From DB')
-
-            final_df['_id'] = final_df['_id'].astype('str')
-            
-            redis_client.set("NEWSDF", final_df.to_json())
-            redis_client.set("NEWSWORDCLOUD", freq_tokens)
-        else:
-            final_df = pd.read_json(redis_client.get("NEWSDF"))
-            freq_tokens = redis_client.get("NEWSWORDCLOUD")
-            logging.warning(f'[session_id: {session_id}] Fetched From Cache')
-            
-    except Exception as e:
-        print(e)
-        final_df = []
-        freq_tokens = ""
-        # raise
-    return final_df, freq_tokens
-
-
-@app.route("/")
-async def index():
-    """
-    Entry point
-    """
-    try:
-        global session_id
-        session_id = uuid.uuid4().hex
-        src_str = ''
-        status_code = 200
-        logging.warning(f'[session_id: {session_id}] Entering the application')
-        final_df, freq_tokens = await fetch_from_db(is_db_fetch_reqd())
-        if len(final_df) == 0:
-            final_df, freq_tokens = await fetch_from_db(1)
-        if len(final_df) == 0:
-            raise Exception("Unable to fetch news")
-        if len(final_df) > 1:
-            
-            final_df["parsed_date"] = [correct_date(date_) for date_ in final_df['parsed_date']]
-            final_df["parsed_date"] = [parser.parse(date_) for date_ in final_df['parsed_date']]
-            final_df["elapsed_time"] =[date_time_parser(date_) for date_ in final_df['parsed_date']]
-            final_df = final_df.loc[final_df["elapsed_time"] <= NEWS_RETENTION_SECONDS, :].copy()
-            final_df["elapsed_time_str"] = final_df["elapsed_time"].apply(elapsed_time_str)
-            final_df.sort_values(by="elapsed_time", inplace=True)
-            src_str = ", ".join(sorted([*final_df['src'].unique()]))
-            final_df['src_time'] = final_df['src'] + ("&nbsp;" * 5) + final_df["elapsed_time_str"]
-            final_df.drop(columns=['_id', 'parsed_date', 'src', 'elapsed_time', 'elapsed_time_str'], inplace=True)
-            final_df.drop_duplicates(subset='description', inplace=True)
-            final_df = final_df.loc[(final_df["title"] != ""), :].copy()
-        else:
-            final_df = pd.DataFrame({'title': '', 'url': '',
-                                     'description': '', 'src_time': ''}, index=[0])
-        
-    except Exception as e:
-        final_df = pd.DataFrame({'title': '', 'url': '',
-                                 'description': '', 'src_time': ''}, index=[0])
-        logging.error(f'[session_id: {session_id}] {traceback.print_exc()}')
-
-    result_str = f'''
-                    <div class="box" id="main">
-                    <form> 
-                    
-                    <div class="banner">
-                    <img src="../static/favicon_new.png" class="logo-img" alt="KSV Muralidhar" />
-                    <h1 style="display:inline-block; vertical-align: middle;">Latest Indian News</h1>
-                    </div>
-                 '''
-    
-    if len(final_df) <= 1:
-        result_str += f'''<div><p class="unavailable">This app is temporarily unavailable</p></div>'''
-        status_code = 500
-    else:
-        last_update_utc = datetime.strptime(redis_client.get('NEWSFETCHTIME'), '%Y-%m-%d %H:%M:%S.%f')
-        last_update_mins = int(np.ceil((datetime.now() - last_update_utc).seconds / 60))
-        last_update_str = f'Updated {last_update_mins} {"minutes" if last_update_mins > 1 else "minute"} ago'
-        result_str += f'<p class="srctxt">News aggregated from <b>{src_str}</b>.<br><br>{last_update_str}&nbsp;&nbsp;&nbsp;&nbsp;<a href="{UK_EDITION_URL}"><b>Switch to UK edition</b></a></p>'
-
-        result_str += '''
-    	<div class="input-container">
-		<input type="text" class="keyword-input" id="keywordInput" placeholder="Search" oninput="filterContent(true)">
-		<div class="clear-btn" id="clearBtn" onclick="clearFilter()">&times;</div>
-        <img src="static/info.png" alt="info" width="18" height="18" align="center" onclick="showSearchInfo()" style="cursor: pointer;">
-		</div>
-      '''
-
-        result_str += f"{freq_tokens} "
-        result_str += '<div class="show-more-word-cloud" onclick=word_cloud_display()><p class="three-dots">...</p></div>'
-
-        result_str += f'''<div style="padding-bottom: 6px; font-size: 12px; font-family: Arial, Helvetica, sans-serif;">
-                         News categories and similar news are AI-generated</div>
-                         <div style="padding-bottom: 10px; font-size: 12px; font-family: Arial, Helvetica, sans-serif; font-weight: bold;">
-                         {len(final_df)} news articles available</div>
-                         '''
-
-        
-        for n, i in final_df.iterrows():  # iterating through the search results
-            href = i["url"]
-            category = i["category"]
-            description = i["description"]
-            url_txt = i["title"]
-            src_time = i["src_time"]
-            sim_news = i['similar_news']
-            result_str += f'''<div class="news-item"><div style="padding-top: 7px;">
-                              <a href="{href}" target="_blank" class="article-category">{category}
-                              </a>
-                              </div>
-                              <div>
-                              <a href="{href}" target="_blank" class="headline">{url_txt}
-                              </a>
-                              </div>
-                              <div>
-                              <a href="{href}" target="_blank" class="description">
-                              {description}
-                              </a>
-                              </div>
-                              <div>
-                              <a href="{href}" target="_blank" class="time">
-                              {src_time}
-                               </a>
-                               </div>
-
-
-                                <div class="container">
-                               <div class="content" style="display: none;">
-                               {sim_news}
-                               </div>
-                               <div class="show-similar-button-container">
-                               <button type="button" class="show-more">Show similar news</button>
-								<button type="button" class="show-less">Hide similar news</button>
-								</div>
-                                </div>
-
-
-                               
-                               <div>
-                               <p></p>
-                               </div></div>
-                               '''
-
-    result_str += '</form></div>'
-    logging.warning(f'[session_id: {session_id}] Successfully rendered template')
-    gc.collect()
-    return await render_template("index.html", body=result_str), status_code
-
-
-if __name__ == "__main__":
-    app.run(host="0.0.0.0", port=7860, workers=5, threads=5) # workers=(2*ncores) + 1, threads= (2 to 4*ncores) + 1
-    
+import numpy as np
+import pandas as pd
+from dateutil import parser
+from quart_cors import cors
+from quart import Quart
+from quart import render_template
+from db_operations.db_operations import DBOperations
+import logging
+import traceback 
+import redis
+import uuid
+from datetime import datetime
+from functools import lru_cache
+import gc
+from word_cloud import get_frequent_words_html
+from config import NEWS_RETENTION_SECONDS, UK_EDITION_URL
+
+
+app = Quart(__name__)
+app = cors(app, allow_origin="*")
+redis_client = redis.Redis(host='localhost', port=6379, decode_responses=True)
+logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s', level=logging.INFO)
+logging.warning(f'Is Redis available?: {redis_client.ping()}') 
+db = DBOperations()
+session_id = None
+
+
+REFRESH_FREQ = 300 # 300 secs = 5 mins
+
+def is_db_fetch_reqd():
+    try:
+        env_news_time = redis_client.get('NEWSFETCHTIME')
+        logging.warning(f'[session_id: {session_id}] fetch_time_env_var: {env_news_time}')
+        fetch_flag = 1
+        if env_news_time is None:
+            redis_client.set("NEWSFETCHTIME", str(datetime.now()))
+            fetch_flag = 1
+    
+        if env_news_time is not None:
+            fetch_time_lapse_seconds = (datetime.now() - datetime.strptime(env_news_time, '%Y-%m-%d %H:%M:%S.%f')).seconds
+            if fetch_time_lapse_seconds <= REFRESH_FREQ: 
+                fetch_flag = 0 
+            else:
+                redis_client.set("NEWSFETCHTIME", str(datetime.now()))
+                fetch_flag = 1
+    except Exception as e:
+        print(e)
+        fetch_flag = 1
+    return fetch_flag
+
+
+def correct_date(x):
+    if (not isinstance(x, str)) or (str(x).find(":") == -1):
+        logging.error(f'[session_id: {session_id}] correct_date() error: {x} is not the right date format')
+        return "2020-11-07 00:36:44+05:30"
+    return x
+
+def date_time_parser(dt):
+    """
+    Computes the minutes elapsed since published time.
+    :param dt: date
+    :return: int, minutes elapsed.
+    """
+    try:
+        return int(np.round((dt.now(dt.tz) - dt).total_seconds() / 60, 0))
+    except:
+        logging.error(f'[session_id: {session_id}] date_time_parser() error: {dt} is not the right date format')
+        return 100000
+
+
+def elapsed_time_str(mins):
+    """
+    Return the time elapsed string from minutes passed as an argument.
+    :param mins: int, minutes elapsed.
+    :return: str, time elapsed string
+    """
+    try:
+        time_str = ''
+        hours = int(mins / 60)
+        days = np.round(mins / (60 * 24), 1)
+        remaining_mins = int(mins - (hours * 60))
+        if days >= 1:
+            time_str = f'{str(days)} days ago'
+            if days == 1:
+                time_str = 'a day ago'
+        elif (days < 1) & (hours < 24) & (mins >= 60):
+            time_str = f'{str(hours)} hours and {str(remaining_mins)} mins ago'
+            if (hours == 1) & (remaining_mins > 1):
+                time_str = f'an hour and {str(remaining_mins)} mins ago'
+            if (hours == 1) & (remaining_mins == 1):
+                time_str = f'an hour and a min ago'
+            if (hours > 1) & (remaining_mins == 1):
+                time_str = f'{str(hours)} hours and a min ago'
+            if (hours > 1) & (remaining_mins == 0):
+                time_str = f'{str(hours)} hours ago'
+            if ((mins / 60) == 1) & (remaining_mins == 0):
+                time_str = 'an hour ago'
+        elif (days < 1) & (hours < 24) & (mins == 0):
+            time_str = 'Just in'
+        else:
+            time_str = f'{str(mins)} minutes ago'
+            if mins == 1:
+                time_str = 'a minute ago'
+        return time_str
+    except:
+        return "-"
+
+
+
+async def fetch_from_db(fetch_flag):
+    try:
+        logging.warning(f'[session_id: {session_id}] fetch_flag: {fetch_flag}')
+        if fetch_flag == 1:
+            final_df = await db.read_news_from_db()
+            freq_tokens = await get_frequent_words_html(final_df)
+            logging.warning(f'[session_id: {session_id}] Fetched From DB')
+
+            final_df['_id'] = final_df['_id'].astype('str')
+            
+            redis_client.set("NEWSDF", final_df.to_json())
+            redis_client.set("NEWSWORDCLOUD", freq_tokens)
+        else:
+            final_df = pd.read_json(redis_client.get("NEWSDF"))
+            freq_tokens = redis_client.get("NEWSWORDCLOUD")
+            logging.warning(f'[session_id: {session_id}] Fetched From Cache')
+            
+    except Exception as e:
+        print(e)
+        final_df = []
+        freq_tokens = ""
+        raise
+    return final_df, freq_tokens
+
+
+@app.route("/")
+async def index():
+    """
+    Entry point
+    """
+    try:
+        global session_id
+        session_id = uuid.uuid4().hex
+        src_str = ''
+        status_code = 200
+        logging.warning(f'[session_id: {session_id}] Entering the application')
+        final_df, freq_tokens = await fetch_from_db(is_db_fetch_reqd())
+        if len(final_df) > 1:
+            
+            final_df["parsed_date"] = [correct_date(date_) for date_ in final_df['parsed_date']]
+            final_df["parsed_date"] = [parser.parse(date_) for date_ in final_df['parsed_date']]
+            final_df["elapsed_time"] =[date_time_parser(date_) for date_ in final_df['parsed_date']]
+            final_df = final_df.loc[final_df["elapsed_time"] <= NEWS_RETENTION_SECONDS, :].copy()
+            final_df["elapsed_time_str"] = final_df["elapsed_time"].apply(elapsed_time_str)
+            final_df.sort_values(by="elapsed_time", inplace=True)
+            src_str = ", ".join(sorted([*final_df['src'].unique()]))
+            final_df['src_time'] = final_df['src'] + ("&nbsp;" * 5) + final_df["elapsed_time_str"]
+            final_df.drop(columns=['_id', 'parsed_date', 'src', 'elapsed_time', 'elapsed_time_str'], inplace=True)
+            final_df.drop_duplicates(subset='description', inplace=True)
+            final_df = final_df.loc[(final_df["title"] != ""), :].copy()
+        else:
+            final_df = pd.DataFrame({'title': '', 'url': '',
+                                     'description': '', 'src_time': ''}, index=[0])
+        
+    except Exception as e:
+        final_df = pd.DataFrame({'title': '', 'url': '',
+                                 'description': '', 'src_time': ''}, index=[0])
+        logging.error(f'[session_id: {session_id}] {traceback.print_exc()}')
+
+    result_str = f'''
+                    <div class="box" id="main">
+                    <form> 
+                    
+                    <div class="banner">
+                    <img src="../static/favicon_new.png" class="logo-img" alt="KSV Muralidhar" />
+                    <h1 style="display:inline-block; vertical-align: middle;">Latest Indian News</h1>
+                    </div>
+                 '''
+    
+    if len(final_df) <= 1:
+        result_str += f'''<div><p class="unavailable">This app is temporarily unavailable</p></div>'''
+        status_code = 500
+    else:
+        last_update_utc = datetime.strptime(redis_client.get('NEWSFETCHTIME'), '%Y-%m-%d %H:%M:%S.%f')
+        last_update_mins = int(np.ceil((datetime.now() - last_update_utc).seconds / 60))
+        last_update_str = f'Updated {last_update_mins} {"minutes" if last_update_mins > 1 else "minute"} ago'
+        result_str += f'<p class="srctxt">News aggregated from <b>{src_str}</b>.<br><br>{last_update_str}&nbsp;&nbsp;&nbsp;&nbsp;<a href="{UK_EDITION_URL}"><b>Switch to UK edition</b></a></p>'
+
+        result_str += '''
+    	<div class="input-container">
+		<input type="text" class="keyword-input" id="keywordInput" placeholder="Search" oninput="filterContent(true)">
+		<div class="clear-btn" id="clearBtn" onclick="clearFilter()">&times;</div>
+        <img src="static/info.png" alt="info" width="18" height="18" align="center" onclick="showSearchInfo()" style="cursor: pointer;">
+		</div>
+      '''
+
+        result_str += f"{freq_tokens} "
+        result_str += '<div class="show-more-word-cloud" onclick=word_cloud_display()><p class="three-dots">...</p></div>'
+
+        result_str += f'''<div style="padding-bottom: 6px; font-size: 12px; font-family: Arial, Helvetica, sans-serif;">
+                         News categories and similar news are AI-generated</div>
+                         <div style="padding-bottom: 10px; font-size: 12px; font-family: Arial, Helvetica, sans-serif; font-weight: bold;">
+                         {len(final_df)} news articles available</div>
+                         '''
+
+        
+        for n, i in final_df.iterrows():  # iterating through the search results
+            href = i["url"]
+            category = i["category"]
+            description = i["description"]
+            url_txt = i["title"]
+            src_time = i["src_time"]
+            sim_news = i['similar_news']
+            result_str += f'''<div class="news-item"><div style="padding-top: 7px;">
+                              <a href="{href}" target="_blank" class="article-category">{category}
+                              </a>
+                              </div>
+                              <div>
+                              <a href="{href}" target="_blank" class="headline">{url_txt}
+                              </a>
+                              </div>
+                              <div>
+                              <a href="{href}" target="_blank" class="description">
+                              {description}
+                              </a>
+                              </div>
+                              <div>
+                              <a href="{href}" target="_blank" class="time">
+                              {src_time}
+                               </a>
+                               </div>
+
+
+                                <div class="container">
+                               <div class="content" style="display: none;">
+                               {sim_news}
+                               </div>
+                               <div class="show-similar-button-container">
+                               <button type="button" class="show-more">Show similar news</button>
+								<button type="button" class="show-less">Hide similar news</button>
+								</div>
+                                </div>
+
+
+                               
+                               <div>
+                               <p></p>
+                               </div></div>
+                               '''
+
+    result_str += '</form></div>'
+    logging.warning(f'[session_id: {session_id}] Successfully rendered template')
+    gc.collect()
+    return await render_template("index.html", body=result_str), status_code
+
+
+if __name__ == "__main__":
+    app.run(host="0.0.0.0", port=7860, workers=5, threads=5) # workers=(2*ncores) + 1, threads= (2 to 4*ncores) + 1
+    

config.py

@@ -1,2 +1,2 @@
-NEWS_RETENTION_SECONDS = 300
-UK_EDITION_URL = "https://ksvmuralidhar-uk-news-aggregator.hf.space"
+NEWS_RETENTION_SECONDS = 300
+UK_EDITION_URL = "https://ksvmuralidhar-uk-news-aggregator.hf.space"

db_operations/db_operations.py

@@ -1,69 +1,69 @@
-import pymongo
-import os
-import pandas as pd
-import logging
-
-
-class DBOperations:
-    """
-    Reads news from MongoDB
-    """
-    def __init__(self):
-        self.url = os.getenv('DB_URL')
-        self.database = "rss_news_db_cat_pred_sim_news"
-        self.collection = "rss_news_cat_pred_sim_news"
-        self.__client = None
-        self.__error = 0
-
-    async def __connect(self):
-        try:
-            self.__client = pymongo.MongoClient(self.url)
-            _ = self.__client.list_database_names()
-        except Exception as conn_exception:
-            self.__error = 1
-            logging.critical(f"Error in DBOperations.connect(): {conn_exception}")
-            self.__client = None
-            raise
-
-    async def __read(self):
-        try:
-            db = self.__client[self.database]
-            coll = db[self.collection]
-            docs = []
-            maxtries = 5
-            ntry = 0
-            
-            while (len(docs) == 0) and (ntry < maxtries):
-                for doc in coll.find():
-                    docs.append(doc)
-                ntry += 1
-                logging.info(f"DB Read try: {ntry}")    
-                
-            rss_df = pd.DataFrame(docs)
-        except Exception as insert_err:
-            self.__error = 1
-            logging.critical(f"Error in DBOperations.read(): {insert_err}")
-            rss_df = pd.DataFrame({'_id': '', 'title': '', 'url': '',
-                                   'description': '', 'parsed_date': '',
-                                   'src': ''}, index=[0])
-        return rss_df
-
-    def __close_connection(self):
-        if self.__client is not None:
-            self.__client.close()
-            self.__client = None
-
-    async def read_news_from_db(self):
-        rss_df = pd.DataFrame({'_id': '', 'title': '', 'url': '',
-                               'description': '', 'parsed_date': '',
-                               'src': ''}, index=[0])
-        if self.url is not None:
-            if self.__error == 0:
-                await self.__connect()
-            if self.__error == 0:
-                rss_df = await self.__read()
-            if self.__error == 0:
-                logging.info("Read Successful")
-            if self.__client is not None:
-                self.__close_connection()
-        return rss_df
+import pymongo
+import os
+import pandas as pd
+import logging
+
+
+class DBOperations:
+    """
+    Reads news from MongoDB
+    """
+    def __init__(self):
+        self.url = os.getenv('DB_URL')
+        self.database = "rss_news_db_cat_pred_sim_news"
+        self.collection = "rss_news_cat_pred_sim_news"
+        self.__client = None
+        self.__error = 0
+
+    async def __connect(self):
+        try:
+            self.__client = pymongo.MongoClient(self.url)
+            _ = self.__client.list_database_names()
+        except Exception as conn_exception:
+            self.__error = 1
+            logging.critical(f"Error in DBOperations.connect(): {conn_exception}")
+            self.__client = None
+            raise
+
+    async def __read(self):
+        try:
+            db = self.__client[self.database]
+            coll = db[self.collection]
+            docs = []
+            maxtries = 5
+            ntry = 0
+            
+            while (len(docs) == 0) and (ntry < maxtries):
+                for doc in coll.find():
+                    docs.append(doc)
+                ntry += 1
+                logging.info(f"DB Read try: {ntry}")    
+                
+            rss_df = pd.DataFrame(docs)
+        except Exception as insert_err:
+            self.__error = 1
+            logging.critical(f"Error in DBOperations.read(): {insert_err}")
+            rss_df = pd.DataFrame({'_id': '', 'title': '', 'url': '',
+                                   'description': '', 'parsed_date': '',
+                                   'src': ''}, index=[0])
+        return rss_df
+
+    def __close_connection(self):
+        if self.__client is not None:
+            self.__client.close()
+            self.__client = None
+
+    async def read_news_from_db(self):
+        rss_df = pd.DataFrame({'_id': '', 'title': '', 'url': '',
+                               'description': '', 'parsed_date': '',
+                               'src': ''}, index=[0])
+        if self.url is not None:
+            if self.__error == 0:
+                await self.__connect()
+            if self.__error == 0:
+                rss_df = await self.__read()
+            if self.__error == 0:
+                logging.info("Read Successful")
+            if self.__client is not None:
+                self.__close_connection()
+        return rss_df

newrelic.ini

@@ -0,0 +1,255 @@
+# ---------------------------------------------------------------------------
+
+#
+# This file configures the New Relic Python Agent.
+#
+# The path to the configuration file should be supplied to the function
+# newrelic.agent.initialize() when the agent is being initialized.
+#
+# The configuration file follows a structure similar to what you would
+# find for Microsoft Windows INI files. For further information on the
+# configuration file format see the Python ConfigParser documentation at:
+#
+#    http://docs.python.org/library/configparser.html
+#
+# For further discussion on the behaviour of the Python agent that can
+# be configured via this configuration file see:
+#
+#    https://docs.newrelic.com/docs/apm/agents/python-agent/configuration/python-agent-configuration/
+#
+
+# ---------------------------------------------------------------------------
+
+# Here are the settings that are common to all environments.
+
+[newrelic]
+
+# You must specify the license key associated with your New
+# Relic account. This may also be set using the NEW_RELIC_LICENSE_KEY
+# environment variable. This key binds the Python Agent's data to
+# your account in the New Relic service. For more information on
+# storing and generating license keys, see
+# https://docs.newrelic.com/docs/apis/intro-apis/new-relic-api-keys/#ingest-license-key
+
+
+# The application name. Set this to be the name of your
+# application as you would like it to show up in New Relic UI.
+# You may also set this using the NEW_RELIC_APP_NAME environment variable.
+# The UI will then auto-map instances of your application into a
+# entry on your home dashboard page. You can also specify multiple
+# app names to group your aggregated data. For further details,
+# please see:
+# https://docs.newrelic.com/docs/apm/agents/manage-apm-agents/app-naming/use-multiple-names-app/
+app_name = news_aggregator
+
+# When "true", the agent collects performance data about your
+# application and reports this data to the New Relic UI at
+# newrelic.com. This global switch is normally overridden for
+# each environment below. It may also be set using the
+# NEW_RELIC_MONITOR_MODE environment variable.
+monitor_mode = true
+
+# Sets the name of a file to log agent messages to. Whatever you
+# set this to, you must ensure that the permissions for the
+# containing directory and the file itself are correct, and
+# that the user that your web application runs as can write out
+# to the file. If not able to out a log file, it is also
+# possible to say "stderr" and output to standard error output.
+# This would normally result in output appearing in your web
+# server log. It can also be set using the NEW_RELIC_LOG
+# environment variable.
+log_file = stdout
+
+# Sets the level of detail of messages sent to the log file, if
+# a log file location has been provided. Possible values, in
+# increasing order of detail, are: "critical", "error", "warning",
+# "info" and "debug". When reporting any agent issues to New
+# Relic technical support, the most useful setting for the
+# support engineers is "debug". However, this can generate a lot
+# of information very quickly, so it is best not to keep the
+# agent at this level for longer than it takes to reproduce the
+# problem you are experiencing. This may also be set using the
+# NEW_RELIC_LOG_LEVEL environment variable.
+log_level = info
+
+# High Security Mode enforces certain security settings, and prevents
+# them from being overridden, so that no sensitive data is sent to New
+# Relic. Enabling High Security Mode means that request parameters are
+# not collected and SQL can not be sent to New Relic in its raw form.
+# To activate High Security Mode, it must be set to 'true' in this
+# local .ini configuration file AND be set to 'true' in the
+# server-side configuration in the New Relic user interface. It can
+# also be set using the NEW_RELIC_HIGH_SECURITY environment variable.
+# For details, see
+# https://docs.newrelic.com/docs/subscriptions/high-security
+high_security = false
+
+# The Python Agent will attempt to connect directly to the New
+# Relic service. If there is an intermediate firewall between
+# your host and the New Relic service that requires you to use a
+# HTTP proxy, then you should set both the "proxy_host" and
+# "proxy_port" settings to the required values for the HTTP
+# proxy. The "proxy_user" and "proxy_pass" settings should
+# additionally be set if proxy authentication is implemented by
+# the HTTP proxy. The "proxy_scheme" setting dictates what
+# protocol scheme is used in talking to the HTTP proxy. This
+# would normally always be set as "http" which will result in the
+# agent then using a SSL tunnel through the HTTP proxy for end to
+# end encryption.
+# See https://docs.newrelic.com/docs/apm/agents/python-agent/configuration/python-agent-configuration/#proxy
+# for information on proxy configuration via environment variables.
+# proxy_scheme = http
+# proxy_host = hostname
+# proxy_port = 8080
+# proxy_user =
+# proxy_pass =
+
+# Capturing request parameters is off by default. To enable the
+# capturing of request parameters, first ensure that the setting
+# "attributes.enabled" is set to "true" (the default value), and
+# then add "request.parameters.*" to the "attributes.include"
+# setting. For details about attributes configuration, please
+# consult the documentation.
+# attributes.include = request.parameters.*
+
+# The transaction tracer captures deep information about slow
+# transactions and sends this to the UI on a periodic basis. The
+# transaction tracer is enabled by default. Set this to "false"
+# to turn it off.
+transaction_tracer.enabled = true
+
+# Threshold in seconds for when to collect a transaction trace.
+# When the response time of a controller action exceeds this
+# threshold, a transaction trace will be recorded and sent to
+# the UI. Valid values are any positive float value, or (default)
+# "apdex_f", which will use the threshold for a dissatisfying
+# Apdex controller action - four times the Apdex T value.
+transaction_tracer.transaction_threshold = apdex_f
+
+# When the transaction tracer is on, SQL statements can
+# optionally be recorded. The recorder has three modes, "off"
+# which sends no SQL, "raw" which sends the SQL statement in its
+# original form, and "obfuscated", which strips out numeric and
+# string literals.
+transaction_tracer.record_sql = obfuscated
+
+# Threshold in seconds for when to collect stack trace for a SQL
+# call. In other words, when SQL statements exceed this
+# threshold, then capture and send to the UI the current stack
+# trace. This is helpful for pinpointing where long SQL calls
+# originate from in an application.
+transaction_tracer.stack_trace_threshold = 0.5
+
+# Determines whether the agent will capture query plans for slow
+# SQL queries. Only supported in MySQL and PostgreSQL. Set this
+# to "false" to turn it off.
+transaction_tracer.explain_enabled = true
+
+# Threshold for query execution time below which query plans
+# will not not be captured. Relevant only when "explain_enabled"
+# is true.
+transaction_tracer.explain_threshold = 0.5
+
+# Space separated list of function or method names in form
+# 'module:function' or 'module:class.function' for which
+# additional function timing instrumentation will be added.
+transaction_tracer.function_trace =
+
+# The error collector captures information about uncaught
+# exceptions or logged exceptions and sends them to UI for
+# viewing. The error collector is enabled by default. Set this
+# to "false" to turn it off. For more details on errors, see
+# https://docs.newrelic.com/docs/apm/agents/manage-apm-agents/agent-data/manage-errors-apm-collect-ignore-or-mark-expected/
+error_collector.enabled = true
+
+# To stop specific errors from reporting to the UI, set this to
+# a space separated list of the Python exception type names to
+# ignore. The exception name should be of the form 'module:class'.
+error_collector.ignore_classes =
+
+# Expected errors are reported to the UI but will not affect the
+# Apdex or error rate. To mark specific errors as expected, set this
+# to a space separated list of the Python exception type names to
+# expected. The exception name should be of the form 'module:class'.
+error_collector.expected_classes =
+
+# Browser monitoring is the Real User Monitoring feature of the UI.
+# For those Python web frameworks that are supported, this
+# setting enables the auto-insertion of the browser monitoring
+# JavaScript fragments.
+browser_monitoring.auto_instrument = true
+
+# A thread profiling session can be scheduled via the UI when
+# this option is enabled. The thread profiler will periodically
+# capture a snapshot of the call stack for each active thread in
+# the application to construct a statistically representative
+# call tree. For more details on the thread profiler tool, see
+# https://docs.newrelic.com/docs/apm/apm-ui-pages/events/thread-profiler-tool/
+thread_profiler.enabled = true
+
+# Your application deployments can be recorded through the
+# New Relic REST API. To use this feature provide your API key
+# below then use the `newrelic-admin record-deploy` command.
+# This can also be set using the NEW_RELIC_API_KEY
+# environment variable.
+# api_key =
+
+# Distributed tracing lets you see the path that a request takes
+# through your distributed system. For more information, please
+# consult our distributed tracing planning guide.
+# https://docs.newrelic.com/docs/transition-guide-distributed-tracing
+distributed_tracing.enabled = true
+
+# This setting enables log decoration, the forwarding of log events,
+# and the collection of logging metrics if these sub-feature
+# configurations are also enabled. If this setting is false, no
+# logging instrumentation features are enabled. This can also be
+# set using the NEW_RELIC_APPLICATION_LOGGING_ENABLED environment
+# variable.
+# application_logging.enabled = true
+
+# If true, the agent captures log records emitted by your application
+# and forwards them to New Relic. `application_logging.enabled` must
+# also be true for this setting to take effect. You can also set
+# this using the NEW_RELIC_APPLICATION_LOGGING_FORWARDING_ENABLED
+# environment variable.
+# application_logging.forwarding.enabled = true
+
+# If true, the agent decorates logs with metadata to link to entities,
+# hosts, traces, and spans. `application_logging.enabled` must also
+# be true for this setting to take effect. This can also be set
+# using the NEW_RELIC_APPLICATION_LOGGING_LOCAL_DECORATING_ENABLED
+# environment variable.
+# application_logging.local_decorating.enabled = true
+
+# If true, the agent captures metrics related to the log lines
+# being sent up by your application. This can also be set
+# using the NEW_RELIC_APPLICATION_LOGGING_METRICS_ENABLED
+# environment variable.
+# application_logging.metrics.enabled = true
+
+
+# ---------------------------------------------------------------------------
+
+#
+# The application environments. These are specific settings which
+# override the common environment settings. The settings related to a
+# specific environment will be used when the environment argument to the
+# newrelic.agent.initialize() function has been defined to be either
+# "development", "test", "staging" or "production".
+#
+
+[newrelic:development]
+monitor_mode = false
+
+[newrelic:test]
+monitor_mode = false
+
+[newrelic:staging]
+app_name =  (Staging)
+monitor_mode = true
+
+[newrelic:production]
+monitor_mode = true
+
+# ---------------------------------------------------------------------------

requirements.txt

@@ -1,15 +1,16 @@
-regex==2021.8.3
-lxml==4.6.3
-numpy==1.21.1
-python-dateutil==2.8.2
-pandas==1.3.1
-requests==2.26.0
-bs4==0.0.1
-gunicorn
-pymongo==4.3.3
-unidecode
-redis
-asyncio
-uvicorn
-Quart
-quart-cors
+regex==2021.8.3
+lxml==4.6.3
+numpy==1.21.1
+python-dateutil==2.8.2
+pandas==1.3.1
+requests==2.26.0
+bs4==0.0.1
+gunicorn
+pymongo==4.3.3
+unidecode
+redis
+newrelic
+asyncio
+uvicorn
+Quart
+quart-cors

start.sh

@@ -1,6 +1,6 @@
-#!/bin/bash
-redis-server --daemonize yes
-redis-cli config set save ""
-redis-cli config set appendonly no
-redis-cli config set stop-writes-on-bgsave-error no
+#!/bin/bash
+redis-server --daemonize yes
+redis-cli config set save ""
+redis-cli config set appendonly no
+redis-cli config set stop-writes-on-bgsave-error no
 gunicorn -b 0.0.0.0:7860 --timeout 120 --worker-class uvicorn.workers.UvicornWorker --workers 5 --threads 5 app:app

static/styles.css

@@ -1,510 +1,510 @@
-html {
-scroll-behavior: smooth;
-}
-
-@media screen and (min-width: 800px) {
-a.headline {
-background-color: #E5E4E2;
-display: block;
-width: relative;
-text-decoration: none;
-color: black;
-line-height: 1.2;
-align: justify;
-border-left: 5px solid transparent;
-border-top: 5px solid transparent;
-border-bottom: 5px solid transparent;
-border-right: 0px;
-font-weight: bold;
-font-size: 18px;
-padding-right: 5px;
-font-family: Arial, Helvetica, sans-serif;
-}
-}
-
-@media screen and (max-width: 800px) {
-a.headline {
-background-color: #E5E4E2;
-display: block;
-width: relative;
-text-decoration: none;
-color: black;
-line-height: 1.2;
-align: justify;
-border-left: 5px solid transparent;
-border-top: 5px solid transparent;
-border-bottom: 5px solid transparent;
-border-right: 0px;
-font-weight: bold;
-font-size: 16.5px;
-padding-right: 5px;
-font-family: Arial, Helvetica, sans-serif;
-}
-}
-
-@media screen and (min-width: 800px) {
-a.description {
-background-color: #E5E4E2;
-align:justify;
-text-align: justify;
-display: block;
-height:100%;
-width: relative;
-text-decoration: none;
-border-left: 5px solid transparent;
-border-top: 0px;
-border-bottom: 7px solid transparent;
-border-right: 0px;
-font-size: 14px;
-padding-right: 5px;
-font-family: Arial, Helvetica, sans-serif;
-color: dimgrey;
-}
-}
-
-@media screen and (max-width: 800px) {
-a.description {
-background-color: #E5E4E2;
-align:justify;
-text-align: justify;
-display: block;
-height:100%;
-width: relative;
-text-decoration: none;
-border-left: 5px solid transparent;
-border-top: 0px;
-border-bottom: 7px solid transparent;
-border-right: 0px;
-font-size: 12.5px;
-padding-right: 5px;
-font-family: Arial, Helvetica, sans-serif;
-color: dimgrey;
-}
-}
-
-@media screen and (min-width: 800px) {
-a.time {
-background-color: #E5E4E2;
-align:justify;
-display: block;
-height:100%;
-width: relative;
-text-decoration: none;
-border-left: 5px solid transparent;
-border-top: 0px;
-border-bottom: 1px solid transparent;
-border-right: 0px;
-padding-right: 5px;
-font-size: 11px;
-padding-bottom: 5px;
-font-family: Arial, Helvetica, sans-serif;
-color: green;
-}
-}
-
-@media screen and (max-width: 800px) {
-a.time {
-background-color: #E5E4E2;
-align:justify;
-display: block;
-height:100%;
-width: relative;
-text-decoration: none;
-border-left: 5px solid transparent;
-border-top: 0px;
-border-bottom: 1px solid transparent;
-border-right: 0px;
-padding-right: 5px;
-font-size: 10px;
-padding-bottom: 5px;
-font-family: Arial, Helvetica, sans-serif;
-color: green;
-}
-}
-
-.box {
-display: flex;
-justify-content: center;
-align-items: center;
-height: inherit;
-padding: 20px;
-}
-@media screen and (min-width: 800px) {
-form {
-width: 50%;
-overflow-x: hidden;
-padding: 20px;
-border-radius: 10px;
-background: #fff;
-box-shadow: 0 0 20px 0 #095484;
-    }}
-
-@media screen and (max-width: 800px) {
-form {
-width: 100%;
-overflow-x: hidden;
-padding: 20px;
-border-radius: 10px;
-background: #fff;
-box-shadow: 0 0 15px 0 #095484;
-}}
-.banner {
-      position: relative;
-      height: 30px;
-/*       background-size: cover; */
-      display: flex;
-/*       justify-content: center; */
-/*       align-items: center; */
-/*       text-align: center; */
-      }
-@media screen and (min-width: 800px) {
-h1 {
-      position: absolute;
-      margin: 0;
-      padding-left: 50px;
-      font-size: 25px;
-      color: black;
-      z-index: 2;
-      font-family: Arial, Helvetica, sans-serif;
-      }
-}
-
-@media screen and (max-width: 800px) {
-h1 {
-      position: absolute;
-      margin: 0;
-      padding-left: 40px;
-      font-size: 24px;
-      color: black;
-      z-index: 2;
-      font-family: Arial, Helvetica, sans-serif;
-      }
-}
-
-p.unavailable {
-background-color: #E5E4E2;
-display: block;
-width: 100%;
-text-decoration: none;
-color: black;
-line-height: 1.2;
-align: justify;
-border-left: 5px solid transparent;
-border-top: 5px solid transparent;
-border-bottom: 5px solid transparent;
-border-right: 0px;
-font-weight: bold;
-font-size: 18px;
-padding-right: 5px;
-font-family: Arial, Helvetica, sans-serif;
-}
-div.news-item{
-	background-color: #E5E4E2;
-/*box-shadow: rgba(0, 0, 0, 0.4) -1px 0px 5px, rgba(0, 0, 0, 0.5) 0px 4px 5px -3px, rgba(0, 0, 0, 0.2) 0px -3px 0px inset;*/
-box-shadow: rgba(0, 0, 0, 0.25) 0px 0px 5px 1px, rgba(0, 0, 0, 0.1) 0px 4px 5px -3px, rgba(0, 0, 0, 0.2) 0px -1px 0px inset;
-
-
-	  }
-div.news-item:hover{
-box-shadow: none;
-}
-
-@media screen and (min-width: 800px) {
-p.srctxt {
-align:justify;
-text-align: justify;
-word-break: break-all;
-font-size: 11px;
-font-family: Arial, Helvetica, sans-serif;
-}
-.logo-img{
-margin-right: 10px;
-vertical-align: center;
-/* position: relative; */
-width: 34px;
-height: 34px;
-
-}
-}
-
-@media screen and (max-width: 800px) {
-p.srctxt {
-align:justify;
-text-align: justify;
-word-break: break-all;
-font-size: 9px;
-font-family: Arial, Helvetica, sans-serif;
-}
-.logo-img{
-margin-right: 10px;
-vertical-align: top;
-/* position: absolute; */
-width: 30px;
-height: 30px;
-}
-}
-
-.float{
-	position:fixed;
-	width:25px;
-	height:25px;
-	bottom:15px;
-	right:12px;
-	background-color: white;
-	border-radius:50%;
-	text-align:center;
-	vertical-align:center;
-	z-index: 99999998;
-	font-size:0;
-	cursor:pointer;
-	animation: beatan 0.8s infinite alternate;
-	
-}
-.top-float{
-	position:fixed;
-	width:25px;
-	height:25px;
-	bottom:52px;
-	right:12px;
-	background-color: white;
-	border-radius:50%;
-	text-align:center;
-	vertical-align:center;
-	z-index: 99999998;
-	font-size:0;
-	cursor:pointer;
-	animation: beatan 0.8s infinite alternate;
-	
-}
-.my-float{
-	margin-top:22px;
-}
-
-@keyframes beatan{
-	to { transform: scale(1.1); }
-}
-
-.loader {
-	position: fixed;
-	left: 0px;
-	top: 0px;
-	width: 100%;
-	height: 100%;
-	z-index: 99999999999;
-	background: url('../static/loader.gif') 50% 50% no-repeat rgb(255,255,255);
-}
-
- .highlight {
-    background-color: yellow;
-    font-weight: bold;
-  }
-
-.input-container {
-	position: relative;
-    padding-bottom: 10px;
-  }
-
-  .keyword-input {
-	
-    border-radius: 5px;
-    transition: border-color 0.3s ease;
-	border: 1px solid silver;
-	width: 10em;
-	height: 1.5em;
-	padding-left: 0.5em;
-	outline: none;
-	overflow: hidden;
-
-  }
-
-  .clear-btn {
-    position: absolute;
-	font-size: 20px;
-    left: 129px;
-    transform: translateY(-105%);
-    cursor: pointer;
-    opacity: 0;
-    transition: opacity 0.3s ease;
-  }
-
-  .clear-btn.show {
-    opacity: 1;
-  }
-
-@media screen and (min-width: 800px) {
-a.article-category {
-background-color: #E5E4E2;
-align:justify;
-display: block;
-height:100%;
-width: relative;
-text-decoration: none;
-border-left: 5px solid transparent;
-border-top: 0px;
-font-weight: bold;
-border-bottom: 1px solid transparent;
-border-right: 0px;
-padding-right: 5px;
-font-size: 11px;
-padding-bottom: 0px;
-font-family: Arial, Helvetica, sans-serif;
-color: green;
-}
-}
-
-@media screen and (max-width: 800px) {
-a.article-category {
-background-color: #E5E4E2;
-align:justify;
-display: block;
-height:100%;
-font-weight: bold;
-width: relative;
-text-decoration: none;
-border-left: 5px solid transparent;
-border-top: 0px;
-border-bottom: 1px solid transparent;
-border-right: 0px;
-padding-right: 5px;
-font-size: 10px;
-padding-bottom: 0px;
-font-family: Arial, Helvetica, sans-serif;
-color: green;
-}
-}
-
-.content {
-    display: none;
-	font-family: Arial, Helvetica, sans-serif;
-	
-  padding-right: 5px;
-  
-  padding-top: 5px;
-  border-left: 5px solid transparent;
-  }
-  
- .container{
- padding-bottom:10px;
- }
- 
- .show-similar-button-container{
- display: flex;
- flex-direction: column;
- align-items: center; 
- }
- 
- .similar-news-item:hover {
- text-decoration: none;
- }
- 
- @media screen and (min-width: 800px) {
-.similar-news-item {
-text-align: justify;
-text-decoration: underline;
-font-size: 14px;
-font-family: Arial, Helvetica, sans-serif;
-color: black;
-display:inline-block;
-padding-bottom: 10px;
-width:100%;
-/*white-space: nowrap;
-overflow: hidden;
-text-overflow: ellipsis;*/
-
-}
-}
-
-@media screen and (max-width: 800px) {
-.similar-news-item {
-text-align: justify;
-text-decoration: underline;
-font-size: 12px;
-font-family: Arial, Helvetica, sans-serif;
-color: black;
-display:inline-block;
-padding-bottom: 8px;
-width:100%;
-/*white-space: nowrap;
-overflow: hidden;
-text-overflow: ellipsis;*/
-}
-}
- 
- 
- 
- .show-more {
-background-color: #E5E4E2;
-font-family: Arial, Helvetica, sans-serif;
-border-radius:4px;
-padding-top:3px;
-padding-bottom:3px;
-padding-left:3px;
-padding-right:3px;
-font-size: 12px;
-display: box;
-border: none;
-
-  }
-  
- .show-more:hover {
-background-color: black;
-color: white;
-  }
- 
- .show-less {
-background-color: #E5E4E2;
-font-family: Arial, Helvetica, sans-serif;
-border-radius:4px;
-padding-top:3px;
-padding-bottom:3px;
-padding-left:3px;
-padding-right:3px;
-font-size: 12px;
-border: none;
-display: none;
-  }
-  
- .show-less:hover {
-background-color: black;
-color: white;
-  }
-  
- .word-cloud-container{
- word-wrap: break-word;
-padding-bottom: 10px;
-
- }
- 
- .wc-tokens{
-font-family: Arial, Helvetica, sans-serif;
-font-size: 13.2px;
-cursor: pointer;
- }
- 
- .wc-tokens:hover{
- text-decoration: underline;
- }
- 
- .word-cloud-section{
-   padding-bottom: 10px;
-	 display: none;
-   word-wrap: break-word;
- }
- 
- .show-more-word-cloud{
- padding-bottom: 23px;
- text-align: center;
- }
- 
- .three-dots{
-	 font-size: 30px;
- margin: 0;
- line-height:0;
- vertical-align: top;
-    padding: 0;
-	cursor: pointer;
- }
- 
- .three-dots:hover{
-	 font-size: 25px;
+html {
+scroll-behavior: smooth;
+}
+
+@media screen and (min-width: 800px) {
+a.headline {
+background-color: #E5E4E2;
+display: block;
+width: relative;
+text-decoration: none;
+color: black;
+line-height: 1.2;
+align: justify;
+border-left: 5px solid transparent;
+border-top: 5px solid transparent;
+border-bottom: 5px solid transparent;
+border-right: 0px;
+font-weight: bold;
+font-size: 18px;
+padding-right: 5px;
+font-family: Arial, Helvetica, sans-serif;
+}
+}
+
+@media screen and (max-width: 800px) {
+a.headline {
+background-color: #E5E4E2;
+display: block;
+width: relative;
+text-decoration: none;
+color: black;
+line-height: 1.2;
+align: justify;
+border-left: 5px solid transparent;
+border-top: 5px solid transparent;
+border-bottom: 5px solid transparent;
+border-right: 0px;
+font-weight: bold;
+font-size: 16.5px;
+padding-right: 5px;
+font-family: Arial, Helvetica, sans-serif;
+}
+}
+
+@media screen and (min-width: 800px) {
+a.description {
+background-color: #E5E4E2;
+align:justify;
+text-align: justify;
+display: block;
+height:100%;
+width: relative;
+text-decoration: none;
+border-left: 5px solid transparent;
+border-top: 0px;
+border-bottom: 7px solid transparent;
+border-right: 0px;
+font-size: 14px;
+padding-right: 5px;
+font-family: Arial, Helvetica, sans-serif;
+color: dimgrey;
+}
+}
+
+@media screen and (max-width: 800px) {
+a.description {
+background-color: #E5E4E2;
+align:justify;
+text-align: justify;
+display: block;
+height:100%;
+width: relative;
+text-decoration: none;
+border-left: 5px solid transparent;
+border-top: 0px;
+border-bottom: 7px solid transparent;
+border-right: 0px;
+font-size: 12.5px;
+padding-right: 5px;
+font-family: Arial, Helvetica, sans-serif;
+color: dimgrey;
+}
+}
+
+@media screen and (min-width: 800px) {
+a.time {
+background-color: #E5E4E2;
+align:justify;
+display: block;
+height:100%;
+width: relative;
+text-decoration: none;
+border-left: 5px solid transparent;
+border-top: 0px;
+border-bottom: 1px solid transparent;
+border-right: 0px;
+padding-right: 5px;
+font-size: 11px;
+padding-bottom: 5px;
+font-family: Arial, Helvetica, sans-serif;
+color: green;
+}
+}
+
+@media screen and (max-width: 800px) {
+a.time {
+background-color: #E5E4E2;
+align:justify;
+display: block;
+height:100%;
+width: relative;
+text-decoration: none;
+border-left: 5px solid transparent;
+border-top: 0px;
+border-bottom: 1px solid transparent;
+border-right: 0px;
+padding-right: 5px;
+font-size: 10px;
+padding-bottom: 5px;
+font-family: Arial, Helvetica, sans-serif;
+color: green;
+}
+}
+
+.box {
+display: flex;
+justify-content: center;
+align-items: center;
+height: inherit;
+padding: 20px;
+}
+@media screen and (min-width: 800px) {
+form {
+width: 50%;
+overflow-x: hidden;
+padding: 20px;
+border-radius: 10px;
+background: #fff;
+box-shadow: 0 0 20px 0 #095484;
+    }}
+
+@media screen and (max-width: 800px) {
+form {
+width: 100%;
+overflow-x: hidden;
+padding: 20px;
+border-radius: 10px;
+background: #fff;
+box-shadow: 0 0 15px 0 #095484;
+}}
+.banner {
+      position: relative;
+      height: 30px;
+/*       background-size: cover; */
+      display: flex;
+/*       justify-content: center; */
+/*       align-items: center; */
+/*       text-align: center; */
+      }
+@media screen and (min-width: 800px) {
+h1 {
+      position: absolute;
+      margin: 0;
+      padding-left: 50px;
+      font-size: 25px;
+      color: black;
+      z-index: 2;
+      font-family: Arial, Helvetica, sans-serif;
+      }
+}
+
+@media screen and (max-width: 800px) {
+h1 {
+      position: absolute;
+      margin: 0;
+      padding-left: 40px;
+      font-size: 24px;
+      color: black;
+      z-index: 2;
+      font-family: Arial, Helvetica, sans-serif;
+      }
+}
+
+p.unavailable {
+background-color: #E5E4E2;
+display: block;
+width: 100%;
+text-decoration: none;
+color: black;
+line-height: 1.2;
+align: justify;
+border-left: 5px solid transparent;
+border-top: 5px solid transparent;
+border-bottom: 5px solid transparent;
+border-right: 0px;
+font-weight: bold;
+font-size: 18px;
+padding-right: 5px;
+font-family: Arial, Helvetica, sans-serif;
+}
+div.news-item{
+	background-color: #E5E4E2;
+/*box-shadow: rgba(0, 0, 0, 0.4) -1px 0px 5px, rgba(0, 0, 0, 0.5) 0px 4px 5px -3px, rgba(0, 0, 0, 0.2) 0px -3px 0px inset;*/
+box-shadow: rgba(0, 0, 0, 0.25) 0px 0px 5px 1px, rgba(0, 0, 0, 0.1) 0px 4px 5px -3px, rgba(0, 0, 0, 0.2) 0px -1px 0px inset;
+
+
+	  }
+div.news-item:hover{
+box-shadow: none;
+}
+
+@media screen and (min-width: 800px) {
+p.srctxt {
+align:justify;
+text-align: justify;
+word-break: break-all;
+font-size: 11px;
+font-family: Arial, Helvetica, sans-serif;
+}
+.logo-img{
+margin-right: 10px;
+vertical-align: center;
+/* position: relative; */
+width: 34px;
+height: 34px;
+
+}
+}
+
+@media screen and (max-width: 800px) {
+p.srctxt {
+align:justify;
+text-align: justify;
+word-break: break-all;
+font-size: 9px;
+font-family: Arial, Helvetica, sans-serif;
+}
+.logo-img{
+margin-right: 10px;
+vertical-align: top;
+/* position: absolute; */
+width: 30px;
+height: 30px;
+}
+}
+
+.float{
+	position:fixed;
+	width:25px;
+	height:25px;
+	bottom:15px;
+	right:12px;
+	background-color: white;
+	border-radius:50%;
+	text-align:center;
+	vertical-align:center;
+	z-index: 99999998;
+	font-size:0;
+	cursor:pointer;
+	animation: beatan 0.8s infinite alternate;
+	
+}
+.top-float{
+	position:fixed;
+	width:25px;
+	height:25px;
+	bottom:52px;
+	right:12px;
+	background-color: white;
+	border-radius:50%;
+	text-align:center;
+	vertical-align:center;
+	z-index: 99999998;
+	font-size:0;
+	cursor:pointer;
+	animation: beatan 0.8s infinite alternate;
+	
+}
+.my-float{
+	margin-top:22px;
+}
+
+@keyframes beatan{
+	to { transform: scale(1.1); }
+}
+
+.loader {
+	position: fixed;
+	left: 0px;
+	top: 0px;
+	width: 100%;
+	height: 100%;
+	z-index: 99999999999;
+	background: url('../static/loader.gif') 50% 50% no-repeat rgb(255,255,255);
+}
+
+ .highlight {
+    background-color: yellow;
+    font-weight: bold;
+  }
+
+.input-container {
+	position: relative;
+    padding-bottom: 10px;
+  }
+
+  .keyword-input {
+	
+    border-radius: 5px;
+    transition: border-color 0.3s ease;
+	border: 1px solid silver;
+	width: 10em;
+	height: 1.5em;
+	padding-left: 0.5em;
+	outline: none;
+	overflow: hidden;
+
+  }
+
+  .clear-btn {
+    position: absolute;
+	font-size: 20px;
+    left: 129px;
+    transform: translateY(-105%);
+    cursor: pointer;
+    opacity: 0;
+    transition: opacity 0.3s ease;
+  }
+
+  .clear-btn.show {
+    opacity: 1;
+  }
+
+@media screen and (min-width: 800px) {
+a.article-category {
+background-color: #E5E4E2;
+align:justify;
+display: block;
+height:100%;
+width: relative;
+text-decoration: none;
+border-left: 5px solid transparent;
+border-top: 0px;
+font-weight: bold;
+border-bottom: 1px solid transparent;
+border-right: 0px;
+padding-right: 5px;
+font-size: 11px;
+padding-bottom: 0px;
+font-family: Arial, Helvetica, sans-serif;
+color: green;
+}
+}
+
+@media screen and (max-width: 800px) {
+a.article-category {
+background-color: #E5E4E2;
+align:justify;
+display: block;
+height:100%;
+font-weight: bold;
+width: relative;
+text-decoration: none;
+border-left: 5px solid transparent;
+border-top: 0px;
+border-bottom: 1px solid transparent;
+border-right: 0px;
+padding-right: 5px;
+font-size: 10px;
+padding-bottom: 0px;
+font-family: Arial, Helvetica, sans-serif;
+color: green;
+}
+}
+
+.content {
+    display: none;
+	font-family: Arial, Helvetica, sans-serif;
+	
+  padding-right: 5px;
+  
+  padding-top: 5px;
+  border-left: 5px solid transparent;
+  }
+  
+ .container{
+ padding-bottom:10px;
+ }
+ 
+ .show-similar-button-container{
+ display: flex;
+ flex-direction: column;
+ align-items: center; 
+ }
+ 
+ .similar-news-item:hover {
+ text-decoration: none;
+ }
+ 
+ @media screen and (min-width: 800px) {
+.similar-news-item {
+text-align: justify;
+text-decoration: underline;
+font-size: 14px;
+font-family: Arial, Helvetica, sans-serif;
+color: black;
+display:inline-block;
+padding-bottom: 10px;
+width:100%;
+/*white-space: nowrap;
+overflow: hidden;
+text-overflow: ellipsis;*/
+
+}
+}
+
+@media screen and (max-width: 800px) {
+.similar-news-item {
+text-align: justify;
+text-decoration: underline;
+font-size: 12px;
+font-family: Arial, Helvetica, sans-serif;
+color: black;
+display:inline-block;
+padding-bottom: 8px;
+width:100%;
+/*white-space: nowrap;
+overflow: hidden;
+text-overflow: ellipsis;*/
+}
+}
+ 
+ 
+ 
+ .show-more {
+background-color: #E5E4E2;
+font-family: Arial, Helvetica, sans-serif;
+border-radius:4px;
+padding-top:3px;
+padding-bottom:3px;
+padding-left:3px;
+padding-right:3px;
+font-size: 12px;
+display: box;
+border: none;
+
+  }
+  
+ .show-more:hover {
+background-color: black;
+color: white;
+  }
+ 
+ .show-less {
+background-color: #E5E4E2;
+font-family: Arial, Helvetica, sans-serif;
+border-radius:4px;
+padding-top:3px;
+padding-bottom:3px;
+padding-left:3px;
+padding-right:3px;
+font-size: 12px;
+border: none;
+display: none;
+  }
+  
+ .show-less:hover {
+background-color: black;
+color: white;
+  }
+  
+ .word-cloud-container{
+ word-wrap: break-word;
+padding-bottom: 10px;
+
+ }
+ 
+ .wc-tokens{
+font-family: Arial, Helvetica, sans-serif;
+font-size: 13.2px;
+cursor: pointer;
+ }
+ 
+ .wc-tokens:hover{
+ text-decoration: underline;
+ }
+ 
+ .word-cloud-section{
+   padding-bottom: 10px;
+	 display: none;
+   word-wrap: break-word;
+ }
+ 
+ .show-more-word-cloud{
+ padding-bottom: 23px;
+ text-align: center;
+ }
+ 
+ .three-dots{
+	 font-size: 30px;
+ margin: 0;
+ line-height:0;
+ vertical-align: top;
+    padding: 0;
+	cursor: pointer;
+ }
+ 
+ .three-dots:hover{
+	 font-size: 25px;
  }

templates/index.html

@@ -1,205 +1,205 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-	
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <link rel="preload" href="../static/loader.gif" as="image">
-	<link rel="preload" href="../static/favicon_new.png" as="image">
-	<link rel="preload" href="../static/refresh_reload_icon.png" as="image">
-<link rel="preload" href="../static/top-icon.png" as="image">
-    <link rel="icon" href="../static/favicon_new.png" type="image/png">
-    
-    <meta charset="UTF-8">
-    <title>Latest Indian News</title>
-    <link rel="stylesheet" href="static/styles.css">
-	<a id="top-loc"></a>
-<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.12.4/jquery.min.js"></script>
-<!--
-  <script>
-$(window).load(function(){
-     $('.loader').fadeOut();
-});
-</script>
--->
-
-<script>
-function filterContent(match_case) {
-	var keyword = document.getElementById("keywordInput").value;
-	if (match_case == false) 
-	{
-		/*var keyword = document.getElementById("keywordInput").value.toLowerCase(); */
-		/*var regex = new RegExp("\\b" + keyword + "\\b", "gi"); */
-	}
-	var clearbtn = document.getElementById("clearBtn");
-	
-	if (keyword !== "") 
-	{
-		clearbtn.style.opacity = 1;
-		var items = document.getElementsByClassName("news-item");
-		for (var i = 0; i < items.length; i++) 
-		{
-			var headline = items[i].querySelector('.headline');
-			var description = items[i].querySelector('.description');
-			if (match_case == true)
-			{
-				var article_category = items[i].querySelector('.article-category');
-				var src_time = items[i].querySelector('.time');
-				var itemText = headline.textContent.concat(" ", description.textContent, " ", article_category.textContent, " ", src_time.textContent)
-			} 
-			else 
-			{
-				var itemText = headline.textContent.concat(" ", description.textContent, " ")
-			}
-	
-			if (match_case == false) 
-			{	var regex = new RegExp("\\b" + keyword + "\\b", "gi");
-				itemText = itemText.toLowerCase();
-				if (regex.test(itemText) == true) 
-				{
-					items[i].style.display = "block";
-					highlightKeyword(headline, keyword, match_case);
-					highlightKeyword(description, keyword, match_case);
-				} 
-				else 
-				{
-					items[i].style.display = "none";
-				}
-			}
-			else
-			{
-				if (itemText.includes(keyword)) 
-				{
-					items[i].style.display = "block";
-					highlightKeyword(headline, keyword, match_case);
-					highlightKeyword(description, keyword, match_case);
-					highlightKeyword(article_category, keyword, match_case);
-					highlightKeyword(src_time, keyword, match_case);
-	  
-				}
-				else
-				{
-					items[i].style.display = "none";
-				}
-			}
-		}
-		} 
-		else
-		{
-			clearFilter();
-		}
-}
-
-function clearFilter() {
-  var items = document.getElementsByClassName("news-item");
-  var clearbtn = document.getElementById("clearBtn");
-  clearbtn.style.opacity=0;
-  for (var i = 0; i < items.length; i++) {
-	var headline = items[i].querySelector('.headline');
-	var description = items[i].querySelector('.description');
-	var article_category = items[i].querySelector('.article-category');
-	var src_time = items[i].querySelector('.time');
-    items[i].style.display = "block";
-    headline.innerHTML = headline.textContent; // Remove highlighting
-	description.innerHTML = description.textContent; // Remove highlighting
-	article_category.innerHTML = article_category.textContent; // Remove highlighting
-	src_time.innerHTML = src_time.textContent; // Remove highlighting
-  }
-
-   document.getElementById("keywordInput").value = ""; // Clear input field
-}
-
-function highlightKeyword(element, keyword, match_case) {
-  var regex = new RegExp(keyword);
-  if (match_case == false)
-  {
-	var regex = new RegExp("\\b" + keyword + "\\b", 'gi');
-  }
-  element.innerHTML = element.textContent.replace(regex, function(match) {
-    return '<span class="highlight">' + match + '</span>';
-  });
-}
-
-</script>
-
-<script>
-document.addEventListener('DOMContentLoaded', function() {
-    const containers = document.querySelectorAll('.container');
-
-    containers.forEach(container => {
-      const content = container.querySelector('.content');
-      const showMoreBtn = container.querySelector('.show-more');
-      const showLessBtn = container.querySelector('.show-less');
-
-      showMoreBtn.addEventListener('click', function() {
-       /* var  similar_news_items = document.getElementsByClassName("content");
-		var  show_less_items = document.getElementsByClassName("show-less");
-		var  show_more_items = document.getElementsByClassName("show-more");
-		for (var i = 0; i < similar_news_items.length; i++) {
-		similar_news_items[i].style.display = 'none';
-		show_more_items[i].style.display = 'block';
-		show_less_items[i].style.display = 'none';
-		}
-       */
-        
-        content.style.display = 'block';
-		content.style.opacity = 1;
-        showMoreBtn.style.display = 'none';
-        showLessBtn.style.display = 'block';
-      });
-
-      showLessBtn.addEventListener('click', function() {
-        document.documentElement.style.scrollBehavior = "auto";
-        var max_h = content.parentElement.parentElement.clientHeight;
-        content.style.display = 'none';
-        showMoreBtn.style.display = 'block';
-        showLessBtn.style.display = 'none';
-        var min_h = content.parentElement.parentElement.clientHeight;
-		$(window).scrollTop($(window).scrollTop() - (max_h - min_h) || 0);
-        document.documentElement.style.scrollBehavior = "smooth";
-      });
-    });
-  });
-</script>
-  
-<script>
-function wc_search(keyword)
-{
-clearFilter();
-document.getElementById("keywordInput").value = keyword;
-filterContent(false);
-}
-
-function word_cloud_display()
-{
-var word_cloud_section = document.getElementById("word-cloud-section-id");
-if (word_cloud_section.style.display == 'block')
-{
-word_cloud_section.style.display = 'none';
-}
-else
-{
-word_cloud_section.style.display = 'block';
-}
-}
-</script>
-
-<script>
-function showSearchInfo() {
-  alert("- Search is case-sensitive.\n- Search for news category (NATION, WORLD, SPORTS, ENTERTAINMENT, BUSINESS, TECHNOLOGY, HEALTH and SCIENCE) to filter news by category.\n- Search for news source (like zeebiz.com, ndtv.com, etc.) to filter news by source.")
-}
-</script>
-  
-	
-</head>
-<body>
-<!--<div class="loader"></div>-->
-{{body | safe}}
-
-<a id="top_theme" class="top-float" onclick="window.scrollTo(0, 0);">
-	<img id="top-theme-icon" alt="_" src="../static/top-icon.png" style="border-radius:50%; border: 3px solid #b9bfc4; background-color: white;" width="25px" height="25px" border="0">
-</a>	
-	<a href="javascript:window.location.reload(true)" id="theme" class="float">
-	<img id="theme-icon" alt="_" src="../static/refresh_reload_icon.png" style="border-radius:50%; border: 3px solid #b9bfc4; background-color: white;" width=25px height=25px border="0" />
-</a>
-</body>
-</html>
+<!DOCTYPE html>
+<html lang="en">
+<head>
+	
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <link rel="preload" href="../static/loader.gif" as="image">
+	<link rel="preload" href="../static/favicon_new.png" as="image">
+	<link rel="preload" href="../static/refresh_reload_icon.png" as="image">
+<link rel="preload" href="../static/top-icon.png" as="image">
+    <link rel="icon" href="../static/favicon_new.png" type="image/png">
+    
+    <meta charset="UTF-8">
+    <title>Latest Indian News</title>
+    <link rel="stylesheet" href="static/styles.css">
+	<a id="top-loc"></a>
+<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.12.4/jquery.min.js"></script>
+<!--
+  <script>
+$(window).load(function(){
+     $('.loader').fadeOut();
+});
+</script>
+-->
+
+<script>
+function filterContent(match_case) {
+	var keyword = document.getElementById("keywordInput").value;
+	if (match_case == false) 
+	{
+		/*var keyword = document.getElementById("keywordInput").value.toLowerCase(); */
+		/*var regex = new RegExp("\\b" + keyword + "\\b", "gi"); */
+	}
+	var clearbtn = document.getElementById("clearBtn");
+	
+	if (keyword !== "") 
+	{
+		clearbtn.style.opacity = 1;
+		var items = document.getElementsByClassName("news-item");
+		for (var i = 0; i < items.length; i++) 
+		{
+			var headline = items[i].querySelector('.headline');
+			var description = items[i].querySelector('.description');
+			if (match_case == true)
+			{
+				var article_category = items[i].querySelector('.article-category');
+				var src_time = items[i].querySelector('.time');
+				var itemText = headline.textContent.concat(" ", description.textContent, " ", article_category.textContent, " ", src_time.textContent)
+			} 
+			else 
+			{
+				var itemText = headline.textContent.concat(" ", description.textContent, " ")
+			}
+	
+			if (match_case == false) 
+			{	var regex = new RegExp("\\b" + keyword + "\\b", "gi");
+				itemText = itemText.toLowerCase();
+				if (regex.test(itemText) == true) 
+				{
+					items[i].style.display = "block";
+					highlightKeyword(headline, keyword, match_case);
+					highlightKeyword(description, keyword, match_case);
+				} 
+				else 
+				{
+					items[i].style.display = "none";
+				}
+			}
+			else
+			{
+				if (itemText.includes(keyword)) 
+				{
+					items[i].style.display = "block";
+					highlightKeyword(headline, keyword, match_case);
+					highlightKeyword(description, keyword, match_case);
+					highlightKeyword(article_category, keyword, match_case);
+					highlightKeyword(src_time, keyword, match_case);
+	  
+				}
+				else
+				{
+					items[i].style.display = "none";
+				}
+			}
+		}
+		} 
+		else
+		{
+			clearFilter();
+		}
+}
+
+function clearFilter() {
+  var items = document.getElementsByClassName("news-item");
+  var clearbtn = document.getElementById("clearBtn");
+  clearbtn.style.opacity=0;
+  for (var i = 0; i < items.length; i++) {
+	var headline = items[i].querySelector('.headline');
+	var description = items[i].querySelector('.description');
+	var article_category = items[i].querySelector('.article-category');
+	var src_time = items[i].querySelector('.time');
+    items[i].style.display = "block";
+    headline.innerHTML = headline.textContent; // Remove highlighting
+	description.innerHTML = description.textContent; // Remove highlighting
+	article_category.innerHTML = article_category.textContent; // Remove highlighting
+	src_time.innerHTML = src_time.textContent; // Remove highlighting
+  }
+
+   document.getElementById("keywordInput").value = ""; // Clear input field
+}
+
+function highlightKeyword(element, keyword, match_case) {
+  var regex = new RegExp(keyword);
+  if (match_case == false)
+  {
+	var regex = new RegExp("\\b" + keyword + "\\b", 'gi');
+  }
+  element.innerHTML = element.textContent.replace(regex, function(match) {
+    return '<span class="highlight">' + match + '</span>';
+  });
+}
+
+</script>
+
+<script>
+document.addEventListener('DOMContentLoaded', function() {
+    const containers = document.querySelectorAll('.container');
+
+    containers.forEach(container => {
+      const content = container.querySelector('.content');
+      const showMoreBtn = container.querySelector('.show-more');
+      const showLessBtn = container.querySelector('.show-less');
+
+      showMoreBtn.addEventListener('click', function() {
+       /* var  similar_news_items = document.getElementsByClassName("content");
+		var  show_less_items = document.getElementsByClassName("show-less");
+		var  show_more_items = document.getElementsByClassName("show-more");
+		for (var i = 0; i < similar_news_items.length; i++) {
+		similar_news_items[i].style.display = 'none';
+		show_more_items[i].style.display = 'block';
+		show_less_items[i].style.display = 'none';
+		}
+       */
+        
+        content.style.display = 'block';
+		content.style.opacity = 1;
+        showMoreBtn.style.display = 'none';
+        showLessBtn.style.display = 'block';
+      });
+
+      showLessBtn.addEventListener('click', function() {
+        document.documentElement.style.scrollBehavior = "auto";
+        var max_h = content.parentElement.parentElement.clientHeight;
+        content.style.display = 'none';
+        showMoreBtn.style.display = 'block';
+        showLessBtn.style.display = 'none';
+        var min_h = content.parentElement.parentElement.clientHeight;
+		$(window).scrollTop($(window).scrollTop() - (max_h - min_h) || 0);
+        document.documentElement.style.scrollBehavior = "smooth";
+      });
+    });
+  });
+</script>
+  
+<script>
+function wc_search(keyword)
+{
+clearFilter();
+document.getElementById("keywordInput").value = keyword;
+filterContent(false);
+}
+
+function word_cloud_display()
+{
+var word_cloud_section = document.getElementById("word-cloud-section-id");
+if (word_cloud_section.style.display == 'block')
+{
+word_cloud_section.style.display = 'none';
+}
+else
+{
+word_cloud_section.style.display = 'block';
+}
+}
+</script>
+
+<script>
+function showSearchInfo() {
+  alert("- Search is case-sensitive.\n- Search for news category (NATION, WORLD, SPORTS, ENTERTAINMENT, BUSINESS, TECHNOLOGY, HEALTH and SCIENCE) to filter news by category.\n- Search for news source (like zeebiz.com, ndtv.com, etc.) to filter news by source.")
+}
+</script>
+  
+	
+</head>
+<body>
+<!--<div class="loader"></div>-->
+{{body | safe}}
+
+<a id="top_theme" class="top-float" onclick="window.scrollTo(0, 0);">
+	<img id="top-theme-icon" alt="_" src="../static/top-icon.png" style="border-radius:50%; border: 3px solid #b9bfc4; background-color: white;" width="25px" height="25px" border="0">
+</a>	
+	<a href="javascript:window.location.reload(true)" id="theme" class="float">
+	<img id="theme-icon" alt="_" src="../static/refresh_reload_icon.png" style="border-radius:50%; border: 3px solid #b9bfc4; background-color: white;" width=25px height=25px border="0" />
+</a>
+</body>
+</html>

word_cloud.py

@@ -1,653 +1,653 @@
-import numpy as np
-import pandas as pd
-import string
-from unidecode import unidecode
-from collections import Counter
-
-
-class TextPreprocessor:
-    def __init__(self, remove_punct: bool = True, remove_digits: bool = True,
-                 remove_stop_words: bool = True,
-                 remove_short_words: bool = False, minlen: int = 1, maxlen: int = 1, top_p: float = None,
-                 bottom_p: float = None):
-        self.remove_punct = remove_punct
-        self.remove_digits = remove_digits
-        self.remove_stop_words = remove_stop_words
-        self.remove_short_words = remove_short_words
-        self.minlen = minlen
-        self.maxlen = maxlen
-        self.top_p = top_p
-        self.bottom_p = bottom_p
-        self.words_to_remove = []
-        self.stop_words = ["'d", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z",
- 'about',
- 'above',
- 'across',
- 'after',
- 'afterwards',
- 'again',
- 'against',
- 'ain',
- 'all',
- 'almost',
- 'alone',
- 'along',
- 'already',
- 'also',
- 'although',
- 'always',
- 'am',
- 'among',
- 'amongst',
- 'amount',
- 'an',
- 'and',
- 'another',
- 'any',
- 'anyhow',
- 'anyone',
- 'anything',
- 'anyway',
- 'anywhere',
- 'are',
- 'around',
- 'as',
- 'at',
- 'back',
- 'be',
- 'became',
- 'because',
- 'become',
- 'becomes',
- 'becoming',
- 'been',
- 'before',
- 'beforehand',
- 'behind',
- 'being',
- 'below',
- 'beside',
- 'besides',
- 'between',
- 'beyond',
- 'both',
- 'bottom',
- 'but',
- 'by',
- 'ca',
- 'call',
- 'can',
- 'cannot',
- 'could',
- 'couldn',
- "couldn't",
- 'd',
- 'did',
- 'do',
- 'does',
- 'doing',
- 'done',
- 'down',
- 'due',
- 'during',
- 'each',
- 'eight',
- 'either',
- 'eleven',
- 'else',
- 'elsewhere',
- 'empty',
- 'enough',
- 'even',
- 'ever',
- 'every',
- 'everyone',
- 'everything',
- 'everywhere',
- 'except',
- 'few',
- 'fifteen',
- 'fifty',
- 'first',
- 'five',
- 'for',
- 'former',
- 'formerly',
- 'forty',
- 'four',
- 'from',
- 'front',
- 'full',
- 'further',
- 'get',
- 'give',
- 'go',
- 'had',
- 'has',
- 'have',
- 'having',
- 'he',
- 'hence',
- 'her',
- 'here',
- 'hereafter',
- 'hereby',
- 'herein',
- 'hereupon',
- 'hers',
- 'herself',
- 'him',
- 'himself',
- 'his',
- 'how',
- 'however',
- 'hundred',
- 'i',
- 'if',
- 'in',
- 'indeed',
- 'into',
- 'is',
- 'it',
- "it's",
- 'its',
- 'itself',
- 'just',
- 'keep',
- 'last',
- 'latter',
- 'latterly',
- 'least',
- 'less',
- 'll',
- 'm',
- 'ma',
- 'made',
- 'make',
- 'many',
-'say',
-'said',
-'says',
-'told',
-'tell',
- 'may',
- 'me',
- 'meanwhile',
- 'might',
- 'mine',
- 'more',
- 'moreover',
- 'most',
- 'mostly',
- 'move',
- 'much',
- 'must',
- 'my',
- 'myself',
- 'name',
- 'namely',
- 'neither',
- 'never',
- 'nevertheless',
- 'next',
- 'nine',
- 'no',
- 'nobody',
- 'none',
- 'noone',
- 'nor',
- 'not',
- 'nothing',
- 'now',
- 'nowhere',
- 'o',
- 'of',
- 'off',
- 'often',
- 'on',
- 'once',
- 'one',
- 'only',
- 'onto',
- 'or',
- 'other',
- 'others',
- 'otherwise',
- 'our',
- 'ours',
- 'ourselves',
- 'out',
- 'over',
- 'own',
- 'part',
- 'per',
- 'perhaps',
- 'please',
- 'put',
- 'quite',
- 'rather',
- 're',
-'rs',
- 'really',
- 'regarding',
- 's',
- 'same',
- 'say',
- 'see',
- 'seem',
- 'seemed',
- 'seeming',
- 'seems',
- 'serious',
- 'several',
- 'shan',
- "shan't",
- 'she',
- "she's",
- 'should',
- "should've",
- 'shouldn',
- "shouldn't",
- 'show',
- 'side',
- 'since',
- 'six',
- 'sixty',
- 'so',
- 'some',
- 'somehow',
- 'someone',
- 'something',
- 'sometime',
- 'sometimes',
- 'somewhere',
- 'still',
- 'such',
- 't',
- 'take',
- 'ten',
- 'than',
- 'that',
- "that'll",
- 'the',
- 'their',
- 'theirs',
- 'them',
- 'themselves',
- 'then',
- 'thence',
- 'there',
- 'thereafter',
- 'thereby',
- 'therefore',
- 'therein',
- 'thereupon',
- 'these',
- 'they',
- 'third',
- 'this',
- 'those',
- 'though',
- 'three',
- 'through',
- 'throughout',
- 'thru',
- 'thus',
- 'to',
- 'together',
- 'too',
- 'top',
- 'toward',
- 'towards',
- 'twelve',
- 'twenty',
- 'two',
- 'under',
- 'unless',
- 'until',
- 'up',
- 'upon',
- 'us',
- 'used',
- 'using',
- 'various',
- 've',
- 'very',
- 'via',
- 'was',
- 'wasn',
- "wasn't",
- 'we',
- 'well',
- 'were',
- 'weren',
- "weren't",
- 'what',
- 'whatever',
- 'when',
- 'whence',
- 'whenever',
- 'where',
- 'whereafter',
- 'whereas',
- 'whereby',
- 'wherein',
- 'whereupon',
- 'wherever',
- 'whether',
- 'which',
- 'while',
- 'whither',
- 'who',
- 'whoever',
- 'whole',
- 'whom',
- 'whose',
- 'why',
- 'will',
- 'with',
- 'within',
- 'without',
- 'won',
- "won't",
- 'would',
- 'wouldn',
- "wouldn't",
- 'y',
- 'yet',
- 'you',
- "you'd",
- "you'll",
- "you're",
- "you've",
- 'your',
- 'yours',
- 'yourself',
- 'yourselves',
- '‘d',
- '‘ll',
- '‘m',
- '‘re',
- '‘s',
- '‘ve',
- '’d',
- '’ll',
- '’m',
- '’re',
-'new',
-'old',
- '’s',
- '’ve']
-
-        self.contraction_to_expansion = {"ain't": "am not",
-                                         "aren't": "are not",
-                                         "can't": "cannot",
-                                         "can't've": "cannot have",
-                                         "'cause": "because",
-                                         "could've": "could have",
-                                         "couldn't": "could not",
-                                         "couldn't've": "could not have",
-                                         "didn't": "did not",
-                                         "doesn't": "does not",
-                                         "don't": "do not",
-                                         "hadn't": "had not",
-                                         "hadn't've": "had not have",
-                                         "hasn't": "has not",
-                                         "haven't": "have not",
-                                         "he'd": "he would",
-                                         "he'd've": "he would have",
-                                         "he'll": "he will",
-                                         "he'll've": "he will have",
-                                         "he's": "he is",
-                                         "how'd": "how did",
-                                         "how'd'y": "how do you",
-                                         "how'll": "how will",
-                                         "how's": "how is",
-                                         "i'd": "i would",
-                                         "i'd've": "i would have",
-                                         "i'll": "i will",
-                                         "i'll've": "i will have",
-                                         "i'm": "i am",
-                                         "i've": "i have",
-                                         "isn't": "is not",
-                                         "it'd": "it had",
-                                         "it'd've": "it would have",
-                                         "it'll": "it will",
-                                         "it'll've": "it will have",
-                                         "it's": "it is",
-                                         "let's": "let us",
-                                         "ma'am": "madam",
-                                         "mayn't": "may not",
-                                         "might've": "might have",
-                                         "mightn't": "might not",
-                                         "mightn't've": "might not have",
-                                         "must've": "must have",
-                                         "mustn't": "must not",
-                                         "mustn't've": "must not have",
-                                         "needn't": "need not",
-                                         "needn't've": "need not have",
-                                         "o'clock": "of the clock",
-                                         "oughtn't": "ought not",
-                                         "oughtn't've": "ought not have",
-                                         "shan't": "shall not",
-                                         "sha'n't": "shall not",
-                                         "shan't've": "shall not have",
-                                         "she'd": "she would",
-                                         "she'd've": "she would have",
-                                         "she'll": "she will",
-                                         "she'll've": "she will have",
-                                         "she's": "she is",
-                                         "should've": "should have",
-                                         "shouldn't": "should not",
-                                         "shouldn't've": "should not have",
-                                         "so've": "so have",
-                                         "so's": "so is",
-                                         "that'd": "that would",
-                                         "that'd've": "that would have",
-                                         "that's": "that is",
-                                         "there'd": "there had",
-                                         "there'd've": "there would have",
-                                         "there's": "there is",
-                                         "they'd": "they would",
-                                         "they'd've": "they would have",
-                                         "they'll": "they will",
-                                         "they'll've": "they will have",
-                                         "they're": "they are",
-                                         "they've": "they have",
-                                         "to've": "to have",
-                                         "wasn't": "was not",
-                                         "we'd": "we had",
-                                         "we'd've": "we would have",
-                                         "we'll": "we will",
-                                         "we'll've": "we will have",
-                                         "we're": "we are",
-                                         "we've": "we have",
-                                         "weren't": "were not",
-                                         "what'll": "what will",
-                                         "what'll've": "what will have",
-                                         "what're": "what are",
-                                         "what's": "what is",
-                                         "what've": "what have",
-                                         "when's": "when is",
-                                         "when've": "when have",
-                                         "where'd": "where did",
-                                         "where's": "where is",
-                                         "where've": "where have",
-                                         "who'll": "who will",
-                                         "who'll've": "who will have",
-                                         "who's": "who is",
-                                         "who've": "who have",
-                                         "why's": "why is",
-                                         "why've": "why have",
-                                         "will've": "will have",
-                                         "won't": "will not",
-                                         "won't've": "will not have",
-                                         "would've": "would have",
-                                         "wouldn't": "would not",
-                                         "wouldn't've": "would not have",
-                                         "y'all": "you all",
-                                         "y'alls": "you alls",
-                                         "y'all'd": "you all would",
-                                         "y'all'd've": "you all would have",
-                                         "y'all're": "you all are",
-                                         "y'all've": "you all have",
-                                         "you'd": "you had",
-                                         "you'd've": "you would have",
-                                         "you'll": "you you will",
-                                         "you'll've": "you you will have",
-                                         "you're": "you are",
-                                         "you've": "you have"
-                                         }
-
-    @staticmethod
-    def __remove_double_whitespaces(string: str):
-        return " ".join(string.split())
-
-    async def __remove_url(self, string_series: pd.Series):
-        """
-        Removes URLs m text
-        :param string_series: pd.Series, input string series
-        :return: pd.Series, cleaned string series
-        """
-        clean_string_series = string_series.str.replace(
-            pat=r"(https?:\/\/(?:www\.|(?!www))[a-zA-Z0-9][a-zA-Z0-9-]+[a-zA-Z0-9]\.[^\s]{2,}|www\.[a-zA-Z0-9][a-zA-Z0-9-]+[a-zA-Z0-9]\.[^\s]{2,}|https?:\/\/(?:www\.|(?!www))[a-zA-Z0-9]+\.[^\s]{2,}|www\.[a-zA-Z0-9]+\.[^\s]{2,})",
-            repl=" ", regex=True).copy()
-        return clean_string_series.map(self.__remove_double_whitespaces)
-
-    async def __expand(self, string_series: pd.Series):
-        """
-        Replaces contractions with expansions. eg. don't wit do not.
-        :param string_series: pd.Series, input string series
-        :return: pd.Series, cleaned string series
-        """
-        clean_string_series = string_series.copy()
-        for c, e in self.contraction_to_expansion.items():
-            clean_string_series = clean_string_series.str.replace(pat=c, repl=e, regex=False).copy()
-        return clean_string_series.map(self.__remove_double_whitespaces)
-
-    async def __remove_punct(self, string_series: pd.Series):
-        """
-       Removes punctuations from the input string.
-       :param string_series: pd.Series, input string series
-       :return: pd.Series, cleaned string series
-       """
-        clean_string_series = string_series.copy()
-        puncts = [r'\n', r'\r', r'\t']
-        puncts.extend(list(string.punctuation))
-        for i in puncts:
-            clean_string_series = clean_string_series.str.replace(pat=i, repl=" ", regex=False).copy()
-        return clean_string_series.map(self.__remove_double_whitespaces)
-
-    async def __remove_digits(self, string_series: pd.Series):
-        """
-       Removes digits from the input string.
-       :param string_series: pd.Series, input string series
-       :return: pd.Series, cleaned string series
-       """
-        clean_string_series = string_series.str.replace(pat=r'\d', repl=" ", regex=True).copy()
-        return clean_string_series.map(self.__remove_double_whitespaces)
-
-    @staticmethod
-    async def __remove_short_words(string_series: pd.Series, minlen: int = 1, maxlen: int = 1):
-        """
-        Reomves words/tokens where minlen <= len <= maxlen.
-        :param string_series: pd.Series, input string series
-        :param minlen: int, minimum length of token to be removed.
-        :param maxlen:  int, maximum length of token to be removed.
-        :return: pd.Series, cleaned string series
-        """
-        clean_string_series = string_series.map(lambda string: " ".join([word for word in string.split() if
-                                                                         (len(word) > maxlen) or (len(word) < minlen)]))
-        return clean_string_series
-
-    async def __remove_stop_words(self, string_series: pd.Series):
-        """
-       Removes stop words from the input string.
-       :param string_series: pd.Series, input string series
-       :return: pd.Series, cleaned string series
-       """
-        def str_remove_stop_words(string: str):
-            stops = self.stop_words
-            return " ".join([token for token in string.split() if token not in stops])
-
-        return string_series.map(str_remove_stop_words)
-
-    async def __remove_top_bottom_words(self, string_series: pd.Series, top_p: int = None,
-                                  bottom_p: int = None, dataset: str = 'train'):
-        """
-        Reomoves top_p percent (frequent) words and bottom_p percent (rare) words.
-        :param string_series: pd.Series, input string series
-        :param top_p: float, percent of frequent words to remove.
-        :param bottom_p: float, percent of rare words to remove.
-        :param dataset: str, "train" for training set, "tesrt" for val/dev/test set.
-        :return: pd.Series, cleaned string series
-        """
-        if dataset == 'train':
-            if top_p is None:
-                top_p = 0
-            if bottom_p is None:
-                bottom_p = 0
-
-            if top_p > 0 or bottom_p > 0:
-                word_freq = pd.Series(" ".join(string_series).split()).value_counts()
-                n_words = len(word_freq)
-
-            if top_p > 0:
-                self.words_to_remove.extend([*word_freq.index[: int(np.ceil(top_p * n_words))]])
-
-            if bottom_p > 0:
-                self.words_to_remove.extend([*word_freq.index[-int(np.ceil(bottom_p * n_words)):]])
-
-        if len(self.words_to_remove) == 0:
-            return string_series
-        else:
-            clean_string_series = string_series.map(lambda string: " ".join([word for word in string.split()
-                                                                             if word not in self.words_to_remove]))
-            return clean_string_series
-
-    async def preprocess(self, string_series: pd.Series, dataset: str = "train"):
-        """
-        Entry point.
-        :param string_series: pd.Series, input string series
-        :param dataset: str, "train" for training set, "tesrt" for val/dev/test set.
-        :return: pd.Series, cleaned string series
-        """
-        string_series = string_series.str.lower().copy()
-        string_series = string_series.map(unidecode).copy()
-        string_series = await self.__remove_url(string_series=string_series)
-        string_series = await self.__expand(string_series=string_series)
-
-        if self.remove_punct:
-            string_series = await self.__remove_punct(string_series=string_series)
-        if self.remove_digits:
-            string_series = await self.__remove_digits(string_series=string_series)
-        if self.remove_stop_words:
-            string_series = await self.__remove_stop_words(string_series=string_series)
-        if self.remove_short_words:
-            string_series = await self.__remove_short_words(string_series=string_series,
-                                                      minlen=self.minlen,
-                                                      maxlen=self.maxlen)
-        string_series = await self.__remove_top_bottom_words(string_series=string_series,
-                                                       top_p=self.top_p,
-                                                       bottom_p=self.bottom_p, dataset=dataset)
-
-        string_series = string_series.str.strip().copy()
-        string_series.replace(to_replace="", value="this is an empty message", inplace=True)
-
-        return string_series
-
-
-async def get_frequent_words_html(df):
-    text_preprocess = TextPreprocessor()
-    preprocessed_txt = await text_preprocess.preprocess(df['title'] + ' ' + df['description'])
-    counter = Counter(' '.join([*preprocessed_txt]).split())
-    
-    freq_tokens_html = '<div class="word-cloud-container">'
-    n = 1
-    for i, j in counter.most_common(25):
-        freq_tokens_html += f'<a class="wc-tokens" onclick=wc_search("{i}")>{i}</a>{"&nbsp;" * np.random.randint(3, 7, 1)[0]}'
-        if n == 5:
-            freq_tokens_html += '<div class="word-cloud-section" id="word-cloud-section-id">'
-        n += 1
-    freq_tokens_html += '</div></div>'
+import numpy as np
+import pandas as pd
+import string
+from unidecode import unidecode
+from collections import Counter
+
+
+class TextPreprocessor:
+    def __init__(self, remove_punct: bool = True, remove_digits: bool = True,
+                 remove_stop_words: bool = True,
+                 remove_short_words: bool = False, minlen: int = 1, maxlen: int = 1, top_p: float = None,
+                 bottom_p: float = None):
+        self.remove_punct = remove_punct
+        self.remove_digits = remove_digits
+        self.remove_stop_words = remove_stop_words
+        self.remove_short_words = remove_short_words
+        self.minlen = minlen
+        self.maxlen = maxlen
+        self.top_p = top_p
+        self.bottom_p = bottom_p
+        self.words_to_remove = []
+        self.stop_words = ["'d", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z",
+ 'about',
+ 'above',
+ 'across',
+ 'after',
+ 'afterwards',
+ 'again',
+ 'against',
+ 'ain',
+ 'all',
+ 'almost',
+ 'alone',
+ 'along',
+ 'already',
+ 'also',
+ 'although',
+ 'always',
+ 'am',
+ 'among',
+ 'amongst',
+ 'amount',
+ 'an',
+ 'and',
+ 'another',
+ 'any',
+ 'anyhow',
+ 'anyone',
+ 'anything',
+ 'anyway',
+ 'anywhere',
+ 'are',
+ 'around',
+ 'as',
+ 'at',
+ 'back',
+ 'be',
+ 'became',
+ 'because',
+ 'become',
+ 'becomes',
+ 'becoming',
+ 'been',
+ 'before',
+ 'beforehand',
+ 'behind',
+ 'being',
+ 'below',
+ 'beside',
+ 'besides',
+ 'between',
+ 'beyond',
+ 'both',
+ 'bottom',
+ 'but',
+ 'by',
+ 'ca',
+ 'call',
+ 'can',
+ 'cannot',
+ 'could',
+ 'couldn',
+ "couldn't",
+ 'd',
+ 'did',
+ 'do',
+ 'does',
+ 'doing',
+ 'done',
+ 'down',
+ 'due',
+ 'during',
+ 'each',
+ 'eight',
+ 'either',
+ 'eleven',
+ 'else',
+ 'elsewhere',
+ 'empty',
+ 'enough',
+ 'even',
+ 'ever',
+ 'every',
+ 'everyone',
+ 'everything',
+ 'everywhere',
+ 'except',
+ 'few',
+ 'fifteen',
+ 'fifty',
+ 'first',
+ 'five',
+ 'for',
+ 'former',
+ 'formerly',
+ 'forty',
+ 'four',
+ 'from',
+ 'front',
+ 'full',
+ 'further',
+ 'get',
+ 'give',
+ 'go',
+ 'had',
+ 'has',
+ 'have',
+ 'having',
+ 'he',
+ 'hence',
+ 'her',
+ 'here',
+ 'hereafter',
+ 'hereby',
+ 'herein',
+ 'hereupon',
+ 'hers',
+ 'herself',
+ 'him',
+ 'himself',
+ 'his',
+ 'how',
+ 'however',
+ 'hundred',
+ 'i',
+ 'if',
+ 'in',
+ 'indeed',
+ 'into',
+ 'is',
+ 'it',
+ "it's",
+ 'its',
+ 'itself',
+ 'just',
+ 'keep',
+ 'last',
+ 'latter',
+ 'latterly',
+ 'least',
+ 'less',
+ 'll',
+ 'm',
+ 'ma',
+ 'made',
+ 'make',
+ 'many',
+'say',
+'said',
+'says',
+'told',
+'tell',
+ 'may',
+ 'me',
+ 'meanwhile',
+ 'might',
+ 'mine',
+ 'more',
+ 'moreover',
+ 'most',
+ 'mostly',
+ 'move',
+ 'much',
+ 'must',
+ 'my',
+ 'myself',
+ 'name',
+ 'namely',
+ 'neither',
+ 'never',
+ 'nevertheless',
+ 'next',
+ 'nine',
+ 'no',
+ 'nobody',
+ 'none',
+ 'noone',
+ 'nor',
+ 'not',
+ 'nothing',
+ 'now',
+ 'nowhere',
+ 'o',
+ 'of',
+ 'off',
+ 'often',
+ 'on',
+ 'once',
+ 'one',
+ 'only',
+ 'onto',
+ 'or',
+ 'other',
+ 'others',
+ 'otherwise',
+ 'our',
+ 'ours',
+ 'ourselves',
+ 'out',
+ 'over',
+ 'own',
+ 'part',
+ 'per',
+ 'perhaps',
+ 'please',
+ 'put',
+ 'quite',
+ 'rather',
+ 're',
+'rs',
+ 'really',
+ 'regarding',
+ 's',
+ 'same',
+ 'say',
+ 'see',
+ 'seem',
+ 'seemed',
+ 'seeming',
+ 'seems',
+ 'serious',
+ 'several',
+ 'shan',
+ "shan't",
+ 'she',
+ "she's",
+ 'should',
+ "should've",
+ 'shouldn',
+ "shouldn't",
+ 'show',
+ 'side',
+ 'since',
+ 'six',
+ 'sixty',
+ 'so',
+ 'some',
+ 'somehow',
+ 'someone',
+ 'something',
+ 'sometime',
+ 'sometimes',
+ 'somewhere',
+ 'still',
+ 'such',
+ 't',
+ 'take',
+ 'ten',
+ 'than',
+ 'that',
+ "that'll",
+ 'the',
+ 'their',
+ 'theirs',
+ 'them',
+ 'themselves',
+ 'then',
+ 'thence',
+ 'there',
+ 'thereafter',
+ 'thereby',
+ 'therefore',
+ 'therein',
+ 'thereupon',
+ 'these',
+ 'they',
+ 'third',
+ 'this',
+ 'those',
+ 'though',
+ 'three',
+ 'through',
+ 'throughout',
+ 'thru',
+ 'thus',
+ 'to',
+ 'together',
+ 'too',
+ 'top',
+ 'toward',
+ 'towards',
+ 'twelve',
+ 'twenty',
+ 'two',
+ 'under',
+ 'unless',
+ 'until',
+ 'up',
+ 'upon',
+ 'us',
+ 'used',
+ 'using',
+ 'various',
+ 've',
+ 'very',
+ 'via',
+ 'was',
+ 'wasn',
+ "wasn't",
+ 'we',
+ 'well',
+ 'were',
+ 'weren',
+ "weren't",
+ 'what',
+ 'whatever',
+ 'when',
+ 'whence',
+ 'whenever',
+ 'where',
+ 'whereafter',
+ 'whereas',
+ 'whereby',
+ 'wherein',
+ 'whereupon',
+ 'wherever',
+ 'whether',
+ 'which',
+ 'while',
+ 'whither',
+ 'who',
+ 'whoever',
+ 'whole',
+ 'whom',
+ 'whose',
+ 'why',
+ 'will',
+ 'with',
+ 'within',
+ 'without',
+ 'won',
+ "won't",
+ 'would',
+ 'wouldn',
+ "wouldn't",
+ 'y',
+ 'yet',
+ 'you',
+ "you'd",
+ "you'll",
+ "you're",
+ "you've",
+ 'your',
+ 'yours',
+ 'yourself',
+ 'yourselves',
+ '‘d',
+ '‘ll',
+ '‘m',
+ '‘re',
+ '‘s',
+ '‘ve',
+ '’d',
+ '’ll',
+ '’m',
+ '’re',
+'new',
+'old',
+ '’s',
+ '’ve']
+
+        self.contraction_to_expansion = {"ain't": "am not",
+                                         "aren't": "are not",
+                                         "can't": "cannot",
+                                         "can't've": "cannot have",
+                                         "'cause": "because",
+                                         "could've": "could have",
+                                         "couldn't": "could not",
+                                         "couldn't've": "could not have",
+                                         "didn't": "did not",
+                                         "doesn't": "does not",
+                                         "don't": "do not",
+                                         "hadn't": "had not",
+                                         "hadn't've": "had not have",
+                                         "hasn't": "has not",
+                                         "haven't": "have not",
+                                         "he'd": "he would",
+                                         "he'd've": "he would have",
+                                         "he'll": "he will",
+                                         "he'll've": "he will have",
+                                         "he's": "he is",
+                                         "how'd": "how did",
+                                         "how'd'y": "how do you",
+                                         "how'll": "how will",
+                                         "how's": "how is",
+                                         "i'd": "i would",
+                                         "i'd've": "i would have",
+                                         "i'll": "i will",
+                                         "i'll've": "i will have",
+                                         "i'm": "i am",
+                                         "i've": "i have",
+                                         "isn't": "is not",
+                                         "it'd": "it had",
+                                         "it'd've": "it would have",
+                                         "it'll": "it will",
+                                         "it'll've": "it will have",
+                                         "it's": "it is",
+                                         "let's": "let us",
+                                         "ma'am": "madam",
+                                         "mayn't": "may not",
+                                         "might've": "might have",
+                                         "mightn't": "might not",
+                                         "mightn't've": "might not have",
+                                         "must've": "must have",
+                                         "mustn't": "must not",
+                                         "mustn't've": "must not have",
+                                         "needn't": "need not",
+                                         "needn't've": "need not have",
+                                         "o'clock": "of the clock",
+                                         "oughtn't": "ought not",
+                                         "oughtn't've": "ought not have",
+                                         "shan't": "shall not",
+                                         "sha'n't": "shall not",
+                                         "shan't've": "shall not have",
+                                         "she'd": "she would",
+                                         "she'd've": "she would have",
+                                         "she'll": "she will",
+                                         "she'll've": "she will have",
+                                         "she's": "she is",
+                                         "should've": "should have",
+                                         "shouldn't": "should not",
+                                         "shouldn't've": "should not have",
+                                         "so've": "so have",
+                                         "so's": "so is",
+                                         "that'd": "that would",
+                                         "that'd've": "that would have",
+                                         "that's": "that is",
+                                         "there'd": "there had",
+                                         "there'd've": "there would have",
+                                         "there's": "there is",
+                                         "they'd": "they would",
+                                         "they'd've": "they would have",
+                                         "they'll": "they will",
+                                         "they'll've": "they will have",
+                                         "they're": "they are",
+                                         "they've": "they have",
+                                         "to've": "to have",
+                                         "wasn't": "was not",
+                                         "we'd": "we had",
+                                         "we'd've": "we would have",
+                                         "we'll": "we will",
+                                         "we'll've": "we will have",
+                                         "we're": "we are",
+                                         "we've": "we have",
+                                         "weren't": "were not",
+                                         "what'll": "what will",
+                                         "what'll've": "what will have",
+                                         "what're": "what are",
+                                         "what's": "what is",
+                                         "what've": "what have",
+                                         "when's": "when is",
+                                         "when've": "when have",
+                                         "where'd": "where did",
+                                         "where's": "where is",
+                                         "where've": "where have",
+                                         "who'll": "who will",
+                                         "who'll've": "who will have",
+                                         "who's": "who is",
+                                         "who've": "who have",
+                                         "why's": "why is",
+                                         "why've": "why have",
+                                         "will've": "will have",
+                                         "won't": "will not",
+                                         "won't've": "will not have",
+                                         "would've": "would have",
+                                         "wouldn't": "would not",
+                                         "wouldn't've": "would not have",
+                                         "y'all": "you all",
+                                         "y'alls": "you alls",
+                                         "y'all'd": "you all would",
+                                         "y'all'd've": "you all would have",
+                                         "y'all're": "you all are",
+                                         "y'all've": "you all have",
+                                         "you'd": "you had",
+                                         "you'd've": "you would have",
+                                         "you'll": "you you will",
+                                         "you'll've": "you you will have",
+                                         "you're": "you are",
+                                         "you've": "you have"
+                                         }
+
+    @staticmethod
+    def __remove_double_whitespaces(string: str):
+        return " ".join(string.split())
+
+    async def __remove_url(self, string_series: pd.Series):
+        """
+        Removes URLs m text
+        :param string_series: pd.Series, input string series
+        :return: pd.Series, cleaned string series
+        """
+        clean_string_series = string_series.str.replace(
+            pat=r"(https?:\/\/(?:www\.|(?!www))[a-zA-Z0-9][a-zA-Z0-9-]+[a-zA-Z0-9]\.[^\s]{2,}|www\.[a-zA-Z0-9][a-zA-Z0-9-]+[a-zA-Z0-9]\.[^\s]{2,}|https?:\/\/(?:www\.|(?!www))[a-zA-Z0-9]+\.[^\s]{2,}|www\.[a-zA-Z0-9]+\.[^\s]{2,})",
+            repl=" ", regex=True).copy()
+        return clean_string_series.map(self.__remove_double_whitespaces)
+
+    async def __expand(self, string_series: pd.Series):
+        """
+        Replaces contractions with expansions. eg. don't wit do not.
+        :param string_series: pd.Series, input string series
+        :return: pd.Series, cleaned string series
+        """
+        clean_string_series = string_series.copy()
+        for c, e in self.contraction_to_expansion.items():
+            clean_string_series = clean_string_series.str.replace(pat=c, repl=e, regex=False).copy()
+        return clean_string_series.map(self.__remove_double_whitespaces)
+
+    async def __remove_punct(self, string_series: pd.Series):
+        """
+       Removes punctuations from the input string.
+       :param string_series: pd.Series, input string series
+       :return: pd.Series, cleaned string series
+       """
+        clean_string_series = string_series.copy()
+        puncts = [r'\n', r'\r', r'\t']
+        puncts.extend(list(string.punctuation))
+        for i in puncts:
+            clean_string_series = clean_string_series.str.replace(pat=i, repl=" ", regex=False).copy()
+        return clean_string_series.map(self.__remove_double_whitespaces)
+
+    async def __remove_digits(self, string_series: pd.Series):
+        """
+       Removes digits from the input string.
+       :param string_series: pd.Series, input string series
+       :return: pd.Series, cleaned string series
+       """
+        clean_string_series = string_series.str.replace(pat=r'\d', repl=" ", regex=True).copy()
+        return clean_string_series.map(self.__remove_double_whitespaces)
+
+    @staticmethod
+    async def __remove_short_words(string_series: pd.Series, minlen: int = 1, maxlen: int = 1):
+        """
+        Reomves words/tokens where minlen <= len <= maxlen.
+        :param string_series: pd.Series, input string series
+        :param minlen: int, minimum length of token to be removed.
+        :param maxlen:  int, maximum length of token to be removed.
+        :return: pd.Series, cleaned string series
+        """
+        clean_string_series = string_series.map(lambda string: " ".join([word for word in string.split() if
+                                                                         (len(word) > maxlen) or (len(word) < minlen)]))
+        return clean_string_series
+
+    async def __remove_stop_words(self, string_series: pd.Series):
+        """
+       Removes stop words from the input string.
+       :param string_series: pd.Series, input string series
+       :return: pd.Series, cleaned string series
+       """
+        def str_remove_stop_words(string: str):
+            stops = self.stop_words
+            return " ".join([token for token in string.split() if token not in stops])
+
+        return string_series.map(str_remove_stop_words)
+
+    async def __remove_top_bottom_words(self, string_series: pd.Series, top_p: int = None,
+                                  bottom_p: int = None, dataset: str = 'train'):
+        """
+        Reomoves top_p percent (frequent) words and bottom_p percent (rare) words.
+        :param string_series: pd.Series, input string series
+        :param top_p: float, percent of frequent words to remove.
+        :param bottom_p: float, percent of rare words to remove.
+        :param dataset: str, "train" for training set, "tesrt" for val/dev/test set.
+        :return: pd.Series, cleaned string series
+        """
+        if dataset == 'train':
+            if top_p is None:
+                top_p = 0
+            if bottom_p is None:
+                bottom_p = 0
+
+            if top_p > 0 or bottom_p > 0:
+                word_freq = pd.Series(" ".join(string_series).split()).value_counts()
+                n_words = len(word_freq)
+
+            if top_p > 0:
+                self.words_to_remove.extend([*word_freq.index[: int(np.ceil(top_p * n_words))]])
+
+            if bottom_p > 0:
+                self.words_to_remove.extend([*word_freq.index[-int(np.ceil(bottom_p * n_words)):]])
+
+        if len(self.words_to_remove) == 0:
+            return string_series
+        else:
+            clean_string_series = string_series.map(lambda string: " ".join([word for word in string.split()
+                                                                             if word not in self.words_to_remove]))
+            return clean_string_series
+
+    async def preprocess(self, string_series: pd.Series, dataset: str = "train"):
+        """
+        Entry point.
+        :param string_series: pd.Series, input string series
+        :param dataset: str, "train" for training set, "tesrt" for val/dev/test set.
+        :return: pd.Series, cleaned string series
+        """
+        string_series = string_series.str.lower().copy()
+        string_series = string_series.map(unidecode).copy()
+        string_series = await self.__remove_url(string_series=string_series)
+        string_series = await self.__expand(string_series=string_series)
+
+        if self.remove_punct:
+            string_series = await self.__remove_punct(string_series=string_series)
+        if self.remove_digits:
+            string_series = await self.__remove_digits(string_series=string_series)
+        if self.remove_stop_words:
+            string_series = await self.__remove_stop_words(string_series=string_series)
+        if self.remove_short_words:
+            string_series = await self.__remove_short_words(string_series=string_series,
+                                                      minlen=self.minlen,
+                                                      maxlen=self.maxlen)
+        string_series = await self.__remove_top_bottom_words(string_series=string_series,
+                                                       top_p=self.top_p,
+                                                       bottom_p=self.bottom_p, dataset=dataset)
+
+        string_series = string_series.str.strip().copy()
+        string_series.replace(to_replace="", value="this is an empty message", inplace=True)
+
+        return string_series
+
+
+async def get_frequent_words_html(df):
+    text_preprocess = TextPreprocessor()
+    preprocessed_txt = await text_preprocess.preprocess(df['title'] + ' ' + df['description'])
+    counter = Counter(' '.join([*preprocessed_txt]).split())
+    
+    freq_tokens_html = '<div class="word-cloud-container">'
+    n = 1
+    for i, j in counter.most_common(25):
+        freq_tokens_html += f'<a class="wc-tokens" onclick=wc_search("{i}")>{i}</a>{"&nbsp;" * np.random.randint(3, 7, 1)[0]}'
+        if n == 5:
+            freq_tokens_html += '<div class="word-cloud-section" id="word-cloud-section-id">'
+        n += 1
+    freq_tokens_html += '</div></div>'
     return freq_tokens_html