Upload files

Dockerfile

@@ -0,0 +1,9 @@
+FROM python:3.7-slim
+WORKDIR /webapp
+COPY . .
+RUN chmod +x /webapp/start.sh
+RUN pip install --upgrade pip
+RUN pip install --no-cache-dir -r requirements.txt
+RUN apt update && apt install -y redis-server
+EXPOSE 7860 6379
+CMD ["/webapp/start.sh"]

README.md

@@ -1,8 +1,8 @@
 ---
 title: News Aggregator
-emoji: 🏢
-colorFrom: purple
-colorTo: red
+emoji: ⚡
+colorFrom: indigo
+colorTo: blue
 sdk: docker
 pinned: false
 license: mit

app.py

@@ -0,0 +1,242 @@
+import numpy as np
+import pandas as pd
+from dateutil import parser
+from flask import Flask, render_template
+from flask_cors import cross_origin, CORS
+from db_operations.db_operations import DBOperations
+import logging
+import traceback 
+import redis
+from datetime import datetime
+from functools import lru_cache
+from word_cloud import get_frequent_words_html
+from config import NEWS_RETENTION_SECONDS
+
+
+app = Flask(__name__)
+CORS(app)
+redis_client = redis.Redis(host='localhost', port=6379, decode_responses=True)
+db = DBOperations()
+
+
+REFRESH_FREQ = 300 # 300 secs = 5 mins
+
+def is_db_fetch_reqd():
+    try:
+        env_news_time = redis_client.get('NEWSFETCHTIME')
+        logging.warning(f'fetch_time_env_var: {env_news_time}')
+        fetch_flag = 1
+        if env_news_time is None:
+            redis_client.set("NEWSFETCHTIME", str(datetime.now()))
+            fetch_flag = 1
+    
+        if env_news_time is not None:
+            fetch_time_lapse_seconds = (datetime.now() - datetime.strptime(env_news_time, '%Y-%m-%d %H:%M:%S.%f')).seconds
+            if fetch_time_lapse_seconds <= REFRESH_FREQ: 
+                fetch_flag = 0 
+            else:
+                redis_client.set("NEWSFETCHTIME", str(datetime.now()))
+                fetch_flag = 1
+    except Exception as e:
+        print(e)
+        fetch_flag = 1
+    return fetch_flag
+
+
+def correct_date(x):
+    if (not isinstance(x, str)) or (str(x).find(":") == -1):
+        logging.warning(f'correct_date() error: {x} is not the right date format')
+        return "2020-11-07 00:36:44+05:30"
+    return x
+
+def date_time_parser(dt):
+    """
+    Computes the minutes elapsed since published time.
+    :param dt: date
+    :return: int, minutes elapsed.
+    """
+    try:
+        return int(np.round((dt.now(dt.tz) - dt).total_seconds() / 60, 0))
+    except:
+        logging.warning(f'date_time_parser() error: {dt} is not the right date format')
+        return 100000
+
+
+def elapsed_time_str(mins):
+    """
+    Return the time elapsed string from minutes passed as an argument.
+    :param mins: int, minutes elapsed.
+    :return: str, time elapsed string
+    """
+    try:
+        time_str = ''
+        hours = int(mins / 60)
+        days = np.round(mins / (60 * 24), 1)
+        remaining_mins = int(mins - (hours * 60))
+        if days >= 1:
+            time_str = f'{str(days)} days ago'
+            if days == 1:
+                time_str = 'a day ago'
+        elif (days < 1) & (hours < 24) & (mins >= 60):
+            time_str = f'{str(hours)} hours and {str(remaining_mins)} mins ago'
+            if (hours == 1) & (remaining_mins > 1):
+                time_str = f'an hour and {str(remaining_mins)} mins ago'
+            if (hours == 1) & (remaining_mins == 1):
+                time_str = f'an hour and a min ago'
+            if (hours > 1) & (remaining_mins == 1):
+                time_str = f'{str(hours)} hours and a min ago'
+            if (hours > 1) & (remaining_mins == 0):
+                time_str = f'{str(hours)} hours ago'
+            if ((mins / 60) == 1) & (remaining_mins == 0):
+                time_str = 'an hour ago'
+        elif (days < 1) & (hours < 24) & (mins == 0):
+            time_str = 'Just in'
+        else:
+            time_str = f'{str(mins)} minutes ago'
+            if mins == 1:
+                time_str = 'a minute ago'
+        return time_str
+    except:
+        return "-"
+
+
+
+def fetch_from_db(fetch_flag):
+    try:
+        logging.warning(f'fetch_flag: {fetch_flag}')
+        if fetch_flag == 1:
+            final_df = db.read_news_from_db()
+            freq_tokens = get_frequent_words_html(final_df)
+            logging.warning('Fetched From DB\n\n')
+
+            final_df['_id'] = final_df['_id'].astype('str')
+            
+            redis_client.set("NEWSDF", final_df.to_json())
+            redis_client.set("NEWSWORDCLOUD", freq_tokens)
+        else:
+            final_df = pd.read_json(redis_client.get("NEWSDF"))
+            freq_tokens = redis_client.get("NEWSWORDCLOUD")
+            logging.warning('Fetched From Cache\n\n')
+            
+    except Exception as e:
+        print(e)
+        final_df = []
+        freq_tokens = ""
+        raise
+    return final_df, freq_tokens
+
+
+@app.route("/")
+@cross_origin()
+def index():
+    """
+    Entry point
+    """
+    try:
+        src_str = ''
+        final_df, freq_tokens = fetch_from_db(is_db_fetch_reqd())
+        if len(final_df) > 1:
+            
+            final_df["parsed_date"] = [correct_date(date_) for date_ in final_df['parsed_date']]
+            final_df["parsed_date"] = [parser.parse(date_) for date_ in final_df['parsed_date']]
+            final_df["elapsed_time"] =[date_time_parser(date_) for date_ in final_df['parsed_date']]
+            final_df = final_df.loc[final_df["elapsed_time"] <= NEWS_RETENTION_SECONDS, :].copy()
+            final_df["elapsed_time_str"] = final_df["elapsed_time"].apply(elapsed_time_str)
+            final_df.sort_values(by="elapsed_time", inplace=True)
+            src_str = ", ".join(sorted([*final_df['src'].unique()]))
+            final_df['src_time'] = final_df['src'] + ("&nbsp;" * 5) + final_df["elapsed_time_str"]
+            final_df.drop(columns=['_id', 'parsed_date', 'src', 'elapsed_time', 'elapsed_time_str'], inplace=True)
+            final_df.drop_duplicates(subset='description', inplace=True)
+            final_df = final_df.loc[(final_df["title"] != ""), :].copy()
+        else:
+            final_df = pd.DataFrame({'title': '', 'url': '',
+                                     'description': '', 'src_time': ''}, index=[0])
+        
+    except Exception as e:
+        final_df = pd.DataFrame({'title': '', 'url': '',
+                                 'description': '', 'src_time': ''}, index=[0])
+        logging.warning(traceback.print_exc())
+
+    result_str = f'''
+                    <div class="box" id="main">
+                    <form> 
+                    
+                    <div class="banner">
+                    <img src="../static/favicon_new.png" class="logo-img" alt="KSV Muralidhar" />
+                    <h1 style="display:inline-block; vertical-align: middle;">Latest News</h1>
+                    </div>
+                 '''
+    
+    if len(final_df) <= 1:
+        result_str += f'''<div><p class="unavailable">This app is temporarily unavailable</p></div>'''
+    else:
+        # last_update_utc = datetime.strptime(os.getenv("NEWSFETCHTIME"), '%Y-%m-%d %H:%M:%S.%f')
+        last_update_utc = datetime.strptime(redis_client.get('NEWSFETCHTIME'), '%Y-%m-%d %H:%M:%S.%f')
+        last_update_utc = last_update_utc.strftime("%Y-%m-%d %H:%M:%S")
+        result_str += f'<p class="srctxt">News aggregated from <b>{src_str}</b>.<br><br>Last updated: {last_update_utc} UTC</p>'
+
+        result_str += '''
+    	<div class="input-container">
+		<input type="text" class="keyword-input" id="keywordInput" placeholder="Search" oninput="filterContent(true)">
+		<div class="clear-btn" id="clearBtn" onclick="clearFilter()">&times;</div>
+		</div>
+      '''
+
+        result_str += f"{freq_tokens} "
+        result_str += '<div class="show-more-word-cloud" onclick=word_cloud_display()><p class="three-dots">...</p></div>'
+
+        result_str += '''<div style="padding-bottom: 10px; font-size: 12px; font-family: Arial, Helvetica, sans-serif;">
+                         News categories and similar news are AI-generated</div>'''
+
+        
+        for n, i in final_df.iterrows():  # iterating through the search results
+            href = i["url"]
+            category = i["category"]
+            description = i["description"]
+            url_txt = i["title"]
+            src_time = i["src_time"]
+            sim_news = i['similar_news']
+            result_str += f'''<div class="news-item"><div style="padding-top: 7px;">
+                              <a href="{href}" target="_blank" class="article-category">{category}
+                              </a>
+                              </div>
+                              <div>
+                              <a href="{href}" target="_blank" class="headline">{url_txt}
+                              </a>
+                              </div>
+                              <div>
+                              <a href="{href}" target="_blank" class="description">
+                              {description}
+                              </a>
+                              </div>
+                              <div>
+                              <a href="{href}" target="_blank" class="time">
+                              {src_time}
+                               </a>
+                               </div>
+
+
+                                <div class="container">
+                               <div class="content" style="display: none;">
+                               {sim_news}
+                               </div>
+                               <div class="show-similar-button-container">
+                               <button type="button" class="show-more">Show similar news</button>
+								<button type="button" class="show-less">Hide similar news</button>
+								</div>
+                                </div>
+
+
+                               
+                               <div>
+                               <p></p>
+                               </div></div>
+                               '''
+
+    result_str += '</form></div>'
+    return render_template("index.html", body=result_str)
+
+
+if __name__ == "__main__":
+    app.run(host="0.0.0.0", port=7860, workers=5, threads=5) # workers=(2*ncores) + 1, threads= (2 to 4*ncores) + 1
+    

config.py

@@ -0,0 +1 @@
+NEWS_RETENTION_SECONDS = 300

db_operations/__init__.py

@@ -0,0 +1 @@
+from db_operations.db_operations import *

db_operations/db_operations.py

@@ -0,0 +1,59 @@
+import pymongo
+import os
+import pandas as pd
+
+
+class DBOperations:
+    """
+    Reads news from MongoDB
+    """
+    def __init__(self):
+        self.url = os.getenv('DB_URL')
+        self.database = "rss_news_db_cat_pred_sim_news"
+        self.collection = "rss_news_cat_pred_sim_news"
+        self.__client = None
+        self.__error = 0
+
+    def __connect(self):
+        try:
+            self.__client = pymongo.MongoClient(self.url)
+            _ = self.__client.list_database_names()
+        except Exception as conn_exception:
+            self.__error = 1
+            self.__client = None
+            raise
+
+    def __read(self):
+        try:
+            db = self.__client[self.database]
+            coll = db[self.collection]
+            docs = []
+            for doc in coll.find():
+                docs.append(doc)
+            rss_df = pd.DataFrame(docs)
+        except Exception as insert_err:
+            self.__error = 1
+            rss_df = pd.DataFrame({'_id': '', 'title': '', 'url': '',
+                                   'description': '', 'parsed_date': '',
+                                   'src': ''}, index=[0])
+        return rss_df
+
+    def __close_connection(self):
+        if self.__client is not None:
+            self.__client.close()
+            self.__client = None
+
+    def read_news_from_db(self):
+        rss_df = pd.DataFrame({'_id': '', 'title': '', 'url': '',
+                               'description': '', 'parsed_date': '',
+                               'src': ''}, index=[0])
+        if self.url is not None:
+            if self.__error == 0:
+                self.__connect()
+            if self.__error == 0:
+                rss_df = self.__read()
+            if self.__error == 0:
+                print("Read Successful")
+            if self.__client is not None:
+                self.__close_connection()
+        return rss_df

requirements.txt

@@ -0,0 +1,12 @@
+regex==2021.8.3
+lxml==4.6.3
+numpy==1.21.1
+python-dateutil==2.8.2
+pandas==1.3.1
+requests==2.26.0
+bs4==0.0.1
+flask==2.2.2
+flask_cors==3.0.10
+gunicorn==20.1.0
+pymongo==4.3.3
+redis

start.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+redis-server --daemonize yes
+gunicorn -b 0.0.0.0:7860 --timeout 120 --workers 5 --threads 5 app:app

static/styles.css

@@ -0,0 +1,510 @@
+html {
+scroll-behavior: smooth;
+}
+
+@media screen and (min-width: 800px) {
+a.headline {
+background-color: #E5E4E2;
+display: block;
+width: relative;
+text-decoration: none;
+color: black;
+line-height: 1.2;
+align: justify;
+border-left: 5px solid transparent;
+border-top: 5px solid transparent;
+border-bottom: 5px solid transparent;
+border-right: 0px;
+font-weight: bold;
+font-size: 18px;
+padding-right: 5px;
+font-family: Arial, Helvetica, sans-serif;
+}
+}
+
+@media screen and (max-width: 800px) {
+a.headline {
+background-color: #E5E4E2;
+display: block;
+width: relative;
+text-decoration: none;
+color: black;
+line-height: 1.2;
+align: justify;
+border-left: 5px solid transparent;
+border-top: 5px solid transparent;
+border-bottom: 5px solid transparent;
+border-right: 0px;
+font-weight: bold;
+font-size: 16.5px;
+padding-right: 5px;
+font-family: Arial, Helvetica, sans-serif;
+}
+}
+
+@media screen and (min-width: 800px) {
+a.description {
+background-color: #E5E4E2;
+align:justify;
+text-align: justify;
+display: block;
+height:100%;
+width: relative;
+text-decoration: none;
+border-left: 5px solid transparent;
+border-top: 0px;
+border-bottom: 7px solid transparent;
+border-right: 0px;
+font-size: 14px;
+padding-right: 5px;
+font-family: Arial, Helvetica, sans-serif;
+color: dimgrey;
+}
+}
+
+@media screen and (max-width: 800px) {
+a.description {
+background-color: #E5E4E2;
+align:justify;
+text-align: justify;
+display: block;
+height:100%;
+width: relative;
+text-decoration: none;
+border-left: 5px solid transparent;
+border-top: 0px;
+border-bottom: 7px solid transparent;
+border-right: 0px;
+font-size: 12.5px;
+padding-right: 5px;
+font-family: Arial, Helvetica, sans-serif;
+color: dimgrey;
+}
+}
+
+@media screen and (min-width: 800px) {
+a.time {
+background-color: #E5E4E2;
+align:justify;
+display: block;
+height:100%;
+width: relative;
+text-decoration: none;
+border-left: 5px solid transparent;
+border-top: 0px;
+border-bottom: 1px solid transparent;
+border-right: 0px;
+padding-right: 5px;
+font-size: 11px;
+padding-bottom: 5px;
+font-family: Arial, Helvetica, sans-serif;
+color: green;
+}
+}
+
+@media screen and (max-width: 800px) {
+a.time {
+background-color: #E5E4E2;
+align:justify;
+display: block;
+height:100%;
+width: relative;
+text-decoration: none;
+border-left: 5px solid transparent;
+border-top: 0px;
+border-bottom: 1px solid transparent;
+border-right: 0px;
+padding-right: 5px;
+font-size: 10px;
+padding-bottom: 5px;
+font-family: Arial, Helvetica, sans-serif;
+color: green;
+}
+}
+
+.box {
+display: flex;
+justify-content: center;
+align-items: center;
+height: inherit;
+padding: 20px;
+}
+@media screen and (min-width: 800px) {
+form {
+width: 50%;
+overflow-x: hidden;
+padding: 20px;
+border-radius: 10px;
+background: #fff;
+box-shadow: 0 0 20px 0 #095484;
+    }}
+
+@media screen and (max-width: 800px) {
+form {
+width: 100%;
+overflow-x: hidden;
+padding: 20px;
+border-radius: 10px;
+background: #fff;
+box-shadow: 0 0 15px 0 #095484;
+}}
+.banner {
+      position: relative;
+      height: 30px;
+/*       background-size: cover; */
+      display: flex;
+/*       justify-content: center; */
+/*       align-items: center; */
+/*       text-align: center; */
+      }
+@media screen and (min-width: 800px) {
+h1 {
+      position: absolute;
+      margin: 0;
+      padding-left: 50px;
+      font-size: 25px;
+      color: black;
+      z-index: 2;
+      font-family: Arial, Helvetica, sans-serif;
+      }
+}
+
+@media screen and (max-width: 800px) {
+h1 {
+      position: absolute;
+      margin: 0;
+      padding-left: 40px;
+      font-size: 24px;
+      color: black;
+      z-index: 2;
+      font-family: Arial, Helvetica, sans-serif;
+      }
+}
+
+p.unavailable {
+background-color: #E5E4E2;
+display: block;
+width: 100%;
+text-decoration: none;
+color: black;
+line-height: 1.2;
+align: justify;
+border-left: 5px solid transparent;
+border-top: 5px solid transparent;
+border-bottom: 5px solid transparent;
+border-right: 0px;
+font-weight: bold;
+font-size: 18px;
+padding-right: 5px;
+font-family: Arial, Helvetica, sans-serif;
+}
+div.news-item{
+	background-color: #E5E4E2;
+/*box-shadow: rgba(0, 0, 0, 0.4) -1px 0px 5px, rgba(0, 0, 0, 0.5) 0px 4px 5px -3px, rgba(0, 0, 0, 0.2) 0px -3px 0px inset;*/
+box-shadow: rgba(0, 0, 0, 0.25) 0px 0px 5px 1px, rgba(0, 0, 0, 0.1) 0px 4px 5px -3px, rgba(0, 0, 0, 0.2) 0px -1px 0px inset;
+
+
+	  }
+div.news-item:hover{
+box-shadow: none;
+}
+
+@media screen and (min-width: 800px) {
+p.srctxt {
+align:justify;
+text-align: justify;
+word-break: break-all;
+font-size: 11px;
+font-family: Arial, Helvetica, sans-serif;
+}
+.logo-img{
+margin-right: 10px;
+vertical-align: center;
+/* position: relative; */
+width: 34px;
+height: 34px;
+
+}
+}
+
+@media screen and (max-width: 800px) {
+p.srctxt {
+align:justify;
+text-align: justify;
+word-break: break-all;
+font-size: 9px;
+font-family: Arial, Helvetica, sans-serif;
+}
+.logo-img{
+margin-right: 10px;
+vertical-align: top;
+/* position: absolute; */
+width: 30px;
+height: 30px;
+}
+}
+
+.float{
+	position:fixed;
+	width:25px;
+	height:25px;
+	bottom:15px;
+	right:12px;
+	background-color: white;
+	border-radius:50%;
+	text-align:center;
+	vertical-align:center;
+	z-index: 99999998;
+	font-size:0;
+	cursor:pointer;
+	animation: beatan 0.8s infinite alternate;
+	
+}
+.top-float{
+	position:fixed;
+	width:25px;
+	height:25px;
+	bottom:52px;
+	right:12px;
+	background-color: white;
+	border-radius:50%;
+	text-align:center;
+	vertical-align:center;
+	z-index: 99999998;
+	font-size:0;
+	cursor:pointer;
+	animation: beatan 0.8s infinite alternate;
+	
+}
+.my-float{
+	margin-top:22px;
+}
+
+@keyframes beatan{
+	to { transform: scale(1.1); }
+}
+
+.loader {
+	position: fixed;
+	left: 0px;
+	top: 0px;
+	width: 100%;
+	height: 100%;
+	z-index: 99999999999;
+	background: url('../static/loader.gif') 50% 50% no-repeat rgb(255,255,255);
+}
+
+ .highlight {
+    background-color: yellow;
+    font-weight: bold;
+  }
+
+.input-container {
+	position: relative;
+    padding-bottom: 10px;
+  }
+
+  .keyword-input {
+	
+    border-radius: 5px;
+    transition: border-color 0.3s ease;
+	border: 1px solid silver;
+	width: 10em;
+	height: 1.5em;
+	padding-left: 0.5em;
+	outline: none;
+	overflow: hidden;
+
+  }
+
+  .clear-btn {
+    position: absolute;
+	font-size: 20px;
+    left: 129px;
+    transform: translateY(-105%);
+    cursor: pointer;
+    opacity: 0;
+    transition: opacity 0.3s ease;
+  }
+
+  .clear-btn.show {
+    opacity: 1;
+  }
+
+@media screen and (min-width: 800px) {
+a.article-category {
+background-color: #E5E4E2;
+align:justify;
+display: block;
+height:100%;
+width: relative;
+text-decoration: none;
+border-left: 5px solid transparent;
+border-top: 0px;
+font-weight: bold;
+border-bottom: 1px solid transparent;
+border-right: 0px;
+padding-right: 5px;
+font-size: 11px;
+padding-bottom: 0px;
+font-family: Arial, Helvetica, sans-serif;
+color: green;
+}
+}
+
+@media screen and (max-width: 800px) {
+a.article-category {
+background-color: #E5E4E2;
+align:justify;
+display: block;
+height:100%;
+font-weight: bold;
+width: relative;
+text-decoration: none;
+border-left: 5px solid transparent;
+border-top: 0px;
+border-bottom: 1px solid transparent;
+border-right: 0px;
+padding-right: 5px;
+font-size: 10px;
+padding-bottom: 0px;
+font-family: Arial, Helvetica, sans-serif;
+color: green;
+}
+}
+
+.content {
+    display: none;
+	font-family: Arial, Helvetica, sans-serif;
+	
+  padding-right: 5px;
+  
+  padding-top: 5px;
+  border-left: 5px solid transparent;
+  }
+  
+ .container{
+ padding-bottom:10px;
+ }
+ 
+ .show-similar-button-container{
+ display: flex;
+ flex-direction: column;
+ align-items: center; 
+ }
+ 
+ .similar-news-item:hover {
+ text-decoration: none;
+ }
+ 
+ @media screen and (min-width: 800px) {
+.similar-news-item {
+text-align: justify;
+text-decoration: underline;
+font-size: 14px;
+font-family: Arial, Helvetica, sans-serif;
+color: black;
+display:inline-block;
+padding-bottom: 10px;
+width:100%;
+/*white-space: nowrap;
+overflow: hidden;
+text-overflow: ellipsis;*/
+
+}
+}
+
+@media screen and (max-width: 800px) {
+.similar-news-item {
+text-align: justify;
+text-decoration: underline;
+font-size: 12px;
+font-family: Arial, Helvetica, sans-serif;
+color: black;
+display:inline-block;
+padding-bottom: 8px;
+width:100%;
+/*white-space: nowrap;
+overflow: hidden;
+text-overflow: ellipsis;*/
+}
+}
+ 
+ 
+ 
+ .show-more {
+background-color: #E5E4E2;
+font-family: Arial, Helvetica, sans-serif;
+border-radius:4px;
+padding-top:3px;
+padding-bottom:3px;
+padding-left:3px;
+padding-right:3px;
+font-size: 12px;
+display: box;
+border: none;
+
+  }
+  
+ .show-more:hover {
+background-color: black;
+color: white;
+  }
+ 
+ .show-less {
+background-color: #E5E4E2;
+font-family: Arial, Helvetica, sans-serif;
+border-radius:4px;
+padding-top:3px;
+padding-bottom:3px;
+padding-left:3px;
+padding-right:3px;
+font-size: 12px;
+border: none;
+display: none;
+  }
+  
+ .show-less:hover {
+background-color: black;
+color: white;
+  }
+  
+ .word-cloud-container{
+ word-wrap: break-word;
+padding-bottom: 10px;
+
+ }
+ 
+ .wc-tokens{
+font-family: Arial, Helvetica, sans-serif;
+font-size: 13.2px;
+cursor: pointer;
+ }
+ 
+ .wc-tokens:hover{
+ text-decoration: underline;
+ }
+ 
+ .word-cloud-section{
+   padding-bottom: 10px;
+	 display: none;
+   word-wrap: break-word;
+ }
+ 
+ .show-more-word-cloud{
+ padding-bottom: 23px;
+ text-align: center;
+ }
+ 
+ .three-dots{
+	 font-size: 30px;
+ margin: 0;
+ line-height:0;
+ vertical-align: top;
+    padding: 0;
+	cursor: pointer;
+ }
+ 
+ .three-dots:hover{
+	 font-size: 25px;
+ }

templates/index.html

@@ -0,0 +1,200 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+	
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <link rel="preload" href="../static/loader.gif" as="image">
+	<link rel="preload" href="../static/favicon_new.png" as="image">
+	<link rel="preload" href="../static/refresh_reload_icon.png" as="image">
+<link rel="preload" href="../static/top-icon.png" as="image">
+    <link rel="icon" href="../static/favicon_new.png" type="image/png">
+    
+    <meta charset="UTF-8">
+    <title>Latest News</title>
+    <link rel="stylesheet" href="static/styles.css">
+	<a id="top-loc"></a>
+<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.12.4/jquery.min.js"></script>
+<script>
+$(window).load(function(){
+     $('.loader').fadeOut();
+});
+</script>
+
+
+
+<script>
+function filterContent(match_case) {
+	var keyword = document.getElementById("keywordInput").value;
+	if (match_case == false) 
+	{
+		/*var keyword = document.getElementById("keywordInput").value.toLowerCase(); */
+		/*var regex = new RegExp("\\b" + keyword + "\\b", "gi"); */
+	}
+	var clearbtn = document.getElementById("clearBtn");
+	
+	if (keyword !== "") 
+	{
+		clearbtn.style.opacity = 1;
+		var items = document.getElementsByClassName("news-item");
+		for (var i = 0; i < items.length; i++) 
+		{
+			var headline = items[i].querySelector('.headline');
+			var description = items[i].querySelector('.description');
+			if (match_case == true)
+			{
+				var article_category = items[i].querySelector('.article-category');
+				var src_time = items[i].querySelector('.time');
+				var itemText = headline.textContent.concat(" ", description.textContent, " ", article_category.textContent, " ", src_time.textContent)
+			} 
+			else 
+			{
+				var itemText = headline.textContent.concat(" ", description.textContent, " ")
+			}
+	
+			if (match_case == false) 
+			{	var regex = new RegExp("\\b" + keyword + "\\b", "gi");
+				itemText = itemText.toLowerCase();
+				if (regex.test(itemText) == true) 
+				{
+					items[i].style.display = "block";
+					highlightKeyword(headline, keyword, match_case);
+					highlightKeyword(description, keyword, match_case);
+				} 
+				else 
+				{
+					items[i].style.display = "none";
+				}
+			}
+			else
+			{
+				if (itemText.includes(keyword)) 
+				{
+					items[i].style.display = "block";
+					highlightKeyword(headline, keyword, match_case);
+					highlightKeyword(description, keyword, match_case);
+					highlightKeyword(article_category, keyword, match_case);
+					highlightKeyword(src_time, keyword, match_case);
+	  
+				}
+				else
+				{
+					items[i].style.display = "none";
+				}
+			}
+		}
+		} 
+		else
+		{
+			clearFilter();
+		}
+}
+
+function clearFilter() {
+  var items = document.getElementsByClassName("news-item");
+  var clearbtn = document.getElementById("clearBtn");
+  clearbtn.style.opacity=0;
+  for (var i = 0; i < items.length; i++) {
+	var headline = items[i].querySelector('.headline');
+	var description = items[i].querySelector('.description');
+	var article_category = items[i].querySelector('.article-category');
+	var src_time = items[i].querySelector('.time');
+    items[i].style.display = "block";
+    headline.innerHTML = headline.textContent; // Remove highlighting
+	description.innerHTML = description.textContent; // Remove highlighting
+	article_category.innerHTML = article_category.textContent; // Remove highlighting
+	src_time.innerHTML = src_time.textContent; // Remove highlighting
+  }
+
+   document.getElementById("keywordInput").value = ""; // Clear input field
+}
+
+function highlightKeyword(element, keyword, match_case) {
+  var regex = new RegExp(keyword);
+  if (match_case == false)
+  {
+	var regex = new RegExp("\\b" + keyword + "\\b", 'gi');
+  }
+  element.innerHTML = element.textContent.replace(regex, function(match) {
+    return '<span class="highlight">' + match + '</span>';
+  });
+}
+
+</script>
+
+<script>
+document.addEventListener('DOMContentLoaded', function() {
+    const containers = document.querySelectorAll('.container');
+
+    containers.forEach(container => {
+      const content = container.querySelector('.content');
+      const showMoreBtn = container.querySelector('.show-more');
+      const showLessBtn = container.querySelector('.show-less');
+
+      showMoreBtn.addEventListener('click', function() {
+       /* var  similar_news_items = document.getElementsByClassName("content");
+		var  show_less_items = document.getElementsByClassName("show-less");
+		var  show_more_items = document.getElementsByClassName("show-more");
+		for (var i = 0; i < similar_news_items.length; i++) {
+		similar_news_items[i].style.display = 'none';
+		show_more_items[i].style.display = 'block';
+		show_less_items[i].style.display = 'none';
+		}
+       */
+        
+        content.style.display = 'block';
+		content.style.opacity = 1;
+        showMoreBtn.style.display = 'none';
+        showLessBtn.style.display = 'block';
+      });
+
+      showLessBtn.addEventListener('click', function() {
+        document.documentElement.style.scrollBehavior = "auto";
+        var max_h = content.parentElement.parentElement.clientHeight;
+        content.style.display = 'none';
+        showMoreBtn.style.display = 'block';
+        showLessBtn.style.display = 'none';
+        var min_h = content.parentElement.parentElement.clientHeight;
+		$(window).scrollTop($(window).scrollTop() - (max_h - min_h) || 0);
+        document.documentElement.style.scrollBehavior = "smooth";
+      });
+    });
+  });
+</script>
+  
+<script>
+function wc_search(keyword)
+{
+clearFilter();
+document.getElementById("keywordInput").value = keyword;
+filterContent(false);
+}
+
+function word_cloud_display()
+{
+var word_cloud_section = document.getElementById("word-cloud-section-id");
+if (word_cloud_section.style.display == 'block')
+{
+word_cloud_section.style.display = 'none';
+}
+else
+{
+word_cloud_section.style.display = 'block';
+}
+}
+</script>
+
+  
+	
+</head>
+<body>
+<div class="loader"></div>
+{{body | safe}}
+
+<a id="top_theme" class="top-float" onclick="window.scrollTo(0, 0);">
+	<img id="top-theme-icon" alt="_" src="../static/top-icon.png" style="border-radius:50%; border: 3px solid #b9bfc4; background-color: white;" width="25px" height="25px" border="0">
+</a>	
+	<a href="javascript:window.location.reload(true)" id="theme" class="float">
+	<img id="theme-icon" alt="_" src="../static/refresh_reload_icon.png" style="border-radius:50%; border: 3px solid #b9bfc4; background-color: white;" width=25px height=25px border="0" />
+</a>
+</body>
+</html>

word_cloud.py

@@ -0,0 +1,653 @@
+import numpy as np
+import pandas as pd
+import string
+from unidecode import unidecode
+from collections import Counter
+
+
+class TextPreprocessor:
+    def __init__(self, remove_punct: bool = True, remove_digits: bool = True,
+                 remove_stop_words: bool = True,
+                 remove_short_words: bool = False, minlen: int = 1, maxlen: int = 1, top_p: float = None,
+                 bottom_p: float = None):
+        self.remove_punct = remove_punct
+        self.remove_digits = remove_digits
+        self.remove_stop_words = remove_stop_words
+        self.remove_short_words = remove_short_words
+        self.minlen = minlen
+        self.maxlen = maxlen
+        self.top_p = top_p
+        self.bottom_p = bottom_p
+        self.words_to_remove = []
+        self.stop_words = ["'d", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z",
+ 'about',
+ 'above',
+ 'across',
+ 'after',
+ 'afterwards',
+ 'again',
+ 'against',
+ 'ain',
+ 'all',
+ 'almost',
+ 'alone',
+ 'along',
+ 'already',
+ 'also',
+ 'although',
+ 'always',
+ 'am',
+ 'among',
+ 'amongst',
+ 'amount',
+ 'an',
+ 'and',
+ 'another',
+ 'any',
+ 'anyhow',
+ 'anyone',
+ 'anything',
+ 'anyway',
+ 'anywhere',
+ 'are',
+ 'around',
+ 'as',
+ 'at',
+ 'back',
+ 'be',
+ 'became',
+ 'because',
+ 'become',
+ 'becomes',
+ 'becoming',
+ 'been',
+ 'before',
+ 'beforehand',
+ 'behind',
+ 'being',
+ 'below',
+ 'beside',
+ 'besides',
+ 'between',
+ 'beyond',
+ 'both',
+ 'bottom',
+ 'but',
+ 'by',
+ 'ca',
+ 'call',
+ 'can',
+ 'cannot',
+ 'could',
+ 'couldn',
+ "couldn't",
+ 'd',
+ 'did',
+ 'do',
+ 'does',
+ 'doing',
+ 'done',
+ 'down',
+ 'due',
+ 'during',
+ 'each',
+ 'eight',
+ 'either',
+ 'eleven',
+ 'else',
+ 'elsewhere',
+ 'empty',
+ 'enough',
+ 'even',
+ 'ever',
+ 'every',
+ 'everyone',
+ 'everything',
+ 'everywhere',
+ 'except',
+ 'few',
+ 'fifteen',
+ 'fifty',
+ 'first',
+ 'five',
+ 'for',
+ 'former',
+ 'formerly',
+ 'forty',
+ 'four',
+ 'from',
+ 'front',
+ 'full',
+ 'further',
+ 'get',
+ 'give',
+ 'go',
+ 'had',
+ 'has',
+ 'have',
+ 'having',
+ 'he',
+ 'hence',
+ 'her',
+ 'here',
+ 'hereafter',
+ 'hereby',
+ 'herein',
+ 'hereupon',
+ 'hers',
+ 'herself',
+ 'him',
+ 'himself',
+ 'his',
+ 'how',
+ 'however',
+ 'hundred',
+ 'i',
+ 'if',
+ 'in',
+ 'indeed',
+ 'into',
+ 'is',
+ 'it',
+ "it's",
+ 'its',
+ 'itself',
+ 'just',
+ 'keep',
+ 'last',
+ 'latter',
+ 'latterly',
+ 'least',
+ 'less',
+ 'll',
+ 'm',
+ 'ma',
+ 'made',
+ 'make',
+ 'many',
+'say',
+'said',
+'says',
+'told',
+'tell',
+ 'may',
+ 'me',
+ 'meanwhile',
+ 'might',
+ 'mine',
+ 'more',
+ 'moreover',
+ 'most',
+ 'mostly',
+ 'move',
+ 'much',
+ 'must',
+ 'my',
+ 'myself',
+ 'name',
+ 'namely',
+ 'neither',
+ 'never',
+ 'nevertheless',
+ 'next',
+ 'nine',
+ 'no',
+ 'nobody',
+ 'none',
+ 'noone',
+ 'nor',
+ 'not',
+ 'nothing',
+ 'now',
+ 'nowhere',
+ 'o',
+ 'of',
+ 'off',
+ 'often',
+ 'on',
+ 'once',
+ 'one',
+ 'only',
+ 'onto',
+ 'or',
+ 'other',
+ 'others',
+ 'otherwise',
+ 'our',
+ 'ours',
+ 'ourselves',
+ 'out',
+ 'over',
+ 'own',
+ 'part',
+ 'per',
+ 'perhaps',
+ 'please',
+ 'put',
+ 'quite',
+ 'rather',
+ 're',
+'rs',
+ 'really',
+ 'regarding',
+ 's',
+ 'same',
+ 'say',
+ 'see',
+ 'seem',
+ 'seemed',
+ 'seeming',
+ 'seems',
+ 'serious',
+ 'several',
+ 'shan',
+ "shan't",
+ 'she',
+ "she's",
+ 'should',
+ "should've",
+ 'shouldn',
+ "shouldn't",
+ 'show',
+ 'side',
+ 'since',
+ 'six',
+ 'sixty',
+ 'so',
+ 'some',
+ 'somehow',
+ 'someone',
+ 'something',
+ 'sometime',
+ 'sometimes',
+ 'somewhere',
+ 'still',
+ 'such',
+ 't',
+ 'take',
+ 'ten',
+ 'than',
+ 'that',
+ "that'll",
+ 'the',
+ 'their',
+ 'theirs',
+ 'them',
+ 'themselves',
+ 'then',
+ 'thence',
+ 'there',
+ 'thereafter',
+ 'thereby',
+ 'therefore',
+ 'therein',
+ 'thereupon',
+ 'these',
+ 'they',
+ 'third',
+ 'this',
+ 'those',
+ 'though',
+ 'three',
+ 'through',
+ 'throughout',
+ 'thru',
+ 'thus',
+ 'to',
+ 'together',
+ 'too',
+ 'top',
+ 'toward',
+ 'towards',
+ 'twelve',
+ 'twenty',
+ 'two',
+ 'under',
+ 'unless',
+ 'until',
+ 'up',
+ 'upon',
+ 'us',
+ 'used',
+ 'using',
+ 'various',
+ 've',
+ 'very',
+ 'via',
+ 'was',
+ 'wasn',
+ "wasn't",
+ 'we',
+ 'well',
+ 'were',
+ 'weren',
+ "weren't",
+ 'what',
+ 'whatever',
+ 'when',
+ 'whence',
+ 'whenever',
+ 'where',
+ 'whereafter',
+ 'whereas',
+ 'whereby',
+ 'wherein',
+ 'whereupon',
+ 'wherever',
+ 'whether',
+ 'which',
+ 'while',
+ 'whither',
+ 'who',
+ 'whoever',
+ 'whole',
+ 'whom',
+ 'whose',
+ 'why',
+ 'will',
+ 'with',
+ 'within',
+ 'without',
+ 'won',
+ "won't",
+ 'would',
+ 'wouldn',
+ "wouldn't",
+ 'y',
+ 'yet',
+ 'you',
+ "you'd",
+ "you'll",
+ "you're",
+ "you've",
+ 'your',
+ 'yours',
+ 'yourself',
+ 'yourselves',
+ '‘d',
+ '‘ll',
+ '‘m',
+ '‘re',
+ '‘s',
+ '‘ve',
+ '’d',
+ '’ll',
+ '’m',
+ '’re',
+'new',
+'old',
+ '’s',
+ '’ve']
+
+        self.contraction_to_expansion = {"ain't": "am not",
+                                         "aren't": "are not",
+                                         "can't": "cannot",
+                                         "can't've": "cannot have",
+                                         "'cause": "because",
+                                         "could've": "could have",
+                                         "couldn't": "could not",
+                                         "couldn't've": "could not have",
+                                         "didn't": "did not",
+                                         "doesn't": "does not",
+                                         "don't": "do not",
+                                         "hadn't": "had not",
+                                         "hadn't've": "had not have",
+                                         "hasn't": "has not",
+                                         "haven't": "have not",
+                                         "he'd": "he would",
+                                         "he'd've": "he would have",
+                                         "he'll": "he will",
+                                         "he'll've": "he will have",
+                                         "he's": "he is",
+                                         "how'd": "how did",
+                                         "how'd'y": "how do you",
+                                         "how'll": "how will",
+                                         "how's": "how is",
+                                         "i'd": "i would",
+                                         "i'd've": "i would have",
+                                         "i'll": "i will",
+                                         "i'll've": "i will have",
+                                         "i'm": "i am",
+                                         "i've": "i have",
+                                         "isn't": "is not",
+                                         "it'd": "it had",
+                                         "it'd've": "it would have",
+                                         "it'll": "it will",
+                                         "it'll've": "it will have",
+                                         "it's": "it is",
+                                         "let's": "let us",
+                                         "ma'am": "madam",
+                                         "mayn't": "may not",
+                                         "might've": "might have",
+                                         "mightn't": "might not",
+                                         "mightn't've": "might not have",
+                                         "must've": "must have",
+                                         "mustn't": "must not",
+                                         "mustn't've": "must not have",
+                                         "needn't": "need not",
+                                         "needn't've": "need not have",
+                                         "o'clock": "of the clock",
+                                         "oughtn't": "ought not",
+                                         "oughtn't've": "ought not have",
+                                         "shan't": "shall not",
+                                         "sha'n't": "shall not",
+                                         "shan't've": "shall not have",
+                                         "she'd": "she would",
+                                         "she'd've": "she would have",
+                                         "she'll": "she will",
+                                         "she'll've": "she will have",
+                                         "she's": "she is",
+                                         "should've": "should have",
+                                         "shouldn't": "should not",
+                                         "shouldn't've": "should not have",
+                                         "so've": "so have",
+                                         "so's": "so is",
+                                         "that'd": "that would",
+                                         "that'd've": "that would have",
+                                         "that's": "that is",
+                                         "there'd": "there had",
+                                         "there'd've": "there would have",
+                                         "there's": "there is",
+                                         "they'd": "they would",
+                                         "they'd've": "they would have",
+                                         "they'll": "they will",
+                                         "they'll've": "they will have",
+                                         "they're": "they are",
+                                         "they've": "they have",
+                                         "to've": "to have",
+                                         "wasn't": "was not",
+                                         "we'd": "we had",
+                                         "we'd've": "we would have",
+                                         "we'll": "we will",
+                                         "we'll've": "we will have",
+                                         "we're": "we are",
+                                         "we've": "we have",
+                                         "weren't": "were not",
+                                         "what'll": "what will",
+                                         "what'll've": "what will have",
+                                         "what're": "what are",
+                                         "what's": "what is",
+                                         "what've": "what have",
+                                         "when's": "when is",
+                                         "when've": "when have",
+                                         "where'd": "where did",
+                                         "where's": "where is",
+                                         "where've": "where have",
+                                         "who'll": "who will",
+                                         "who'll've": "who will have",
+                                         "who's": "who is",
+                                         "who've": "who have",
+                                         "why's": "why is",
+                                         "why've": "why have",
+                                         "will've": "will have",
+                                         "won't": "will not",
+                                         "won't've": "will not have",
+                                         "would've": "would have",
+                                         "wouldn't": "would not",
+                                         "wouldn't've": "would not have",
+                                         "y'all": "you all",
+                                         "y'alls": "you alls",
+                                         "y'all'd": "you all would",
+                                         "y'all'd've": "you all would have",
+                                         "y'all're": "you all are",
+                                         "y'all've": "you all have",
+                                         "you'd": "you had",
+                                         "you'd've": "you would have",
+                                         "you'll": "you you will",
+                                         "you'll've": "you you will have",
+                                         "you're": "you are",
+                                         "you've": "you have"
+                                         }
+
+    @staticmethod
+    def __remove_double_whitespaces(string: str):
+        return " ".join(string.split())
+
+    def __remove_url(self, string_series: pd.Series):
+        """
+        Removes URLs m text
+        :param string_series: pd.Series, input string series
+        :return: pd.Series, cleaned string series
+        """
+        clean_string_series = string_series.str.replace(
+            pat=r"(https?:\/\/(?:www\.|(?!www))[a-zA-Z0-9][a-zA-Z0-9-]+[a-zA-Z0-9]\.[^\s]{2,}|www\.[a-zA-Z0-9][a-zA-Z0-9-]+[a-zA-Z0-9]\.[^\s]{2,}|https?:\/\/(?:www\.|(?!www))[a-zA-Z0-9]+\.[^\s]{2,}|www\.[a-zA-Z0-9]+\.[^\s]{2,})",
+            repl=" ", regex=True).copy()
+        return clean_string_series.map(self.__remove_double_whitespaces)
+
+    def __expand(self, string_series: pd.Series):
+        """
+        Replaces contractions with expansions. eg. don't wit do not.
+        :param string_series: pd.Series, input string series
+        :return: pd.Series, cleaned string series
+        """
+        clean_string_series = string_series.copy()
+        for c, e in self.contraction_to_expansion.items():
+            clean_string_series = clean_string_series.str.replace(pat=c, repl=e, regex=False).copy()
+        return clean_string_series.map(self.__remove_double_whitespaces)
+
+    def __remove_punct(self, string_series: pd.Series):
+        """
+       Removes punctuations from the input string.
+       :param string_series: pd.Series, input string series
+       :return: pd.Series, cleaned string series
+       """
+        clean_string_series = string_series.copy()
+        puncts = [r'\n', r'\r', r'\t']
+        puncts.extend(list(string.punctuation))
+        for i in puncts:
+            clean_string_series = clean_string_series.str.replace(pat=i, repl=" ", regex=False).copy()
+        return clean_string_series.map(self.__remove_double_whitespaces)
+
+    def __remove_digits(self, string_series: pd.Series):
+        """
+       Removes digits from the input string.
+       :param string_series: pd.Series, input string series
+       :return: pd.Series, cleaned string series
+       """
+        clean_string_series = string_series.str.replace(pat=r'\d', repl=" ", regex=True).copy()
+        return clean_string_series.map(self.__remove_double_whitespaces)
+
+    @staticmethod
+    def __remove_short_words(string_series: pd.Series, minlen: int = 1, maxlen: int = 1):
+        """
+        Reomves words/tokens where minlen <= len <= maxlen.
+        :param string_series: pd.Series, input string series
+        :param minlen: int, minimum length of token to be removed.
+        :param maxlen:  int, maximum length of token to be removed.
+        :return: pd.Series, cleaned string series
+        """
+        clean_string_series = string_series.map(lambda string: " ".join([word for word in string.split() if
+                                                                         (len(word) > maxlen) or (len(word) < minlen)]))
+        return clean_string_series
+
+    def __remove_stop_words(self, string_series: pd.Series):
+        """
+       Removes stop words from the input string.
+       :param string_series: pd.Series, input string series
+       :return: pd.Series, cleaned string series
+       """
+        def str_remove_stop_words(string: str):
+            stops = self.stop_words
+            return " ".join([token for token in string.split() if token not in stops])
+
+        return string_series.map(str_remove_stop_words)
+
+    def __remove_top_bottom_words(self, string_series: pd.Series, top_p: int = None,
+                                  bottom_p: int = None, dataset: str = 'train'):
+        """
+        Reomoves top_p percent (frequent) words and bottom_p percent (rare) words.
+        :param string_series: pd.Series, input string series
+        :param top_p: float, percent of frequent words to remove.
+        :param bottom_p: float, percent of rare words to remove.
+        :param dataset: str, "train" for training set, "tesrt" for val/dev/test set.
+        :return: pd.Series, cleaned string series
+        """
+        if dataset == 'train':
+            if top_p is None:
+                top_p = 0
+            if bottom_p is None:
+                bottom_p = 0
+
+            if top_p > 0 or bottom_p > 0:
+                word_freq = pd.Series(" ".join(string_series).split()).value_counts()
+                n_words = len(word_freq)
+
+            if top_p > 0:
+                self.words_to_remove.extend([*word_freq.index[: int(np.ceil(top_p * n_words))]])
+
+            if bottom_p > 0:
+                self.words_to_remove.extend([*word_freq.index[-int(np.ceil(bottom_p * n_words)):]])
+
+        if len(self.words_to_remove) == 0:
+            return string_series
+        else:
+            clean_string_series = string_series.map(lambda string: " ".join([word for word in string.split()
+                                                                             if word not in self.words_to_remove]))
+            return clean_string_series
+
+    def preprocess(self, string_series: pd.Series, dataset: str = "train"):
+        """
+        Entry point.
+        :param string_series: pd.Series, input string series
+        :param dataset: str, "train" for training set, "tesrt" for val/dev/test set.
+        :return: pd.Series, cleaned string series
+        """
+        string_series = string_series.str.lower().copy()
+        string_series = string_series.map(unidecode).copy()
+        string_series = self.__remove_url(string_series=string_series)
+        string_series = self.__expand(string_series=string_series)
+
+        if self.remove_punct:
+            string_series = self.__remove_punct(string_series=string_series)
+        if self.remove_digits:
+            string_series = self.__remove_digits(string_series=string_series)
+        if self.remove_stop_words:
+            string_series = self.__remove_stop_words(string_series=string_series)
+        if self.remove_short_words:
+            string_series = self.__remove_short_words(string_series=string_series,
+                                                      minlen=self.minlen,
+                                                      maxlen=self.maxlen)
+        string_series = self.__remove_top_bottom_words(string_series=string_series,
+                                                       top_p=self.top_p,
+                                                       bottom_p=self.bottom_p, dataset=dataset)
+
+        string_series = string_series.str.strip().copy()
+        string_series.replace(to_replace="", value="this is an empty message", inplace=True)
+
+        return string_series
+
+
+def get_frequent_words_html(df):
+    text_preprocess = TextPreprocessor()
+    preprocessed_txt = text_preprocess.preprocess(df['title'] + ' ' + df['description'])
+    counter = Counter(' '.join([*preprocessed_txt]).split())
+    
+    freq_tokens_html = '<div class="word-cloud-container">'
+    n = 1
+    for i, j in counter.most_common(25):
+        freq_tokens_html += f'<a class="wc-tokens" onclick=wc_search("{i}")>{i}</a>{"&nbsp;" * np.random.randint(3, 7, 1)[0]}'
+        if n == 5:
+            freq_tokens_html += '<div class="word-cloud-section" id="word-cloud-section-id">'
+        n += 1
+    freq_tokens_html += '</div></div>'
+    return freq_tokens_html