Adds linter and fixes linting

app.py

@@ -5,9 +5,7 @@ st.set_page_config(
     page_icon="🔎",
     layout="wide",
     initial_sidebar_state="expanded",
-    menu_items={
-        'About': "https://github.com/ugm2/neural-search-demo"
-    }
+    menu_items={"About": "https://github.com/ugm2/neural-search-demo"},
 )
 
 from streamlit_option_menu import option_menu

core/pipelines.py

@@ -9,9 +9,10 @@ from haystack.nodes.retriever import DensePassageRetriever, TfidfRetriever
 from haystack.nodes.preprocessor import PreProcessor
 import streamlit as st
 
+
 @st.cache(allow_output_mutation=True)
 def keyword_search(
-    index='documents',
+    index="documents",
 ):
     document_store = InMemoryDocumentStore(index=index)
     keyword_retriever = TfidfRetriever(document_store=(document_store))
@@ -31,16 +32,25 @@ def keyword_search(
     # INDEXING PIPELINE
     index_pipeline = Pipeline()
     index_pipeline.add_node(processor, name="Preprocessor", inputs=["File"])
-    index_pipeline.add_node(keyword_retriever, name="TfidfRetriever", inputs=["Preprocessor"])
+    index_pipeline.add_node(
+        keyword_retriever, name="TfidfRetriever", inputs=["Preprocessor"]
+    )
     index_pipeline.add_node(
         document_store, name="DocumentStore", inputs=["TfidfRetriever"]
     )
 
     return search_pipeline, index_pipeline
 
-@st.cache(hash_funcs={tokenizers.Tokenizer: lambda _: None, tokenizers.AddedToken: lambda _: None}, allow_output_mutation=True)
+
+@st.cache(
+    hash_funcs={
+        tokenizers.Tokenizer: lambda _: None,
+        tokenizers.AddedToken: lambda _: None,
+    },
+    allow_output_mutation=True,
+)
 def dense_passage_retrieval(
-    index='documents',
+    index="documents",
     query_embedding_model="facebook/dpr-question_encoder-single-nq-base",
     passage_embedding_model="facebook/dpr-ctx_encoder-single-nq-base",
 ):

core/search_index.py

@@ -6,9 +6,9 @@ def format_docs(documents):
     """Given a list of documents, format the documents and return the documents and doc ids."""
     db_docs: list = []
     for doc in documents:
-        doc_id = doc['id'] if doc['id'] is not None else str(uuid.uuid4())
+        doc_id = doc["id"] if doc["id"] is not None else str(uuid.uuid4())
         db_doc = {
-            "content": doc['text'],
+            "content": doc["text"],
             "content_type": "text",
             "id": str(uuid.uuid4()),
             "meta": {"id": doc_id},
@@ -16,11 +16,13 @@ def format_docs(documents):
         db_docs.append(Document(**db_doc))
     return db_docs, [doc.meta["id"] for doc in db_docs]
 
+
 def index(documents, pipeline):
     documents, doc_ids = format_docs(documents)
     pipeline.run(documents=documents)
     return doc_ids
 
+
 def search(queries, pipeline):
     results = []
     matches_queries = pipeline.run_batch(queries=queries)
@@ -35,10 +37,12 @@ def search(queries, pipeline):
                     "text": res.content,
                     "score": res.score,
                     "id": res.meta["id"],
-                    "fragment_id": res.id
+                    "fragment_id": res.id,
                 }
             )
         if not score_is_empty:
-            query_results = sorted(query_results, key=lambda x: x["score"], reverse=True)
+            query_results = sorted(
+                query_results, key=lambda x: x["score"], reverse=True
+            )
         results.append(query_results)
-    return results
+    return results

interface/components.py

@@ -3,36 +3,47 @@ import core.pipelines as pipelines_functions
 from inspect import getmembers, isfunction
 from networkx.drawing.nx_agraph import to_agraph
 
+
 def component_select_pipeline(container):
-    pipeline_names, pipeline_funcs = list(zip(*getmembers(pipelines_functions, isfunction)))
-    pipeline_names = [' '.join([n.capitalize() for n in name.split('_')]) for name in pipeline_names]
+    pipeline_names, pipeline_funcs = list(
+        zip(*getmembers(pipelines_functions, isfunction))
+    )
+    pipeline_names = [
+        " ".join([n.capitalize() for n in name.split("_")]) for name in pipeline_names
+    ]
     with container:
         selected_pipeline = st.selectbox(
-            'Select pipeline',
+            "Select pipeline",
             pipeline_names,
-            index=pipeline_names.index('Keyword Search') if 'Keyword Search' in pipeline_names else 0
+            index=pipeline_names.index("Keyword Search")
+            if "Keyword Search" in pipeline_names
+            else 0,
         )
-        st.session_state['search_pipeline'], \
-            st.session_state['index_pipeline'] = \
-                pipeline_funcs[pipeline_names.index(selected_pipeline)]()
+        (
+            st.session_state["search_pipeline"],
+            st.session_state["index_pipeline"],
+        ) = pipeline_funcs[pipeline_names.index(selected_pipeline)]()
+
 
 def component_show_pipeline(container, pipeline):
     """Draw the pipeline"""
-    with st.expander('Show pipeline'):
+    with st.expander("Show pipeline"):
         graphviz = to_agraph(pipeline.graph)
         graphviz.layout("dot")
         st.graphviz_chart(graphviz.string())
-    
+
+
 def component_show_search_result(container, results):
     with container:
         for idx, document in enumerate(results):
             st.markdown(f"### Match {idx+1}")
             st.markdown(f"**Text**: {document['text']}")
             st.markdown(f"**Document**: {document['id']}")
-            if document['score'] is not None:
+            if document["score"] is not None:
                 st.markdown(f"**Score**: {document['score']:.3f}")
             st.markdown("---")
 
+
 def component_text_input(container):
     """Draw the Text Input widget"""
     with container:
@@ -48,7 +59,6 @@ def component_text_input(container):
                 else:
                     break
         corpus = [
-            {"text": doc["text"], "id": doc_id}
-            for doc_id, doc in enumerate(texts)
+            {"text": doc["text"], "id": doc_id} for doc_id, doc in enumerate(texts)
         ]
-        return corpus
+        return corpus

interface/pages.py

@@ -1,7 +1,12 @@
 import streamlit as st
 from streamlit_option_menu import option_menu
 from core.search_index import index, search
-from interface.components import component_show_pipeline, component_show_search_result, component_text_input
+from interface.components import (
+    component_show_pipeline,
+    component_show_search_result,
+    component_text_input,
+)
+
 
 def page_landing_page(container):
     with container:
@@ -22,33 +27,34 @@ def page_landing_page(container):
             "\n  - Include file/url indexing"
             "\n  - [Optional] Include text to audio to read responses"
         )
-        
+
+
 def page_search(container):
     with container:
         st.title("Query me!")
-        
+
         ## SEARCH ##
         query = st.text_input("Query")
-        
-        component_show_pipeline(container, st.session_state['search_pipeline'])
-        
+
+        component_show_pipeline(container, st.session_state["search_pipeline"])
+
         if st.button("Search"):
-            st.session_state['search_results'] = search(
+            st.session_state["search_results"] = search(
                 queries=[query],
-                pipeline=st.session_state['search_pipeline'],
+                pipeline=st.session_state["search_pipeline"],
             )
-        if 'search_results' in st.session_state:
+        if "search_results" in st.session_state:
             component_show_search_result(
-                container=container,
-                results=st.session_state['search_results'][0]
+                container=container, results=st.session_state["search_results"][0]
             )
-        
+
+
 def page_index(container):
     with container:
         st.title("Index time!")
-        
-        component_show_pipeline(container, st.session_state['index_pipeline'])
-        
+
+        component_show_pipeline(container, st.session_state["index_pipeline"])
+
         input_funcs = {
             "Raw Text": (component_text_input, "card-text"),
         }
@@ -60,15 +66,15 @@ def page_index(container):
             default_index=0,
             orientation="horizontal",
         )
-        
+
         corpus = input_funcs[selected_input][0](container)
-        
+
         if len(corpus) > 0:
             index_results = None
             if st.button("Index"):
                 index_results = index(
                     corpus,
-                    st.session_state['index_pipeline'],
+                    st.session_state["index_pipeline"],
                 )
             if index_results:
-                st.write(index_results)
+                st.write(index_results)

linter.sh

@@ -0,0 +1 @@
+python -m black app.py interface core

requirements.txt

@@ -1,4 +1,5 @@
 streamlit
 streamlit_option_menu
 farm-haystack
-pygraphviz
+pygraphviz
+black