add support for scispacy

app.py

@@ -2,6 +2,8 @@ import streamlit as st
 import textract
 import tempfile
 import spacy
+import subprocess
+import scispacy
 from spacy.tokens import DocBin, Doc, Span
 from collections import Counter
 import srsly
@@ -16,12 +18,17 @@ if 'query' not in st.session_state:
     st.session_state['query'] = ''
 
 @st.cache
-def download_model(select_model:str):
-    try:
-        spacy.cli.download(select_model)
+def download_model(language:str, select_model:str):
+    if language == 'Science':
+        urls = srsly.read_json('scispacy.json')
+        subprocess.run(['pip', 'install', f'{urls[select_model]}'])
         return True
-    except Exception as e:
-        return False
+    else:
+        try:
+            spacy.cli.download(select_model)
+            return True
+        except Exception as e:
+            return False
 
 def search_docs(query:str, documents:list[Doc], nlp) -> list[Span]:
     terms = query.split('|')
@@ -47,7 +54,7 @@ language = st.selectbox("Language", languages, index=len(models.keys())-1, help=
 if language:
     select_model = st.selectbox("Model", models[language], help="spaCy model")
     if select_model:
-        model_downloaded = download_model(select_model)
+        model_downloaded = download_model(language, select_model)
 
         if model_downloaded:
 

models.json

@@ -144,6 +144,18 @@
         "sv_core_news_md",
         "sv_core_news_lg"
         
-    ]     
+    ],
+    "Science":[
+        "",
+        "en_core_sci_sm",
+        "en_core_sci_md",
+        "en_ner_craft_md",
+        "en_ner_jnlpba_md",
+        "en_ner_bc5cdr_md",
+        "en_ner_bionlp13cg_md",
+        "en_core_sci_lg",
+        "en_core_sci_scibert"
+        
+    ]          
 
 }

requirements.txt

@@ -1,4 +1,139 @@
+altair==4.2.0
+argcomplete==1.10.3
+argon2-cffi==21.3.0
+argon2-cffi-bindings==21.2.0
+asttokens==2.0.7
+attrs==22.1.0
+backcall==0.2.0
+beautifulsoup4==4.8.2
+bleach==5.0.1
+blinker==1.5
+blis==0.7.8
+cachetools==5.2.0
+catalogue==2.0.8
+certifi==2022.6.15
+cffi==1.15.1
+chardet==3.0.4
+charset-normalizer==2.1.0
+click==8.0.4
+commonmark==0.9.1
+compressed-rtf==1.0.6
+conllu==4.5.2
+cymem==2.0.6
+debugpy==1.6.2
+decorator==5.1.1
+defusedxml==0.7.1
+docx2txt==0.8
+ebcdic==1.1.1
+en-core-sci-sm==0.5.0
+en-core-web-sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.2.0/en_core_web_sm-3.2.0-py3-none-any.whl
+entrypoints==0.4
+executing==0.9.1
+extract-msg==0.28.7
+fastjsonschema==2.16.1
+gitdb==4.0.9
+GitPython==3.1.27
+idna==3.3
+IMAPClient==2.1.0
+importlib-metadata==4.12.0
+ipykernel==6.15.1
+ipython==8.4.0
+ipython-genutils==0.2.0
+ipywidgets==7.7.1
+jedi==0.18.1
+Jinja2==3.1.2
+joblib==1.1.0
+jsonschema==4.9.1
+jupyter-client==7.3.4
+jupyter-core==4.11.1
+jupyterlab-pygments==0.2.2
+jupyterlab-widgets==1.1.1
+langcodes==3.3.0
+lxml==4.9.1
+MarkupSafe==2.1.1
+matplotlib-inline==0.1.3
+mistune==0.8.4
+murmurhash==1.0.7
+nbclient==0.6.6
+nbconvert==6.5.1
+nbformat==5.4.0
+nest-asyncio==1.5.5
+nmslib==2.1.1
+notebook==6.4.12
+numpy==1.23.1
+olefile==0.46
+packaging==21.3
+pandas==1.4.3
+pandocfilters==1.5.0
+parso==0.8.3
+pathy==0.6.2
+pdfminer.six==20191110
+pexpect==4.8.0
+pickleshare==0.7.5
+Pillow==9.2.0
+preshed==3.0.6
+prometheus-client==0.14.1
+prompt-toolkit==3.0.30
+protobuf==3.20.1
+psutil==5.9.1
+ptyprocess==0.7.0
+pure-eval==0.2.2
+pyarrow==9.0.0
+pybind11==2.6.1
+pycparser==2.21
+pycryptodome==3.15.0
+pydantic==1.8.2
+pydeck==0.7.1
+Pygments==2.12.0
+Pympler==1.0.1
+pyparsing==3.0.9
+pyrsistent==0.18.1
+pysbd==0.3.4
+python-dateutil==2.8.2
+python-pptx==0.6.21
+pytz==2022.1
+pytz-deprecation-shim==0.1.0.post0
+pyzmq==23.2.0
+requests==2.28.1
+rich==12.5.1
+scikit-learn==1.1.2
+scipy==1.9.0
+scispacy==0.5.0
+semver==2.13.0
+Send2Trash==1.8.0
+six==1.12.0
+smart-open==5.2.1
+smmap==5.0.0
+sortedcontainers==2.4.0
+soupsieve==2.3.2.post1
+spacy==3.2.4
+spacy-legacy==3.0.9
+spacy-loggers==1.0.3
+SpeechRecognition==3.8.1
+srsly==2.4.4
+stack-data==0.3.0
 streamlit==1.11.1
+terminado==0.15.0
 textract==1.6.5
-spacy==3.4.1
-#spacy-streamlit==1.0.4
+thinc==8.0.17
+threadpoolctl==3.1.0
+tinycss2==1.1.1
+toml==0.10.2
+toolz==0.12.0
+tornado==6.2
+tqdm==4.64.0
+traitlets==5.3.0
+typer==0.4.2
+typing_extensions==4.3.0
+tzdata==2022.1
+tzlocal==4.2
+urllib3==1.26.11
+validators==0.20.0
+wasabi==0.10.1
+watchdog==2.1.9
+wcwidth==0.2.5
+webencodings==0.5.1
+widgetsnbextension==3.6.1
+xlrd==1.2.0
+XlsxWriter==3.0.3
+zipp==3.8.1

scispacy.json

@@ -0,0 +1,10 @@
+{
+    "en_core_sci_sm": "https://s3-us-west-2.amazonaws.com/ai2-s2-scispacy/releases/v0.5.0/en_core_sci_sm-0.5.0.tar.gz",
+    "en_core_sci_md": "https://s3-us-west-2.amazonaws.com/ai2-s2-scispacy/releases/v0.5.0/en_core_sci_md-0.5.0.tar.gz",
+    "en_ner_craft_md": "https://s3-us-west-2.amazonaws.com/ai2-s2-scispacy/releases/v0.5.0/en_ner_craft_md-0.5.0.tar.gz",
+    "en_ner_jnlpba_md": "https://s3-us-west-2.amazonaws.com/ai2-s2-scispacy/releases/v0.5.0/en_ner_jnlpba_md-0.5.0.tar.gz",
+    "en_ner_bc5cdr_md": "https://s3-us-west-2.amazonaws.com/ai2-s2-scispacy/releases/v0.5.0/en_ner_bc5cdr_md-0.5.0.tar.gz",
+    "en_ner_bionlp13cg_md": "https://s3-us-west-2.amazonaws.com/ai2-s2-scispacy/releases/v0.5.0/en_ner_bionlp13cg_md-0.5.0.tar.gz",
+    "en_core_sci_lg": "https://s3-us-west-2.amazonaws.com/ai2-s2-scispacy/releases/v0.5.0/en_core_sci_lg-0.5.0.tar.gz",
+    "en_core_sci_scibert": "https://s3-us-west-2.amazonaws.com/ai2-s2-scispacy/releases/v0.5.0/en_core_sci_scibert-0.5.0.tar.gz"
+}