Upload 5 files

app.py

@@ -0,0 +1,378 @@
+import os
+import numpy as np
+import pandas as pd
+import streamlit as st
+
+import scanpy as sc
+
+#import mpld3
+import matplotlib.pyplot as plt
+import seaborn as sns
+
+
+import streamlit.components.v1 as components
+
+from IPython.display import Markdown as md
+
+from functions import pathway_analyses
+
+# SMALL_SIZE = 2
+# MEDIUM_SIZE = 2
+# BIGGER_SIZE = 2
+
+# plt.rc('font', size=SMALL_SIZE)          # controls default text sizes
+# plt.rc('axes', titlesize=SMALL_SIZE)     # fontsize of the axes title
+# plt.rc('axes', labelsize=MEDIUM_SIZE)    # fontsize of the x and y labels
+# plt.rc('xtick', labelsize=SMALL_SIZE)    # fontsize of the tick labels
+# plt.rc('ytick', labelsize=SMALL_SIZE)    # fontsize of the tick labels
+# plt.rc('legend', fontsize=SMALL_SIZE)    # legend fontsize
+# plt.rc('figure', titlesize=BIGGER_SIZE)  # fontsize of the figure title
+
+sc.settings.set_figure_params(dpi=80, facecolor='white')
+
+#disable st.pyplot warning
+st.set_page_config(layout="wide") 
+st.markdown(
+    """
+<style>
+.streamlit-expanderHeader {
+    font-size: x-large;
+}
+</style>
+""",
+    unsafe_allow_html=True,
+)
+m=st.markdown("""
+
+<style>
+
+div.stTitle {
+
+font-size:40px;
+
+}
+
+</style>"""
+,unsafe_allow_html=True)
+
+st.set_option('deprecation.showPyplotGlobalUse', False)
+
+#load Data
+cwd=os.getcwd()+'/'#+'data/'
+@st.cache_data
+def get_data():
+    if 'adata_annot' not in st.session_state:
+        adata_annot = sc.read_h5ad(cwd+'multiregion_brainaging_annotated.h5ad')
+        st.session_state['adata_annot'] = adata_annot
+        if 'genes_list' not in st.session_state:        
+            genes=adata_annot.var.index
+            #genes_list=sorted(genes.unique())
+            st.session_state['genes_list'] = sorted(genes.unique())
+        if 'cell_type' not in st.session_state:
+            #cell_type=diff_fdr[diff_fdr.type=='cell_type']['tissue']
+            #cell_type=sorted(cell_type.unique())
+            anno=adata_annot.obs.new_anno
+            #cell_type=sorted(anno.unique())
+            st.session_state['cell_type'] = sorted(anno.unique())
+        if 'broad_type' not in st.session_state:        
+            broad_celltype=adata_annot.obs.broad_celltype
+            #broad_type=sorted(broad_type.unique())
+            st.session_state['broad_type'] = sorted(broad_celltype.unique())
+            
+    #Also load Go Terms
+    if 'go_table' not in st.session_state:        
+        bp = pathway_analyses.read_pathways('pathway_databases/GO_Biological_Process_2021.txt')
+
+        # cy = pathway_analyses.read_pathways('pathway_databases/HumanCyc_2016.txt')
+        # ke = pathway_analyses.read_pathways('pathway_databases/KEGG_2019_Human.txt')
+        # re = pathway_analyses.read_pathways('pathway_databases/Reactome_2016.txt')
+
+        # all_paths = pd.concat([bp, cy, ke, re], join='outer', axis=0, ignore_index=True)
+        # all_paths.set_index(0, inplace=True)
+        # all_paths.fillna("", inplace=True)
+        # all_paths_dict = all_paths.to_dict(orient='index')
+
+
+        go_bp_paths = bp.set_index(0)
+        go_bp_paths.fillna("", inplace=True)
+        go_bp_paths_dict = go_bp_paths.to_dict(orient='index')
+
+
+        gene_set_by_path = {key: [val for val in value.values() if val != ""] for key, value in go_bp_paths_dict.items()}
+        gene_set_by_path = pd.DataFrame.from_dict(gene_set_by_path, orient='index').transpose()     
+        st.session_state['path_ways']=gene_set_by_path.columns
+        st.session_state['go_table']=gene_set_by_path
+#done load Data
+
+
+
+#st.title('Single nuclei atlas of human aging in brain regions')
+st.title('Brain Age Browser')
+
+#genes_list,adata_annot=get_data()
+
+get_data()
+
+tab1, tab2,readme = st.tabs(["Gene Expression by CellType", "Age associations for multiple genes", "README"])
+data = np.random.randn(10, 1)
+with tab1:
+    with st.form(key='columns_in_form'):
+        #c1, c2, c3 = st.columns([4,4,2])
+        c1, c2 = st.columns(2)
+        with c1:
+            selected_gene = st.selectbox(
+            'Please select a gene',
+            st.session_state['genes_list'])
+        with c2:
+            selected_celltype = st.selectbox(
+            'Please select CellType',
+            st.session_state['cell_type']
+            )
+        # with c3:
+        #     plot_choice = st.checkbox(
+        #     "Which Plots",
+        #     ('Gene','Old/Young'))
+            
+            
+
+        Updated=st.form_submit_button(label = 'Go')
+    if not isinstance(selected_gene, type(None)) and not isinstance(selected_celltype, type(None)) and Updated:
+
+
+        # fig11, axx1 = plt.subplots()
+        # sc.pl.umap(st.session_state['adata_annot'], color='new_anno', title='', legend_loc='on data',legend_fontsize='4', frameon=False,show=False, ax=axx1)
+        # st.pyplot(plt.gcf().set_size_inches(4, 4))     
+
+        col1,col2= st.columns([1,1])
+        with col1:
+            fig11, axx11 = plt.subplots(figsize=(5,5))
+            sc.pl.umap(st.session_state['adata_annot'], color='new_anno', title='', legend_loc='on data',legend_fontsize='8', frameon=False,show=False, ax=axx11)
+            st.pyplot(fig11)   
+                 
+        with col2:
+            fig12, axx12 = plt.subplots(figsize=(5,5))
+            #sc.pl.umap(st.session_state['adata_annot'], color='new_anno', title='', legend_loc='on data', frameon=False,show=False, ax=axx2)
+            sc.pl.umap(st.session_state['adata_annot'], color=selected_gene, title=selected_gene, legend_loc='best', frameon=False,show=False,legend_fontsize='xx-small', ax=axx12)#,vmax='p99')
+            #plt.xticks(rotation = 45)
+            st.pyplot(fig12)        
+
+        #Subset Younv and Old
+        adata_Young = st.session_state['adata_annot'][st.session_state['adata_annot'].obs['Age_group']=='young']
+        adata_Old = st.session_state['adata_annot'][st.session_state['adata_annot'].obs['Age_group']=='old']
+        
+        #Young/Old  but for cell_type
+        adata_YoungAst = adata_Young[adata_Young.obs['new_anno']==selected_celltype]
+        adata_OldAst = adata_Old[adata_Old.obs['new_anno']==selected_celltype]
+        
+        # # #Young/Old  but for cell_type
+        # # adata_YoungAst = adata_Young[adata_Young.obs['broad_celltype']==selected_celltype]
+        # # adata_OldAst = adata_Old[adata_Old.obs['broad_celltype']==selected_celltype]
+        
+        #Young
+        dot_size=.05
+        col1,col2= st.columns([1,1])
+        
+        with col1:
+            
+            #st.markdown('<div style="text-align: center;">**Young**</div>', unsafe_allow_html=True)
+            str_title='Young: '+selected_gene
+            #st.markdown("<h3 style='text-align: center; color: red;'>str_title</h3>", unsafe_allow_html=True)
+            st.markdown("# {} ".format(str_title))#,align_text='center')
+            #md("# {} ".format(str_title))
+
+            fig21, axx21 = plt.subplots(figsize=(1,1))
+            #sc.pl.umap(adata_Young, color=selected_gene, title="Young: "+selected_gene, legend_loc='right margin', color_map='viridis',frameon=False,show=False,size=dot_size, legend_fontsize='4',colorbar_loc=None,ax=axx21)  
+            sc.pl.umap(adata_Young, color=selected_gene, title="", legend_loc='right margin', color_map='viridis',frameon=False,show=False,size=dot_size, legend_fontsize='xx-small',colorbar_loc=None,ax=axx21)  
+            #st.pyplot(fig21)  
+            st.pyplot(plt.gcf())        
+        with col2:
+            str_title='Young: '+selected_gene+" ("+selected_celltype+")"
+            st.markdown("# {} ".format(str_title))#,align_text='center')
+            fig22, axx22 = plt.subplots(figsize=(1,1))
+            #sc.pl.umap(st.session_state['adata_annot'], color='new_anno', title='', legend_loc='on data', frameon=False,show=False, ax=axx2)
+            #sc.pl.umap(st.session_state['adata_annot'], color=selected_gene, title=selected_gene, legend_loc='best', frameon=False,show=False, ax=axx2)#,vmax='p99')
+            sc.pl.umap(adata_YoungAst, color=selected_gene, title="", legend_loc='right margin', color_map='viridis', frameon=False,show=False,size=dot_size,legend_fontsize='xx-small',colorbar_loc=None, ax=axx22)  
+            #sc.pl.umap(adata_Old, color=selected_gene, title="Old: "+selected_gene, legend_loc='right margin', color_map='viridis', frameon=False,show=False, ax=axx22)  
+            #plt.xticks(rotation = 45)
+            #st.pyplot(fig22)        
+            st.pyplot(plt.gcf())  
+        #Old
+        col1,col2= st.columns([1,1])
+        with col1:
+            str_title='Old: '+selected_gene+" ("+selected_celltype+")"
+            st.markdown("# {} ".format(str_title))#,align_text='center')
+            
+            fig31, axx31 = plt.subplots(figsize=(1,1))
+            #sc.pl.umap(st.session_state['adata_annot'], color='new_anno', title='', legend_loc='on data',legend_fontsize='8', frameon=False,show=False, ax=axx1)
+            sc.pl.umap(adata_Old, color=selected_gene, title="", legend_loc='right margin', color_map='viridis', frameon=False,show=False,size=dot_size,legend_fontsize='xx-small', colorbar_loc="bottom",ax=axx31)  
+            st.pyplot(fig31)        
+        with col2:
+            str_title='Old: '+selected_gene+" ("+selected_celltype+")"
+            st.markdown("# {} ".format(str_title))#,align_text='center')
+            
+            fig32, axx32 = plt.subplots(figsize=(1,1))
+            #sc.pl.umap(st.session_state['adata_annot'], color='new_anno', title='', legend_loc='on data', frameon=False,show=False, ax=axx2)
+            #sc.pl.umap(st.session_state['adata_annot'], color=selected_gene, title=selected_gene, legend_loc='best', frameon=False,show=False, ax=axx2)#,vmax='p99')
+            sc.pl.umap(adata_OldAst, color=selected_gene, title="", legend_loc='right margin', color_map='viridis', frameon=False,show=False,size=dot_size,legend_fontsize='xx-small', colorbar_loc="bottom",ax=axx32)  
+            #plt.xticks(rotation = 45)
+            st.pyplot(fig32)        
+        
+        # fig, ax = plt.subplots(3,2)
+        # sc.pl.umap(st.session_state['adata_annot'], color='new_anno', title='', legend_loc='on data', frameon=False,show=False, ax=ax[0,0])
+        # sc.pl.umap(st.session_state['adata_annot'], color=selected_gene, title=selected_gene, legend_loc='best', frameon=False,show=False, ax=ax[0,1],vmax='p99')
+        
+        # #Subset Younv and Old
+        # adata_Young = st.session_state['adata_annot'][st.session_state['adata_annot'].obs['Age_group']=='young']
+        # adata_Old = st.session_state['adata_annot'][st.session_state['adata_annot'].obs['Age_group']=='old']
+        # sc.pl.umap(adata_Young, color=selected_gene, title="Young: "+selected_gene, legend_loc='right margin', color_map='viridis',frameon=False,show=False, ax=ax[1,0])  
+        # sc.pl.umap(adata_Old, color=selected_gene, title="Old: "+selected_gene, legend_loc='right margin', color_map='viridis', frameon=False,show=False, ax=ax[2,0])  
+        
+        # # #Young/Old  but for cell_type
+        # # adata_YoungAst = adata_Young[adata_Young.obs['broad_celltype']==selected_celltype]
+        # # adata_OldAst = adata_Old[adata_Old.obs['broad_celltype']==selected_celltype]
+
+        # #Young/Old  but for cell_type
+        # adata_YoungAst = adata_Young[adata_Young.obs['new_anno']==selected_celltype]
+        # adata_OldAst = adata_Old[adata_Old.obs['new_anno']==selected_celltype]
+        
+        
+        # sc.pl.umap(adata_YoungAst, color=selected_gene, title=selected_celltype, legend_loc='right margin', color_map='viridis', frameon=False,show=False, ax=ax[1,1])  
+        # sc.pl.umap(adata_OldAst, color=selected_gene, title=selected_celltype, legend_loc='right margin', color_map='viridis', frameon=False,show=False, ax=ax[2,1])  
+        
+        # #sc.pl.umap(st.session_state['adata_annot'], color='Brain_region', title='Brain Region', legend_loc='right margin', frameon=False,show=False, ax=ax[1,1])  
+        # #sc.pl.umap(st.session_state['adata_annot'], color='Age_group', title='Age Group', legend_loc='right margin', frameon=False,show=False, ax=ax[2,0])  
+        # #sc.pl.umap(st.session_state['adata_annot'], color=selected_celltype, title=selected_celltype, legend_loc='on data', frameon=False,show=False, ax=ax[2,1])  
+        
+        
+        # st.pyplot(plt.gcf().set_size_inches(15, 30))
+                
+
+
+with tab2:
+    with st.form(key='multiselect_form'):
+        c1, c2, c3 = st.columns([4,4,2])
+        with c1:
+            multi_genes = st.multiselect(
+            'Select Genes List',
+            st.session_state['genes_list'])
+        with c2:
+            go_term = st.selectbox(
+            'Select GO Term',
+            st.session_state['path_ways'])
+        with c3:
+            Choice = st.radio(
+            "",
+            ('Gene Set','GO Term'))
+ 
+        Updated_tab2=st.form_submit_button(label = 'Show Results')
+    if not isinstance(multi_genes, type(None)) and Updated_tab2:
+        if Choice=='Gene Set':
+            multi_genes = np.sort(multi_genes)
+        else:
+            multi_genes=st.session_state['go_table'].loc[:,go_term]
+            multi_genes=multi_genes.dropna().values
+        #multi_genes=['WNT3', 'VPS13C', 'VAMP4', 'UBTF', 'UBAP2', 'TMEM175', 'TMEM163', 'SYT17', 'STK39', 'SPPL2B', 'SIPA1L2', 'SH3GL2', 'SCARB2', 'SCAF11', 'RPS6KL1', 'RPS12', 'RIT2', 'RIMS1', 'RETREG3', 'PMVK', 'PAM', 'NOD2', 'MIPOL1', 'MEX3C', 'MED12L', 'MCCC1', 'MBNL2', 'MAPT', 'LRRK2', 'KRTCAP2', 'KCNS3', 'KCNIP3', 'ITGA8', 'IP6K2', 'GPNMB', 'GCH1', 'GBA', 'FYN', 'FCGR2A', 'FBRSL1', 'FAM49B', 'FAM171A2', 'ELOVL7', 'DYRK1A', 'DNAH17', 'DLG2', 'CTSB', 'CRLS1', 'CRHR1', 'CLCN3', 'CHRNB1', 'CAMK2D', 'CAB39L', 'BRIP1', 'BIN3', 'ASXL3', 'SNCA']
+        #########
+        
+        #sns.clustermap(st.session_state['adata_annot'], figsize=(14,12),
+         #      pivot_kws={'index': 'country', 
+          #                        'columns': 'year',
+           #                       'values': 'lifeExp'})
+        # col1,col2= st.columns([1,1])
+        # #fig_szx=2*len(st.session_state['cell_type'])
+        # #fig_szy=100*len(multi_genes)
+        # with col1:
+        #     figa, axxaa = plt.subplots(figsize=(5, 5))
+        #     #sc.pl.umap(st.session_state['adata_annot'], color='new_anno', title='', legend_loc='on data',legend_fontsize='8', frameon=False,show=False, ax=axx11)
+        #     axxaa=sc.pl.clustermap(st.session_state['adata_annot'], obs_keys=multi_genes) #,'new_anno',size_title='Fraction of\n Expressing Cells',colorbar_title='Mean\nExpression',cmap='BuPu',swap_axes=True,show=False,vmax=5)
+        #     #st.pyplot(fig11)      
+        #     #st.pyplot(plt.gcf().set_size_inches(fig_szx, fig_szy))  
+        #     st.pyplot(plt.gcf())  
+        
+        
+        
+        col1,col2= st.columns([1,1])
+        #fig_szx=2*len(st.session_state['cell_type'])
+        #fig_szy=100*len(multi_genes)
+        with col1:
+            fig11, axx11 = plt.subplots(figsize=(5, 5))
+            #sc.pl.umap(st.session_state['adata_annot'], color='new_anno', title='', legend_loc='on data',legend_fontsize='8', frameon=False,show=False, ax=axx11)
+            axx11=sc.pl.dotplot(st.session_state['adata_annot'], multi_genes,'new_anno',size_title='Fraction of\n Expressing Cells',colorbar_title='Mean\nExpression',cmap='BuPu',swap_axes=True,show=False,vmax=5)
+            #st.pyplot(fig11)      
+            #st.pyplot(plt.gcf().set_size_inches(fig_szx, fig_szy))  
+            st.pyplot(plt.gcf())  
+        with col2:
+            fig12, axx12 = plt.subplots(figsize=(5, 5))
+            #sc.pl.umap(st.session_state['adata_annot'], color='new_anno', title='', legend_loc='on data', frameon=False,show=False, ax=axx2)
+            #sc.pl.umap(st.session_state['adata_annot'], color=selected_gene, title=selected_gene, legend_loc='best', frameon=False,show=False,legend_fontsize='xx-small', ax=axx12)#,vmax='p99')
+            axx12=sc.pl.heatmap(st.session_state['adata_annot'], multi_genes, groupby='new_anno', vmin=-1, vmax=1, cmap='BuPu', dendrogram=True, swap_axes=True)#,ax=ax2)
+            #plt.xticks(rotation = 45)
+            #st.pyplot(fig12)      
+            #st.pyplot(plt.gcf().set_size_inches(fig_szx, fig_szy))    
+            st.pyplot(plt.gcf())   
+            
+        
+        #######
+        
+        #multi_genes=['WNT3', 'VPS13C', 'VAMP4', 'UBTF', 'UBAP2', 'TMEM175', 'TMEM163', 'SYT17', 'STK39', 'SPPL2B', 'SIPA1L2', 'SH3GL2', 'SCARB2', 'SCAF11', 'RPS6KL1', 'RPS12', 'RIT2', 'RIMS1', 'RETREG3', 'PMVK', 'PAM', 'NOD2', 'MIPOL1', 'MEX3C', 'MED12L', 'MCCC1', 'MBNL2', 'MAPT', 'LRRK2', 'KRTCAP2', 'KCNS3', 'KCNIP3', 'ITGA8', 'IP6K2', 'GPNMB', 'GCH1', 'GBA', 'FYN', 'FCGR2A', 'FBRSL1', 'FAM49B', 'FAM171A2', 'ELOVL7', 'DYRK1A', 'DNAH17', 'DLG2', 'CTSB', 'CRLS1', 'CRHR1', 'CLCN3', 'CHRNB1', 'CAMK2D', 'CAB39L', 'BRIP1', 'BIN3', 'ASXL3', 'SNCA']
+        #multi_genes=np.sort(multi_genes)
+        # fig, ax1 = plt.subplots(1,2)
+        # sc.pl.dotplot(st.session_state['adata_annot'], multi_genes,'new_anno',size_title='Fraction of\n Expressing Cells',colorbar_title='Mean\nExpression',cmap='RdBu_r',show=False, ax=ax1[0])
+        # st.pyplot(plt.gcf().set_size_inches(10, 10))
+        # fig, ax2 = plt.subplots(1,2)
+        # ax2=sc.pl.heatmap(st.session_state['adata_annot'], multi_genes, 'new_anno', vmin=-1, vmax=1, cmap='RdBu_r', dendrogram=True, swap_axes=True)
+        # st.pyplot(plt.gcf().set_size_inches(10, 10))
+        #ax[0]=sc.pl.dotplot(st.session_state['adata_annot'],multi_genes,'new_anno',show=False)
+        #fig, (ax1, ax2, ax3) = plt.subplots(1, 3, figsize=(20,4), gridspec_kw={'wspace':0.9})
+     
+     
+     
+        #commented these-working ones
+     
+        # fig, (ax1) = plt.subplots(1, 1, figsize=(20,4), gridspec_kw={'wspace':0.9})
+        # #ax = plt.subplot()
+        # ax1_dict=sc.pl.dotplot(st.session_state['adata_annot'], multi_genes,'new_anno',size_title='Fraction of\n Expressing Cells',colorbar_title='Mean\nExpression',cmap='BuPu',swap_axes=True,show=False, ax=ax1,vmax=5)
+        # #ax_dict=sc.pl.dotplot(st.session_state['adata_annot'], multi_genes,'new_anno',size_title='Fraction of\n Expressing Cells',colorbar_title='Mean\nExpression',cmap='RdBu_r',swap_axes=True,show=False, ax=ax)
+        # st.pyplot(plt.gcf().set_size_inches(10, 15))
+        # #ax2_dict=sc.pl.dotplot(st.session_state['adata_annot'], multi_genes,'Sex',size_title='Fraction of\n Expressing Cells',colorbar_title='Mean\nExpression',cmap='RdBu_r',swap_axes=True,show=False, ax=ax2)
+        # fig, (ax2) = plt.subplots(1, 1, figsize=(20,4), gridspec_kw={'wspace':0.9})
+        # #ax2_dict=sc.pl.matrixplot(st.session_state['adata_annot'], multi_genes, 'new_anno', vmin=-1, vmax=1, show=False, cmap='BuPu',dendrogram=True, swap_axes=True, ax=ax2)
+        
+        # #sc.pl.heatmap(adata_annot, genes_lst, groupby='new_anno', vmin=-1, vmax=1, cmap='RdBu_r', dendrogram=True, swap_axes=True, figsize=(11,4))
+        # ax2_dict=sc.pl.heatmap(st.session_state['adata_annot'], multi_genes, groupby='new_anno', vmin=-1, vmax=1, cmap='BuPu', dendrogram=True, swap_axes=True)#,ax=ax2)
+        
+        # st.pyplot(plt.gcf().set_size_inches(10, 15))
+        
+    
+with readme:
+    expander = st.expander("How to use this app")   
+    #st.header('How to use this app')
+    expander.markdown('Please select **Results Menue** checkbox from the sidebar')
+    expander.markdown('Select a Gene from the dropdown list')
+    expander.markdown('A table showing all reference gudies from three LISTS will appear in the main panel')
+    expander.markdown('To see results for each of the selected reference guide from ListA, ListB and ListC, Please select respective checkbox')
+    expander.markdown('Results are shown as two tables, **MATCHED** and **MUTATED** guides tables and **NOT FOUND** table if guides are not found in GRCh38 and LR reference fasta files')
+    expander.markdown('**MATCHED** guides table shows the genomic postion in GRCh38 and LR Fasta file along other fields. **If a guide is found in GRCh38 but not in LR fasta, then corresponding columns will be NA**')
+    expander.markdown('**MUTATED** guides table shows the genomic postion in GRCh38 and LR Fasta file along other fields. **If a guide is found in GRCh38 but not in LR fasta, then corresponding columns will be NA**')
+    
+    expander1 = st.expander('Introduction')
+    
+    expander1.markdown(
+        """ This app helps navigate all probable genomic **miss-matched/Mutations (upto 2 bp)** for a given sgRNA (from 3 lists of CRISPRi dual sgRNA libraries) in GRCh38 reference fasta and a Reference fasta generated from BAM generated against KOLF2.1J longread data.
+            """
+            )
+    expander1.markdown('Merged bam file was converted to fasta file using following steps:')
+    expander1.markdown('- samtools mpileup to generate bcf file')
+    expander1.markdown('- bcftools to generate vcf file')
+    expander1.markdown('- bcftools consensus to generate fasta file')
+    expander1.markdown('A GPU based [Cas-OFFinder](http://www.rgenome.net/cas-offinder/) tool was used to find off-target sequences (upto 2 miss-matched) for each geiven reference guide against GRCh38 and LR fasta references.')
+
+css = '''
+<style>
+    .stTabs [data-baseweb="tab-list"] button [data-testid="stMarkdownContainer"] p {
+    font-size:1.5rem;
+    }
+</style>
+'''
+
+st.markdown(css, unsafe_allow_html=True)

env.yml

@@ -0,0 +1,111 @@
+name: scBrainApp
+channels:
+  - conda-forge
+  - bioconda
+  - defaults
+dependencies:
+  - ca-certificates=2023.5.7=h8857fd0_0
+  - libcxx=16.0.5=hd57cbcb_0
+  - libffi=3.3=h046ec9c_2
+  - libsqlite=3.42.0=h58db7d2_0
+  - libzlib=1.2.13=h8a1eda9_5
+  - ncurses=6.4=hf0c8a7f_0
+  - openssl=1.1.1u=h8a1eda9_0
+  - pip=23.1.2=pyhd8ed1ab_0
+  - python=3.9.0=h4f09611_5_cpython
+  - readline=8.2=h9e318b2_1
+  - setuptools=67.7.2=pyhd8ed1ab_0
+  - sqlite=3.42.0=h2b0dec6_0
+  - tk=8.6.12=h5dbffcc_0
+  - wheel=0.40.0=pyhd8ed1ab_0
+  - xz=5.2.6=h775f41a_0
+  - zlib=1.2.13=h8a1eda9_5
+  - pip:
+      - altair==5.0.1
+      - anndata==0.9.1
+      - appnope==0.1.3
+      - asttokens==2.2.1
+      - attrs==23.1.0
+      - backcall==0.2.0
+      - blinker==1.6.2
+      - cachetools==5.3.1
+      - certifi==2023.5.7
+      - charset-normalizer==3.1.0
+      - click==8.1.3
+      - contourpy==1.0.7
+      - cycler==0.11.0
+      - decorator==5.1.1
+      - decoupler==1.4.0
+      - executing==1.2.0
+      - fonttools==4.40.0
+      - gitdb==4.0.10
+      - gitpython==3.1.31
+      - h5py==3.8.0
+      - idna==3.4
+      - importlib-metadata==6.6.0
+      - importlib-resources==5.12.0
+      - ipython==8.14.0
+      - jedi==0.18.2
+      - jinja2==3.1.2
+      - joblib==1.2.0
+      - jsonschema==4.17.3
+      - kiwisolver==1.4.4
+      - llvmlite==0.40.1rc1
+      - markdown-it-py==3.0.0
+      - markupsafe==2.1.3
+      - matplotlib==3.7.1
+      - matplotlib-inline==0.1.6
+      - mdurl==0.1.2
+      - natsort==8.3.1
+      - networkx==3.1
+      - numba==0.57.0
+      - numpy==1.24.3
+      - packaging==23.1
+      - pandas==2.0.2
+      - parso==0.8.3
+      - patsy==0.5.3
+      - pexpect==4.8.0
+      - pickleshare==0.7.5
+      - pillow==9.5.0
+      - prompt-toolkit==3.0.38
+      - protobuf==4.23.2
+      - ptyprocess==0.7.0
+      - pure-eval==0.2.2
+      - pyarrow==12.0.1
+      - pydeck==0.8.1b0
+      - pygments==2.15.1
+      - pympler==1.0.1
+      - pynndescent==0.5.10
+      - pyparsing==3.0.9
+      - pyrsistent==0.19.3
+      - python-dateutil==2.8.2
+      - pytz==2023.3
+      - pytz-deprecation-shim==0.1.0.post0
+      - requests==2.31.0
+      - rich==13.4.2
+      - scanpy==1.9.3
+      - scikit-learn==1.2.2
+      - scipy==1.10.1
+      - seaborn==0.12.2
+      - session-info==1.0.0
+      - six==1.16.0
+      - smmap==5.0.0
+      - stack-data==0.6.2
+      - statsmodels==0.14.0
+      - stdlib-list==0.8.0
+      - streamlit==1.23.1
+      - tenacity==8.2.2
+      - threadpoolctl==3.1.0
+      - toml==0.10.2
+      - toolz==0.12.0
+      - tornado==6.3.2
+      - tqdm==4.65.0
+      - traitlets==5.9.0
+      - typing-extensions==4.6.3
+      - tzdata==2023.3
+      - tzlocal==4.3
+      - umap-learn==0.5.3
+      - urllib3==2.0.3
+      - validators==0.20.0
+      - wcwidth==0.2.6
+      - zipp==3.15.0

functions/pathway_analyses.py

@@ -0,0 +1,1015 @@
+
+import scipy
+import warnings
+#import anndata2ri
+import pandas as pd
+import scanpy as sc
+import numpy as np
+import seaborn as sb
+import decoupler as dc
+from scipy import sparse
+from anndata import AnnData
+from tabnanny import verbose
+import matplotlib.pyplot as plt
+#from gsva_prep import prep_gsva
+from typing import Optional, Union
+from matplotlib.pyplot import rcParams
+#from statsmodels.stats.multitest import multipletests
+#from sklearn.model_selection import train_test_split
+#from rpy2.robjects.conversion import localconverter
+
+
+
+
+def rescale_matrix(S, log_scale=False):
+    """
+    Sums cell-level counts by factors in label vector
+
+    Parameters
+    ----------
+    S : np.ndarray, scipy.sparse.csr_matrix or pandas.DataFrame
+        Matrix with read counts (gene x cell)
+    log_scale : bool, optional (default: False)
+        Whether to log-transform the rescaled matrix
+
+    Returns
+    -------
+    B : np.ndarray or scipy.sparse.csr_matrix
+        Scaled and log-transformed matrix
+    """
+    if isinstance(S, pd.DataFrame):
+        S = S.values
+    elif isinstance(S, np.ndarray):
+        pass
+    elif isinstance(S, scipy.sparse.csr_matrix):
+        S = S.toarray()
+    else:
+        raise ValueError('Input S must be a pandas.DataFrame, numpy.ndarray or scipy.sparse.csr_matrix')
+        
+    cs = np.sum(S, axis=0)
+    cs[cs == 0] = 1
+    B = np.median(cs) * (S / cs)
+    if log_scale:
+        B = np.log1p(B)
+    return B
+
+def normalize_default(adata, log_scale=True):
+    """
+    Normalizes gene expression matrix by total count and scales by median
+
+    Parameters
+    ----------
+    adata : AnnData
+        Annotated data matrix.
+    log_scale : bool, optional (default: True)
+        Whether to log-transform the rescaled matrix.
+
+    Returns
+    -------
+    adata : AnnData
+        Annotated data matrix with normalized and scaled expression values.
+    """
+    if 'counts' in adata.layers.keys():
+        print('normalizaing data using count data in .layers["counts] ')
+        S = adata.layers['counts']
+    else:
+        print('normaling data using count data in .X')
+        S = adata.X 
+    B = rescale_matrix(S, log_scale=log_scale)
+    adata.X = B
+    return adata
+
+
+def normalize_matrix(
+    X: Union[np.ndarray, sparse.spmatrix],
+    top_features_frac: float = 1.0,
+    scale_factor: Union[str, float, int, np.ndarray, None] = "median",
+    transformation: Union[str, None] = "log",
+    anchor_features: Union[np.ndarray, None] = None,
+) -> Union[np.ndarray, sparse.spmatrix]:
+
+    X = X.astype(dtype=np.float64)
+
+    # Which features (i.e. genes) should we use to compute library sizes?
+    if anchor_features is not None:
+        lib_sizes = np.array(np.mean(X[:, anchor_features], axis=1))
+    else:
+        if top_features_frac < 1.0:
+            universality = np.array(np.mean(X > 0, axis=0))
+            selected_features = np.flatnonzero(universality > (1 - top_features_frac))
+            lib_sizes = np.array(np.mean(X[:, selected_features], axis=1))
+        else:
+            lib_sizes = np.array(np.mean(X, axis=1))
+
+    # Note: mean as opposed to sum
+
+    # Normalize library sizes
+    if isinstance(X, sparse.spmatrix):
+        X_scaled = X.multiply(1 / lib_sizes)
+    else:
+        try:
+            X_scaled = X / lib_sizes
+        except ValueError:
+            lib_sizes = np.reshape(lib_sizes, (-1, 1))
+            X_scaled = X / lib_sizes
+
+    # scale normalized columns
+    if scale_factor == "median":
+        kappa = np.median(np.array(np.sum(X, axis=1) / np.sum(X_scaled, axis=1)))
+        X_scaled_norm = X_scaled * kappa
+    elif isinstance(scale_factor, (int, float)):
+        X_scaled_norm = X_scaled * scale_factor
+    elif isinstance(scale_factor, np.ndarray):
+        if sparse.issparse(X_scaled):
+            X_scaled_norm = X_scaled.multiply(scale_factor)
+        else:
+            X_scaled_norm = X_scaled / scale_factor
+
+    # For compatibility with C
+    if sparse.issparse(X_scaled_norm):
+        X_scaled_norm = sparse.csc_matrix(X_scaled_norm)
+
+    # Post-transformation
+    if transformation == "log":
+        X_scaled_norm_trans = np.log1p(X_scaled_norm)
+    elif transformation == "tukey":
+        if sparse.issparse(X_scaled_norm):
+            nnz_idx = X_scaled_norm.nonzero()
+            ii = nnz_idx[0]
+            jj = nnz_idx[1]
+            vv = X_scaled_norm[ii, jj]
+            vv_transformed = np.sqrt(vv) + np.sqrt(1 + vv)
+            X_scaled_norm[ii, jj] = vv_transformed
+        else:
+            X_scaled_norm[X_scaled_norm < 0] = 0
+            vv = X_scaled_norm[X_scaled_norm != 0]
+            vv_transformed = np.sqrt(vv) + np.sqrt(1 + vv)
+            X_scaled_norm[X_scaled_norm != 0] = vv_transformed
+        
+    # elif transformation == "lsi":
+    #     if sparse.issparse(X_scaled_norm):
+    #         X_scaled_norm_trans = _an.LSI(X_scaled_norm)
+    #     else:
+    #         X_scaled_norm_sp = sparse.csc_matrix(X_scaled_norm)
+    #         X_scaled_norm_trans = _an.LSI(X_scaled_norm_sp).toarray()
+    else:
+        X_scaled_norm_trans = X_scaled_norm
+
+    return X_scaled_norm_trans
+
+
+def normalize_actionet(
+    adata: AnnData,
+    layer_key: Optional[str] = None,
+    layer_key_out: Optional[str] = None,
+    top_features_frac: float = 1.0,
+    scale_factor: Union[str, float, int, np.ndarray, None] = "median",
+    transformation: Union[str, None] = "log",
+    anchor_features: Union[np.ndarray, None] = None,
+    copy: Optional[bool] = False,
+) -> Optional[AnnData]:
+    adata = adata.copy() if copy else adata
+
+    if "metadta" in adata.uns.keys():
+        if "norm_method" in adata.uns["metadata"].keys():  # Already normalized? leave it alone!
+            # return adata if copy else None
+            warnings.warn("AnnData object is prenormalized. Please make sure to use the right assay.")
+
+    if layer_key is None and "input_assay" in adata.uns["metadata"].keys():
+        layer_key = adata.uns["metadata"]["input_assay"]
+
+    if layer_key is not None:
+        if layer_key not in adata.layers.keys():
+            raise ValueError("Did not find adata.layers['" + layer_key + "']. ")
+        S = adata.layers[layer_key]
+    else:
+        S = adata.X
+
+    if sparse.issparse(S):
+        UE = set(S.data)
+    else:
+        UE = set(S.flatten())
+
+    nonint_count = len(UE.difference(set(np.arange(0, max(UE) + 1))))
+    if 0 < nonint_count:
+        warnings.warn("Input [count] assay has non-integer values, which looks like a normalized matrix. Please make sure to use the right assay.")
+
+    S = normalize_matrix(
+        S,
+        anchor_features=anchor_features,
+        top_features_frac=top_features_frac,
+        scale_factor=scale_factor,
+        transformation=transformation,
+    )
+
+    adata.uns["metadata"] = {}
+    adata.uns["metadata"]["norm_method"] = "default_top%.2f_%s" % (
+        top_features_frac,
+        transformation,
+    )
+
+    if layer_key_out is not None:
+        adata.uns["metadata"]["default_assay"] = layer_key_out
+        adata.layers[layer_key_out] = S
+    else:
+        adata.uns["metadata"]["default_assay"] = None
+        adata.X = S
+
+    return adata if copy else None
+
+def read_pathways(filename):
+    with open(filename, 'r') as temp_f:
+        col_count = [ len(l.split("\t")) for l in temp_f.readlines() ]
+    column_names = [i for i in range(0, max(col_count))]
+    ### Read csv
+    return pd.read_csv(filename, header=None, delimiter="\t", names=column_names)
+
+
+
+def filter_expressed_genes_by_celltype(adata: AnnData, 
+                                      threshold: float=0.05,
+                                      filter_genes_from: str='singlecell', 
+                                      subject_id: str='Subject'):
+    """
+
+        Function to filter expressed genes by cell type based on a threshold
+
+    Parameters:
+    -----------
+    adata : AnnData object
+        Annotated Data matrix with rows representing genes and columns representing cells.
+    threshold : float, optional (default=0.05)
+        The threshold to use for filtering expressed genes based on the minimum number of cells they are detected in.
+    filter_genes_from: str, optional (default=`singlecell`)
+        Whether to filter genes that meet threshold in pseudobulk data or singlecell data.
+    subject_id (str): a string indicating the column containing individual identifiers.
+
+
+    Returns:
+    --------
+    expressed_genes_per_celltype : pandas DataFrame
+        A dataframe where the rows are the gene names and columns are the cell types, 
+        containing only the genes that are expressed in at least the specified percentage of cells for each cell type.
+
+
+    """
+
+    # Initialize empty dictionaries to store the expressed genes and gene sets per cell type
+    expressed_genes_per_celltype = {}
+    gene_set_per_celltype = {}
+
+    if filter_genes_from=='pseudobulk':
+        # Get pseudo-bulk profile
+        adata = dc.get_pseudobulk(adata,
+                                sample_col=subject_id,
+                                groups_col='cell_type',
+                                layer='counts',
+                                mode='sum',
+                                min_cells=0,
+                                min_counts=0
+                                )
+        # Loop through each unique cell type in the input AnnData object
+
+        for cell_type in adata.obs.cell_type.unique():
+
+            expressed_genes_per_celltype[cell_type] = dc.filter_by_prop(adata[adata.obs['cell_type']==cell_type],
+                                                                      min_prop=threshold)
+    
+    elif filter_genes_from=='singlecell':
+        # Loop through each unique cell type in the input AnnData object
+
+        for cell_type in adata.obs.cell_type.unique():
+
+            # Calculate the number of cells based on the specified threshold
+            percent = threshold
+            num_cells = round(percent*len(adata[adata.obs['cell_type']==cell_type]))
+
+            # Filter genes based on minimum number of cells and store the resulting gene names
+            expressed_genes_per_celltype[cell_type], _ = sc.pp.filter_genes(adata[adata.obs.cell_type==cell_type].layers['counts'],
+                                                                            min_cells=num_cells, inplace=False)
+            expressed_genes_per_celltype[cell_type] = list(adata.var_names[expressed_genes_per_celltype[cell_type]])
+
+    # Convert the dictionary of expressed genes per cell type to a Pandas DataFrame
+    expressed_genes_per_celltype = pd.DataFrame.from_dict(expressed_genes_per_celltype, orient='index').transpose()
+
+    return expressed_genes_per_celltype
+
+
+def filter_lowly_exp_genes(expressed: pd.DataFrame,
+                           all_paths: pd.DataFrame,
+                           threshold: float = 0.33):
+
+    """
+    Filters lowly expressed gene sets based on a threshold and pathway membership.
+
+    Parameters:
+    -----------
+    expressed: pandas.DataFrame
+        A DataFrame of expressed genes with cell types as columns and gene IDs as rows.
+    all_paths: pandas.DataFrame
+        A DataFrame of gene sets with pathways as columns and gene IDs as rows.
+    threshold: float, optional (default=0.33)
+        A proportion threshold used to filter gene sets based on their expression in each cell type.
+
+    Returns:
+    --------
+    gene_set_per_celltype: dict of pandas.DataFrame
+        A dictionary of gene sets per cell type, with cell type names as keys and gene set dataframes as values.
+        Each gene set dataframe has three columns: 'description', 'member', and 'name'.
+    """
+
+    # Initialize empty dictionaries to store the gene sets and gene sets per cell type
+    gene_set = {}
+    gene_set_per_celltype = {}
+
+    # Loop through each cell type in the input Pandas DataFrame of expressed genes
+    for cell_type in expressed.columns:
+        # Determine which pathways have a proportion of genes above the specified threshold
+        index = [sum(all_paths[x].isin(expressed[cell_type]))/len(all_paths[x]) > threshold for x in all_paths.columns]
+        # Filter pathways based on threshold and store the resulting gene sets
+        p = all_paths.loc[:, index]
+        x = {y: pd.Series(list(set(expressed[cell_type]).intersection(set(p[y])))) for y in p.columns}
+        x = {k: v for k, v in x.items() if not v.empty}
+        gene_set[cell_type] = x
+
+        # Convert the gene sets to Pandas DataFrames and store them in a dictionary by cell type
+        gene_set_per_celltype[cell_type] = pd.DataFrame(columns=['description', 'member', 'name'])    
+        for pathway, gene_list in gene_set[cell_type].items():
+
+            df = pd.DataFrame(columns=['description', 'member', 'name'])  
+            df['member'] = gene_list
+            df['name'] = pathway
+            df['description'] = pathway.split(" ")[-1]                                                          
+            gene_set_per_celltype[cell_type] = pd.concat([gene_set_per_celltype[cell_type], df], join='outer', ignore_index=True)
+
+        # Sort the resulting gene sets by description and member
+        gene_set_per_celltype[cell_type].sort_index(axis=1, inplace=True)
+        gene_set_per_celltype[cell_type].sort_index(axis=0, inplace=True)
+
+
+    return gene_set_per_celltype
+
+
+def get_ind_level_ave(adata: AnnData, subject_id: str = 'Subject', method: str = "agg_x_num",
+                     expressed_genes_per_celltype: dict = {}, filter_genes_at_threshold: bool = True):
+    """
+    Get averaged expression data for each cell type and individual in an AnnData object.
+    
+
+    Args:
+
+        adata (AnnData): An AnnData object with read counts (gene x cell).
+        subject_id (str): a string indicating the column containing individual identifiers.
+        method (str): a string indicating the method to be used. The default is "agg_x_num".
+        filter_genes_at_threshold (bool): A boolean indicating whether to filter genes based on threshold. The default is True.
+        expressed_genes_per_celltype (float): A dictionary of the genes to be filtered for each celltype.
+        
+    Returns:
+
+        Dictionary: A dictionary of data frames with averaged expression data for each cell type and individual.
+
+    """
+
+    if method == "agg_x_norm":
+
+        avs_logcounts_cellxind = {}
+        # loop over each unique cell type in the annotation metadata
+        for cell_type in adata.obs.cell_type.unique():
+
+            # filter genes based on threshold
+            if filter_genes_at_threshold:
+                adata_temp = adata[adata.obs.cell_type==cell_type].copy()
+                # sc.pp.filter_genes(adata_temp, min_cells=gene_celltype_threshold*adata_temp.n_obs)
+                adata_temp = adata_temp[:, adata_temp.var_names.isin(expressed_genes_per_celltype[cell_type].tolist())]
+            else:
+                adata_temp = adata.copy()
+
+            # Get pseudo-bulk profile
+            pdata = dc.get_pseudobulk(adata_temp, sample_col=subject_id, groups_col='cell_type', layer='counts', mode='sum',
+                                    min_cells=0, min_counts=0)
+            
+            # genes = dc.filter_by_prop(pdata, min_prop=0.05, min_smpls=1)
+            # pdata = pdata[:, genes].copy()
+
+            # Normalize and log transform
+
+            # sc.pp.normalize_total(pdata, 1e06)
+            # sc.pp.log1p(pdata)
+
+            pdata.layers['counts'] = pdata.X
+            pdata = normalize_actionet(pdata, layer_key = 'counts', layer_key_out = None, 
+                                top_features_frac = 1.0, scale_factor = "median", 
+                                transformation = "log", anchor_features = None, copy = True)
+
+            # Store the log-normalized, averaged expression data for each individual and cell type
+            avs_logcounts_cellxind[cell_type] = pd.DataFrame(pdata.X.T, columns=pdata.obs[subject_id], index=pdata.var_names)
+            
+            del adata_temp, pdata
+            
+    elif method == 'norm_x_agg':
+
+        def sum_counts(counts, label, cell_labels, gene_labels):
+
+            """
+            Sums cell-level counts by factors in label vector.
+            
+            Args:
+                counts (AnnData): An AnnData object with read counts (gene x cell).
+                label (pd.DataFrame): Variable of interest by which to sum counts.
+                cell_labels (pd.Index): Vector of cell labels.
+                gene_labels (pd.Index): Vector of gene labels.
+                
+            Returns:
+                Dictionary: A dictionary with the following keys:
+                    - 'summed_counts': A data frame with summed counts.
+                    - 'ncells': A data frame with the number of cells used per summation.
+            """
+            # Create a data frame with the label vector and add a column of 1s for counting.
+            label_df = pd.DataFrame(label)
+            label_df.columns = ['ID']
+            label_df['index'] = 1
+
+            # Add a column for cell type and pivot the data frame to create a matrix of counts.
+            label_df['celltype'] = cell_labels
+            label_df = label_df.pivot_table(index='celltype', columns='ID', values='index', aggfunc=np.sum, fill_value=0)
+            label_df = label_df.astype(float)
+
+            # Multiply the counts matrix by the gene expression matrix to get summed counts.
+            summed_counts = pd.DataFrame(counts.X.T @ label_df.values, index = gene_labels, columns= label_df.columns)
+
+            # Sum the number of cells used for each summation.
+            ncells = label_df.sum()
+
+            # Return the summed counts and number of cells as a dictionary.
+            return {'summed_counts': summed_counts, 'ncells': ncells}
+
+        
+        # Get metadata from the AnnData object.
+        meta = adata.obs # Get metadata
+
+
+        # Create a data frame of labels by combining cell type and individual metadata fields.
+        # Sum counts by individual
+        labels = pd.DataFrame(meta['cell_type'].astype(str) + '_' + meta[subject_id].astype(str), columns=['individual'])
+
+        # Sum counts by individual and store the results in a dictionary.
+        summed_logcounts_cellxind = sum_counts(adata, labels, adata.obs_names, adata.var_names)
+
+        # Calculate averages for each cell type and individual and store the results in a dictionary.
+        # Get averages corresponding to both count matrices
+        avs_logcounts = np.array(summed_logcounts_cellxind['summed_counts'].values) / np.array(summed_logcounts_cellxind['ncells'].values)
+        # avs_logcounts = np.array(summed_logcounts_cellxind['summed_counts'].values)
+        avs_logcounts = pd.DataFrame(avs_logcounts, index = summed_logcounts_cellxind['summed_counts'].index,
+                                                columns=summed_logcounts_cellxind['summed_counts'].columns)
+        
+
+        # Split the averages by cell type and individual and store the results in a dictionary.
+        # Split column names into two parts: cell type and individual
+        x = [col.split('_') for col in avs_logcounts.columns]
+        celltype = [col[0] for col in x]
+        individual = [col[1] for col in x]
+
+        # Get unique cell types in the dataset
+        celltype_unique = np.unique(celltype)
+
+        # Create an empty dictionary to store the average counts for each cell type and individual    
+        avs_by_ind_out = {}
+
+        # Loop over the unique cell types and subset the average counts for each cell type and individual
+        for i in celltype_unique:
+            index = np.array(celltype)==i
+            df = avs_logcounts.loc[:, index]
+            df.columns = np.array(individual)[index]
+            avs_by_ind_out[i] = df
+            
+            if filter_genes_at_threshold:
+                # num_cells = round(gene_celltype_threshold*len(adata[adata.obs['cell_type']==cell_type]))
+                # # Filter genes based on minimum number of cells and store the resulting gene names
+                # gene_mask, _ = sc.pp.filter_genes(adata[adata.obs.cell_type==cell_type].layers['counts'],
+                #                                                                 min_cells=num_cells, 
+                #                                                                 inplace=False)
+                # genes = list(adata.var_names[gene_mask])
+                avs_by_ind_out[i] = avs_by_ind_out[i].loc[expressed_genes_per_celltype[i], :]
+            else:
+                adata = adata.copy()
+        # Store the dictionary of average counts for each cell type and individual    
+        avs_logcounts_cellxind = avs_by_ind_out
+
+        # Return the dictionary of average counts for each cell type and individual
+
+    return  avs_logcounts_cellxind
+
+
+def plot_and_select_top_deps(all_pathways: pd.DataFrame(), 
+                            list_of_paths_to_annotate: list = [], 
+                            save_name='cell_type_specific',
+                            save_prefix: str = 'mathys_pfc', 
+                            filter: bool=False,
+                            cell_type_specific: bool = True,
+                            test_name: str = ''): 
+
+    if cell_type_specific:
+        # Plot certain cell_type specific pathways 
+        collated_df = pd.DataFrame(all_pathways.groupby(all_pathways.index).agg({'score_adj': list, 'celltype': list, 
+                                    'logFC': list, 'P.Value': list, 'shortened': list, 'highlight': list}))
+        # filter pathways only expressed in one cell type
+        mask = collated_df["celltype"].apply(len) == 1
+        df = collated_df[mask]
+
+        # create pathway by cell type pivot table
+        scores_table = pd.pivot_table(all_pathways, values='score_adj', index='pathway', columns='celltype')
+        scores_table = scores_table.loc[df.index]
+        scores_table['shortened'] = df.shortened.apply(lambda x: x[0])
+        scores_table['highlight'] = df.highlight.apply(lambda x: x[0])
+        scores_table.sort_values(by=[cell_type for cell_type in all_pathways.celltype.unique()], inplace=True)
+
+        # drop pathways with same shortened names ??
+        scores_table = scores_table.drop_duplicates(subset='shortened', keep='first')
+
+        ###### Plot Cell type specific data
+
+        if filter:
+            xticks = ['Excitatory', 'Inhibitory', 'Astrocyte', 'Oligodendrocyte', 'OPC', 'Microglia', 'Endothelial']
+
+            # select only pathways that should be visualized
+            shortened_names = scores_table[scores_table.shortened.isin(list_of_paths_to_annotate)]['shortened']
+            scores_table = scores_table[scores_table.shortened.isin(list_of_paths_to_annotate)]
+
+            n_rows = len(scores_table)
+
+            fig, ax1 = plt.subplots(1, 1, figsize=(0.5, n_rows*0.095), sharex=False, layout='constrained')
+            fig.tight_layout()
+    
+            # order table by cell type name
+            # scores_table = scores_table.reindex(columns=['Excitatory', 'Inhibitory', 'Astrocyte', 'Oligodendrocyte',
+            #                                                     'OPC', 'Microglia'])
+            scores_table = scores_table[xticks]
+
+            g1 = sb.heatmap(scores_table, cmap='bwr', center=0, vmin=-2.5, vmax=2.5, robust=False, annot=None, fmt='.1g', 
+                            linewidths=0.15, linecolor='black', annot_kws=None, cbar_kws={'shrink': 0.2},
+                            cbar_ax=None, square=False,ax=ax1, xticklabels=xticks, yticklabels=shortened_names, mask=None,) 
+
+
+            cax = g1.figure.axes[-1]
+
+            g1.set_title(f'Select Cell-type-specific Pathways in {test_name.split("_")[0]}- vs {test_name.split("_")[-1]}-pathology',
+                          fontsize=3)           
+            g1.set_ylabel('')
+            g1.set_xlabel('')
+
+            ax1.tick_params(axis='both', which='major', labelsize=4, length=1.5, width=0.5)
+            cax.tick_params(labelsize=4, length=1.5, width=0.5, which="major")
+
+            plt.tight_layout()
+            plt.savefig(f'results/{test_name}/{save_prefix}_filtered_{save_name}_diff_exp_paths.pdf', bbox_inches='tight')
+            plt.show(block=False)
+
+        else:
+            xticks = ['Excitatory', 'Inhibitory', 'Astrocyte', 'Oligodendrocyte', 'OPC', 'Microglia', 'Endothelial']
+
+
+            scores_table = scores_table[scores_table.shortened!='None']     
+            yticklabels = scores_table['shortened']
+            # order table by cell type name
+            
+            scores_table = scores_table[xticks]
+
+            n_rows = len(scores_table)
+
+            fig, ax1 = plt.subplots(1, 1, figsize=(0.5, n_rows*0.095), sharex=False, layout='constrained')
+            fig.tight_layout()
+
+            g1 = sb.heatmap(scores_table, cmap='bwr', center=0, vmin=-2.5, vmax=2.5, robust=False, annot=None, fmt='.1g', 
+                            linewidths=0.07, linecolor='black', annot_kws=None, cbar_kws={'shrink': 0.1},
+                            cbar_ax=None, square=False, ax=ax1, xticklabels=xticks, yticklabels=yticklabels, mask=None,) 
+
+
+            cax = g1.figure.axes[-1]
+
+            g1.set_title(f'All Cell-type-specific Pathways in {test_name.split("_")[0]}- vs {test_name.split("_")[-1]}-pathology', 
+                         fontsize=3)           
+            g1.set_ylabel('')
+            g1.set_xlabel('')
+
+            ax1.tick_params(axis='both', which='major', labelsize=2, length=1.5, width=0.25)
+            cax.tick_params(labelsize=4, length=1.5, width=0.25, which="major")
+
+            plt.tight_layout()
+            #plt.savefig(f'../results/{test_name}/{save_prefix}_all_{save_name}_diff_exp_paths.pdf', bbox_inches='tight')
+            plt.savefig(f'results/{test_name}/{save_prefix}_all_{save_name}_diff_exp_paths.pdf', bbox_inches='tight')
+            plt.show(block=False)
+
+
+    else:
+        # Plot certain cell_type specific pathways 
+        collated_df = pd.DataFrame(all_pathways.groupby(all_pathways.index).agg({'score_adj': list, 'celltype': list, 
+                                    'logFC': list, 'P.Value': list, 'shortened': list, 'highlight': list}))
+        # filte pathways only expressed in one cell type
+        mask = collated_df["celltype"].apply(len) > 1
+        df = collated_df[mask]
+
+        # create pathway by cell type pivot table
+        scores_table = pd.pivot_table(all_pathways, values='score_adj', index='pathway', columns='celltype')
+        scores_table = scores_table.loc[df.index]
+        scores_table['shortened'] = df.shortened.apply(lambda x: x[0])
+        scores_table['highlight'] = df.highlight.apply(lambda x: x[0])
+        scores_table.sort_values(by=[cell_type for cell_type in all_pathways.celltype.unique()], inplace=True)
+
+        # drop pathways with same shortened names ??
+        scores_table = scores_table.drop_duplicates(subset='shortened', keep='first')
+
+        ###### Plot Cell type specific data
+
+        if filter:
+            xticks = ['Excitatory', 'Inhibitory', 'Astrocyte', 'Oligodendrocyte', 'OPC', 'Microglia', 'Endothelial']
+
+            # select only pathways that should be visualized
+            shortened_names = scores_table[scores_table.shortened.isin(list_of_paths_to_annotate)]['shortened']
+            scores_table = scores_table[scores_table.shortened.isin(list_of_paths_to_annotate)]
+        
+            # order table by cell type name
+            scores_table = scores_table[xticks]
+
+            n_rows = len(scores_table)
+
+            fig, ax1 = plt.subplots(1, 1, figsize=(0.5, n_rows*0.095), sharex=False, layout='constrained')
+            fig.tight_layout()
+
+            g1 = sb.heatmap(scores_table, cmap='bwr', center=0, vmin=-2.5, vmax=2.5, robust=False, annot=None, fmt='.1g', 
+                            linewidths=0.15, linecolor='black', annot_kws=None, cbar_kws={'shrink': 0.2},
+                            cbar_ax=None, square=False,ax=ax1, xticklabels=xticks, yticklabels=shortened_names, mask=None,) 
+
+            cax = g1.figure.axes[-1]
+
+            g1.set_title(f'Select Shared Pathways in {test_name.split("_")[0]}- vs {test_name.split("_")[-1]}-pathology', fontsize=3)           
+            g1.set_ylabel('')
+            g1.set_xlabel('')
+
+            ax1.tick_params(axis='both', which='major', labelsize=4, length=1.5, width=0.5)
+            cax.tick_params(labelsize=4, length=1.5, width=0.5, which="major")
+
+            plt.tight_layout()
+            plt.savefig(f'results/{test_name}/{save_prefix}_filtered_{save_name}_diff_exp_paths.pdf', bbox_inches='tight')
+            plt.show(block=False)
+
+        else:
+            xticks = ['Excitatory', 'Inhibitory', 'Astrocyte', 'Oligodendrocyte', 'OPC', 'Microglia', 'Endothelial']
+
+            scores_table = scores_table[scores_table.shortened!='None']     
+            yticklabels = scores_table['shortened']
+            # order table by cell type name
+            
+            scores_table = scores_table[xticks]
+
+            n_rows = len(scores_table)
+
+            fig, ax1 = plt.subplots(1, 1, figsize=(0.5, n_rows*0.095), sharex=False, layout='constrained')
+            fig.tight_layout()
+
+            g1 = sb.heatmap(scores_table, cmap='bwr', center=0, vmin=-2.5, vmax=2.5, robust=False, annot=None, fmt='.1g', 
+                            linewidths=0.07, linecolor='black', annot_kws=None, cbar_kws={'shrink': 0.1},
+                            cbar_ax=None, square=False, ax=ax1, xticklabels=xticks, yticklabels=yticklabels, mask=None,) 
+
+            cax = g1.figure.axes[-1]
+
+            g1.set_title(f'All Broad Pathways in {test_name.split("_")[0]}- vs {test_name.split("_")[-1]}-pathology', fontsize=3)           
+            g1.set_ylabel('')
+            g1.set_xlabel('')
+
+            ax1.tick_params(axis='both', which='major', labelsize=2, length=1.5, width=0.25)
+            cax.tick_params(labelsize=4, length=1.5, width=0.25, which="major")
+
+            plt.tight_layout()
+            plt.savefig(f'results/{test_name}/{save_prefix}_all_{save_name}_diff_exp_paths.pdf', bbox_inches='tight')
+            plt.show(block=False)
+
+    return 
+
+
+def multi_study_pathway_overlap(pathway_scores: dict = {},
+                                filtered_pathways: list = [],
+                                cell_types: list = ["Excitatory", "Inhibitory", "Astrocyte",
+                                                     "Microglia", "Oligodendrocyte", "OPC", "Endothelial"],
+                                test_name: str = 'ad_vs_no',
+                                top_n: int = 10,
+                                pathways: list = [],
+                                filter: bool = False,
+                                save_suffix: str = 'ad_vs_no',
+                                method: str = 'cell_type_overlap'):
+
+    """
+    This function generates a heatmap of the overlapping pathways across multiple studies. The heatmap displays the adjusted
+    pathway scores across different cell types for each pathway in each study. The function also returns a dictionary of
+    filtered scores that contain only the overlapping pathways across the studies.
+
+    Parameters:
+    -----------
+    pathway_scores : dict
+        A dictionary of pathway scores for different studies.
+    filtered_pathways : list, optional
+        A list of pathways to be used as a filter.
+    cell_types : list, optional
+        A list of cell types to be included in the heatmap. Default is ["Excitatory", "Inhibitory", "Astrocyte",
+        "Microglia", "Oligodendrocyte", "OPC", "Endothelial"].
+    test_name : str, optional
+        The name of the test being compared. Default is 'ad_vs_no'.
+    top_n : int, optional
+        The number of top pathways to be included in the heatmap. Default is 10.
+    pathways : list, optional
+        A list of pathways to be included in the heatmap. If not empty, only these pathways will be included in the
+        heatmap. Default is [].
+    filter : bool, optional
+        If True, the function will filter out pathways that are not present in the filtered_pathways list. Default is
+        False.
+    save_suffix : str, optional
+        A suffix to be added to the output file name. Default is 'ad_vs_no'.
+    method : str, optional
+        The method used to generate the overlap. 'cell_type_overlap' will generate the overlap based on cell type.
+        'global_overlap' will generate the overlap based on all pathways in the studies. Default is 'cell_type_overlap'.
+
+    Returns:
+    --------
+    filtered_scores : dict
+        A dictionary of pathway scores for the overlapping pathways across the studies.
+
+    Examples:
+    ---------
+    >>> multi_study_pathway_overlap(pathway_scores, filtered_pathways=['pathway1', 'pathway2'],
+                                        cell_types=['Excitatory', 'Astrocyte'], test_name='ad_vs_no', filter=True)
+    """
+
+    
+    for i, study in enumerate(pathway_scores.keys()):
+        pathway_scores[study][test_name] = pathway_scores[study][test_name][pathway_scores[study][test_name].celltype.isin(cell_types)]
+    
+    if method == "cell_type_overlap":
+        overlap = []
+        for cell_type in cell_types:
+            eval_string = []
+            for i, study in enumerate(pathway_scores.keys()):
+                eval_string.append(f'set(pathway_scores["{study}"]["{test_name}"][pathway_scores["{study}"]["{test_name}"].celltype=="{cell_type}"].pathway)')
+
+            eval_string = '&'.join(eval_string)
+            overlap.extend(list(eval(eval_string)))
+
+    elif method == "global_overlap":
+        overlap = []
+        eval_string = []
+        for i, study in enumerate(pathway_scores.keys()):
+            eval_string.append(f'set(pathway_scores["{study}"]["{test_name}"].pathway)')
+
+        eval_string = '&'.join(eval_string)
+        overlap.extend(list(eval(eval_string)))
+            
+
+    if filter:
+        n_rows = len(set(filtered_pathways) & set(overlap))
+    else:
+        n_rows = len(overlap)
+        
+    fig, axs = plt.subplots(1, 3, figsize=(3.5, n_rows*0.095), gridspec_kw={'width_ratios':[0.85, 0.85, 1]}, sharex=False,
+                                sharey=True, layout='constrained')
+    fig.tight_layout()
+
+    filtered_scores = {}
+    shortened_names = {}
+
+    for i, study in enumerate(pathway_scores.keys()):
+        filtered_scores[study] = pathway_scores[study][test_name][pathway_scores[study][test_name].pathway.isin(overlap)]
+        filtered_scores[study] = pd.pivot_table(filtered_scores[study], values='score_adj', index='pathway', columns='celltype')
+        filtered_scores[study] = filtered_scores[study][cell_types]
+
+        if filter:
+            filtered_scores[study] = filtered_scores[study].loc[filtered_scores[study].index.isin(filtered_pathways)]
+
+        shortened_names[study] = [' '.join(name.split(" ")[:-1]) for name in filtered_scores[study].index]
+        # shortened_names[study] = filtered_scores[study].index
+
+        cbar=True if study==list(pathway_scores.keys())[-1] else False
+        g1 = sb.heatmap(filtered_scores[study], cmap='bwr', center=0, vmin=-2.5, vmax=2.5, robust=False, annot=None, fmt='.1g', 
+                        linewidths=0.015, linecolor='black', annot_kws=None, cbar_kws={'shrink': 0.2}, cbar=cbar,
+                        cbar_ax=None, square=False, ax=axs[i], xticklabels=cell_types, yticklabels=shortened_names[study], mask=None,) 
+
+        axs[i].tick_params(axis='both', which='major', labelsize=2.5, length=1.5, width=0.5)
+
+        g1.set_title(study.split('_')[-1].upper(), fontsize=3)           
+        g1.set_ylabel('', fontsize=4)
+        g1.set_xlabel('')
+
+    cax = g1.figure.axes[-1]
+    cax.tick_params(labelsize=4, length=1.5, width=0.5, which="major")
+
+    # plt.tight_layout()
+    # if filter:  
+    #     plt.savefig(f'../results/pathway_meta_analysis/filtered_overlap_pathway_diff_exp_patterns_{save_suffix}.pdf', bbox_inches='tight')
+    # else:
+
+    plt.suptitle(f"{test_name.split('_')[0].capitalize()}- vs {test_name.split('_')[-1]}-pathology", fontsize=4)
+
+    if filter:
+        plt.savefig(f'results/{test_name}/multi_study_pathway_overlap_filtered.pdf', bbox_inches='tight')       
+    else:
+        plt.savefig(f'results/{test_name}/multi_study_pathway_overlap_all.pdf', bbox_inches='tight')       
+    plt.show(block=False)  
+    
+    return filtered_scores
+
+
+def save_plot(fig, ax, save):
+    if save is not None:
+        if ax is not None:
+            if fig is not None:
+                fig.savefig(save, bbox_inches='tight')
+            else:
+                raise ValueError("fig is None, cannot save figure.")
+        else:
+            raise ValueError("ax is None, cannot save figure.")
+        
+
+def check_if_matplotlib(return_mpl=False):
+    if not return_mpl:
+        try:
+            import matplotlib.pyplot as plt
+        except Exception:
+            raise ImportError('matplotlib is not installed. Please install it with: pip install matplotlib')
+        return plt
+    else:
+        try:
+            import matplotlib as mpl
+        except Exception:
+            raise ImportError('matplotlib is not installed. Please install it with: pip install matplotlib')
+        return mpl
+
+
+def check_if_seaborn():
+    try:
+        import seaborn as sns
+    except Exception:
+        raise ImportError('seaborn is not installed. Please install it with: pip install seaborn')
+    return sns
+
+
+def check_if_adjustText():
+    try:
+        import adjustText as at
+    except Exception:
+        raise ImportError('adjustText is not installed. Please install it with: pip install adjustText')
+    return at
+
+
+def filter_limits(df, sign_limit=None, lFCs_limit=None):
+        
+    """
+    Filters a DataFrame by limits of the absolute value of the columns pvals and logFCs.
+
+    Parameters
+    ----------
+    df : pd.DataFrame
+        The input DataFrame to be filtered.
+    sign_limit : float, None
+        The absolute value limit for the p-values. If None, defaults to infinity.
+    lFCs_limit : float, None
+        The absolute value limit for the logFCs. If None, defaults to infinity.
+
+    Returns
+    -------
+    pd.DataFrame
+        The filtered DataFrame.
+    """
+
+    # Define limits if not defined
+    if sign_limit is None:
+        sign_limit = np.inf
+    if lFCs_limit is None:
+        lFCs_limit = np.inf
+
+    # Filter by absolute value limits
+    msk_sign = df['pvals'] < np.abs(sign_limit)
+    msk_lFCs = np.abs(df['logFCs']) < np.abs(lFCs_limit)
+    df = df.loc[msk_sign & msk_lFCs]
+
+    return df
+
+
+def plot_volcano(data, x, y, x_label, y_label='-log10(pvals)', annotate=True, 
+                    annot_by='top', names=[], 
+                    top=5, sign_thr=0.05, lFCs_thr=0.5, sign_limit=None, lFCs_limit=None,
+                    figsize=(7, 5), dpi=100, ax=None, return_fig=False, save=None, 
+                    fontsizes={"on_plot": 4}):
+    """
+    Plot logFC and p-values from a long formated data-frame.
+
+    Parameters
+    ----------
+    data : pd.DataFrame
+        Results of DEA in long format.
+    x : str
+        Column name of data storing the logFCs.
+    y : str
+        Columns name of data storing the p-values.
+    x_label: str
+        Aternate name for LogFC to be included in plot. If None, defaults to x
+    y_label: str
+        Aternate name for p-values to be included in plot. If None, defaults to y
+    annotate: bool
+        Whether to annotate labels.
+    annot_by: str
+        Determines how to annotate the plot for top features. It can be either 'top' or 'name'. 
+        If set to 'top', the top top differentially expressed features will be annotated. If set to 'name',
+        only the features specified in names will be annotated.
+    names: list[]:
+        A list of feature names to be annotated in the plot. Only used if annot_by is set to 'name'.
+    top : int
+        Number of top differentially expressed features to show.
+    sign_thr : float
+        Significance threshold for p-values.
+    lFCs_thr : float
+        Significance threshold for logFCs.
+    sign_limit : float
+        Limit of p-values to plot in -log10.
+    lFCs_limit : float
+        Limit of logFCs to plot in absolute value.
+    figsize : tuple
+        Figure size.
+    dpi : int
+        DPI resolution of figure.
+    ax : Axes, None
+        A matplotlib axes object. If None returns new figure.
+    return_fig : bool
+        Whether to return a Figure object or not.
+    save : str, None
+        Path to where to save the plot. Infer the filetype if ending on {`.pdf`, `.png`, `.svg`}.
+
+    Returns
+    -------
+    fig : Figure, None
+        If return_fig, returns Figure object.
+    """
+
+
+    if x_label is None:
+        x_label = x
+
+    if y_label is None:
+        y_label = y
+
+    # Load plotting packages
+    plt = check_if_matplotlib()
+    at = check_if_adjustText()
+
+    # Transform sign_thr
+    sign_thr = -np.log10(sign_thr)
+
+    # Extract df
+    df = data.copy()
+    df['logFCs'] = df[x]
+    df['pvals'] = -np.log10(df[y])
+
+    # Filter by limits
+    df = filter_limits(df, sign_limit=sign_limit, lFCs_limit=lFCs_limit)
+
+    # Define color by up or down regulation and significance
+    df['weight'] = 'gray'
+    up_msk = (df['logFCs'] >= lFCs_thr) & (df['pvals'] >= sign_thr)
+    dw_msk = (df['logFCs'] <= -lFCs_thr) & (df['pvals'] >= sign_thr)
+    df.loc[up_msk, 'weight'] = '#D62728'
+    df.loc[dw_msk, 'weight'] = '#1F77B4'
+
+    # Plot
+    fig = None
+    if ax is None:
+        fig, ax = plt.subplots(1, 1, figsize=figsize, dpi=dpi)
+
+    n = df.shape[0]
+    size = 120000 / (100*n)
+
+    df.plot.scatter(x='logFCs', y='pvals', c='weight', sharex=False, ax=ax, s=size)
+
+    # Draw sign lines
+    ax.axhline(y=sign_thr, linestyle='--', color="black")
+    ax.axvline(x=lFCs_thr, linestyle='--', color="black")
+    ax.axvline(x=-lFCs_thr, linestyle='--', color="black")
+
+    # Plot top sign features
+    signs = df[up_msk | dw_msk].sort_values('pvals', ascending=False)
+
+    # Add labels
+    ax.set_ylabel(y_label)
+    ax.set_xlabel(x_label)
+    
+    if annotate:
+        if annot_by == 'top':
+            signs = signs.iloc[:top]
+        elif annot_by == 'name':
+            signs = signs.loc[signs.index.isin(names)]
+        
+        texts = []
+        for x, y, s in zip(signs['logFCs'], signs['pvals'], signs.index):
+            texts.append(ax.text(x, y, s, fontsize=fontsizes['on_plot']))
+        if len(texts) > 0:
+            at.adjust_text(texts, arrowprops=dict(arrowstyle='-', color='black'), ax=ax)
+
+    save_plot(fig, ax, save)
+
+    if return_fig:
+        return fig