Upload 46 files

This is the first raw knowledge base of OmicVerse

ovrawm/t_anno_trans.txt

@@ -0,0 +1,171 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# # Celltype annotation transfer in multi-omics
+# 
+# In the field of multi-omics research, transferring cell type annotations from one data modality to another is a crucial step. For instance, when annotating cell types in single-cell ATAC sequencing (scATAC-seq) data, it's often desirable to leverage the cell type labels already annotated in single-cell RNA sequencing (scRNA-seq) data. This process involves integrating information from both scRNA-seq and scATAC-seq data modalities.
+# 
+# GLUE is a prominent algorithm used for cross-modality integration, allowing researchers to combine data from different omics modalities effectively. However, GLUE does not inherently provide a method for transferring cell type labels from scRNA-seq to scATAC-seq data. To address this limitation, an approach was implemented in the omicverse platform using K-nearest neighbor (KNN) graphs.
+# 
+# The KNN graph-based approach likely involves constructing KNN graphs separately for scRNA-seq and scATAC-seq data. In these graphs, each cell is connected to its K nearest neighbors based on certain similarity metrics, which could be calculated using gene expression profiles in scRNA-seq and accessibility profiles in scATAC-seq. Once these graphs are constructed, the idea is to transfer the cell type labels from the scRNA-seq side to the scATAC-seq side by assigning labels to scATAC-seq cells based on the labels of their KNN neighbors in the scRNA-seq graph.
+# 
+# Colab_Reproducibility：https://colab.research.google.com/drive/1aIMmSgyIw-PGjJ65WvMgz4Ob3EtoK_UV?usp=sharing
+
+# In[3]:
+
+
+import omicverse as ov
+import matplotlib.pyplot as plt
+import scanpy as sc
+ov.ov_plot_set()
+
+
+# ## Loading the data preprocessed with GLUE
+# 
+# Here, we use two output files from the GLUE cross-modal integration, and their common feature is that they both have the `obsm['X_glue']` layer. And the rna have been annotated.
+
+# In[4]:
+
+
+rna=sc.read("data/analysis_lymph/rna-emb.h5ad")
+atac=sc.read("data/analysis_lymph/atac-emb.h5ad")
+
+
+# We can visualize the intergrated effect of GLUE with UMAP
+
+# In[5]:
+
+
+import scanpy as sc
+combined=sc.concat([rna,atac],merge='same')
+combined
+
+
+# In[6]:
+
+
+combined.obsm['X_mde']=ov.utils.mde(combined.obsm['X_glue'])
+
+
+# We can see that the two layers are correctly aligned
+
+# In[8]:
+
+
+ov.utils.embedding(combined,
+               basis='X_mde',
+               color='domain',
+                title='Layers',
+                show=False,
+                palette=ov.utils.red_color,
+                frameon='small'
+               )
+
+
+# And the RNA modality has an already annotated cell type label on it
+
+# In[22]:
+
+
+ov.utils.embedding(rna,
+               basis='X_mde',
+               color='major_celltype',
+                title='Cell type',
+                show=False,
+                #palette=ov.utils.red_color,
+                frameon='small'
+               )
+
+
+# ## Celltype transfer
+# 
+# We train a knn nearest neighbour classifier using `X_glue` features
+
+# In[13]:
+
+
+knn_transformer=ov.utils.weighted_knn_trainer(
+    train_adata=rna,
+    train_adata_emb='X_glue',
+    n_neighbors=15,
+)
+
+
+# In[14]:
+
+
+labels,uncert=ov.utils.weighted_knn_transfer(
+    query_adata=atac,
+    query_adata_emb='X_glue',
+    label_keys='major_celltype',
+    knn_model=knn_transformer,
+    ref_adata_obs=rna.obs,
+)
+
+
+# We migrate the training results of the KNN classifier to atac. `unc` stands for uncertainty, with higher uncertainty demonstrating lower migration accuracy, suggesting that the cell in question may be a double-fate signature or some other type of cell.
+
+# In[15]:
+
+
+atac.obs["transf_celltype"]=labels.loc[atac.obs.index,"major_celltype"]
+atac.obs["transf_celltype_unc"]=uncert.loc[atac.obs.index,"major_celltype"]
+
+
+# In[24]:
+
+
+atac.obs["major_celltype"]=atac.obs["transf_celltype"].copy()
+
+
+# In[27]:
+
+
+ov.utils.embedding(atac,
+               basis='X_umap',
+               color=['transf_celltype_unc','transf_celltype'],
+                #title='Cell type Un',
+                show=False,
+                palette=ov.palette()[11:],
+                frameon='small'
+               )
+
+
+# ## Visualization
+# 
+# We can merge atac and rna after migration annotation and observe on the umap plot whether the cell types are consistent after merging the modalities.
+
+# In[28]:
+
+
+import scanpy as sc
+combined1=sc.concat([rna,atac],merge='same')
+combined1
+
+
+# In[29]:
+
+
+combined1.obsm['X_mde']=ov.utils.mde(combined1.obsm['X_glue'])
+
+
+# We found that the annotation was better, suggesting that the KNN nearest-neighbour classifier we constructed can effectively migrate cell type labels from RNA to ATAC.
+
+# In[31]:
+
+
+ov.utils.embedding(combined1,
+               basis='X_mde',
+               color=['domain','major_celltype'],
+                title=['Layers','Cell type'],
+                show=False,
+                palette=ov.palette()[11:],
+                frameon='small'
+               )
+
+
+# In[ ]:
+
+
+
+

ovrawm/t_aucell.txt

@@ -0,0 +1,93 @@
+import omicverse as ov
+import scanpy as sc
+import scvelo as scv
+
+ov.utils.ov_plot_set()
+
+ov.utils.download_pathway_database()
+ov.utils.download_geneid_annotation_pair()
+
+adata = scv.datasets.pancreas()
+adata
+
+adata.X.max()
+
+sc.pp.normalize_total(adata, target_sum=1e4)
+sc.pp.log1p(adata)
+
+adata.X.max()
+
+pathway_dict=ov.utils.geneset_prepare('genesets/GO_Biological_Process_2021.txt',organism='Mouse')
+
+##Assest one geneset
+geneset_name='response to vitamin (GO:0033273)'
+ov.single.geneset_aucell(adata,
+                            geneset_name=geneset_name,
+                            geneset=pathway_dict[geneset_name])
+sc.pl.embedding(adata,
+                basis='umap',
+          color=["{}_aucell".format(geneset_name)])
+
+##Assest more than one geneset
+geneset_names=['response to vitamin (GO:0033273)','response to vitamin D (GO:0033280)']
+ov.single.pathway_aucell(adata,
+                            pathway_names=geneset_names,
+                            pathways_dict=pathway_dict)
+sc.pl.embedding(adata,
+                basis='umap',
+                color=[i+'_aucell' for i in geneset_names])
+
+##Assest test geneset
+ov.single.geneset_aucell(adata,
+                            geneset_name='Sox',
+                            geneset=['Sox17', 'Sox4', 'Sox7', 'Sox18', 'Sox5'])
+sc.pl.embedding(adata,
+                basis='umap',
+          color=["Sox_aucell"])
+
+##Assest all pathways
+adata_aucs=ov.single.pathway_aucell_enrichment(adata,
+                                                  pathways_dict=pathway_dict,
+                                                  num_workers=8)
+
+adata_aucs.obs=adata[adata_aucs.obs.index].obs
+adata_aucs.obsm=adata[adata_aucs.obs.index].obsm
+adata_aucs.obsp=adata[adata_aucs.obs.index].obsp
+adata_aucs
+
+adata_aucs.write_h5ad('data/pancreas_auce.h5ad',compression='gzip')
+
+adata_aucs=sc.read('data/pancreas_auce.h5ad')
+
+sc.pl.embedding(adata_aucs,
+                basis='umap',
+          color=geneset_names)
+
+#adata_aucs.uns['log1p']['base']=None
+sc.tl.rank_genes_groups(adata_aucs, 'clusters', method='t-test',n_genes=100)
+sc.pl.rank_genes_groups_dotplot(adata_aucs,groupby='clusters',
+                                cmap='Spectral_r',
+                                standard_scale='var',n_genes=3)
+
+degs = sc.get.rank_genes_groups_df(adata_aucs, group='Beta', key='rank_genes_groups', log2fc_min=2, 
+                                    pval_cutoff=0.05)['names'].squeeze()
+degs
+
+import matplotlib.pyplot as plt
+#fig, axes = plt.subplots(4,3,figsize=(12,9))
+axes=sc.pl.embedding(adata_aucs,ncols=3,
+                basis='umap',show=False,return_fig=True,wspace=0.55,hspace=0.65,
+                color=['clusters']+degs.values.tolist(),
+                title=[ov.utils.plot_text_set(i,3,20)for i in ['clusters']+degs.values.tolist()])
+
+axes.tight_layout()
+
+adata.uns['log1p']['base']=None
+sc.tl.rank_genes_groups(adata, 'clusters', method='t-test',n_genes=100)
+
+res=ov.single.pathway_enrichment(adata,pathways_dict=pathway_dict,organism='Mouse',
+                                     group_by='clusters',plot=True)
+
+ax=ov.single.pathway_enrichment_plot(res,plot_title='Enrichment',cmap='Reds',
+                                         xticklabels=True,cbar=False,square=True,vmax=10,
+                                         yticklabels=True,cbar_kws={'label': '-log10(qvalue)','shrink': 0.5,})

ovrawm/t_bulk_combat.txt

@@ -0,0 +1,205 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# # Batch correction in Bulk RNA-seq or microarray data
+# 
+# Variability in datasets are not only the product of biological processes: they are also the product of technical biases (Lander et al, 1999). ComBat is one of the most widely used tool for correcting those technical biases called batch effects.
+# 
+# pyComBat (Behdenna et al, 2020) is a new Python implementation of ComBat (Johnson et al, 2007), a software widely used for the adjustment of batch effects in microarray data. While the mathematical framework is strictly the same, pyComBat:
+# 
+# - has similar results in terms of batch effects correction;
+# - is as fast or faster than the R implementation of ComBat and;
+# - offers new tools for the community to participate in its development.
+# 
+# Paper: [pyComBat, a Python tool for batch effects correction in high-throughput molecular data using empirical Bayes methods](https://doi.org/10.1101/2020.03.17.995431)
+# 
+# Code: https://github.com/epigenelabs/pyComBat
+# 
+# Colab_Reproducibility：https://colab.research.google.com/drive/121bbIiI3j4pTZ3yA_5p8BRkRyGMMmNAq?usp=sharing
+
+# In[7]:
+
+
+import anndata
+import pandas as pd
+import omicverse as ov
+ov.ov_plot_set()
+
+
+# ## Loading dataset
+# 
+# This minimal usage example illustrates how to use pyComBat in a default setting, and shows some results on ovarian cancer data, freely available on NCBI’s [Gene Expression Omnibus](https://www.ncbi.nlm.nih.gov/geo/), namely:
+# 
+# - GSE18520
+# - GSE66957
+# - GSE69428
+# 
+# The corresponding expression files are available on [GitHub](https://github.com/epigenelabs/pyComBat/tree/master/data).
+
+# In[15]:
+
+
+dataset_1 = pd.read_pickle("data/combat/GSE18520.pickle")
+adata1=anndata.AnnData(dataset_1.T)
+adata1.obs['batch']='1'
+adata1
+
+
+# In[16]:
+
+
+dataset_2 = pd.read_pickle("data/combat/GSE66957.pickle")
+adata2=anndata.AnnData(dataset_2.T)
+adata2.obs['batch']='2'
+adata2
+
+
+# In[17]:
+
+
+dataset_3 = pd.read_pickle("data/combat/GSE69428.pickle")
+adata3=anndata.AnnData(dataset_3.T)
+adata3.obs['batch']='3'
+adata3
+
+
+# We use the concat function to join the three datasets together and take the intersection for the same genes
+
+# In[18]:
+
+
+adata=anndata.concat([adata1,adata2,adata3],merge='same')
+adata
+
+
+# ## Removing batch effect
+
+# In[31]:
+
+
+ov.bulk.batch_correction(adata,batch_key='batch')
+
+
+# ## Saving results
+# 
+# Raw datasets
+
+# In[70]:
+
+
+raw_data=adata.to_df().T
+raw_data.head()
+
+
+# Removing Batch datasets
+
+# In[71]:
+
+
+removing_data=adata.to_df(layer='batch_correction').T
+removing_data.head()
+
+
+# save
+
+# In[ ]:
+
+
+raw_data.to_csv('raw_data.csv')
+removing_data.to_csv('removing_data.csv')
+
+
+# You can also save adata object
+
+# In[ ]:
+
+
+adata.write_h5ad('adata_batch.h5ad',compression='gzip')
+#adata=ov.read('adata_batch.h5ad')
+
+
+# ## Compare the dataset before and after correction
+# 
+# We specify three different colours for three different datasets
+
+# In[51]:
+
+
+color_dict={
+    '1':ov.utils.red_color[1],
+    '2':ov.utils.blue_color[1],
+    '3':ov.utils.green_color[1],
+}
+
+
+# In[57]:
+
+
+fig,ax=plt.subplots( figsize = (20,4))
+bp=plt.boxplot(adata.to_df().T,patch_artist=True)
+for i,batch in zip(range(adata.shape[0]),adata.obs['batch']):
+    bp['boxes'][i].set_facecolor(color_dict[batch])
+ax.axis(False)
+plt.show()
+
+
+# In[58]:
+
+
+fig,ax=plt.subplots( figsize = (20,4))
+bp=plt.boxplot(adata.to_df(layer='batch_correction').T,patch_artist=True)
+for i,batch in zip(range(adata.shape[0]),adata.obs['batch']):
+    bp['boxes'][i].set_facecolor(color_dict[batch])
+ax.axis(False)
+plt.show()
+
+
+# In addition to using boxplots to observe the effect of batch removal, we can also use PCA to observe the effect of batch removal
+
+# In[59]:
+
+
+adata.layers['raw']=adata.X.copy()
+
+
+# We first calculate the PCA on the raw dataset
+
+# In[60]:
+
+
+ov.pp.pca(adata,layer='raw',n_pcs=50)
+adata
+
+
+# We then calculate the PCA on the batch_correction dataset
+
+# In[61]:
+
+
+ov.pp.pca(adata,layer='batch_correction',n_pcs=50)
+adata
+
+
+# In[62]:
+
+
+ov.utils.embedding(adata,
+                  basis='raw|original|X_pca',
+                  color='batch',
+                  frameon='small')
+
+
+# In[63]:
+
+
+ov.utils.embedding(adata,
+                  basis='batch_correction|original|X_pca',
+                  color='batch',
+                  frameon='small')
+
+
+# In[ ]:
+
+
+
+

ovrawm/t_cellanno.txt

@@ -0,0 +1,378 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# # Celltype auto annotation with SCSA
+# Single-cell transcriptomics allows the analysis of thousands of cells in a single experiment and the identification of novel cell types, states and dynamics in a variety of tissues and organisms. Standard experimental protocols and analytical workflows have been developed to create single-cell transcriptomic maps from tissues. 
+# 
+# This tutorial focuses on how to interpret this data to identify cell types, states, and other biologically relevant patterns with the goal of creating annotated cell maps.
+# 
+# Paper: [SCSA: A Cell Type Annotation Tool for Single-Cell RNA-seq Data](https://doi.org/10.3389/fgene.2020.00490)
+# 
+# Code: https://github.com/bioinfo-ibms-pumc/SCSA
+# 
+# Colab_Reproducibility：https://colab.research.google.com/drive/1BC6hPS0CyBhNu0BYk8evu57-ua1bAS0T?usp=sharing
+# 
+# <div class="admonition warning">
+#   <p class="admonition-title">Note</p>
+#   <p>
+#     The annotation with SCSA can't be used in rare celltype annotations
+#   </p>
+# </div>
+# 
+# ![scsa](https://www.frontiersin.org/files/Articles/524690/fgene-11-00490-HTML/image_m/fgene-11-00490-g001.jpg)
+
+# In[1]:
+
+
+import omicverse as ov
+print(f'omicverse version:{ov.__version__}')
+import scanpy as sc
+print(f'scanpy version:{sc.__version__}')
+ov.ov_plot_set()
+
+
+# ## Loading data
+# 
+# The data consist of 3k PBMCs from a Healthy Donor and are freely available from 10x Genomics ([here](http://cf.10xgenomics.com/samples/cell-exp/1.1.0/pbmc3k/pbmc3k_filtered_gene_bc_matrices.tar.gz) from this [webpage](https://support.10xgenomics.com/single-cell-gene-expression/datasets/1.1.0/pbmc3k)). On a unix system, you can uncomment and run the following to download and unpack the data. The last line creates a directory for writing processed data.
+# 
+
+# In[2]:
+
+
+# !mkdir data
+# !wget http://cf.10xgenomics.com/samples/cell-exp/1.1.0/pbmc3k/pbmc3k_filtered_gene_bc_matrices.tar.gz -O data/pbmc3k_filtered_gene_bc_matrices.tar.gz
+# !cd data; tar -xzf pbmc3k_filtered_gene_bc_matrices.tar.gz
+# !mkdir write
+
+
+# Read in the count matrix into an AnnData object, which holds many slots for annotations and different representations of the data. It also comes with its own HDF5-based file format: `.h5ad`.
+
+# In[3]:
+
+
+adata = sc.read_10x_mtx(
+    'data/filtered_gene_bc_matrices/hg19/',  # the directory with the `.mtx` file
+    var_names='gene_symbols',                # use gene symbols for the variable names (variables-axis index)
+    cache=True)                              # write a cache file for faster subsequent reading
+
+
+# ## Data preprocessing
+# 
+# Here, we use `ov.single.scanpy_lazy` to preprocess the raw data of scRNA-seq, it included filter the doublets cells, normalizing counts per cell, log1p, extracting highly variable genes, and cluster of cells calculation. 
+# 
+# But if you want to experience step-by-step preprocessing, we also provide more detailed preprocessing steps here, please refer to our [preprocess chapter](https://omicverse.readthedocs.io/en/latest/Tutorials-single/t_preprocess/) for a detailed explanation.
+# 
+# We stored the raw counts in `count` layers, and the raw data in `adata.raw.to_adata()`.
+
+# In[4]:
+
+
+#adata=ov.single.scanpy_lazy(adata)
+
+#quantity control
+adata=ov.pp.qc(adata,
+              tresh={'mito_perc': 0.05, 'nUMIs': 500, 'detected_genes': 250})
+#normalize and high variable genes (HVGs) calculated
+adata=ov.pp.preprocess(adata,mode='shiftlog|pearson',n_HVGs=2000,)
+
+#save the whole genes and filter the non-HVGs
+adata.raw = adata
+adata = adata[:, adata.var.highly_variable_features]
+
+#scale the adata.X
+ov.pp.scale(adata)
+
+#Dimensionality Reduction
+ov.pp.pca(adata,layer='scaled',n_pcs=50)
+
+#Neighbourhood graph construction
+sc.pp.neighbors(adata, n_neighbors=15, n_pcs=50,
+               use_rep='scaled|original|X_pca')
+
+#clusters
+sc.tl.leiden(adata)
+
+#Dimensionality Reduction for visualization(X_mde=X_umap+GPU)
+adata.obsm["X_mde"] = ov.utils.mde(adata.obsm["scaled|original|X_pca"])
+adata
+
+
+# ## Cell annotate automatically
+# 
+# We create a pySCSA object from the `adata`, and we need to set some parameter to annotate correctly.
+# 
+# In normal annotate, we set `celltype`=`'normal'` and `target`=`'cellmarker'` or `'panglaodb'` to perform the cell annotate.
+# 
+# But in cancer annotate, we need to set the `celltype`=`'cancer'` and `target`=`'cancersea'` to perform the cell annotate.
+# 
+# <div class="admonition note">
+#   <p class="admonition-title">Note</p>
+#   <p>
+#     The annotation with SCSA need to download the database at first. It can be downloaded automatically. But sometimes you will have problems with network errors.
+#   </p>
+# </div>
+# 
+# The database can be downloaded from [figshare](https://figshare.com/ndownloader/files/41369037) or [Google Drive](https://drive.google.com/drive/folders/1pqyuCp8mTXDFRGUkX8iDdPAg45JHvheF?usp=sharing). And you need to set parameter `model_path`=`'path'`
+
+# In[5]:
+
+
+scsa=ov.single.pySCSA(adata=adata,
+                      foldchange=1.5,
+                      pvalue=0.01,
+                      celltype='normal',
+                      target='cellmarker',
+                      tissue='All',
+                      model_path='temp/pySCSA_2023_v2_plus.db'                    
+)
+
+
+# In the previous cell clustering we used the leiden algorithm, so here we specify that the type is set to leiden. if you are using louvain, please change it. And, we will annotate all clusters, if you only want to annotate a few of the classes, please follow `'[1]'`, `'[1,2,3]'`, `'[...]'` Enter in the format.
+# 
+# `rank_rep` means the `sc.tl.rank_genes_groups(adata, clustertype, method='wilcoxon')`, if we provided the `rank_genes_groups` in adata.uns, `rank_rep` can be set as `False`
+
+# In[6]:
+
+
+anno=scsa.cell_anno(clustertype='leiden',
+               cluster='all',rank_rep=True)
+
+
+# We can query only the better annotated results
+
+# In[7]:
+
+
+scsa.cell_auto_anno(adata,key='scsa_celltype_cellmarker')
+
+
+# We can also use `panglaodb` as target to annotate the celltype
+
+# In[8]:
+
+
+scsa=ov.single.pySCSA(adata=adata,
+                          foldchange=1.5,
+                          pvalue=0.01,
+                          celltype='normal',
+                          target='panglaodb',
+                          tissue='All',
+                          model_path='temp/pySCSA_2023_v2_plus.db'
+                          
+)
+
+
+# In[9]:
+
+
+res=scsa.cell_anno(clustertype='leiden',
+               cluster='all',rank_rep=True)
+
+
+# We can query only the better annotated results
+
+# In[10]:
+
+
+scsa.cell_anno_print()
+
+
+# In[11]:
+
+
+scsa.cell_auto_anno(adata,key='scsa_celltype_panglaodb')
+
+
+# Here, we introduce the dimensionality reduction visualisation function `ov.utils.embedding`, which is similar to `scanpy.pl.embedding`, except that when we set `frameon='small'`, we scale the axes to the bottom-left corner and scale the colourbar to the bottom-right corner.
+# 
+# - adata: the anndata object
+# - basis: the visualized embedding stored in adata.obsm
+# - color: the visualized obs/var
+# - legend_loc: the location of legend, if you set None, it will be visualized in right.
+# - frameon: it can be set `small`, False or None
+# - legend_fontoutline: the outline in the text of legend.
+# - palette: Different categories of colours, we have a number of different colours preset in omicverse, including `ov.utils.palette()`, `ov.utils.red_color`, `ov.utils.blue_color`, `ov.utils.green_color`, `ov. utils.orange_color`. The preset colours can help you achieve a more beautiful visualisation.
+
+# In[12]:
+
+
+ov.utils.embedding(adata,
+                   basis='X_mde',
+                   color=['leiden','scsa_celltype_cellmarker','scsa_celltype_panglaodb'], 
+                   legend_loc='on data', 
+                   frameon='small',
+                   legend_fontoutline=2,
+                   palette=ov.utils.palette()[14:],
+                  )
+
+
+# If you want to draw stacked histograms of cell type proportions, you first need to colour the groups you intend to draw using `ov.utils.embedding`. Then use `ov.utils.plot_cellproportion` to specify the groups you want to plot, and you can see a plot of cell proportions in the different groups
+
+# In[13]:
+
+
+#Randomly designate the first 1000 cells as group B and the rest as group A
+adata.obs['group']='A'
+adata.obs.loc[adata.obs.index[:1000],'group']='B'
+#Colored
+ov.utils.embedding(adata,
+                   basis='X_mde',
+                   color=['group'], 
+                   frameon='small',legend_fontoutline=2,
+                   palette=ov.utils.red_color,
+                  )
+
+
+# In[14]:
+
+
+ov.utils.plot_cellproportion(adata=adata,celltype_clusters='scsa_celltype_cellmarker',
+                    visual_clusters='group',
+                    visual_name='group',figsize=(2,4))
+
+
+# Of course, we also provide another downscaled visualisation of the graph using `ov.utils.plot_embedding_celltype`
+
+# In[15]:
+
+
+ov.utils.plot_embedding_celltype(adata,figsize=None,basis='X_mde',
+                            celltype_key='scsa_celltype_cellmarker',
+                            title='            Cell type',
+                            celltype_range=(2,6),
+                            embedding_range=(4,10),)
+
+
+# We calculated the ratio of observed to expected cell numbers (Ro/e) for each cluster in different tissues to quantify the tissue preference of each cluster (Guo et al., 2018; Zhang et al., 2018). The expected cell num- bers for each combination of cell clusters and tissues were obtained from the chi-square test. One cluster was identified as being enriched in a specific tissue if Ro/e>1.
+# 
+# The Ro/e function was wrote by `Haihao Zhang`.
+
+# In[16]:
+
+
+roe=ov.utils.roe(adata,sample_key='group',cell_type_key='scsa_celltype_cellmarker')
+
+
+# In[40]:
+
+
+import seaborn as sns
+import matplotlib.pyplot as plt
+fig, ax = plt.subplots(figsize=(2,4))
+
+transformed_roe = roe.copy()
+transformed_roe = transformed_roe.applymap(
+    lambda x: '+++' if x >= 2 else ('++' if x >= 1.5 else ('+' if x >= 1 else '+/-')))
+
+sns.heatmap(roe, annot=transformed_roe, cmap='RdBu_r', fmt='', 
+            cbar=True, ax=ax,vmin=0.5,vmax=1.5,cbar_kws={'shrink':0.5})
+plt.xticks(fontsize=12)
+plt.yticks(fontsize=12)
+
+plt.xlabel('Group',fontsize=13)
+plt.ylabel('Cell type',fontsize=13)
+plt.title('Ro/e',fontsize=13)
+
+
+# ## Cell annotate manually
+# 
+# In order to compare the accuracy of our automatic annotations, we will here use marker genes to manually annotate the cluster and compare the accuracy of the pySCSA and manual.
+# 
+# We need to prepare a marker's dict at first
+
+# In[38]:
+
+
+res_marker_dict={
+    'Megakaryocyte':['ITGA2B','ITGB3'],
+    'Dendritic cell':['CLEC10A','IDO1'],
+    'Monocyte' :['S100A8','S100A9','LST1',],
+    'Macrophage':['CSF1R','CD68'],
+    'B cell':['MS4A1','CD79A','MZB1',],
+    'NK/NKT cell':['GNLY','KLRD1'],
+    'CD8+T cell':['CD8A','CD8B'],
+    'Treg':['CD4','CD40LG','IL7R','FOXP3','IL2RA'],
+    'CD4+T cell':['PTPRC','CD3D','CD3E'],
+
+}
+
+
+# We then calculated the expression of marker genes in each cluster and the fraction
+
+# In[39]:
+
+
+sc.tl.dendrogram(adata,'leiden')
+sc.pl.dotplot(adata, res_marker_dict, 'leiden', 
+              dendrogram=True,standard_scale='var')
+
+
+# Based on the dotplot, we name each cluster according `ov.single.scanpy_cellanno_from_dict`
+
+# In[40]:
+
+
+# create a dictionary to map cluster to annotation label
+cluster2annotation = {
+     '0': 'T cell',
+     '1': 'T cell',
+     '2': 'Monocyte',#Germ-cell(Oid)
+     '3': 'B cell',#Germ-cell(Oid)
+     '4': 'T cell',
+     '5': 'Macrophage',
+     '6': 'NKT cells',
+     '7': 'T cell',
+    '8':'Monocyte',
+    '9':'Dendritic cell',
+    '10':'Megakaryocyte',
+
+}
+ov.single.scanpy_cellanno_from_dict(adata,anno_dict=cluster2annotation,
+                                       clustertype='leiden')
+
+
+# ## Compare the pySCSA and Manual
+# 
+# We can see that the auto-annotation results are almost identical to the manual annotation, the only difference is between monocyte and macrophages, but in the previous auto-annotation results, pySCSA gives the option of `monocyte|macrophage`, so it can be assumed that pySCSA performs better on the pbmc3k data
+
+# In[52]:
+
+
+ov.utils.embedding(adata,
+                   basis='X_mde',
+                   color=['major_celltype','scsa_celltype_cellmarker'], 
+                   legend_loc='on data', frameon='small',legend_fontoutline=2,
+                   palette=ov.utils.palette()[14:],
+                  )
+
+
+# We can use `get_celltype_marker` to obtain the marker of each celltype
+
+# In[42]:
+
+
+marker_dict=ov.single.get_celltype_marker(adata,clustertype='scsa_celltype_cellmarker')
+marker_dict.keys()
+
+
+# In[43]:
+
+
+marker_dict['B cell']
+
+
+# ## The tissue name in database
+# 
+# For annotation of cell types in specific tissues, we can query the tissues available in the database using `get_model_tissue`.
+
+# In[44]:
+
+
+scsa.get_model_tissue()
+
+
+# In[ ]:
+
+
+
+

ovrawm/t_cellfate.txt

@@ -0,0 +1,218 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# #  Identify the driver regulators of cell fate decisions
+# CEFCON is a computational tool for deciphering driver regulators of cell fate decisions from single-cell RNA-seq data. It takes a prior gene interaction network and expression profiles from scRNA-seq data associated with a given developmental trajectory as inputs, and consists of three main components, including cell-lineage-specific gene regulatory network (GRN) construction, driver regulator identification and regulon-like gene module (RGM) identification.
+# 
+# Check out [(Wang et al., Nature Communications, 2023)](https://www.nature.com/articles/s41467-023-44103-3) for the detailed methods and applications.
+# 
+# Code: [https://github.com/WPZgithub/CEFCON](https://github.com/WPZgithub/CEFCON)
+# 
+
+# In[1]:
+
+
+import omicverse as ov
+#print(f"omicverse version: {ov.__version__}")
+import scanpy as sc
+#print(f"scanpy version: {sc.__version__}")
+import pandas as pd
+from tqdm.auto import tqdm
+ov.plot_set()
+
+
+# # Data loading and processing
+# Here, we use the mouse hematopoiesis data provided by [Nestorowa et al. (2016, Blood).](https://doi.org/10.1182/blood-2016-05-716480)
+# 
+# **The scRNA-seq data requires processing to extract lineage information for the CEFCON analysis.** Please refer to the [original notebook](https://github.com/WPZgithub/CEFCON/blob/e74d2d248b88fb3349023d1a97d3cc8a52cc4060/notebooks/preprocessing_nestorowa16_data.ipynb) for detailed instructions on preprocessing scRNA-seq data.
+
+# In[2]:
+
+
+adata = ov.single.mouse_hsc_nestorowa16()
+adata
+
+
+# CEFCON fully exploit an available global and **context-free gene interaction network** as prior knowledge, from which we extract the cell-lineage-specific gene interactions according to the gene expression profiles derived from scRNA-seq data associated with a given developmental trajectory. 
+# 
+# You can download the prior network in the [zenodo](https://zenodo.org/records/8013900). **CEFCON only provides the prior network for human and mosue data anaylsis**. For other species, you should provide the prior network mannully.
+# 
+# The author of CEFCON has provided several prior networks here; however, 'nichenet' yields the best results.
+
+# In[3]:
+
+
+prior_network = ov.single.load_human_prior_interaction_network(dataset='nichenet') 
+
+
+# **In the scRNA-seq analysis of human data, you should not run this step. Running it may change the gene symbol and result in errors.**
+# 
+# 
+# 
+# 
+
+# In[4]:
+
+
+# Convert the gene symbols of the prior gene interaction network to the mouse gene symbols
+prior_network = ov.single.convert_human_to_mouse_network(prior_network,server_name='asia') 
+prior_network
+
+
+# In[12]:
+
+
+prior_network.to_csv('result/combined_network_Mouse.txt.gz',sep='\t')
+
+
+# Alternatively, you can directly specify the file path of the input prior interaction network and import the specified file.
+
+# In[3]:
+
+
+#prior_network = './Reference_Networks/combined_network_Mouse.txt'
+prior_network=ov.read('result/combined_network_Mouse.txt.gz',index_col=0)
+
+
+# # Training CEFCON model
+# 
+# We recommend using GRUOBI to solve the integer linear programming (ILP) problem when identifying driver genes. GUROBI is a commercial solver that requires licenses to run. Thankfully, it provides free licenses in academia, as well as trial licenses outside academia. If there is no problem about the licenses, you need to install the `gurobipy` package.
+# 
+# If difficulties arise while using GUROBI, the non-commercial solver, SCIP, will be employed as an alternative. But the use of SCIP does not come with a guarantee of achieving a successful solutio
+# 
+# **By default, the program will verify the availability of GRUOBI. If GRUOBI is not accessible, it will automatically switch the solver to SCIP.**
+# 
+
+# In[4]:
+
+
+CEFCON_obj = ov.single.pyCEFCON(adata, prior_network, repeats=5, solver='GUROBI')
+CEFCON_obj
+
+
+# Construct cell-lineage-specific GRNs
+
+# In[5]:
+
+
+CEFCON_obj.preprocess()
+
+
+# Lineage-by-lineage computation:
+
+# In[6]:
+
+
+CEFCON_obj.train()
+
+
+# In[9]:
+
+
+# Idenytify driver regulators for each lineage
+CEFCON_obj.predicted_driver_regulators()
+
+
+# We can find out the driver regulators identified by CEFCON.
+
+# In[10]:
+
+
+CEFCON_obj.cefcon_results_dict['E_pseudotime'].driver_regulator.head()
+
+
+# In[11]:
+
+
+CEFCON_obj.predicted_RGM()
+
+
+# # Downstream analysis
+
+# In[12]:
+
+
+CEFCON_obj.cefcon_results_dict['E_pseudotime']
+
+
+# In[13]:
+
+
+lineage = 'E_pseudotime'
+result = CEFCON_obj.cefcon_results_dict[lineage]
+
+
+# Plot gene embedding clusters
+
+# In[20]:
+
+
+gene_ad=sc.AnnData(result.gene_embedding)
+sc.pp.neighbors(gene_ad, n_neighbors=30, use_rep='X')
+# Higher resolutions lead to more communities, while lower resolutions lead to fewer communities.
+sc.tl.leiden(gene_ad, resolution=1)
+sc.tl.umap(gene_ad, n_components=2, min_dist=0.3)
+
+
+# In[27]:
+
+
+ov.utils.embedding(gene_ad,basis='X_umap',legend_loc='on data',
+                       legend_fontsize=8, legend_fontoutline=2,
+                  color='leiden',frameon='small',title='Leiden clustering using CEFCON\nderived gene embeddings')
+
+
+# Plot influence scores of driver regulators
+
+# In[40]:
+
+
+import matplotlib.pyplot as plt
+import seaborn as sns
+data_for_plot = result.driver_regulator[result.driver_regulator['is_driver_regulator']]
+data_for_plot = data_for_plot[0:20]
+
+plt.figure(figsize=(2, 20 * 0.2))
+sns.set_theme(style='ticks', font_scale=0.5)
+
+ax = sns.barplot(x='influence_score', y=data_for_plot.index, data=data_for_plot, orient='h',
+                 palette=sns.color_palette(f"ch:start=.5,rot=-.5,reverse=1,dark=0.4", n_colors=20))
+ax.set_title(result.name)
+ax.set_xlabel('Influence score')
+ax.set_ylabel('Driver regulators')
+
+ax.spines['left'].set_position(('outward', 10))
+ax.spines['bottom'].set_position(('outward', 10))
+plt.xticks(fontsize=12)
+plt.yticks(fontsize=12)
+
+plt.grid(False)
+#设置spines可视化情况
+ax.spines['top'].set_visible(False)
+ax.spines['right'].set_visible(False)
+ax.spines['bottom'].set_visible(True)
+ax.spines['left'].set_visible(True)
+
+plt.title('E_pseudotime',fontsize=12)
+plt.xlabel('Influence score',fontsize=12)
+plt.ylabel('Driver regulon',fontsize=12)
+
+sns.despine()
+
+
+# In[41]:
+
+
+result.plot_driver_genes_Venn()
+
+
+# Plot heat map of the activity matrix of RGMs
+
+# In[42]:
+
+
+adata_lineage = adata[adata.obs_names[adata.obs[result.name].notna()],:]
+
+result.plot_RGM_activity_heatmap(cell_label=adata_lineage.obs['cell_type_finely'],
+                                 type='out',col_cluster=True,bbox_to_anchor=(1.48, 0.25))
+

ovrawm/t_cellfate_gene.txt

@@ -0,0 +1,468 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# # Timing-associated genes analysis with cellfategenie
+# 
+# In our single-cell analysis, we analyse the underlying temporal state in the cell, which we call pseudotime. and identifying the genes associated with pseudotime becomes the key to unravelling models of gene dynamic regulation. In traditional analysis, we would use correlation coefficients, or gene dynamics model fitting. The correlation coefficient approach will have a preference for genes at the beginning and end of the time series, and the gene dynamics model requires RNA velocity information. Unbiased identification of chronosequence-related genes, as well as the need for no additional dependency information, has become a challenge in current chronosequence analyses.
+# 
+# Here, we developed CellFateGenie, which first removes potential noise from the data through metacells, and then constructs an adaptive ridge regression model to find the minimum set of genes needed to satisfy the timing fit.CellFateGenie has similar accuracy to gene dynamics models while eliminating preferences for the start and end of the time series.
+# 
+# Colab_Reproducibility：https://colab.research.google.com/drive/1Q1Sk5lGCBGBWS5Bs2kncAq9ZbjaDzSR4?usp=sharing
+
+# In[1]:
+
+
+import omicverse as ov
+import scvelo as scv
+import matplotlib.pyplot as plt
+ov.ov_plot_set()
+
+
+# ## Data preprocessed
+# 
+# We using dataset of dentategyrus in scvelo to demonstrate the timing-associated genes analysis. Firstly, We use `ov.pp.qc` and `ov.pp.preprocess` to preprocess the dataset.
+# 
+# Then we use `ov.pp.scale` and `ov.pp.pca` to analysis the principal component of the data
+
+# In[18]:
+
+
+adata = scv.datasets.dentategyrus()
+adata
+
+
+# In[19]:
+
+
+adata=ov.pp.qc(adata,
+              tresh={'mito_perc': 0.15, 'nUMIs': 500, 'detected_genes': 250},
+              )
+
+
+# In[20]:
+
+
+ov.utils.store_layers(adata,layers='counts')
+adata
+
+
+# In[21]:
+
+
+adata=ov.pp.preprocess(adata,mode='shiftlog|pearson',
+                       n_HVGs=2000)
+
+
+# In[22]:
+
+
+adata.raw = adata
+adata = adata[:, adata.var.highly_variable_features]
+adata
+
+
+# In[23]:
+
+
+ov.pp.scale(adata)
+ov.pp.pca(adata,layer='scaled',n_pcs=50)
+
+adata.obsm["X_mde_pca"] = ov.utils.mde(adata.obsm["scaled|original|X_pca"])
+
+
+# In[24]:
+
+
+adata=adata.raw.to_adata()
+
+
+# In[25]:
+
+
+fig, ax = plt.subplots(figsize=(3,3))
+ov.utils.embedding(adata,
+                basis='X_mde_pca',frameon='small',
+                color=['clusters'],show=False,ax=ax)
+
+
+# ## Meta-cells calculated
+# 
+# To reduce the noisy of the raw dataset and improve the accuracy of the regrssion model. We using `SEACells` to perform the Meta-cells calculated.
+
+# In[451]:
+
+
+import SEACells
+adata=adata[adata.obs['clusters']!='Endothelial']
+model = SEACells.core.SEACells(adata, 
+                  build_kernel_on='scaled|original|X_pca', 
+                  n_SEACells=200, 
+                  n_waypoint_eigs=10,
+                  convergence_epsilon = 1e-5)
+
+
+# In[452]:
+
+
+model.construct_kernel_matrix()
+M = model.kernel_matrix
+# Initialize archetypes
+model.initialize_archetypes()
+
+
+# In[453]:
+
+
+model.fit(min_iter=10, max_iter=50)
+
+
+# The model will stop early, we can use `model.step` to force the model run additional iterations. Usually, 100 iters can get the best metacells.
+
+# In[454]:
+
+
+# Check for convergence 
+get_ipython().run_line_magic('matplotlib', 'inline')
+model.plot_convergence()
+
+
+# In[489]:
+
+
+# You can force the model to run additional iterations step-wise using the .step() function
+print(f'Run for {len(model.RSS_iters)} iterations')
+for _ in range(10):
+    model.step()
+print(f'Run for {len(model.RSS_iters)} iterations')
+
+
+# In[490]:
+
+
+# Check for convergence 
+get_ipython().run_line_magic('matplotlib', 'inline')
+model.plot_convergence()
+
+
+# In[491]:
+
+
+get_ipython().run_line_magic('matplotlib', 'inline')
+SEACells.plot.plot_2D(adata, key='X_mde_pca', colour_metacells=False,
+                     figsize=(4,4),cell_size=20,title='Dentategyrus Metacells',
+                     )
+
+
+# We notice the shape of raw anndata not consistent with the HVGs anndata.
+
+# In[492]:
+
+
+adata.raw=adata.copy()
+
+
+# And we use `SEACells.core.summarize_by_soft_SEACell` to get the normalized metacells
+
+# In[493]:
+
+
+SEACell_soft_ad = SEACells.core.summarize_by_soft_SEACell(adata, model.A_, 
+                                                          celltype_label='clusters',
+                                                          summarize_layer='raw', minimum_weight=0.05)
+SEACell_soft_ad
+
+
+# We visualized the metacells with PCA and UMAP
+
+# In[494]:
+
+
+import scanpy as sc
+SEACell_soft_ad.raw=SEACell_soft_ad.copy()
+sc.pp.highly_variable_genes(SEACell_soft_ad, n_top_genes=2000, inplace=True)
+SEACell_soft_ad=SEACell_soft_ad[:,SEACell_soft_ad.var.highly_variable]
+
+
+# In[495]:
+
+
+ov.pp.scale(SEACell_soft_ad)
+ov.pp.pca(SEACell_soft_ad,layer='scaled',n_pcs=50)
+sc.pp.neighbors(SEACell_soft_ad, use_rep='scaled|original|X_pca')
+sc.tl.umap(SEACell_soft_ad)
+
+
+# And we can use the raw color of anndata.
+
+# In[496]:
+
+
+SEACell_soft_ad.obs['celltype']=SEACell_soft_ad.obs['celltype'].astype('category')
+SEACell_soft_ad.obs['celltype']=SEACell_soft_ad.obs['celltype'].cat.reorder_categories(adata.obs['clusters'].cat.categories)
+SEACell_soft_ad.uns['celltype_colors']=adata.uns['clusters_colors']
+
+
+# In[15]:
+
+
+import matplotlib.pyplot as plt
+fig, ax = plt.subplots(figsize=(3,3))
+ov.utils.embedding(SEACell_soft_ad,
+                   basis='X_umap',
+                   color=["celltype"],
+                   title='Meta Celltype',
+                   frameon='small',
+                   legend_fontsize=12,
+                   #palette=ov.utils.palette()[11:],
+                   ax=ax,
+                   show=False)
+
+
+# ## Pseudotime calculated
+# 
+# Accurately calculating the pseudotime in metacells is another challenge we need to face, here we use pyVIA to complete the calculation of the pseudotime. Since the metacell has only 200 cells, we may not get proper proposed time series results by using the default parameters of pyVIA, so we manually adjust the relevant parameters.
+# 
+# We need to set `jac_std_global`, `too_big_factor` and `knn` manually. If you know the origin cells, set the `root_user` is helpful too.
+
+# In[ ]:
+
+
+v0 = ov.single.pyVIA(adata=SEACell_soft_ad,adata_key='scaled|original|X_pca',
+                         adata_ncomps=50, basis='X_umap',
+                         clusters='celltype',knn=10, root_user=['nIPC','Neuroblast'],
+                         dataset='group', 
+                         random_seed=112,is_coarse=True, 
+                         preserve_disconnected=True,
+                         piegraph_arrow_head_width=0.05,piegraph_edgeweight_scalingfactor=2.5,
+                         gene_matrix=SEACell_soft_ad.X,velo_weight=0.5,
+                         edgebundle_pruning_twice=False, edgebundle_pruning=0.15, 
+                         jac_std_global=0.05,too_big_factor=0.05,
+                         cluster_graph_pruning_std=1,
+                         time_series=False,
+                        )
+
+v0.run()
+
+
+# In[500]:
+
+
+v0.get_pseudotime(SEACell_soft_ad)
+
+
+# In[17]:
+
+
+#v0.get_pseudotime(SEACell_soft_ad)
+import matplotlib.pyplot as plt
+fig, ax = plt.subplots(figsize=(3,3))
+ov.utils.embedding(SEACell_soft_ad,
+                   basis='X_umap',
+                   color=["pt_via"],
+                   title='Pseudotime',
+                   frameon='small',
+                   cmap='Reds',
+                   #size=40,
+                   legend_fontsize=12,
+                   #palette=ov.utils.palette()[11:],
+                   ax=ax,
+                   show=False)
+
+
+# Now we save the result of metacells for under analysis.
+
+# In[502]:
+
+
+SEACell_soft_ad.write_h5ad('data/tutorial_meta_den.h5ad',compression='gzip')
+
+
+# In[2]:
+
+
+SEACell_soft_ad=ov.utils.read('data/tutorial_meta_den.h5ad')
+
+
+# ## Timing-associated genes analysis
+# 
+# We have encapsulated the cellfategenie algorithm into omicverse, and we can simply use omicverse to analysis.
+
+# In[3]:
+
+
+cfg_obj=ov.single.cellfategenie(SEACell_soft_ad,pseudotime='pt_via')
+cfg_obj.model_init()
+
+
+# We used Adaptive Threshold Regression to calculate the minimum number of gene sets that would have the same accuracy as the regression model constructed for all genes.
+
+# In[4]:
+
+
+cfg_obj.ATR(stop=500,flux=0.01)
+
+
+# In[5]:
+
+
+fig,ax=cfg_obj.plot_filtering(color='#5ca8dc')
+ax.set_title('Dentategyrus Metacells\nCellFateGenie')
+
+
+# In[6]:
+
+
+res=cfg_obj.model_fit()
+
+
+# ## Visualization
+# 
+# We prepared a series of function to visualize the result. we can use `plot_color_fitting` to observe the different cells how to transit with the pseudotime.
+
+# In[7]:
+
+
+cfg_obj.plot_color_fitting(type='raw',cluster_key='celltype')
+
+
+# In[8]:
+
+
+cfg_obj.plot_color_fitting(type='filter',cluster_key='celltype')
+
+
+# ## Kendalltau test
+# 
+# We can further narrow down the set of genes that satisfy the maximum regression coefficient. We used the kendalltau test to calculate the trend significance for each gene.
+
+# In[9]:
+
+
+kt_filter=cfg_obj.kendalltau_filter()
+kt_filter.head()
+
+
+# In[10]:
+
+
+var_name=kt_filter.loc[kt_filter['pvalue']<kt_filter['pvalue'].mean()].index.tolist()
+gt_obj=ov.single.gene_trends(SEACell_soft_ad,'pt_via',var_name)
+gt_obj.calculate(n_convolve=10)
+
+
+# In[11]:
+
+
+print(f"Dimension: {len(var_name)}")
+
+
+# In[12]:
+
+
+fig,ax=gt_obj.plot_trend(color=ov.utils.blue_color[3])
+ax.set_title(f'Dentategyrus meta\nCellfategenie',fontsize=13)
+
+
+# In[14]:
+
+
+g=ov.utils.plot_heatmap(SEACell_soft_ad,var_names=var_name,
+                  sortby='pt_via',col_color='celltype',
+                 n_convolve=10,figsize=(1,6),show=False)
+
+g.fig.set_size_inches(2, 6)
+g.fig.suptitle('CellFateGenie',x=0.25,y=0.83,
+               horizontalalignment='left',fontsize=12,fontweight='bold')
+g.ax_heatmap.set_yticklabels(g.ax_heatmap.get_yticklabels(),fontsize=12)
+plt.show()
+
+
+# ## Fate Genes
+# 
+# Unlike traditional proposed timing analyses, CellFateGenie can also access key genes/gene sets during fate transitions
+
+# In[26]:
+
+
+gt_obj.cal_border_cell(SEACell_soft_ad,'pt_via','celltype')
+
+
+# In[27]:
+
+
+bordgene_dict=gt_obj.get_multi_border_gene(SEACell_soft_ad,'celltype',
+                                          threshold=0.5)
+
+
+# We use `Granule immature` and `Granule mature` to try calculated the fate related genes.
+
+# In[30]:
+
+
+gt_obj.get_border_gene(SEACell_soft_ad,'celltype','Granule immature','Granule mature',
+                      threshold=0.5)
+
+
+# In comparison to the `get_border_gene` function, the `get_special_border_gene` function serves the purpose of extracting exclusive fate information from two specific cell types. However, it operates with a higher degree of stringency.
+
+# In[33]:
+
+
+gt_obj.get_special_border_gene(SEACell_soft_ad,'celltype','Granule immature','Granule mature')
+
+
+# We can visualize these genes.
+
+# In[36]:
+
+
+import matplotlib.pyplot as plt
+g=ov.utils.plot_heatmap(SEACell_soft_ad,
+                        var_names=gt_obj.get_border_gene(SEACell_soft_ad,'celltype','Granule immature','Granule mature'),
+                  sortby='pt_via',col_color='celltype',yticklabels=True,
+                 n_convolve=10,figsize=(1,6),show=False)
+
+g.fig.set_size_inches(2, 4)
+g.ax_heatmap.set_yticklabels(g.ax_heatmap.get_yticklabels(),fontsize=12)
+plt.show()
+
+
+# Similiarly, we can use `get_special_kernel_gene` or `get_kernel_gene` to obtain the celltype special genes.
+
+# In[37]:
+
+
+gt_obj.get_special_kernel_gene(SEACell_soft_ad,'celltype','Granule immature')
+
+
+# In[42]:
+
+
+gt_obj.get_kernel_gene(SEACell_soft_ad,
+                       'celltype','Granule immature',
+                       threshold=0.3,
+                      num_gene=10)
+
+
+# In[43]:
+
+
+import matplotlib.pyplot as plt
+g=ov.utils.plot_heatmap(SEACell_soft_ad,
+                        var_names=gt_obj.get_kernel_gene(SEACell_soft_ad,
+                       'celltype','Granule immature',
+                       threshold=0.3,
+                      num_gene=10),
+                  sortby='pt_via',col_color='celltype',yticklabels=True,
+                 n_convolve=10,figsize=(1,6),show=False)
+
+g.fig.set_size_inches(2, 4)
+g.ax_heatmap.set_yticklabels(g.ax_heatmap.get_yticklabels(),fontsize=12)
+plt.show()
+
+
+# In[ ]:
+
+
+
+

ovrawm/t_cellfate_genesets.txt

@@ -0,0 +1,181 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# # Timing-associated geneset analysis with cellfategenie
+# 
+# In our single-cell analysis, we analyse the underlying temporal state in the cell, which we call pseudotime. and identifying the genes associated with pseudotime becomes the key to unravelling models of gene dynamic regulation. In traditional analysis, we would use correlation coefficients, or gene dynamics model fitting. The correlation coefficient approach will have a preference for genes at the beginning and end of the time series, and the gene dynamics model requires RNA velocity information. Unbiased identification of chronosequence-related genes, as well as the need for no additional dependency information, has become a challenge in current chronosequence analyses.
+# 
+# Here, we developed CellFateGenie, which first removes potential noise from the data through metacells, and then constructs an adaptive ridge regression model to find the minimum set of genes needed to satisfy the timing fit.CellFateGenie has similar accuracy to gene dynamics models while eliminating preferences for the start and end of the time series.
+# 
+# We provided the AUCell to evaluate the geneset of adata
+# 
+# Colab_Reproducibility：https://colab.research.google.com/drive/1upcKKZHsZMS78eOliwRAddbaZ9ICXSrc?usp=sharing
+
+# In[ ]:
+
+
+import omicverse as ov
+import scvelo as scv
+import matplotlib.pyplot as plt
+ov.ov_plot_set()
+
+
+# ## Data preprocessed
+# 
+# We using dataset of dentategyrus in scvelo to demonstrate the timing-associated genes analysis. Firstly, We use `ov.pp.qc` and `ov.pp.preprocess` to preprocess the dataset.
+# 
+# Then we use `ov.pp.scale` and `ov.pp.pca` to analysis the principal component of the data
+
+# In[2]:
+
+
+adata=ov.read('data/tutorial_meta_den.h5ad')
+adata=adata.raw.to_adata()
+adata
+
+
+# ## Genesets evaluata
+
+# In[3]:
+
+
+import omicverse as ov
+pathway_dict=ov.utils.geneset_prepare('../placenta/genesets/GO_Biological_Process_2021.txt',organism='Mouse')
+len(pathway_dict.keys())
+
+
+# In[ ]:
+
+
+##Assest all pathways
+adata_aucs=ov.single.pathway_aucell_enrichment(adata,
+                                                pathways_dict=pathway_dict,
+                                                num_workers=8)
+
+
+# In[11]:
+
+
+adata_aucs.obs=adata[adata_aucs.obs.index].obs
+adata_aucs.obsm=adata[adata_aucs.obs.index].obsm
+adata_aucs.obsp=adata[adata_aucs.obs.index].obsp
+adata_aucs.uns=adata[adata_aucs.obs.index].uns
+
+adata_aucs
+
+
+# ## Timing-associated genes analysis
+# 
+# We have encapsulated the cellfategenie algorithm into omicverse, and we can simply use omicverse to analysis.
+
+# In[12]:
+
+
+cfg_obj=ov.single.cellfategenie(adata_aucs,pseudotime='pt_via')
+cfg_obj.model_init()
+
+
+# We used Adaptive Threshold Regression to calculate the minimum number of gene sets that would have the same accuracy as the regression model constructed for all genes.
+
+# In[13]:
+
+
+cfg_obj.ATR(stop=500)
+
+
+# In[14]:
+
+
+fig,ax=cfg_obj.plot_filtering(color='#5ca8dc')
+ax.set_title('Dentategyrus Metacells\nCellFateGenie')
+
+
+# In[15]:
+
+
+res=cfg_obj.model_fit()
+
+
+# ## Visualization
+# 
+# We prepared a series of function to visualize the result. we can use `plot_color_fitting` to observe the different cells how to transit with the pseudotime.
+
+# In[16]:
+
+
+cfg_obj.plot_color_fitting(type='raw',cluster_key='celltype')
+
+
+# In[17]:
+
+
+cfg_obj.plot_color_fitting(type='filter',cluster_key='celltype')
+
+
+# ## Kendalltau test
+# 
+# We can further narrow down the set of genes that satisfy the maximum regression coefficient. We used the kendalltau test to calculate the trend significance for each gene.
+
+# In[18]:
+
+
+kt_filter=cfg_obj.kendalltau_filter()
+kt_filter.head()
+
+
+# In[21]:
+
+
+var_name=kt_filter.loc[kt_filter['pvalue']<kt_filter['pvalue'].mean()].index.tolist()
+gt_obj=ov.single.gene_trends(adata_aucs,'pt_via',var_name)
+gt_obj.calculate(n_convolve=10)
+
+
+# In[22]:
+
+
+print(f"Dimension: {len(var_name)}")
+
+
+# In[23]:
+
+
+fig,ax=gt_obj.plot_trend(color=ov.utils.blue_color[3])
+ax.set_title(f'Dentategyrus meta\nCellfategenie',fontsize=13)
+
+
+# In[25]:
+
+
+g=ov.utils.plot_heatmap(adata_aucs,var_names=var_name,
+                  sortby='pt_via',col_color='celltype',
+                 n_convolve=10,figsize=(1,6),show=False)
+
+g.fig.set_size_inches(2, 6)
+g.fig.suptitle('CellFateGenie',x=0.25,y=0.83,
+               horizontalalignment='left',fontsize=12,fontweight='bold')
+g.ax_heatmap.set_yticklabels(g.ax_heatmap.get_yticklabels(),fontsize=12)
+plt.show()
+
+
+# In[32]:
+
+
+gw_obj1=ov.utils.geneset_wordcloud(adata=adata_aucs[:,var_name],
+                                  cluster_key='celltype',pseudotime='pt_via',figsize=(3,6))
+gw_obj1.get()
+
+
+# In[33]:
+
+
+g=gw_obj1.plot_heatmap(figwidth=6,cmap='RdBu_r')
+plt.suptitle('CellFateGenie',x=0.18,y=0.95,
+               horizontalalignment='left',fontsize=12,fontweight='bold')
+
+
+# In[ ]:
+
+
+
+

ovrawm/t_cellphonedb.txt

@@ -0,0 +1,439 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# # Cell interaction with CellPhoneDB
+# 
+# CellPhoneDB is a publicly available repository of curated receptors, ligands and their interactions in HUMAN. CellPhoneDB can be used to search for a particular ligand/receptor, or interrogate your own single-cell transcriptomics data. 
+# 
+# We made three improvements in integrating the CellPhoneDB algorithm in OmicVerse:
+# 
+# - We have added a tutorial on analysing `anndata` based on any `anndata`.
+# - We added prettier heatmaps, chord diagrams and network diagrams for visualising relationships between cells.
+# - We added visualisation of ligand receptor proteins in different groups
+# 
+# Paper: [Single-cell reconstruction of the early maternal–fetal interface in humans](https://www.nature.com/articles/s41586-018-0698-6)
+# 
+# Code: https://github.com/ventolab/CellphoneDB
+# 
+# This notebook will demonstrate how to use CellPhoneDB on scRNA data and visualize it.
+
+# In[1]:
+
+
+import scanpy as sc
+import matplotlib.pyplot as plt
+import pandas as pd
+import numpy as np
+import omicverse as ov
+import os
+
+ov.plot_set()
+#print(f'cellphonedb version{cellphonedb.__version__}')
+
+
+# ## The EVT Data
+# 
+# Th EVT data have finished the celltype annotation, it can be download from the tutorial of CellPhoneDB.
+# 
+# Download: https://github.com/ventolab/CellphoneDB/blob/master/notebooks/data_tutorial.zip
+# 
+
+# In[2]:
+
+
+adata=sc.read('data/cpdb/normalised_log_counts.h5ad')
+adata=adata[adata.obs['cell_labels'].isin(['eEVT','iEVT','EVT_1','EVT_2','DC','dNK1','dNK2','dNK3',
+                                          'VCT','VCT_CCC','VCT_fusing','VCT_p','GC','SCT'])]
+adata
+
+
+# In[3]:
+
+
+ov.pl.embedding(adata,
+               basis='X_umap',
+               color='cell_labels',
+               frameon='small',
+               palette=ov.pl.red_color+ov.pl.blue_color+ov.pl.green_color+ov.pl.orange_color+ov.pl.purple_color)
+
+
+# In[4]:
+
+
+adata.X.max()
+
+
+# We can clearly see that the maximum value of the data is a floating point number less than 10. The fact that the maximum value is not an integer means that it has been normalised, and the fact that it is less than 10 means that it has been logarithmised. Note that our data cannot be `scaled`.
+
+# ## Export the anndata object
+# 
+# As the input to CellPhoneDB only requires the expression matrix and cell type, we extracted only the expression matrix and cell type from adata for the next step of analysis
+
+# In[5]:
+
+
+sc.pp.filter_cells(adata, min_genes=200)
+sc.pp.filter_genes(adata, min_cells=3)
+adata1=sc.AnnData(adata.X,obs=pd.DataFrame(index=adata.obs.index),
+                          var=pd.DataFrame(index=adata.var.index))
+adata1.write_h5ad('data/cpdb/norm_log.h5ad',compression='gzip')
+adata1
+
+
+# ## Export the meta info of cells
+# 
+# we construct a `DataFrame` object to export the meta info of cells. In EVT adata object, the celltypes were stored in the `obs['cell_labels']`
+
+# In[6]:
+
+
+#meta导出
+df_meta = pd.DataFrame(data={'Cell':list(adata[adata1.obs.index].obs.index),
+                             'cell_type':[ i for i in adata[adata1.obs.index].obs['cell_labels']]
+                            })
+df_meta.set_index('Cell', inplace=True)
+df_meta.to_csv('data/cpdb/meta.tsv', sep = '\t')
+
+
+# ## Cell interaction analysis
+# 
+# Now, we prepare the meta info of cells `meta.tsv` and matrix of scRNA-eq `norm_log.h5ad`, we can use the method of CellPhoneDB to calculate the interaction of each celltype in scRNA-seq data. 
+# 
+# Importantly, to avoid a series of bugs, we set the absolute path for CellPhoneDB analysis. we use `os.getcwd() ` to get the path now analysis.
+
+# In[7]:
+
+
+import os
+os.getcwd() 
+
+
+# Another thing to note is that we need to download the `cellphonedb.zip` file from `cellphonedb-data` for further analysis. I have placed it in the `data/CPDB` directory, but you can place it in any path you are interested in
+# 
+# Downloads: https://github.com/ventolab/cellphonedb-data
+
+# In[8]:
+
+
+cpdb_file_path = '/Users/fernandozeng/Desktop/analysis/cellphonedb-data/cellphonedb.zip'
+meta_file_path = os.getcwd()+'/data/cpdb/meta.tsv'
+counts_file_path = os.getcwd()+'/data/cpdb/norm_log.h5ad'
+microenvs_file_path = None
+active_tf_path = None
+out_path =os.getcwd()+'/data/cpdb/test_cellphone'
+
+
+# Now we run `cpdb_statistical_analysis_method` to predicted the cell interaction in scRNA-seq
+
+# In[9]:
+
+
+from cellphonedb.src.core.methods import cpdb_statistical_analysis_method
+
+cpdb_results = cpdb_statistical_analysis_method.call(
+    cpdb_file_path = cpdb_file_path,                 # mandatory: CellphoneDB database zip file.
+    meta_file_path = meta_file_path,                 # mandatory: tsv file defining barcodes to cell label.
+    counts_file_path = counts_file_path,             # mandatory: normalized count matrix - a path to the counts file, or an in-memory AnnData object
+    counts_data = 'hgnc_symbol',                     # defines the gene annotation in counts matrix.
+    active_tfs_file_path = active_tf_path,           # optional: defines cell types and their active TFs.
+    microenvs_file_path = microenvs_file_path,       # optional (default: None): defines cells per microenvironment.
+    score_interactions = True,                       # optional: whether to score interactions or not. 
+    iterations = 1000,                               # denotes the number of shufflings performed in the analysis.
+    threshold = 0.1,                                 # defines the min % of cells expressing a gene for this to be employed in the analysis.
+    threads = 10,                                     # number of threads to use in the analysis.
+    debug_seed = 42,                                 # debug randome seed. To disable >=0.
+    result_precision = 3,                            # Sets the rounding for the mean values in significan_means.
+    pvalue = 0.05,                                   # P-value threshold to employ for significance.
+    subsampling = False,                             # To enable subsampling the data (geometri sketching).
+    subsampling_log = False,                         # (mandatory) enable subsampling log1p for non log-transformed data inputs.
+    subsampling_num_pc = 100,                        # Number of componets to subsample via geometric skectching (dafault: 100).
+    subsampling_num_cells = 1000,                    # Number of cells to subsample (integer) (default: 1/3 of the dataset).
+    separator = '|',                                 # Sets the string to employ to separate cells in the results dataframes "cellA|CellB".
+    debug = False,                                   # Saves all intermediate tables employed during the analysis in pkl format.
+    output_path = out_path,                          # Path to save results.
+    output_suffix = None                             # Replaces the timestamp in the output files by a user defined string in the  (default: None).
+    )
+
+
+# In[10]:
+
+
+ov.utils.save(cpdb_results,'data/cpdb/gex_cpdb_test.pkl')
+
+
+# In[5]:
+
+
+cpdb_results=ov.utils.load('data/cpdb/gex_cpdb_test.pkl')
+
+
+# ## Network of celltype analysis
+# 
+# It is worth noting that we will be using ov for all downstream analysis, starting with cell network analysis, where we provide the `ov.single.cpdb_network_cal` function to extract interactions, and the `ov.single.cpdb_plot_network` function for very elegant visualization
+
+# In[6]:
+
+
+interaction=ov.single.cpdb_network_cal(adata = adata,
+        pvals = cpdb_results['pvalues'],
+        celltype_key = "cell_labels",)   
+
+
+# In[7]:
+
+
+interaction['interaction_edges'].head()
+
+
+# In[8]:
+
+
+ov.plot_set()
+
+
+# In[9]:
+
+
+fig, ax = plt.subplots(figsize=(4,4)) 
+ov.pl.cpdb_heatmap(adata,interaction['interaction_edges'],celltype_key='cell_labels',
+                   fontsize=11,
+          ax=ax,legend_kws={'fontsize':12,'bbox_to_anchor':(5, -0.9),'loc':'center left',})
+
+
+# In[10]:
+
+
+fig, ax = plt.subplots(figsize=(2,4)) 
+ov.pl.cpdb_heatmap(adata,interaction['interaction_edges'],celltype_key='cell_labels',
+                   source_cells=['EVT_1','EVT_2','dNK1','dNK2','dNK3'],
+          ax=ax,legend_kws={'fontsize':12,'bbox_to_anchor':(5, -0.9),'loc':'center left',})
+
+
+# In[11]:
+
+
+fig=ov.pl.cpdb_chord(adata,interaction['interaction_edges'],celltype_key='cell_labels',
+          count_min=60,fontsize=12,padding=50,radius=100,save=None,)
+fig.show()
+
+
+# In[12]:
+
+
+fig, ax = plt.subplots(figsize=(4,4)) 
+ov.pl.cpdb_network(adata,interaction['interaction_edges'],celltype_key='cell_labels',
+             counts_min=60,
+            nodesize_scale=5,
+                  ax=ax)
+
+
+# In[13]:
+
+
+fig, ax = plt.subplots(figsize=(4,4)) 
+ov.pl.cpdb_network(adata,interaction['interaction_edges'],celltype_key='cell_labels',
+            counts_min=60,
+            nodesize_scale=5,
+            source_cells=['EVT_1','EVT_2','dNK1','dNK2','dNK3'],
+            ax=ax)
+
+
+# In[14]:
+
+
+fig, ax = plt.subplots(figsize=(4,4)) 
+ov.pl.cpdb_network(adata,interaction['interaction_edges'],celltype_key='cell_labels',
+            counts_min=60,
+            nodesize_scale=5,
+            target_cells=['EVT_1','EVT_2','dNK1','dNK2','dNK3'],
+            ax=ax)
+
+
+# In[15]:
+
+
+ov.single.cpdb_plot_network(adata=adata,
+                  interaction_edges=interaction['interaction_edges'],
+                  celltype_key='cell_labels',
+                  nodecolor_dict=None,title='EVT Network',
+                  edgeswidth_scale=25,nodesize_scale=10,
+                  pos_scale=1,pos_size=10,figsize=(6,6),
+                  legend_ncol=3,legend_bbox=(0.8,0.2),legend_fontsize=10)
+
+
+# Sometimes, the whole network you don't want to use for analysis, the sub-network is useful for analysis. we can exacted the sub-network from it.
+# 
+# We need to exacted the sub-interaction first, we assumed that the five celltypes `['EVT_1','EVT_2','dNK1','dNK2','dNK3']` which is interested
+
+# In[16]:
+
+
+sub_i=interaction['interaction_edges']
+sub_i=sub_i.loc[sub_i['SOURCE'].isin(['EVT_1','EVT_2','dNK1','dNK2','dNK3'])]
+sub_i=sub_i.loc[sub_i['TARGET'].isin(['EVT_1','EVT_2','dNK1','dNK2','dNK3'])]
+
+
+# Then, we exacted the sub-anndata object
+
+# In[17]:
+
+
+sub_adata=adata[adata.obs['cell_labels'].isin(['EVT_1','EVT_2','dNK1','dNK2','dNK3'])]
+sub_adata
+
+
+# Now we plot the sub-interaction network between the cells in scRNA-seq
+
+# In[18]:
+
+
+ov.single.cpdb_plot_network(adata=sub_adata,
+                  interaction_edges=sub_i,
+                  celltype_key='cell_labels',
+                  nodecolor_dict=None,title='Sub-EVT Network',
+                  edgeswidth_scale=25,nodesize_scale=1,
+                  pos_scale=1,pos_size=10,figsize=(5,5),
+                  legend_ncol=3,legend_bbox=(0.8,0.2),legend_fontsize=10)
+
+
+# In[19]:
+
+
+fig=ov.pl.cpdb_chord(sub_adata,sub_i,celltype_key='cell_labels',
+          count_min=10,fontsize=12,padding=60,radius=100,save=None,)
+fig.show()
+
+
+# In[20]:
+
+
+fig, ax = plt.subplots(figsize=(4,4)) 
+ov.pl.cpdb_network(sub_adata,sub_i,celltype_key='cell_labels',
+             counts_min=10,
+            nodesize_scale=5,
+                  ax=ax)
+
+
+# In[21]:
+
+
+fig, ax = plt.subplots(figsize=(3,3)) 
+ov.pl.cpdb_heatmap(sub_adata,sub_i,celltype_key='cell_labels',
+          ax=ax,legend_kws={'fontsize':12,'bbox_to_anchor':(5, -0.9),'loc':'center left',})
+
+
+# ## The ligand-receptor exacted
+# 
+# We can set EVT as ligand or receptor to exacted the ligand-receptor proteins from the result of CellPhoneDB.
+
+# 
+# 
+# The most important step is that we need to extract the results of the analysis with eEVT as the ligand, and here we use ov's function `ov.single.cpdb_exact_target`,`ov.single.cpdb_exact_source` to do this
+
+# In[22]:
+
+
+sub_means=ov.single.cpdb_exact_target(cpdb_results['means'],['eEVT','iEVT'])
+sub_means=ov.single.cpdb_exact_source(sub_means,['dNK1','dNK2','dNK3'])
+sub_means.head() 
+
+
+# In[23]:
+
+
+ov.pl.cpdb_interacting_heatmap(adata=adata,
+                         celltype_key='cell_labels',
+                            means=cpdb_results['means'],
+                            pvalues=cpdb_results['pvalues'],
+                            source_cells=['dNK1','dNK2','dNK3'],
+                            target_cells=['eEVT','iEVT'],
+                            plot_secret=True,
+                            min_means=3,
+                            nodecolor_dict=None,
+                            ax=None,
+                            figsize=(2,6),
+                            fontsize=10,)
+
+
+# Sometimes we care about the expression of ligand in SOURCE and receptor in TARGET, we provide another function for getting the expression situation
+
+# In[24]:
+
+
+ov.pl.cpdb_group_heatmap(adata=adata,
+                         celltype_key='cell_labels',
+                            means=cpdb_results['means'],
+                            cmap={'Target':'Blues','Source':'Reds'},
+                            source_cells=['dNK1','dNK2','dNK3'],
+                            target_cells=['eEVT','iEVT'],
+                            plot_secret=True,
+                            min_means=3,
+                            nodecolor_dict=None,
+                            ax=None,
+                            figsize=(2,6),
+                            fontsize=10,)
+
+
+# We can also build Ligand, Receptor, SOURCE, and TARGET into a regulatory network, which is interesting.
+
+# In[25]:
+
+
+ov.pl.cpdb_interacting_network(adata=adata,
+                         celltype_key='cell_labels',
+                            means=cpdb_results['means'],
+                            source_cells=['dNK1','dNK2','dNK3'],
+                            target_cells=['eEVT','iEVT'],
+                            means_min=1,
+                             means_sum_min=1,        
+                            nodecolor_dict=None,
+                            ax=None,
+                            figsize=(6,6),
+                            fontsize=10)
+
+
+# Sometimes we want to analyse ligand-receptor pathway enrichment or function, so we need to extract ligand-receptor pairs from the significant ligand-receptors filtered out above, and omicverse provides an easy function `ov.single.cpdb_interaction_filtered` to do this here
+
+# In[40]:
+
+
+sub_means=sub_means.loc[~sub_means['gene_a'].isnull()]
+sub_means=sub_means.loc[~sub_means['gene_b'].isnull()]
+enrichr_genes=sub_means['gene_a'].tolist()+sub_means['gene_b'].tolist()
+
+
+# A tutorial on enrichment you can find in the Bulk chapter of tutorials:
+# 
+# https://omicverse.readthedocs.io/en/latest/Tutorials-bulk/t_deg/ or https://starlitnightly.github.io/omicverse/Tutorials-bulk/t_deg/
+
+# In[ ]:
+
+
+pathway_dict=ov.utils.geneset_prepare('genesets/GO_Biological_Process_2023.txt',organism='Human')
+
+
+# In[14]:
+
+
+#deg_genes=dds.result.loc[dds.result['sig']!='normal'].index.tolist()
+enr=ov.bulk.geneset_enrichment(gene_list=enrichr_genes,
+                                pathways_dict=pathway_dict,
+                                pvalue_type='auto',
+                                organism='human')
+
+
+# In[20]:
+
+
+ov.plot_set()
+ov.bulk.geneset_plot(enr,figsize=(2,4),fig_title='GO-Bio(EVT)',
+                    cax_loc=[2, 0.45, 0.5, 0.02],num=8,
+                    bbox_to_anchor_used=(-0.25, -13),custom_ticks=[10,100],
+                    cmap='Greens')
+
+
+# In[ ]:
+
+
+
+

ovrawm/t_cluster.txt

@@ -0,0 +1,312 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# # Clustering space
+# 
+# In this tutorial, we will explore how to run the Supervised clustering, unsupervised clustering, and amortized Latent Dirichlet Allocation (LDA) model implementation in `omicverse` with `GaussianMixture`,`Leiden/Louvain` and `MiRA`. 
+# 
+# In scRNA-seq data analysis, we describe cellular structure in our dataset with finding cell identities that relate to known cell states or cell cycle stages. This process is usually called cell identity annotation. For this purpose, we structure cells into clusters to infer the identity of similar cells. Clustering itself is a common unsupervised machine learning problem. 
+# 
+# LDA is a topic modelling method first introduced in the natural language processing field. By treating each cell as a document and each gene expression count as a word, we can carry over the method to the single-cell biology field.
+# 
+# Below, we will train the model over a dataset, plot the topics over a UMAP of the reference set, and inspect the topics for characteristic gene sets.
+# 
+# ## Preprocess data
+# 
+# As an example, we apply differential kinetic analysis to dentate gyrus neurogenesis, which comprises multiple heterogeneous subpopulations.
+# 
+# Colab_Reproducibility：https://colab.research.google.com/drive/1d_szq-y-g7O0C5rJgK22XC7uWTUNrYpK?usp=sharing
+
+# In[1]:
+
+
+import omicverse as ov
+import scanpy as sc
+import scvelo as scv
+ov.plot_set()
+
+
+# In[2]:
+
+
+import scvelo as scv
+adata=scv.datasets.dentategyrus()
+adata
+
+
+# In[3]:
+
+
+adata=ov.pp.preprocess(adata,mode='shiftlog|pearson',n_HVGs=3000,)
+adata.raw = adata
+adata = adata[:, adata.var.highly_variable_features]
+ov.pp.scale(adata)
+ov.pp.pca(adata,layer='scaled',n_pcs=50)
+
+
+# Let us inspect the contribution of single PCs to the total variance in the data. This gives us information about how many PCs we should consider in order to compute the neighborhood relations of cells. In our experience, often a rough estimate of the number of PCs does fine.
+
+# In[4]:
+
+
+ov.utils.plot_pca_variance_ratio(adata)
+
+
+# ## Unsupervised clustering
+# 
+# The Leiden algorithm is as an improved version of the Louvain algorithm which outperformed other clustering methods for single-cell RNA-seq data analysis ([Du et al., 2018, Freytag et al., 2018, Weber and Robinson, 2016]). Since the Louvain algorithm is no longer maintained, using Leiden instead is preferred.
+# 
+# We, therefore, propose to use the Leiden algorithm[Traag et al., 2019] on single-cell k-nearest-neighbour (KNN) graphs to cluster single-cell datasets.
+# 
+# Leiden creates clusters by taking into account the number of links between cells in a cluster versus the overall expected number of links in the dataset. 
+# 
+# Here, we set `method='leiden'` to cluster the cells using `Leiden`
+# 
+
+# In[5]:
+
+
+sc.pp.neighbors(adata, n_neighbors=15, n_pcs=50,
+               use_rep='scaled|original|X_pca')
+ov.utils.cluster(adata,method='leiden',resolution=1)
+
+
+# In[6]:
+
+
+ov.utils.embedding(adata,basis='X_umap',
+                   color=['clusters','leiden'],
+                   frameon='small',wspace=0.5)
+
+
+# We can also set `method='louvain'` to cluster the cells using `Louvain`
+
+# In[7]:
+
+
+sc.pp.neighbors(adata, n_neighbors=15, n_pcs=50,
+               use_rep='scaled|original|X_pca')
+ov.utils.cluster(adata,method='louvain',resolution=1)
+
+
+# In[8]:
+
+
+ov.utils.embedding(adata,basis='X_umap',
+                   color=['clusters','louvain'],
+                   frameon='small',wspace=0.5)
+
+
+# ## Supervised clustering
+# 
+# In addition to clustering using unsupervised clustering methods, we can also try supervised clustering methods, such as Gaussian mixture model clustering, which is a supervised clustering algorithm that works better in machine learning
+# 
+# Gaussian mixture models can be used to cluster unlabeled data in much the same way as k-means. There are, however, a couple of advantages to using Gaussian mixture models over k-means.
+# 
+# First and foremost, k-means does not account for variance. By variance, we are referring to the width of the bell shape curve.
+# 
+# The second difference between k-means and Gaussian mixture models is that the former performs hard classification whereas the latter performs soft classification. In other words, k-means tells us what data point belong to which cluster but won’t provide us with the probabilities that a given data point belongs to each of the possible clusters.
+# 
+# Here, we set `method='GMM'` to cluster the cells using `GaussianMixture`,`n_components` means the PCs to be used in clustering, `covariance_type` means the Gaussian Mixture Models (`diagonal`, `spherical`, `tied` and `full` covariance matrices supported). More arguments could found in [`sklearn.mixture.GaussianMixture`](http://scikit-learn.org/stable/modules/generated/sklearn.mixture.GaussianMixture.html)
+# 
+
+# In[9]:
+
+
+ov.utils.cluster(adata,use_rep='scaled|original|X_pca',
+                 method='GMM',n_components=21,
+                 covariance_type='full',tol=1e-9, max_iter=1000, )
+
+
+# In[10]:
+
+
+ov.utils.embedding(adata,basis='X_umap',
+                   color=['clusters','gmm_cluster'],
+                   frameon='small',wspace=0.5)
+
+
+# ## Latent Dirichlet Allocation (LDA) model implementation
+# 
+# Topic models, like Latent Dirichlet Allocation (LDA), have traditionally been used to decompose a corpus of text into topics - or themes - composed of words that often appear together in documents. Documents, in turn, are modeled as a mixture of topics based on the words they contain.
+# 
+# MIRA extends these ideas to single-cell genomics data, where topics are groups of genes that are co-expressed or cis-regulatory elements that are co-accessible, and cells are a mixture of these regulatory modules. 
+# 
+# Here, we used `ov.utils.LDA_topic` to construct the model of MiRA.
+# 
+# Particularly, and at a minimum, we must tell the model
+# 
+# - feature_type: what type of features we are working with (either “expression” or “accessibility”)
+# - highly_variable_key: which .var key to find our highly variable genes
+# - counts_layer: which layer to get the raw counts from.
+# - categorical_covariates, continuous_covariates: Technical variables influencing the generative process of the data. For example, a categorical technical factor may be the cells’ batch of origin, as shown here. A continous technical factor might be % of mitchondrial reads. For unbatched data, ignore these parameters.
+# - learning_rate: for larger datasets, the default of 1e-3, 0.1 usually works well.
+
+# In[11]:
+
+
+LDA_obj=ov.utils.LDA_topic(adata,feature_type='expression',
+                  highly_variable_key='highly_variable_features',
+                 layers='counts',batch_key=None,learning_rate=1e-3)
+
+
+# This method works by instantiating a special version of the CODAL model with far too many topics, which are gradually pruned if that topic is not needed to describe the data. The function returns the maximum contribution of each topic to any cell in the dataset. The predicted number of topics is given by the elbo of the maximum contribution curve, minus 1. A rule of thumb is that the last valid topic to include in the model is followed by a drop-off, after which all subsequent topics hover between 0.-0.05 maximum contributions.
+# 
+# We set `NUM_TOPICS` to six to try.
+
+# In[12]:
+
+
+LDA_obj.plot_topic_contributions(6)
+
+
+# We can observe that there are 13 TOPICs to be above the threshold line, so we set the new NUM_TOPIC to 12
+
+# In[13]:
+
+
+LDA_obj.predicted(13)
+
+
+# One can plot the distribution of topics across cells to see how the latent space reflects changes in cell state:
+
+# In[14]:
+
+
+ov.plot_set()
+ov.utils.embedding(adata, basis='X_umap',color = LDA_obj.model.topic_cols, cmap='BuPu', ncols=4,
+           add_outline=True,  frameon='small',)
+
+
+# In[15]:
+
+
+ov.utils.embedding(adata,basis='X_umap',
+                   color=['clusters','LDA_cluster'],
+                   frameon='small',wspace=0.5)
+
+
+# Here we are, proposing another idea of categorisation. We use cells with LDA greater than 0.4 as a primitive class, and then train a random forest classification model, and then use the random forest classification model to classify cells with LDA less than 0.5 to get a more accurate
+
+# In[20]:
+
+
+LDA_obj.get_results_rfc(adata,use_rep='scaled|original|X_pca',
+                        LDA_threshold=0.4,num_topics=13)
+
+
+# In[21]:
+
+
+ov.utils.embedding(adata,basis='X_umap',
+                   color=['LDA_cluster_rfc','LDA_cluster_clf'],
+                   frameon='small',wspace=0.5)
+
+
+# ## cNMF
+# 
+# More detail could be found in https://starlitnightly.github.io/omicverse/Tutorials-single/t_cnmf/
+
+# In[32]:
+
+
+adata.X.toarray()
+
+
+# In[ ]:
+
+
+import numpy as np
+## Initialize the cnmf object that will be used to run analyses
+cnmf_obj = ov.single.cNMF(adata,components=np.arange(5,11), n_iter=20, seed=14, num_highvar_genes=2000,
+                          output_dir='example_dg1/cNMF', name='dg_cNMF')
+## Specify that the jobs are being distributed over a single worker (total_workers=1) and then launch that worker
+cnmf_obj.factorize(worker_i=0, total_workers=4)
+cnmf_obj.combine(skip_missing_files=True)
+cnmf_obj.k_selection_plot(close_fig=False)
+
+
+# In[35]:
+
+
+selected_K = 7
+density_threshold = 2.00
+cnmf_obj.consensus(k=selected_K, 
+                   density_threshold=density_threshold, 
+                   show_clustering=True, 
+                   close_clustergram_fig=False)
+result_dict = cnmf_obj.load_results(K=selected_K, density_threshold=density_threshold)
+cnmf_obj.get_results(adata,result_dict)
+
+
+# In[36]:
+
+
+ov.pl.embedding(adata, basis='X_umap',color=result_dict['usage_norm'].columns,
+           use_raw=False, ncols=3, vmin=0, vmax=1,frameon='small')
+
+
+# In[40]:
+
+
+cnmf_obj.get_results_rfc(adata,result_dict,
+                         use_rep='scaled|original|X_pca',
+                        cNMF_threshold=0.5)
+
+
+# In[41]:
+
+
+ov.pl.embedding(
+    adata,
+    basis="X_umap",
+    color=['cNMF_cluster_rfc','cNMF_cluster_clf'],
+    frameon='small',
+    #title="Celltypes",
+    #legend_loc='on data',
+    legend_fontsize=14,
+    legend_fontoutline=2,
+    #size=10,
+    #legend_loc=True, 
+    add_outline=False, 
+    #add_outline=True,
+    outline_color='black',
+    outline_width=1,
+    show=False,
+)
+
+
+# ## Evaluation the clustering space
+# 
+# Rand index adjusted for chance. The Rand Index computes a similarity measure between two clusterings by considering all pairs of samples and counting pairs that are assigned in the same or different clusters in the predicted and true clusterings.
+
+# In[42]:
+
+
+from sklearn.metrics.cluster import adjusted_rand_score
+ARI = adjusted_rand_score(adata.obs['clusters'], adata.obs['leiden'])
+print('Leiden, Adjusted rand index = %.2f' %ARI)
+
+ARI = adjusted_rand_score(adata.obs['clusters'], adata.obs['louvain'])
+print('Louvain, Adjusted rand index = %.2f' %ARI)
+
+ARI = adjusted_rand_score(adata.obs['clusters'], adata.obs['gmm_cluster'])
+print('GMM, Adjusted rand index = %.2f' %ARI)
+
+ARI = adjusted_rand_score(adata.obs['clusters'], adata.obs['LDA_cluster'])
+print('LDA, Adjusted rand index = %.2f' %ARI)
+
+ARI = adjusted_rand_score(adata.obs['clusters'], adata.obs['LDA_cluster_rfc'])
+print('LDA_rfc, Adjusted rand index = %.2f' %ARI)
+
+ARI = adjusted_rand_score(adata.obs['clusters'], adata.obs['LDA_cluster_clf'])
+print('LDA_clf, Adjusted rand index = %.2f' %ARI)
+
+ARI = adjusted_rand_score(adata.obs['clusters'], adata.obs['cNMF_cluster_rfc'])
+print('cNMF_rfc, Adjusted rand index = %.2f' %ARI)
+
+ARI = adjusted_rand_score(adata.obs['clusters'], adata.obs['cNMF_cluster_clf'])
+print('cNMF_clf, Adjusted rand index = %.2f' %ARI)
+
+
+# We can find that the LDA topic model is the most effective among the above clustering algorithms, but it also takes the longest computation time and requires GPU resources. We notice that the Gaussian mixture model is second only to the LDA topic model. The GMM will be a great choice in omicverse's future clustering algorithms for spatial transcriptomics.

ovrawm/t_cluster_space.txt

@@ -0,0 +1,399 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# # Spatial clustering and denoising expressions
+# 
+# Spatial clustering, which shares an analogy with single-cell clustering, has expanded the scope of tissue physiology studies from cell-centroid to structure-centroid with spatially resolved transcriptomics (SRT) data.
+# 
+# Here, we presented four spatial clustering methods in OmicVerse.
+# 
+# We made three improvements in integrating the `GraphST`,`BINARY`,`CAST` and `STAGATE` algorithm in OmicVerse:
+# - We removed the preprocessing that comes with `GraphST` and used the preprocessing consistent with all SRTs in OmicVerse
+# - We optimised the dimensional display of `GraphST`, and PCA is considered a self-contained computational step.
+# - We implemented `mclust` using Python, removing the R language dependency.
+# - We provided a unified interface `ov.space.cluster`, the user can use the function interface at once to complete all the simultaneous
+# 
+# If you found this tutorial helpful, please cite `GraphST`,`BINARY`,`CAST` and `STAGATE` and `OmicVerse`:
+# 
+# - Long, Y., Ang, K.S., Li, M. et al. Spatially informed clustering, integration, and deconvolution of spatial transcriptomics with GraphST. Nat Commun 14, 1155 (2023). https://doi.org/10.1038/s41467-023-36796-3
+# - Lin S, Cui Y, Zhao F, Yang Z, Song J, Yao J, et al. Complete spatially resolved gene expression is not necessary for identifying spatial domains. Cell Genomics. 2024;4:100565.
+# - Tang, Z., Luo, S., Zeng, H. et al. Search and match across spatial omics samples at single-cell resolution. Nat Methods 21, 1818–1829 (2024). https://doi.org/10.1038/s41592-024-02410-7
+# - Dong, K., Zhang, S. Deciphering spatial domains from spatially resolved transcriptomics with an adaptive graph attention auto-encoder. Nat Commun 13, 1739 (2022). https://doi.org/10.1038/s41467-022-29439-6
+# 
+# 
+
+# In[1]:
+
+
+import omicverse as ov
+#print(f"omicverse version: {ov.__version__}")
+import scanpy as sc
+#print(f"scanpy version: {sc.__version__}")
+ov.plot_set()
+
+
+# ## Preprocess data
+# 
+# Here we present our re-analysis of 151676 sample of the dorsolateral prefrontal cortex (DLPFC) dataset. Maynard et al. has manually annotated DLPFC layers and white matter (WM) based on the morphological features and gene markers.
+# 
+# This tutorial demonstrates how to identify spatial domains on 10x Visium data using STAGATE. The processed data are available at https://github.com/LieberInstitute/spatialLIBD. We downloaded the manual annotation from the spatialLIBD package and provided at https://drive.google.com/drive/folders/10lhz5VY7YfvHrtV40MwaqLmWz56U9eBP?usp=sharing.
+
+# In[2]:
+
+
+adata = sc.read_visium(path='data', count_file='151676_filtered_feature_bc_matrix.h5')
+adata.var_names_make_unique()
+
+
+# <div class="admonition warning">
+#   <p class="admonition-title">Note</p>
+#   <p>
+#     We introduced the spatial special svg calculation module prost in omicverse versions greater than `1.6.0` to replace scanpy's HVGs, if you want to use scanpy's HVGs you can set mode=`scanpy` in `ov.space.svg` or use the following code.
+#   </p>
+# </div>
+# 
+# ```python
+# #adata=ov.pp.preprocess(adata,mode='shiftlog|pearson',n_HVGs=3000,target_sum=1e4)
+# #adata.raw = adata
+# #adata = adata[:, adata.var.highly_variable_features]
+# ```
+
+# In[3]:
+
+
+sc.pp.calculate_qc_metrics(adata, inplace=True)
+adata = adata[:,adata.var['total_counts']>100]
+adata=ov.space.svg(adata,mode='prost',n_svgs=3000,target_sum=1e4,platform="visium",)
+adata
+
+
+# In[5]:
+
+
+adata.write('data/cluster_svg.h5ad',compression='gzip')
+
+
+# In[2]:
+
+
+adata=ov.read('data/cluster_svg.h5ad',compression='gzip')
+
+
+# (Optional) We read the ground truth area of our spatial data
+# 
+# This step is not mandatory to run, in the tutorial, it's just to demonstrate the accuracy of our clustering effect, and in your own tasks, there is often no Ground_truth
+
+# In[3]:
+
+
+# read the annotation
+import pandas as pd
+import os
+Ann_df = pd.read_csv(os.path.join('data', '151676_truth.txt'), sep='\t', header=None, index_col=0)
+Ann_df.columns = ['Ground Truth']
+adata.obs['Ground Truth'] = Ann_df.loc[adata.obs_names, 'Ground Truth']
+sc.pl.spatial(adata, img_key="hires", color=["Ground Truth"])
+
+
+# ## Method1: GraphST
+# 
+# GraphST was rated as one of the best spatial clustering algorithms on Nature Method 2024.04, so we first tried to call GraphST for spatial domain identification in OmicVerse.
+
+# In[4]:
+
+
+methods_kwargs={}
+methods_kwargs['GraphST']={
+    'device':'cuda:0',
+    'n_pcs':30
+}
+
+adata=ov.space.clusters(adata,
+                  methods=['GraphST'],
+                  methods_kwargs=methods_kwargs,
+                  lognorm=1e4)
+
+
+# In[11]:
+
+
+ov.utils.cluster(adata,use_rep='graphst|original|X_pca',method='mclust',n_components=10,
+                 modelNames='EEV', random_state=112,
+                )
+adata.obs['mclust_GraphST'] = ov.utils.refine_label(adata, radius=50, key='mclust') 
+adata.obs['mclust_GraphST']=adata.obs['mclust_GraphST'].astype('category')
+
+
+# In[12]:
+
+
+res=ov.space.merge_cluster(adata,groupby='mclust_GraphST',use_rep='graphst|original|X_pca',
+                  threshold=0.2,plot=True)
+
+
+# In[13]:
+
+
+sc.pl.spatial(adata, color=['mclust_GraphST','mclust_GraphST_tree','mclust','Ground Truth'])
+
+
+# We can also use `mclust_R` to cluster the spatial domain, but this method need to install `rpy2` at first.
+# 
+# The use of the mclust algorithm requires the rpy2 package and the mclust package. See https://pypi.org/project/rpy2/ and https://cran.r-project.org/web/packages/mclust/index.html for detail.
+
+# In[14]:
+
+
+ov.utils.cluster(adata,use_rep='graphst|original|X_pca',method='mclust_R',n_components=10,
+                 random_state=42,
+                )
+adata.obs['mclust_R_GraphST'] = ov.utils.refine_label(adata, radius=30, key='mclust_R') 
+adata.obs['mclust_R_GraphST']=adata.obs['mclust_R_GraphST'].astype('category')
+res=ov.space.merge_cluster(adata,groupby='mclust_R_GraphST',use_rep='graphst|original|X_pca',
+                  threshold=0.2,plot=True)
+
+
+# In[15]:
+
+
+sc.pl.spatial(adata, color=['mclust_R_GraphST','mclust_R_GraphST_tree','mclust','Ground Truth'])
+
+
+# ## Method2: BINARY
+# 
+# BINARY outperforms existing methods across various SRT data types while using significantly less input information.
+# 
+# If your data is very large, or very sparse, I believe BINARY would be a great choice.
+
+# In[3]:
+
+
+methods_kwargs={}
+methods_kwargs['BINARY']={
+    'use_method':'KNN',
+    'cutoff':6,
+    'obs_key':'BINARY_sample',
+    'use_list':None,
+    'pos_weight':10,
+    'device':'cuda:0',
+    'hidden_dims':[512, 30],
+    'n_epochs': 1000,
+    'lr':  0.001,
+    'key_added': 'BINARY',
+    'gradient_clipping': 5,
+    'weight_decay': 0.0001,
+    'verbose': True,
+    'random_seed':0,
+    'lognorm':1e4,
+    'n_top_genes':2000,
+}
+adata=ov.space.clusters(adata,
+                  methods=['BINARY'],
+                 methods_kwargs=methods_kwargs)
+
+
+# if you want to use R's `mclust`, you can use `ov.utils.cluster`.
+# 
+# But you need to install `rpy2` and `mclust` at first.
+
+# In[4]:
+
+
+ov.utils.cluster(adata,use_rep='BINARY',method='mclust_R',n_components=10,
+                 random_state=42,
+                )
+adata.obs['mclust_BINARY'] = ov.utils.refine_label(adata, radius=30, key='mclust_R') 
+adata.obs['mclust_BINARY']=adata.obs['mclust_BINARY'].astype('category')
+
+
+# In[5]:
+
+
+res=ov.space.merge_cluster(adata,groupby='mclust_BINARY',use_rep='BINARY',
+                  threshold=0.01,plot=True)
+
+
+# In[6]:
+
+
+sc.pl.spatial(adata, color=['mclust_BINARY','mclust_BINARY_tree','mclust','Ground Truth'])
+
+
+# In[10]:
+
+
+ov.utils.cluster(adata,use_rep='BINARY',method='mclust',n_components=10,
+                 modelNames='EEV', random_state=42,
+                )
+adata.obs['mclustpy_BINARY'] = ov.utils.refine_label(adata, radius=30, key='mclust') 
+adata.obs['mclustpy_BINARY']=adata.obs['mclustpy_BINARY'].astype('category')
+
+
+# In[13]:
+
+
+adata.obs['mclustpy_BINARY']=adata.obs['mclustpy_BINARY'].astype('category')
+res=ov.space.merge_cluster(adata,groupby='mclustpy_BINARY',use_rep='BINARY',
+                  threshold=0.013,plot=True)
+
+
+# In[14]:
+
+
+sc.pl.spatial(adata, color=['mclustpy_BINARY','mclustpy_BINARY_tree','mclust','Ground Truth'])
+#adata.obs['mclust_BINARY'] = ov.utils.refine_label(adata, radius=30, key='mclust') 
+#adata.obs['mclust_BINARY']=adata.obs['mclust_BINARY'].astype('category')
+
+
+# ## Method3: STAGATE
+# 
+# STAGATE is designed for spatial clustering and denoising expressions of spatial resolved transcriptomics (ST) data.
+# 
+# STAGATE learns low-dimensional latent embeddings with both spatial information and gene expressions via a graph attention auto-encoder. The method adopts an attention mechanism in the middle layer of the encoder and decoder, which adaptively learns the edge weights of spatial neighbor networks, and further uses them to update the spot representation by collectively aggregating information from its neighbors. The latent embeddings and the reconstructed expression profiles can be used to downstream tasks such as spatial domain identification, visualization, spatial trajectory inference, data denoising and 3D expression domain extraction.
+# 
+# Dong, Kangning, and Shihua Zhang. “Deciphering spatial domains from spatially resolved transcriptomics with an adaptive graph attention auto-encoder.” Nature Communications 13.1 (2022): 1-12.
+# 
+# 
+# Here, we used `ov.space.pySTAGATE` to construct a STAGATE object to train the model. 
+# 
+
+# In[12]:
+
+
+methods_kwargs={}
+methods_kwargs['STAGATE']={
+    'num_batch_x':3,'num_batch_y':2,
+    'spatial_key':['X','Y'],'rad_cutoff':200,
+    'num_epoch':1000,'lr':0.001,
+    'weight_decay':1e-4,'hidden_dims':[512, 30],
+    'device':'cuda:0',
+    #'n_top_genes':2000,
+}
+
+adata=ov.space.clusters(adata,
+                  methods=['STAGATE'],
+                 methods_kwargs=methods_kwargs)
+
+
+# In[36]:
+
+
+ov.utils.cluster(adata,use_rep='STAGATE',method='mclust_R',n_components=10,
+                 random_state=112,
+                )
+adata.obs['mclust_R_STAGATE'] = ov.utils.refine_label(adata, radius=30, key='mclust_R') 
+adata.obs['mclust_R_STAGATE']=adata.obs['mclust_R_STAGATE'].astype('category')
+res=ov.space.merge_cluster(adata,groupby='mclust_R_STAGATE',use_rep='STAGATE',
+                  threshold=0.005,plot=True)
+
+
+# In[37]:
+
+
+sc.pl.spatial(adata, color=['mclust_R_STAGATE','mclust_R_STAGATE_tree','mclust_R','Ground Truth'])
+
+
+# ### Denoising
+
+# In[52]:
+
+
+adata.var.sort_values('PI',ascending=False).head(5)
+
+
+# In[53]:
+
+
+plot_gene = 'MBP'
+import matplotlib.pyplot as plt
+fig, axs = plt.subplots(1, 2, figsize=(8, 4))
+sc.pl.spatial(adata, img_key="hires", color=plot_gene, show=False, ax=axs[0], title='RAW_'+plot_gene, vmax='p99')
+sc.pl.spatial(adata, img_key="hires", color=plot_gene, show=False, ax=axs[1], title='STAGATE_'+plot_gene, layer='STAGATE_ReX', vmax='p99')
+
+
+# ## Method4: CAST
+# 
+# CAST would be a great algorithm if your spatial transcriptome is at single-cell resolution and in multiple slices.
+
+# In[38]:
+
+
+methods_kwargs={}
+methods_kwargs['CAST']={
+    'output_path_t':'result/CAST_gas/output',
+    'device':'cuda:0',
+    'gpu_t':0
+}
+adata=ov.space.clusters(adata,
+                  methods=['CAST'],
+                 methods_kwargs=methods_kwargs)
+
+
+# In[39]:
+
+
+ov.utils.cluster(adata,use_rep='X_cast',method='mclust',n_components=10,
+                 modelNames='EEV', random_state=42,
+                )
+adata.obs['mclust_CAST'] = ov.utils.refine_label(adata, radius=50, key='mclust') 
+adata.obs['mclust_CAST']=adata.obs['mclust_CAST'].astype('category')
+
+
+# In[40]:
+
+
+res=ov.space.merge_cluster(adata,groupby='mclust_CAST',use_rep='X_cast',
+                  threshold=0.1,plot=True)
+
+
+# In[41]:
+
+
+sc.pl.spatial(adata, color=['mclust_CAST','mclust_CAST_tree','mclust','Ground Truth'])
+
+
+# In[42]:
+
+
+adata
+
+
+# ## Evaluate cluster
+# 
+# We use ARI to evaluate the scoring of our clusters against the true values
+# 
+# While it appears that STAGATE works best, note that this is only on this dataset.
+# - If your data is spot-level resolution, GraphST, BINARY and STAGATE would be good algorithms to use
+# - BINARY and CAST would be good algorithms if your data is NanoString or other single-cell resolution
+
+# In[50]:
+
+
+from sklearn.metrics.cluster import adjusted_rand_score
+
+obs_df = adata.obs.dropna()
+#GraphST
+ARI = adjusted_rand_score(obs_df['mclust_GraphST'], obs_df['Ground Truth'])
+print('mclust_GraphST: Adjusted rand index = %.2f' %ARI)
+
+ARI = adjusted_rand_score(obs_df['mclust_R_GraphST'], obs_df['Ground Truth'])
+print('mclust_R_GraphST: Adjusted rand index = %.2f' %ARI)
+
+ARI = adjusted_rand_score(obs_df['mclust_R_STAGATE'], obs_df['Ground Truth'])
+print('mclust_STAGATE: Adjusted rand index = %.2f' %ARI)
+
+ARI = adjusted_rand_score(obs_df['mclust_BINARY'], obs_df['Ground Truth'])
+print('mclust_BINARY: Adjusted rand index = %.2f' %ARI)
+
+ARI = adjusted_rand_score(obs_df['mclustpy_BINARY'], obs_df['Ground Truth'])
+print('mclustpy_BINARY: Adjusted rand index = %.2f' %ARI)
+
+ARI = adjusted_rand_score(obs_df['mclust_CAST'], obs_df['Ground Truth'])
+print('mclust_CAST: Adjusted rand index = %.2f' %ARI)
+
+
+# In[ ]:
+
+
+
+

ovrawm/t_cnmf.txt

@@ -0,0 +1,331 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# # Consensus Non-negative Matrix factorization (cNMF)
+# 
+# cNMF is an analysis pipeline for inferring gene expression programs from single-cell RNA-Seq (scRNA-Seq) data.
+# 
+# It takes a count matrix (N cells X G genes) as input and produces a (K x G) matrix of gene expression programs (GEPs) and a (N x K) matrix specifying the usage of each program for each cell in the data. You can read more about the method in the [github](https://github.com/dylkot/cNMF) and check out examples on dentategyrus.
+
+# In[1]:
+
+
+import scanpy as sc
+import omicverse as ov
+ov.plot_set()
+
+
+# ## Loading dataset
+# 
+# Here, we use the dentategyrus dataset as an example for cNMF.
+
+# In[2]:
+
+
+import scvelo as scv
+adata=scv.datasets.dentategyrus()
+
+
+# In[3]:
+
+
+get_ipython().run_cell_magic('time', '', "adata=ov.pp.preprocess(adata,mode='shiftlog|pearson',n_HVGs=2000,)\nadata\n")
+
+
+# In[23]:
+
+
+ov.pp.scale(adata)
+ov.pp.pca(adata)
+
+
+# In[4]:
+
+
+import matplotlib.pyplot as plt
+from matplotlib import patheffects
+fig, ax = plt.subplots(figsize=(4,4))
+ov.pl.embedding(
+    adata,
+    basis="X_umap",
+    color=['clusters'],
+    frameon='small',
+    title="Celltypes",
+    #legend_loc='on data',
+    legend_fontsize=14,
+    legend_fontoutline=2,
+    #size=10,
+    ax=ax,
+    #legend_loc=True, 
+    add_outline=False, 
+    #add_outline=True,
+    outline_color='black',
+    outline_width=1,
+    show=False,
+)
+
+
+# ## Initialize and Training model
+
+# In[5]:
+
+
+import numpy as np
+## Initialize the cnmf object that will be used to run analyses
+cnmf_obj = ov.single.cNMF(adata,components=np.arange(5,11), n_iter=20, seed=14, num_highvar_genes=2000,
+                          output_dir='example_dg/cNMF', name='dg_cNMF')
+
+
+# In[6]:
+
+
+## Specify that the jobs are being distributed over a single worker (total_workers=1) and then launch that worker
+cnmf_obj.factorize(worker_i=0, total_workers=2)
+
+
+# In[7]:
+
+
+cnmf_obj.combine(skip_missing_files=True)
+
+
+# ## Compute the stability and error at each choice of K to see if a clear choice jumps out.
+# 
+# Please note that the maximum stability solution is not always the best choice depending on the application. However it is often a good starting point even if you have to investigate several choices of K
+
+# In[8]:
+
+
+cnmf_obj.k_selection_plot(close_fig=False)
+
+
+# In this range, K=7 gave the most stable solution so we will begin by looking at that.
+# 
+# The next step computes the consensus solution for a given choice of K. We first run it without any outlier filtering to see what that looks like. Setting the density threshold to anything >= 2.00 (the maximum possible distance between two unit vectors) ensures that nothing will be filtered.
+# 
+# Then we run the consensus with a filter for outliers determined based on inspecting the histogram of distances between components and their nearest neighbors
+
+# In[9]:
+
+
+selected_K = 7
+density_threshold = 2.00
+
+
+# In[10]:
+
+
+cnmf_obj.consensus(k=selected_K, 
+                   density_threshold=density_threshold, 
+                   show_clustering=True, 
+                   close_clustergram_fig=False)
+
+
+# The above consensus plot shows that there is a substantial degree of concordance between the replicates with a few outliers. An outlier threshold of 0.1 seems appropriate
+
+# In[11]:
+
+
+density_threshold = 0.10
+
+
+# In[12]:
+
+
+cnmf_obj.consensus(k=selected_K, 
+                   density_threshold=density_threshold, 
+                   show_clustering=True, 
+                   close_clustergram_fig=False)
+
+
+# ## Visualization the result
+
+# In[13]:
+
+
+import seaborn as sns
+import matplotlib.pyplot as plt
+from matplotlib import patheffects
+
+from matplotlib import gridspec
+import matplotlib.pyplot as plt
+
+width_ratios = [0.2, 4, 0.5, 10, 1]
+height_ratios = [0.2, 4]
+fig = plt.figure(figsize=(sum(width_ratios), sum(height_ratios)))
+gs = gridspec.GridSpec(len(height_ratios), len(width_ratios), fig,
+                        0.01, 0.01, 0.98, 0.98,
+                       height_ratios=height_ratios,
+                       width_ratios=width_ratios,
+                       wspace=0, hspace=0)
+            
+D = cnmf_obj.topic_dist[cnmf_obj.spectra_order, :][:, cnmf_obj.spectra_order]
+dist_ax = fig.add_subplot(gs[1,1], xscale='linear', yscale='linear',
+                                      xticks=[], yticks=[],xlabel='', ylabel='',
+                                      frameon=True)
+dist_im = dist_ax.imshow(D, interpolation='none', cmap='viridis',
+                         aspect='auto', rasterized=True)
+
+left_ax = fig.add_subplot(gs[1,0], xscale='linear', yscale='linear', xticks=[], yticks=[],
+                xlabel='', ylabel='', frameon=True)
+left_ax.imshow(cnmf_obj.kmeans_cluster_labels.values[cnmf_obj.spectra_order].reshape(-1, 1),
+                            interpolation='none', cmap='Spectral', aspect='auto',
+                            rasterized=True)
+
+top_ax = fig.add_subplot(gs[0,1], xscale='linear', yscale='linear', xticks=[], yticks=[],
+                xlabel='', ylabel='', frameon=True)
+top_ax.imshow(cnmf_obj.kmeans_cluster_labels.values[cnmf_obj.spectra_order].reshape(1, -1),
+                  interpolation='none', cmap='Spectral', aspect='auto',
+                    rasterized=True)
+
+cbar_gs = gridspec.GridSpecFromSubplotSpec(3, 3, subplot_spec=gs[1, 2],
+                                   wspace=0, hspace=0)
+cbar_ax = fig.add_subplot(cbar_gs[1,2], xscale='linear', yscale='linear',
+    xlabel='', ylabel='', frameon=True, title='Euclidean\nDistance')
+cbar_ax.set_title('Euclidean\nDistance',fontsize=12)
+vmin = D.min().min()
+vmax = D.max().max()
+fig.colorbar(dist_im, cax=cbar_ax,
+        ticks=np.linspace(vmin, vmax, 3),
+        )
+cbar_ax.set_yticklabels(cbar_ax.get_yticklabels(),fontsize=12)
+
+
+# In[14]:
+
+
+density_filter = cnmf_obj.local_density.iloc[:, 0] < density_threshold
+fig, hist_ax = plt.subplots(figsize=(4,4))
+
+#hist_ax = fig.add_subplot(hist_gs[0,0], xscale='linear', yscale='linear',
+ #   xlabel='', ylabel='', frameon=True, title='Local density histogram')
+hist_ax.hist(cnmf_obj.local_density.values, bins=np.linspace(0, 1, 50))
+hist_ax.yaxis.tick_right()
+
+xlim = hist_ax.get_xlim()
+ylim = hist_ax.get_ylim()
+if density_threshold < xlim[1]:
+    hist_ax.axvline(density_threshold, linestyle='--', color='k')
+    hist_ax.text(density_threshold  + 0.02, ylim[1] * 0.95, 'filtering\nthreshold\n\n', va='top')
+hist_ax.set_xlim(xlim)
+hist_ax.set_xlabel('Mean distance to k nearest neighbors\n\n%d/%d (%.0f%%) spectra above threshold\nwere removed prior to clustering'%(sum(~density_filter), len(density_filter), 100*(~density_filter).mean()))
+hist_ax.set_title('Local density histogram')
+
+
+# ## Explode the cNMF result
+# 
+# We can load the results for a cNMF run with a given K and density filtering threshold like below
+
+# In[15]:
+
+
+result_dict = cnmf_obj.load_results(K=selected_K, density_threshold=density_threshold)
+
+
+# In[16]:
+
+
+result_dict['usage_norm'].head()
+
+
+# In[17]:
+
+
+result_dict['gep_scores'].head()
+
+
+# In[18]:
+
+
+result_dict['gep_tpm'].head()
+
+
+# In[19]:
+
+
+result_dict['top_genes'].head()
+
+
+# We can extract cell classes directly based on the highest cNMF in each cell, but this has the disadvantage that it will lead to mixed cell classes if the heterogeneity of our data is not as strong as it should be.
+
+# In[20]:
+
+
+cnmf_obj.get_results(adata,result_dict)
+
+
+# In[21]:
+
+
+ov.pl.embedding(adata, basis='X_umap',color=result_dict['usage_norm'].columns,
+           use_raw=False, ncols=3, vmin=0, vmax=1,frameon='small')
+
+
+# In[24]:
+
+
+ov.pl.embedding(
+    adata,
+    basis="X_umap",
+    color=['cNMF_cluster'],
+    frameon='small',
+    #title="Celltypes",
+    #legend_loc='on data',
+    legend_fontsize=14,
+    legend_fontoutline=2,
+    #size=10,
+    #legend_loc=True, 
+    add_outline=False, 
+    #add_outline=True,
+    outline_color='black',
+    outline_width=1,
+    show=False,
+)
+
+
+# Here we are, proposing another idea of categorisation. We use cells with cNMF greater than 0.5 as a primitive class, and then train a random forest classification model, and then use the random forest classification model to classify cells with cNMF less than 0.5 to get a more accurate
+
+# In[25]:
+
+
+cnmf_obj.get_results_rfc(adata,result_dict,
+                         use_rep='scaled|original|X_pca',
+                        cNMF_threshold=0.5)
+
+
+# In[27]:
+
+
+ov.pl.embedding(
+    adata,
+    basis="X_umap",
+    color=['cNMF_cluster_rfc','cNMF_cluster_clf'],
+    frameon='small',
+    #title="Celltypes",
+    #legend_loc='on data',
+    legend_fontsize=14,
+    legend_fontoutline=2,
+    #size=10,
+    #legend_loc=True, 
+    add_outline=False, 
+    #add_outline=True,
+    outline_color='black',
+    outline_width=1,
+    show=False,
+)
+
+
+# In[25]:
+
+
+plot_genes=[]
+for i in result_dict['top_genes'].columns:
+    plot_genes+=result_dict['top_genes'][i][:3].values.reshape(-1).tolist()
+
+
+# In[26]:
+
+
+sc.pl.dotplot(adata,plot_genes,
+              "cNMF_cluster", dendrogram=False,standard_scale='var',)
+

ovrawm/t_commot_flowsig.txt

@@ -0,0 +1,395 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# # Spatial Communication
+# 
+# Spatial communication is a point of interest for us for the Spatial Transcriptomics Society, and we would like to find the conduction process of spatial communication.
+# 
+# Here, we introduce two method integrated in omicverse named `COMMOT` and `flowsig`.
+# 
+# We made three improvements in integrating the `COMMOT` and `flowsig` algorithm in OmicVerse:
+# 
+# - We reduced the installation conflict of `COMMOT` and `flowsig`, user only need to update OmicVerse to the latest version.
+# - We optimized the visualization of `COMMOT` and `flowsig` and unified the data preprocessing process so that users don't need to struggle with different data processing flows.
+# - We have fixed some bugs that could occur during function.
+# 
+# If you found this tutorial helpful, please cite `COMMOT`, `flowsig` and OmicVerse:
+# 
+# - Cang, Z., Zhao, Y., Almet, A.A. et al. Screening cell–cell communication in spatial transcriptomics via collective optimal transport. Nat Methods 20, 218–228 (2023). https://doi.org/10.1038/s41592-022-01728-4
+# - Almet, A.A., Tsai, YC., Watanabe, M. et al. Inferring pattern-driving intercellular flows from single-cell and spatial transcriptomics. Nat Methods (2024). https://doi.org/10.1038/s41592-024-02380-w
+
+# In[1]:
+
+
+import omicverse as ov
+#print(f"omicverse version: {ov.__version__}")
+import scanpy as sc
+#print(f"scanpy version: {sc.__version__}")
+ov.plot_set()
+
+
+# ## Preprocess data
+# 
+# Here we present our re-analysis of 151676 sample of the dorsolateral prefrontal cortex (DLPFC) dataset. Maynard et al. has manually annotated DLPFC layers and white matter (WM) based on the morphological features and gene markers.
+# 
+# This tutorial demonstrates how to identify spatial domains on 10x Visium data using STAGATE. The processed data are available at https://github.com/LieberInstitute/spatialLIBD. We downloaded the manual annotation from the spatialLIBD package and provided at https://drive.google.com/drive/folders/10lhz5VY7YfvHrtV40MwaqLmWz56U9eBP?usp=sharing.
+
+# In[40]:
+
+
+adata = sc.read_visium(path='data', count_file='151676_filtered_feature_bc_matrix.h5')
+adata.var_names_make_unique()
+
+
+# <div class="admonition warning">
+#   <p class="admonition-title">Note</p>
+#   <p>
+#     We introduced the spatial special svg calculation module prost in omicverse versions greater than `1.6.0` to replace scanpy's HVGs, if you want to use scanpy's HVGs you can set mode=`scanpy` in `ov.space.svg` or use the following code.
+#   </p>
+# </div>
+# 
+# ```python
+# #adata=ov.pp.preprocess(adata,mode='shiftlog|pearson',n_HVGs=3000,target_sum=1e4)
+# #adata.raw = adata
+# #adata = adata[:, adata.var.highly_variable_features]
+# ```
+
+# In[ ]:
+
+
+sc.pp.calculate_qc_metrics(adata, inplace=True)
+adata = adata[:,adata.var['total_counts']>100]
+adata=ov.space.svg(adata,mode='prost',n_svgs=3000,target_sum=1e4,platform="visium",)
+adata
+
+
+# In[ ]:
+
+
+adata.write('data/cluster_svg.h5ad',compression='gzip')
+
+
+# In[3]:
+
+
+#adata=ov.read('data/cluster_svg.h5ad',compression='gzip')
+
+
+# ## Communication Analysis with COMMOT
+# 
+# ### Spatial communication inference
+# 
+# We will use the CellChatDB ligand-receptor database here. Only the secreted signaling LR pairs will be used.
+# 
+# Jin, Suoqin, et al. “Inference and analysis of cell-cell communication using CellChat.” Nature communications 12.1 (2021): 1-20.
+
+# In[42]:
+
+
+df_cellchat = ov.externel.commot.pp.ligand_receptor_database(species='human', 
+                                                             signaling_type='Secreted Signaling', 
+                                                             database='CellChat')
+print(df_cellchat.shape)
+
+
+# We then filter the LR pairs to keep only the pairs with both ligand and receptor expressed in at least 5% of the spots.
+
+# In[43]:
+
+
+df_cellchat_filtered = ov.externel.commot.pp.filter_lr_database(df_cellchat, 
+                                                                adata, 
+                                                                min_cell_pct=0.05)
+print(df_cellchat_filtered.shape)
+
+
+# Now perform spatial communication inference for these 250 ligand-receptor pairs with a spatial distance limit of 500. CellChat database considers heteromeric units. The signaling results are stored as spot-by-spot matrices in the obsp slots. For example, the score for spot i signaling to spot j through the LR pair can be retrieved from `adata.obsp['commot-cellchat-Wnt4-Fzd4_Lrp6'][i,j]`.
+
+# In[44]:
+
+
+ov.externel.commot.tl.spatial_communication(adata,
+                            database_name='cellchat', 
+                            df_ligrec=df_cellchat_filtered, 
+                            dis_thr=500, heteromeric=True, 
+                            pathway_sum=True)
+
+
+# (Optional) We read the ground truth area of our spatial data
+# 
+# This step is not mandatory to run, in the tutorial, it's just to demonstrate the accuracy of our clustering effect, and in your own tasks, there is often no Ground_truth
+# 
+# <div class="admonition warning">
+#   <p class="admonition-title">Note</p>
+#   <p>
+#     You can also use Celltype and other annotated results in adata.obs, here is just a randomly selected type, there is no particular significance, in order to facilitate the visualization and study the signal
+#   </p>
+# </div>
+
+# In[45]:
+
+
+# read the annotation
+import pandas as pd
+import os
+Ann_df = pd.read_csv(os.path.join('data', '151676_truth.txt'), sep='\t', header=None, index_col=0)
+Ann_df.columns = ['Ground_Truth']
+adata.obs['Ground_Truth'] = Ann_df.loc[adata.obs_names, 'Ground_Truth']
+Layer_color=['#283b5c', '#d8e17b', '#838e44', '#4e8991', '#d08c35', '#511a3a',
+       '#c2c2c2', '#dfc648']
+sc.pl.spatial(adata, img_key="hires", color=["Ground_Truth"],palette=Layer_color)
+
+
+# ### Visualize the communication signal in spatial space
+# 
+# Determine the spatial direction of a signaling pathway, for example, the FGF pathway. The interpolated signaling directions for where the signals are sent by the spots and where the signals received by the spots are from are stored in `adata.obsm['commot_sender_vf-cellchat-FGF']` and `adata.obsm['commot_receiver_vf-cellchat-FGF']`, respectively.
+# 
+# Taken together, our findings indicate that FGF signaling is crucial for cortical folding in gyrencephalic mammals and is a pivotal upstream regulator of the production of OSVZ progenitors. FGF signaling is the first signaling pathway found to regulate cortical folding.
+
+# In[46]:
+
+
+ct_color_dict=dict(zip(adata.obs['Ground_Truth'].cat.categories,
+                      adata.uns['Ground_Truth_colors']))
+
+
+# In[47]:
+
+
+adata.uns['commot-cellchat-info']['df_ligrec'].head()
+
+
+# In[48]:
+
+
+import matplotlib.pyplot as plt
+scale=0.000008
+k=5
+goal_pathway='FGF'
+ov.externel.commot.tl.communication_direction(adata, database_name='cellchat', pathway_name=goal_pathway, k=k)
+ov.externel.commot.pl.plot_cell_communication(adata, database_name='cellchat', 
+                                              pathway_name='FGF', plot_method='grid', 
+                                              background_legend=True,
+                                              scale=scale, ndsize=8, grid_density=0.4, 
+                                              summary='sender', background='cluster', 
+                                              clustering='Ground_Truth', 
+                                              cluster_cmap=ct_color_dict,
+                                              cmap='Alphabet',
+                                              normalize_v = True, normalize_v_quantile=0.995)
+plt.title(f'Pathway:{goal_pathway}',fontsize=13)
+#plt.savefig('figures/TLE/TLE_cellchat_all_FGF.png',dpi=300,bbox_inches='tight')
+#fig.savefig('pdf/TLE/control_cellchat_all_FGF.pdf',dpi=300,bbox_inches='tight')
+
+
+# In[49]:
+
+
+adata.write('data/151676_commot.h5ad',compression='gzip')
+
+
+# In[2]:
+
+
+adata=ov.read('data/151676_commot.h5ad')
+adata
+
+
+# ## Communication signal inference with flowsig
+# 
+# ### Construct GEMs
+# We now construct gene expression modules (GEMs) from the unnormalised count data. For ST data, we use `NMF`.
+
+# In[3]:
+
+
+adata.layers['normalized'] = adata.X.copy()
+
+# We construct 10 gene expression modules using the raw cell count.
+ov.externel.flowsig.pp.construct_gems_using_nmf(adata,
+                                n_gems = 10,
+                                layer_key = 'counts',
+                                                   )
+
+
+# If you want to study the genes in a GEM, we provide the `ov.externel.flowsig.ul.get_top_gem_genes` function for getting the genes in a specific GEM.
+
+# In[4]:
+
+
+goal_gem='GEM-5'
+gem_gene=ov.externel.flowsig.ul.get_top_gem_genes(adata=adata,
+                                            gems=[goal_gem],
+                                         n_genes=100,
+                                         gene_type='all',
+                                        method = 'nmf',
+                                        )
+gem_gene.head()
+
+
+# ### Construct the flow expression matrices
+# 
+# We construct augmented flow expression matrices for each condition that measure three types of variables:
+# 
+# 1. Intercellular signal inflow, i.e., how much of a signal did a cell receive. For ST data, signal inflow is constructed by summing the received signals for each significant ligand inferred by COMMOT.
+# 2. GEMs, which encapsulate intracellular information processing. We define these as cellwise membership to the GEM.
+# Intercellular signal outflow, i.e., how much of a signal did a cell send. These are simply ligand gene expression.
+# 3. The kay assumption of flowsig is that all intercellular information flows are directed from signal inflows to GEMs, from one GEM to another GEM, and from GEMs to signal outflows.
+# 
+# For spatial data, we use COMMOT output directly to construct signal inflow expression and do not need knowledge about TF databases.
+
+# In[5]:
+
+
+commot_output_key = 'commot-cellchat'
+# We first construct the potential cellular flows from the commot output
+ov.externel.flowsig.pp.construct_flows_from_commot(adata,
+                                commot_output_key,
+                                gem_expr_key = 'X_gem',
+                                scale_gem_expr = True,
+                                flowsig_network_key = 'flowsig_network',
+                                flowsig_expr_key = 'X_flow')
+
+
+# For spatial data, we retain spatially informative variables, which we determine by calculating the Moran's I value for signal inflow and signal outflow variables. In case the spatial graph has not been calculated for this data yet, FlowSig will do so, meaning that we need to specify both the coordinate type, grid or generic, and in the case of the former, n_neighs, which in this case, is 8.
+# 
+# Flow expression variables are defined to be spatially informative if their Moran's I value is above a specified threshold.
+
+# In[6]:
+
+
+# Then we subset for "spatially flowing" inflows and outflows
+ov.externel.flowsig.pp.determine_informative_variables(adata,  
+                                    flowsig_expr_key = 'X_flow',
+                                    flowsig_network_key = 'flowsig_network',
+                                    spatial = True,
+                                    moran_threshold = 0.15,
+                                    coord_type = 'grid',
+                                    n_neighbours = 8,
+                                    library_key = None)
+
+
+# ### Learn intercellular flows
+# 
+# For spatial data, where there are far fewer "control vs. perturbed" studies, we use the GSP method, which uses conditional independence testing and a greedy algorithm to learn the CPDAG containing directed arcs and undirected edges.
+# 
+# For spatial data, we cannot bootstrap by resampling across individual cells because we would lose the additional layer of correlation contained in the spatial data. Rather, we divide the tissue up into spatial "blocks" and resample within blocks. This is known as block bootstrapping.
+# 
+# To calculate the blocks, we used scikit-learn's k-means clustering method to generate 20 roughly equally sized spatial blocks.
+
+# In[9]:
+
+
+from sklearn.cluster import KMeans
+import pandas as pd
+
+kmeans = KMeans(n_clusters=10, random_state=0).fit(adata.obsm['spatial'])
+adata.obs['spatial_kmeans'] = pd.Series(kmeans.labels_, dtype='category').values
+
+
+# We use these blocks to learn the spatial intercellular flows.
+
+# In[ ]:
+
+
+# # Now we are ready to learn the network
+ov.externel.flowsig.tl.learn_intercellular_flows(adata,
+                        flowsig_key = 'flowsig_network',
+                        flow_expr_key = 'X_flow',
+                        use_spatial = True,
+                        block_key = 'spatial_kmeans',
+                        n_jobs = 4,
+                        n_bootstraps = 500)
+
+
+# ### Partially validate intercellular flow network
+# 
+# Finally, we will remove any "false positive" edges. Noting that the CPDAG contains directed arcs and undirected arcs we do two things.
+# 
+# First, we remove directed arcs that are not oriented from signal inflow to GEM, GEM to GEM, or from GEM to signal outflow and for undirected edges, we reorient them so that they obey the previous directionalities.
+
+# In[8]:
+
+
+# This part is key for reducing false positives
+ov.externel.flowsig.tl.apply_biological_flow(adata,
+                        flowsig_network_key = 'flowsig_network',
+                        adjacency_key = 'adjacency',
+                        validated_key = 'validated')
+
+
+# Second, we will remove directed arcs whose bootstrapped frequencies are below a specified edge threshold as well as undirected edges whose total bootstrapped frequencies are below the same threshold. Because we did not have perturbation data, we specify a more stringent edge threshold.
+# 
+
+# In[26]:
+
+
+edge_threshold = 0.7
+
+ov.externel.flowsig.tl.filter_low_confidence_edges(adata,
+                                edge_threshold = edge_threshold,
+                                flowsig_network_key = 'flowsig_network',
+                                adjacency_key = 'adjacency_validated',
+                                filtered_key = 'filtered')
+
+
+# In[27]:
+
+
+adata.write('data/cortex_commot_flowsig.h5ad',compression='gzip')
+
+
+# In[2]:
+
+
+#adata=ov.read('data/cortex_commot_flowsig.h5ad')
+
+
+# ## Visualize the result of flowsig
+# 
+# We can construct the directed NetworkX DiGraph object from adjacency_validated_filtered.
+
+# In[3]:
+
+
+flow_network = ov.externel.flowsig.tl.construct_intercellular_flow_network(adata,
+                                                        flowsig_network_key = 'flowsig_network',
+                                                        adjacency_key = 'adjacency_validated_filtered')
+
+
+# ### Cell-specific GEM
+# 
+# The first thing we need to be concerned about is which GEM, exactly, is relevant to the cell I want to study. Here, we use dotplot to visualize the expression of GEM in different cell types.
+
+# In[4]:
+
+
+flowsig_expr_key='X_gem'
+X_flow = adata.obsm[flowsig_expr_key]
+adata_subset = sc.AnnData(X=X_flow)
+adata_subset.obs = adata.obs
+adata_subset.var.index =[f'GEM-{i}' for i in range(1,len(adata_subset.var)+1)]
+
+
+# In[5]:
+
+
+import matplotlib.pyplot as plt
+ax=sc.pl.dotplot(adata_subset, adata_subset.var.index, groupby='Ground_Truth', 
+              dendrogram=True,standard_scale='var',cmap='Reds',show=False)
+color_dict=dict(zip(adata.obs['Ground_Truth'].cat.categories,adata.uns['Ground_Truth_colors']))
+
+
+# ### Visualize the flowsig network
+# 
+# We fixed the network function provided by the author and provided a better visualization.
+
+# In[7]:
+
+
+ov.pl.plot_flowsig_network(flow_network=flow_network,
+                         gem_plot=['GEM-2','GEM-7','GEM-1','GEM-3','GEM-4','GEM-5'],
+                        figsize=(8,4),
+                     curve_awarg={'eps':2},
+                      node_shape={'GEM':'^','Sender':'o','Receptor':'o'},
+                          ylim=(-0.5,0.5),xlim=(-3,3))
+

ovrawm/t_cytotrace.txt

@@ -0,0 +1,110 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# # Prediction of absolute developmental potential using CytoTrace2
+# 
+# CytoTRACE 2 is a computational method for predicting cellular potency categories and absolute developmental potential from single-cell RNA-sequencing data.
+# 
+# Potency categories in the context of CytoTRACE 2 classify cells based on their developmental potential, ranging from totipotent and pluripotent cells with broad differentiation potential to lineage-restricted oligopotent, multipotent and unipotent cells capable of producing varying numbers of downstream cell types, and finally, differentiated cells, ranging from mature to terminally differentiated phenotypes.
+# 
+# We made three improvements in integrating the CytoTrace2 algorithm in OmicVerse:
+# 
+# - No additional packages to install, including R
+# - We fixed a bug in multi-threaded pools to avoid potential error reporting
+# - Native support for `anndata`, you don't need to export `input_file` and `annotation_file`.
+# 
+# If you found this tutorial helpful, please cite CytoTrace2 and OmicVerse:
+# 
+# Kang, M., Armenteros, J. J. A., Gulati, G. S., Gleyzer, R., Avagyan, S., Brown, E. L., Zhang, W., Usmani, A., Earland, N., Wu, Z., Zou, J., Fields, R. C., Chen, D. Y., Chaudhuri, A. A., & Newman, A. M. (2024). Mapping single-cell developmental potential in health and disease with interpretable deep learning. bioRxiv : the preprint server for biology, 2024.03.19.585637. https://doi.org/10.1101/2024.03.19.585637
+
+# In[1]:
+
+
+import omicverse as ov
+ov.plot_set()
+
+
+# ## Preprocess data
+# 
+# As an example, we apply differential kinetic analysis to dentate gyrus neurogenesis, which comprises multiple heterogeneous subpopulations.
+
+# In[2]:
+
+
+import scvelo as scv
+adata=scv.datasets.dentategyrus()
+adata
+
+
+# In[4]:
+
+
+get_ipython().run_cell_magic('time', '', "adata=ov.pp.preprocess(adata,mode='shiftlog|pearson',n_HVGs=2000,)\nadata\n")
+
+
+# ## Predict cytotrace2
+# 
+# We need to import the two pre-trained models from CytoTrace2, see the download links for the models:
+# 
+# - Figshare:
+#   https://figshare.com/ndownloader/files/47258749
+# 
+# - or Github:
+#     https://github.com/digitalcytometry/cytotrace2/tree/main/cytotrace2_python/cytotrace2_py/resources/17_models_weights
+#     https://github.com/digitalcytometry/cytotrace2/tree/main/cytotrace2_python/cytotrace2_py/resources/5_models_weights
+# 
+# All parameters are explained as follows:
+# - adata: AnnData object containing the scRNA-seq data.
+# - use_model_dir: Path to the directory containing the pre-trained model files.
+# - species: The species of the input data. Default is "mouse".
+# - batch_size: The number of cells to process in each batch. Default is 10000.
+# - smooth_batch_size: The number of cells to process in each batch for smoothing. Default is 1000.
+# - disable_parallelization: If True, disable parallel processing. Default is False.
+# - max_cores: Maximum number of CPU cores to use for parallel processing. If None, all available cores will be used. Default is None.
+# - max_pcs: Maximum number of principal components to use. Default is 200.
+# - seed: Random seed for reproducibility. Default is 14.
+# - output_dir: Directory to save the results. Default is 'cytotrace2_results'.
+
+# In[5]:
+
+
+results =  ov.single.cytotrace2(adata,
+    use_model_dir="cymodels/5_models_weights",
+    species="mouse",
+    batch_size = 10000,
+    smooth_batch_size = 1000,
+    disable_parallelization = False,
+    max_cores = None,
+    max_pcs = 200,
+    seed = 14,
+    output_dir = 'cytotrace2_results'
+)
+
+
+# ## Visualizing
+# 
+# Visualizing the results we can directly compare the predicted potency scores with the known developmental stage of the cells, seeing how the predictions meticulously align with the known biology. Take a look!
+
+# In[8]:
+
+
+ov.utils.embedding(adata,basis='X_umap',
+                   color=['clusters','CytoTRACE2_Score'],
+                   frameon='small',cmap='Reds',wspace=0.55)
+
+
+# - Left: demonstrates the distribution of different cell types in UMAP space.
+# - Right: demonstrates the CytoTRACE 2 scores of different cell types; cells with high scores are generally considered to have a higher pluripotency or undifferentiated state.
+
+# In[9]:
+
+
+ov.utils.embedding(adata,basis='X_umap',
+                   color=['CytoTRACE2_Potency','CytoTRACE2_Relative'],
+                   frameon='small',cmap='Reds',wspace=0.55)
+
+
+# - Potency category:
+#     The UMAP embedding plot of predicted potency category reflects the discrete classification of cells into potency categories, taking possible values of Differentiated, Unipotent, Oligopotent, Multipotent, Pluripotent, and Totipotent.
+# - Relative order:
+#     UMAP embedding of predicted relative order, which is based on absolute predicted potency scores normalized to the range 0 (more differentiated) to 1 (less differentiated). Provides the relative ordering of cells by developmental potential

ovrawm/t_deg.txt

@@ -0,0 +1,323 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# # Different Expression Analysis
+# 
+# An important task of bulk rna-seq analysis is the different expression , which we can perform with omicverse. For different expression analysis, ov change the `gene_id` to `gene_name` of matrix first. When our dataset existed the batch effect, we can use the SizeFactors of DEseq2 to normalize it, and use `t-test` of `wilcoxon` to calculate the p-value of genes. Here we demonstrate this pipeline with a matrix from `featureCounts`. The same pipeline would generally be used to analyze any collection of RNA-seq tasks. 
+# 
+# Colab_Reproducibility：https://colab.research.google.com/drive/1q5lDfJepbtvNtc1TKz-h4wGUifTZ3i0_?usp=sharing
+
+# In[1]:
+
+
+import omicverse as ov
+import scanpy as sc
+import matplotlib.pyplot as plt
+
+ov.plot_set()
+
+
+# ## Geneset Download
+# 
+# When we need to convert a gene id, we need to prepare a mapping pair file. Here we have pre-processed 6 genome gtf files and generated mapping pairs including `T2T-CHM13`, `GRCh38`, `GRCh37`, `GRCm39`, `danRer7`, and `danRer11`. If you need to convert other id_mapping, you can generate your own mapping using gtf Place the files in the `genesets` directory.
+
+# In[2]:
+
+
+ov.utils.download_geneid_annotation_pair()
+
+
+# Note that this dataset has not been processed in any way and is only exported by `featureCounts`, and Sequence alignment was performed from the genome file of CRCm39
+# 
+# sample data can be download from: https://raw.githubusercontent.com/Starlitnightly/omicverse/master/sample/counts.txt
+
+# In[3]:
+
+
+#data=pd.read_csv('https://raw.githubusercontent.com/Starlitnightly/omicverse/master/sample/counts.txt',index_col=0,sep='\t',header=1)
+data=ov.read('data/counts.txt',index_col=0,header=1)
+#replace the columns `.bam` to `` 
+data.columns=[i.split('/')[-1].replace('.bam','') for i in data.columns]
+data.head()
+
+
+# ## ID mapping
+# 
+# We performed the gene_id mapping by the mapping pair file `GRCm39` downloaded before.
+
+# In[4]:
+
+
+data=ov.bulk.Matrix_ID_mapping(data,'genesets/pair_GRCm39.tsv')
+data.head()
+
+
+# ## Different expression analysis with ov
+# 
+# We can do differential expression analysis very simply by ov, simply by providing an expression matrix. To run DEG, we simply need to:
+# 
+# - Read the raw count by featureCount or any other qualify methods.
+# - Create an ov DEseq object.
+
+# In[5]:
+
+
+dds=ov.bulk.pyDEG(data)
+
+
+# We notes that the gene_name mapping before exist some duplicates, we will process the duplicate indexes to retain only the highest expressed genes
+
+# In[6]:
+
+
+dds.drop_duplicates_index()
+print('... drop_duplicates_index success')
+
+
+# We also need to remove the batch effect of the expression matrix, `estimateSizeFactors` of DEseq2 to be used to normalize our matrix
+
+# In[7]:
+
+
+dds.normalize()
+print('... estimateSizeFactors and normalize success')
+
+
+# Now we can calculate the different expression gene from matrix, we need to input the treatment and control groups
+
+# In[8]:
+
+
+treatment_groups=['4-3','4-4']
+control_groups=['1--1','1--2']
+result=dds.deg_analysis(treatment_groups,control_groups,method='ttest')
+result.head()
+
+
+# One important thing is that we do not filter out low expression genes when processing DEGs, and in future versions I will consider building in the corresponding processing.
+
+# In[9]:
+
+
+print(result.shape)
+result=result.loc[result['log2(BaseMean)']>1]
+print(result.shape)
+
+
+# We also need to set the threshold of Foldchange, we prepare a method named `foldchange_set` to finish. This function automatically calculates the appropriate threshold based on the log2FC distribution, but you can also enter it manually.
+
+# In[10]:
+
+
+# -1 means automatically calculates
+dds.foldchange_set(fc_threshold=-1,
+                   pval_threshold=0.05,
+                   logp_max=6)
+
+
+# ## Visualize the DEG result and specific genes
+# 
+# To visualize the DEG result, we use `plot_volcano` to do it. This fuction can visualize the gene interested or high different expression genes. There are some parameters you need to input:
+# 
+# - title: The title of volcano
+# - figsize: The size of figure
+# - plot_genes: The genes you interested
+# - plot_genes_num: If you don't have interested genes, you can auto plot it.
+
+# In[11]:
+
+
+dds.plot_volcano(title='DEG Analysis',figsize=(4,4),
+                 plot_genes_num=8,plot_genes_fontsize=12,)
+
+
+# To visualize the specific genes, we only need to use the `dds.plot_boxplot` function to finish it.
+
+# In[12]:
+
+
+dds.plot_boxplot(genes=['Ckap2','Lef1'],treatment_groups=treatment_groups,
+                control_groups=control_groups,figsize=(2,3),fontsize=12,
+                 legend_bbox=(2,0.55))
+
+
+# In[13]:
+
+
+dds.plot_boxplot(genes=['Ckap2'],treatment_groups=treatment_groups,
+                control_groups=control_groups,figsize=(2,3),fontsize=12,
+                 legend_bbox=(2,0.55))
+
+
+# ## Pathway enrichment analysis by ov
+# 
+# Here we use the `gseapy` package, which included the GSEA analysis and Enrichment. We have optimised the output of the package and given some better looking graph drawing functions
+# 
+# Similarly, we need to download the pathway/genesets first. Five genesets we prepare previously, you can use `ov.utils.download_pathway_database()` to download automatically. Besides, you can download the pathway you interested from enrichr: https://maayanlab.cloud/Enrichr/#libraries
+
+# In[37]:
+
+
+ov.utils.download_pathway_database()
+
+
+# In[14]:
+
+
+pathway_dict=ov.utils.geneset_prepare('genesets/WikiPathways_2019_Mouse.txt',organism='Mouse')
+
+
+# Note that the `pvalue_type` we set to `auto`, this is because when the genesets we enrichment if too small, use the `adjusted pvalue` we can't get the correct result. So you can set `adjust` or `raw` to get the significant geneset.
+# 
+# If you didn't have internet, please set `background` to all genes expressed in rna-seq,like:
+# 
+# ```python
+# enr=ov.bulk.geneset_enrichment(gene_list=deg_genes,
+#                                 pathways_dict=pathway_dict,
+#                                 pvalue_type='auto',
+#                                 background=dds.result.index.tolist(),
+#                                 organism='mouse')
+# ```
+
+# In[15]:
+
+
+deg_genes=dds.result.loc[dds.result['sig']!='normal'].index.tolist()
+enr=ov.bulk.geneset_enrichment(gene_list=deg_genes,
+                                pathways_dict=pathway_dict,
+                                pvalue_type='auto',
+                                organism='mouse')
+
+
+# To visualize the enrichment, we use `geneset_plot` to finish it
+
+# In[21]:
+
+
+ov.bulk.geneset_plot(enr,figsize=(2,5),fig_title='Wiki Pathway enrichment',
+                    cax_loc=[2, 0.45, 0.5, 0.02],
+                    bbox_to_anchor_used=(-0.25, -13),node_diameter=10,
+                     custom_ticks=[5,7],text_knock=3,
+                    cmap='Reds')
+
+
+# ## Multi pathway enrichment
+# 
+# In addition to pathway enrichment for a single database, OmicVerse supports enriching and visualizing multiple pathways at the same time, which is implemented using [`pyComplexHeatmap`](https://dingwb.github.io/PyComplexHeatmap/build/html/notebooks/gene_enrichment_analysis.html), and Citetation is welcome!
+
+# In[22]:
+
+
+pathway_dict=ov.utils.geneset_prepare('genesets/GO_Biological_Process_2023.txt',organism='Mouse')
+enr_go_bp=ov.bulk.geneset_enrichment(gene_list=deg_genes,
+                                pathways_dict=pathway_dict,
+                                pvalue_type='auto',
+                                organism='mouse')
+pathway_dict=ov.utils.geneset_prepare('genesets/GO_Molecular_Function_2023.txt',organism='Mouse')
+enr_go_mf=ov.bulk.geneset_enrichment(gene_list=deg_genes,
+                                pathways_dict=pathway_dict,
+                                pvalue_type='auto',
+                                organism='mouse')
+pathway_dict=ov.utils.geneset_prepare('genesets/GO_Cellular_Component_2023.txt',organism='Mouse')
+enr_go_cc=ov.bulk.geneset_enrichment(gene_list=deg_genes,
+                                pathways_dict=pathway_dict,
+                                pvalue_type='auto',
+                                organism='mouse')
+
+
+# In[167]:
+
+
+enr_dict={'BP':enr_go_bp,
+         'MF':enr_go_mf,
+         'CC':enr_go_cc}
+colors_dict={
+    'BP':ov.pl.red_color[1],
+    'MF':ov.pl.green_color[1],
+    'CC':ov.pl.blue_color[1],
+}
+                
+ov.bulk.geneset_plot_multi(enr_dict,colors_dict,num=3,
+                   figsize=(2,5),
+                   text_knock=3,fontsize=8,
+                    cmap='Reds'
+                  )
+
+
+# In[166]:
+
+
+def geneset_plot_multi(enr_dict,colors_dict,num:int=5,fontsize=10,
+                        fig_title:str='',fig_xlabel:str='Fractions of genes',
+                        figsize:tuple=(2,4),cmap:str='YlGnBu',
+                        text_knock:int=5,text_maxsize:int=20,ax=None,
+                        ):
+    from PyComplexHeatmap import HeatmapAnnotation,DotClustermapPlotter,anno_label,anno_simple,AnnotationBase
+    for key in enr_dict.keys():
+        enr_dict[key]['Type']=key
+    enr_all=pd.concat([enr_dict[i].iloc[:num] for i in enr_dict.keys()],axis=0)
+    enr_all['Term']=[ov.utils.plot_text_set(i.split('(')[0],text_knock=text_knock,text_maxsize=text_maxsize) for i in enr_all.Term.tolist()]
+    enr_all.index=enr_all.Term
+    enr_all['Term1']=[i for i in enr_all.index.tolist()]
+    del enr_all['Term']
+
+    colors=colors_dict
+
+    left_ha = HeatmapAnnotation(
+                          label=anno_label(enr_all.Type, merge=True,rotation=0,colors=colors,relpos=(1,0.8)),
+                          Category=anno_simple(enr_all.Type,cmap='Set1',
+                                           add_text=False,legend=False,colors=colors),
+                           axis=0,verbose=0,label_kws={'rotation':45,'horizontalalignment':'left','visible':False})
+    right_ha = HeatmapAnnotation(
+                              label=anno_label(enr_all.Term1, merge=True,rotation=0,relpos=(0,0.5),arrowprops=dict(visible=True),
+                                               colors=enr_all.assign(color=enr_all.Type.map(colors)).set_index('Term1').color.to_dict(),
+                                              fontsize=fontsize,luminance=0.8,height=2),
+                               axis=0,verbose=0,#label_kws={'rotation':45,'horizontalalignment':'left'},
+                                orientation='right')
+    if ax==None:
+        fig, ax = plt.subplots(figsize=figsize) 
+    else:
+        ax=ax
+    #plt.figure(figsize=figsize)
+    cm = DotClustermapPlotter(data=enr_all, x='fraction',y='Term1',value='logp',c='logp',s='num',
+                              cmap=cmap,
+                              row_cluster=True,#col_cluster=True,#hue='Group',
+                              #cmap={'Group1':'Greens','Group2':'OrRd'},
+                              vmin=-1*np.log10(0.1),vmax=-1*np.log10(1e-10),
+                              #colors={'Group1':'yellowgreen','Group2':'orange'},
+                              #marker={'Group1':'*','Group2':'$\\ast$'},
+                              show_rownames=True,show_colnames=False,row_dendrogram=False,
+                              col_names_side='top',row_names_side='right',
+                              xticklabels_kws={'labelrotation': 30, 'labelcolor': 'blue','labelsize':fontsize},
+                              #yticklabels_kws={'labelsize':10},
+                              #top_annotation=col_ha,left_annotation=left_ha,right_annotation=right_ha,
+                              left_annotation=left_ha,right_annotation=right_ha,
+                              spines=False,
+                              row_split=enr_all.Type,# row_split_gap=1,
+                              #col_split=df_col.Group,col_split_gap=0.5,
+                              verbose=1,legend_gap=10,
+                              #dot_legend_marker='*',
+                              
+                              xlabel='Fractions of genes',xlabel_side="bottom",
+                              xlabel_kws=dict(labelpad=8,fontweight='normal',fontsize=fontsize+2),
+                              # xlabel_bbox_kws=dict(facecolor=facecolor)
+                             )
+    tesr=plt.gcf().axes
+    for ax in plt.gcf().axes:
+        if hasattr(ax, 'get_xlabel'):
+            if ax.get_xlabel() == 'Fractions of genes':  # 假设 colorbar 有一个特定的标签
+                cbar = ax
+                cbar.grid(False)
+            if ax.get_ylabel() == 'logp':  # 假设 colorbar 有一个特定的标签
+                cbar = ax
+                cbar.tick_params(labelsize=fontsize+2)
+                cbar.set_ylabel(r'$−Log_{10}(P_{adjusted})$',fontsize=fontsize+2)
+                cbar.grid(False)
+    return ax
+
+
+# In[ ]:
+
+
+
+

ovrawm/t_deseq2.txt

@@ -0,0 +1,237 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# # Different Expression Analysis with DEseq2
+# 
+# An important task of bulk rna-seq analysis is the different expression , which we can perform with omicverse. For different expression analysis, ov change the `gene_id` to `gene_name` of matrix first. 
+# 
+# Now we can use `PyDEseq2` to perform DESeq2 analysis like R
+# 
+# Paper: [PyDESeq2: a python package for bulk RNA-seq differential expression analysis](https://www.biorxiv.org/content/10.1101/2022.12.14.520412v1)
+# 
+# Code: https://github.com/owkin/PyDESeq2
+# 
+# Colab_Reproducibility：https://colab.research.google.com/drive/1fZS-v0zdIYkXrEoIAM1X5kPoZVfVvY5h?usp=sharing
+
+# In[1]:
+
+
+import omicverse as ov
+ov.utils.ov_plot_set()
+
+
+# Note that this dataset has not been processed in any way and is only exported by `featureCounts`, and Sequence alignment was performed from the genome file of CRCm39
+
+# In[2]:
+
+
+data=ov.utils.read('https://raw.githubusercontent.com/Starlitnightly/Pyomic/master/sample/counts.txt',index_col=0,header=1)
+#replace the columns `.bam` to `` 
+data.columns=[i.split('/')[-1].replace('.bam','') for i in data.columns]
+data.head()
+
+
+# ## ID mapping
+# 
+# We performed the gene_id mapping by the mapping pair file `GRCm39` downloaded before.
+
+# In[ ]:
+
+
+ov.utils.download_geneid_annotation_pair()
+
+
+# In[3]:
+
+
+data=ov.bulk.Matrix_ID_mapping(data,'genesets/pair_GRCm39.tsv')
+data.head()
+
+
+# ## Different expression analysis with ov
+# 
+# We can do differential expression analysis very simply by ov, simply by providing an expression matrix. To run DEG, we simply need to:
+# 
+# - Read the raw count by featureCount or any other qualify methods.
+# - Create an ov DEseq object.
+
+# In[4]:
+
+
+dds=ov.bulk.pyDEG(data)
+
+
+# We notes that the gene_name mapping before exist some duplicates, we will process the duplicate indexes to retain only the highest expressed genes
+
+# In[5]:
+
+
+dds.drop_duplicates_index()
+print('... drop_duplicates_index success')
+
+
+# Now we can calculate the different expression gene from matrix, we need to input the treatment and control groups
+
+# In[6]:
+
+
+treatment_groups=['4-3','4-4']
+control_groups=['1--1','1--2']
+result=dds.deg_analysis(treatment_groups,control_groups,method='DEseq2')
+
+
+# One important thing is that we do not filter out low expression genes when processing DEGs, and in future versions I will consider building in the corresponding processing.
+
+# In[7]:
+
+
+print(result.shape)
+result=result.loc[result['log2(BaseMean)']>1]
+print(result.shape)
+
+
+# We also need to set the threshold of Foldchange, we prepare a method named `foldchange_set` to finish. This function automatically calculates the appropriate threshold based on the log2FC distribution, but you can also enter it manually.
+
+# In[8]:
+
+
+# -1 means automatically calculates
+dds.foldchange_set(fc_threshold=-1,
+                   pval_threshold=0.05,
+                   logp_max=10)
+
+
+# ## Visualize the DEG result and specific genes
+# 
+# To visualize the DEG result, we use `plot_volcano` to do it. This fuction can visualize the gene interested or high different expression genes. There are some parameters you need to input:
+# 
+# - title: The title of volcano
+# - figsize: The size of figure
+# - plot_genes: The genes you interested
+# - plot_genes_num: If you don't have interested genes, you can auto plot it.
+
+# In[9]:
+
+
+dds.plot_volcano(title='DEG Analysis',figsize=(4,4),
+                 plot_genes_num=8,plot_genes_fontsize=12,)
+
+
+# To visualize the specific genes, we only need to use the `dds.plot_boxplot` function to finish it.
+
+# In[10]:
+
+
+dds.plot_boxplot(genes=['Ckap2','Lef1'],treatment_groups=treatment_groups,
+                control_groups=control_groups,figsize=(2,3),fontsize=12,
+                 legend_bbox=(2,0.55))
+
+
+# In[11]:
+
+
+dds.plot_boxplot(genes=['Ckap2'],treatment_groups=treatment_groups,
+                control_groups=control_groups,figsize=(2,3),fontsize=12,
+                 legend_bbox=(2,0.55))
+
+
+# ## Pathway enrichment analysis by Pyomic
+# 
+# Here we use the `gseapy` package, which included the GSEA analysis and Enrichment. We have optimised the output of the package and given some better looking graph drawing functions
+# 
+# Similarly, we need to download the pathway/genesets first. Five genesets we prepare previously, you can use `Pyomic.utils.download_pathway_database()` to download automatically. Besides, you can download the pathway you interested from enrichr: https://maayanlab.cloud/Enrichr/#libraries
+
+# In[13]:
+
+
+ov.utils.download_pathway_database()
+
+
+# In[14]:
+
+
+pathway_dict=ov.utils.geneset_prepare('genesets/WikiPathways_2019_Mouse.txt',organism='Mouse')
+
+
+# To perform the GSEA analysis, we need to ranking the genes at first. Using `dds.ranking2gsea` can obtain a ranking gene's matrix sorted by -log10(padj).
+# 
+# $Metric=\frac{-log_{10}(padj)}{sign(log2FC)}$
+
+# In[15]:
+
+
+rnk=dds.ranking2gsea()
+
+
+# We used `ov.bulk.pyGSEA` to construst a GSEA object to perform enrichment.
+
+# In[16]:
+
+
+gsea_obj=ov.bulk.pyGSEA(rnk,pathway_dict)
+
+
+# In[17]:
+
+
+enrich_res=gsea_obj.enrichment()
+
+
+# The results are stored in the `enrich_res` attribute.
+
+# In[18]:
+
+
+gsea_obj.enrich_res.head()
+
+
+# To visualize the enrichment, we use `plot_enrichment` to do.
+# - num: The number of enriched terms to plot. Default is 10.
+# - node_size: A list of integers defining the size of nodes in the plot. Default is [5,10,15].
+# - cax_loc: The location of the colorbar on the plot. Default is 2.
+# - cax_fontsize: The fontsize of the colorbar label. Default is 12.
+# - fig_title: The title of the plot. Default is an empty string.
+# - fig_xlabel: The label of the x-axis. Default is 'Fractions of genes'.
+# - figsize: The size of the plot. Default is (2,4).
+# - cmap: The colormap to use for the plot. Default is 'YlGnBu'.
+
+# In[19]:
+
+
+gsea_obj.plot_enrichment(num=10,node_size=[10,20,30],
+                        cax_fontsize=12,
+                        fig_title='Wiki Pathway Enrichment',fig_xlabel='Fractions of genes',
+                        figsize=(2,4),cmap='YlGnBu',
+                        text_knock=2,text_maxsize=30,
+                        cax_loc=[2.5, 0.45, 0.5, 0.02],
+                          bbox_to_anchor_used=(-0.25, -13),node_diameter=10,)
+
+
+# Not only the basic analysis, pyGSEA also help us to visualize the term with Ranked and Enrichment Score. 
+# 
+# We can select the number of term to plot, which stored in `gsea_obj.enrich_res.index`, the `0` is `Complement and Coagulation Cascades WP449` and the `1` is `Matrix Metalloproteinases WP441`
+
+# In[20]:
+
+
+gsea_obj.enrich_res.index[:5]
+
+
+# We can set the `gene_set_title` to change the title of GSEA plot
+
+# In[22]:
+
+
+fig=gsea_obj.plot_gsea(term_num=1,
+                  gene_set_title='Matrix Metalloproteinases',
+                  figsize=(3,4),
+                  cmap='RdBu_r',
+                  title_fontsize=14,
+                  title_y=0.95)
+
+
+# In[ ]:
+
+
+
+

ovrawm/t_gptanno.txt

@@ -0,0 +1,330 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# # Automatic cell type annotation with GPT/Other
+# 
+# GPTCelltype, an open-source R software package to facilitate cell type annotation by GPT-4. 
+# 
+# We made three improvements in integrating the `GPTCelltype` algorithm in OmicVerse:
+# 
+# - **Native support for Python**: Since GPTCelltype is an R language package, in order to make it conform to scverse's anndata ecosystem, we have rewritten the whole function so that it works perfectly under Python.
+# - **More model support**: We provide more big models to choose from outside of Openai, e.g. Qwen(通义千问), Kimi, and more model support is available through the parameter `base_url` or `model_name`.
+# 
+# If you found this tutorial helpful, please cite GPTCelltype and OmicVerse:
+# 
+# Hou, W. and Ji, Z., 2023. Reference-free and cost-effective automated cell type annotation with GPT-4 in single-cell RNA-seq analysis. [Nature Methods, 2024 March 25](https://link.springer.com/article/10.1038/s41592-024-02235-4?utm_source=rct_congratemailt&utm_medium=email&utm_campaign=oa_20240325&utm_content=10.1038/s41592-024-02235-4).
+
+# In[3]:
+
+
+import omicverse as ov
+print(f'omicverse version:{ov.__version__}')
+import scanpy as sc
+print(f'scanpy version:{sc.__version__}')
+ov.ov_plot_set()
+
+
+# ## Loading data
+# 
+# The data consist of 3k PBMCs from a Healthy Donor and are freely available from 10x Genomics ([here](http://cf.10xgenomics.com/samples/cell-exp/1.1.0/pbmc3k/pbmc3k_filtered_gene_bc_matrices.tar.gz) from this [webpage](https://support.10xgenomics.com/single-cell-gene-expression/datasets/1.1.0/pbmc3k)). On a unix system, you can uncomment and run the following to download and unpack the data. The last line creates a directory for writing processed data.
+# 
+
+# In[21]:
+
+
+# !mkdir data
+# !wget http://cf.10xgenomics.com/samples/cell-exp/1.1.0/pbmc3k/pbmc3k_filtered_gene_bc_matrices.tar.gz -O data/pbmc3k_filtered_gene_bc_matrices.tar.gz
+# !cd data; tar -xzf pbmc3k_filtered_gene_bc_matrices.tar.gz
+# !mkdir write
+
+
+# Read in the count matrix into an AnnData object, which holds many slots for annotations and different representations of the data. It also comes with its own HDF5-based file format: `.h5ad`.
+
+# In[3]:
+
+
+adata = sc.read_10x_mtx(
+    'data/filtered_gene_bc_matrices/hg19/',  # the directory with the `.mtx` file
+    var_names='gene_symbols',                # use gene symbols for the variable names (variables-axis index)
+    cache=True)                              # write a cache file for faster subsequent reading
+
+
+# ## Data preprocessing
+# 
+# Here, we use `ov.single.scanpy_lazy` to preprocess the raw data of scRNA-seq, it included filter the doublets cells, normalizing counts per cell, log1p, extracting highly variable genes, and cluster of cells calculation. 
+# 
+# But if you want to experience step-by-step preprocessing, we also provide more detailed preprocessing steps here, please refer to our [preprocess chapter](https://omicverse.readthedocs.io/en/latest/Tutorials-single/t_preprocess/) for a detailed explanation.
+# 
+# We stored the raw counts in `count` layers, and the raw data in `adata.raw.to_adata()`.
+
+# In[ ]:
+
+
+#adata=ov.single.scanpy_lazy(adata)
+
+#quantity control
+adata=ov.pp.qc(adata,
+              tresh={'mito_perc': 0.05, 'nUMIs': 500, 'detected_genes': 250})
+#normalize and high variable genes (HVGs) calculated
+adata=ov.pp.preprocess(adata,mode='shiftlog|pearson',n_HVGs=2000,)
+
+#save the whole genes and filter the non-HVGs
+adata.raw = adata
+adata = adata[:, adata.var.highly_variable_features]
+
+#scale the adata.X
+ov.pp.scale(adata)
+
+#Dimensionality Reduction
+ov.pp.pca(adata,layer='scaled',n_pcs=50)
+
+#Neighbourhood graph construction
+sc.pp.neighbors(adata, n_neighbors=15, n_pcs=50,
+               use_rep='scaled|original|X_pca')
+
+#clusters
+sc.tl.leiden(adata)
+
+#find marker
+sc.tl.dendrogram(adata,'leiden',use_rep='scaled|original|X_pca')
+sc.tl.rank_genes_groups(adata, 'leiden', use_rep='scaled|original|X_pca',
+                        method='wilcoxon',use_raw=False,)
+
+#Dimensionality Reduction for visualization(X_mde=X_umap+GPU)
+adata.obsm["X_mde"] = ov.utils.mde(adata.obsm["scaled|original|X_pca"])
+adata
+
+
+# In[5]:
+
+
+ov.pl.embedding(adata,
+                   basis='X_mde',
+                   color=['leiden'], 
+                   legend_loc='on data', 
+                   frameon='small',
+                   legend_fontoutline=2,
+                   palette=ov.utils.palette()[14:],
+                  )
+
+
+# ## GPT Celltype
+# 
+# gptcelltype supports dictionary format input, we provide `omicverse.single.get_celltype_marker` to get the marker genes for each cell type as a dictionary.
+
+# #### Using genes manually
+# 
+# We can manually define a dictionary to determine the accuracy of the output
+
+# In[25]:
+
+
+import os
+all_markers={'cluster1':['CD3D','CD3E'],
+            'cluster2':['MS4A1']}
+
+os.environ['AGI_API_KEY'] = 'sk-**'  # Replace with your actual API key
+result = ov.single.gptcelltype(all_markers, tissuename='PBMC', speciename='human',
+                      model='qwen-plus', provider='qwen',
+                      topgenenumber=5)
+result
+
+
+# #### Get Genes for Each Cluster Automatically
+# 
+
+# In[14]:
+
+
+all_markers=ov.single.get_celltype_marker(adata,clustertype='leiden',rank=True,
+                                          key='rank_genes_groups',
+                                          foldchange=2,topgenenumber=5)
+all_markers
+
+
+# ### Option 1. Through OpenAI API (`provider` or `base_url`)
+# 
+# Use `ov.single.gptcelltype` function to annotate cell types.
+# 
+# You can simply set `provider` (or `base_url`) and `provider` parameters to provide the function with base url and the exact model which are required for model calling.
+
+# In[17]:
+
+
+import os
+os.environ['AGI_API_KEY'] = 'sk-**'  # Replace with your actual API key
+result = ov.single.gptcelltype(all_markers, tissuename='PBMC', speciename='human',
+                      model='qwen-plus', provider='qwen',
+                      topgenenumber=5)
+result
+
+
+# We can keep only the cell type of the output and remove other irrelevant information.
+
+# In[18]:
+
+
+new_result={}
+for key in result.keys():
+    new_result[key]=result[key].split(': ')[-1].split(' (')[0].split('. ')[1]
+new_result
+
+
+# In[19]:
+
+
+adata.obs['gpt_celltype'] = adata.obs['leiden'].map(new_result).astype('category')
+
+
+# In[20]:
+
+
+ov.pl.embedding(adata,
+                   basis='X_mde',
+                   color=['leiden','gpt_celltype'], 
+                   legend_loc='on data', 
+                   frameon='small',
+                   legend_fontoutline=2,
+                   palette=ov.utils.palette()[14:],
+                  )
+
+
+# ## More models
+# 
+# Our implementation of `gptcelltype` in `omicverse` supports almost all large language models that support the `openai` api format.
+
+# In[27]:
+
+
+all_markers={'cluster1':['CD3D','CD3E'],
+            'cluster2':['MS4A1']}
+
+
+# ### Openai
+# 
+# The OpenAI API uses API keys for authentication. You can create API keys at a user or service account level. Service accounts are tied to a "bot" individual and should be used to provision access for production systems. Each API key can be scoped to one of the following,
+# 
+# - [User keys](https://platform.openai.com/account/api-keys) - Our legacy keys. Provides access to all organizations and all projects that user has been added to; access API Keys to view your available keys. We highly advise transitioning to project keys for best security practices, although access via this method is currently still supported.
+# 
+# - Please select the model you need to use: [list of supported models](https://platform.openai.com/docs/models).
+# 
+
+# In[28]:
+
+
+os.environ['AGI_API_KEY'] = 'sk-**'  # Replace with your actual API key
+result = ov.single.gptcelltype(all_markers, tissuename='PBMC', speciename='human',
+                      model='gpt-4o', provider='openai',
+                      topgenenumber=5)
+result
+
+
+# ### Qwen(通义千问)
+# 
+# - Enabled DashScope service and obtained API-KEY: [Enabled DashScope and created API-KEY](https://help.aliyun.com/zh/dashscope/developer-reference/activate-dashscope-and-create-an-api-key).
+# 
+# - We recommend you to configure API-KEY in environment variable to reduce the risk of API-KEY leakage, please refer to Configuring API-KEY through Environment Variable for the configuration method, you can also configure API-KEY in code, but the risk of leakage will be increased.
+# 
+# - Please select the model you need to use: [list of supported models](https://help.aliyun.com/zh/dashscope/developer-reference/compatibility-of-openai-with-dashscope/?spm=a2c4g.11186623.0.i6#eadfc13038jd5).
+# 
+# **简体中文**
+# 
+# - 已开通灵积模型服务并获得API-KEY：[开通DashScope并创建API-KEY](https://help.aliyun.com/zh/dashscope/developer-reference/activate-dashscope-and-create-an-api-key)。
+# 
+# - 我们推荐您将API-KEY配置到环境变量中以降低API-KEY的泄漏风险，配置方法可参考通过环境变量配置API-KEY。您也可以在代码中配置API-KEY，但是泄漏风险会提高。
+# 
+# - 请选择您需要使用的模型：[支持的模型列表](https://help.aliyun.com/zh/dashscope/developer-reference/compatibility-of-openai-with-dashscope/?spm=a2c4g.11186623.0.i6#eadfc13038jd5)。
+# 
+
+# In[26]:
+
+
+os.environ['AGI_API_KEY'] = 'sk-**'  # Replace with your actual API key
+result = ov.single.gptcelltype(all_markers, tissuename='PBMC', speciename='human',
+                      model='qwen-plus', provider='qwen',
+                      topgenenumber=5)
+result
+
+
+# ### Kimi(月之暗面)
+# 
+# - You will need a Dark Side of the Moon API key to use our service. You can create an API key in [Console](https://platform.moonshot.cn/console).
+# 
+# - Please select the model you need to use: [List of supported models](https://platform.moonshot.cn/docs/pricing#%E4%BA%A7%E5%93%81%E5%AE%9A%E4%BB%B7)
+# 
+# **简体中文**
+# 
+# - 你需要一个 月之暗面的 API 密钥��使用我们的服务。你可以在[控制台](https://platform.moonshot.cn/console)中创建一个 API 密钥。
+# 
+# - 请选择您需要使用的模型：[支持的模型列表](https://platform.moonshot.cn/docs/pricing#%E4%BA%A7%E5%93%81%E5%AE%9A%E4%BB%B7)
+
+# In[28]:
+
+
+os.environ['AGI_API_KEY'] = 'sk-**'  # Replace with your actual API key
+result = ov.single.gptcelltype(all_markers, tissuename='PBMC', speciename='human',
+                      model='moonshot-v1-8k', provider='kimi',
+                      topgenenumber=5)
+result
+
+
+# #### Other Models
+# 
+# You can manually set the `base_url` parameter to specify other models that need to be used, note that the model needs to support Openai's parameters. Three examples are provided here (when you specify the `base_url` parameter, the `provider` parameter will be invalid):
+# 
+# ```python
+# if provider == 'openai':
+#     base_url = "https://api.openai.com/v1/"
+# elif provider == 'kimi':
+#     base_url = "https://api.moonshot.cn/v1"
+# elif provider == 'qwen':
+#     base_url = "https://dashscope.aliyuncs.com/compatible-mode/v1"
+# ```
+
+# In[30]:
+
+
+os.environ['AGI_API_KEY'] = 'sk-**'  # Replace with your actual API key
+result = ov.single.gptcelltype(all_markers, tissuename='PBMC', speciename='human',
+                      model='moonshot-v1-8k', base_url="https://api.moonshot.cn/v1",
+                      topgenenumber=5)
+result
+
+
+# ### Option 2. Through Local LLM (`model_name`)
+
+# Use `ov.single.gptcelltype_local` function to annotate cell types.
+# 
+# You can simply set the `model_name` parameter.
+
+# In[5]:
+
+
+anno_model = 'path/to/your/local/LLM'  # '~/models/Qwen2-7B-Instruct'
+
+result = ov.single.gptcelltype_local(all_markers, tissuename='PBMC', speciename='human', 
+                     model_name=anno_model, topgenenumber=5)
+result
+
+
+# Note that you may encounter network problems that prevent you from downloading LLMs.
+# 
+# In this case, please refer to https://zhuanlan.zhihu.com/p/663712983
+
+# In[ ]:
+
+
+
+
+
+# In[ ]:
+
+
+
+
+
+# In[ ]:
+
+
+
+

ovrawm/t_mapping.txt

@@ -0,0 +1,191 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# # Mapping single-cell profile onto spatial profile
+# 
+# Tangram is a method for mapping single-cell (or single-nucleus) gene expression data onto spatial gene expression data. Tangram takes as input a single-cell dataset and a spatial dataset, collected from the same anatomical region/tissue type. Via integration, Tangram creates new spatial data by aligning the scRNAseq profiles in space. This allows to project every annotation in the scRNAseq (e.g. cell types, program usage) on space.
+# 
+# The most common application of Tangram is to resolve cell types in space. Another usage is to correct gene expression from spatial data: as scRNA-seq data are less prone to dropout than (e.g.) Visium or Slide-seq, the “new” spatial data generated by Tangram resolve many more genes. As a result, we can visualize program usage in space, which can be used for ligand-receptor pair discovery or, more generally, cell-cell communication mechanisms. If cell segmentation is available, Tangram can be also used for deconvolution of spatial data. If your single cell are multimodal, Tangram can be used to spatially resolve other modalities, such as chromatin accessibility.
+# 
+# Biancalani, T., Scalia, G., Buffoni, L. et al. Deep learning and alignment of spatially resolved single-cell transcriptomes with Tangram. Nat Methods 18, 1352–1362 (2021). https://doi.org/10.1038/s41592-021-01264-7
+# 
+# ![img](https://tangram-sc.readthedocs.io/en/latest/_images/tangram_overview.png)
+
+# In[1]:
+
+
+import omicverse as ov
+#print(f"omicverse version: {ov.__version__}")
+import scanpy as sc
+#print(f"scanpy version: {sc.__version__}")
+ov.utils.ov_plot_set()
+
+
+# ## Prepared scRNA-seq
+# 
+# Published scRNA-seq datasets of lymph nodes have typically lacked an adequate representation of germinal centre-associated immune cell populations due to age of patient donors. We, therefore, include scRNA-seq datasets spanning lymph nodes, spleen and tonsils in our single-cell reference to ensure that we captured the full diversity of immune cell states likely to exist in the spatial transcriptomic dataset.
+# 
+# Here we download this dataset, import into anndata and change variable names to ENSEMBL gene identifiers.
+# 
+# Link: https://cell2location.cog.sanger.ac.uk/paper/integrated_lymphoid_organ_scrna/RegressionNBV4Torch_57covariates_73260cells_10237genes/sc.h5ad
+
+# In[2]:
+
+
+adata_sc=ov.read('data/sc.h5ad')
+import matplotlib.pyplot as plt
+fig, ax = plt.subplots(figsize=(3,3))
+ov.utils.embedding(
+    adata_sc,
+    basis="X_umap",
+    color=['Subset'],
+    title='Subset',
+    frameon='small',
+    #ncols=1,
+    wspace=0.65,
+    #palette=ov.utils.pyomic_palette()[11:],
+    show=False,
+    ax=ax
+)
+
+
+# For data quality control and preprocessing, we can easily use omicverse's own preprocessing functions to do so
+
+# In[3]:
+
+
+print("RAW",adata_sc.X.max())
+adata_sc=ov.pp.preprocess(adata_sc,mode='shiftlog|pearson',n_HVGs=3000,target_sum=1e4)
+adata_sc.raw = adata_sc
+adata_sc = adata_sc[:, adata_sc.var.highly_variable_features]
+print("Normalize",adata_sc.X.max())
+
+
+# ## Prepared stRNA-seq
+# 
+# First let’s read spatial Visium data from 10X Space Ranger output. Here we use lymph node data generated by 10X and presented in [Kleshchevnikov et al (section 4, Fig 4)](https://www.biorxiv.org/content/10.1101/2020.11.15.378125v1). This dataset can be conveniently downloaded and imported using scanpy. See [this tutorial](https://cell2location.readthedocs.io/en/latest/notebooks/cell2location_short_demo.html) for a more extensive and practical example of data loading (multiple visium samples).
+
+# In[5]:
+
+
+adata = sc.datasets.visium_sge(sample_id="V1_Human_Lymph_Node")
+adata.obs['sample'] = list(adata.uns['spatial'].keys())[0]
+adata.var_names_make_unique()
+
+
+# We used the same pre-processing steps as for scRNA-seq
+# 
+# <div class="admonition warning">
+#   <p class="admonition-title">Note</p>
+#   <p>
+#     We introduced the spatial special svg calculation module prost in omicverse versions greater than `1.6.0` to replace scanpy's HVGs, if you want to use scanpy's HVGs you can set mode=`scanpy` in `ov.space.svg` or use the following code.
+#   </p>
+# </div>
+# 
+# ```python
+# #adata=ov.pp.preprocess(adata,mode='shiftlog|pearson',n_HVGs=3000,target_sum=1e4)
+# #adata.raw = adata
+# #adata = adata[:, adata.var.highly_variable_features]
+# ```
+
+# In[6]:
+
+
+sc.pp.calculate_qc_metrics(adata, inplace=True)
+adata = adata[:,adata.var['total_counts']>100]
+adata=ov.space.svg(adata,mode='prost',n_svgs=3000,target_sum=1e4,platform="visium",)
+adata.raw = adata
+adata = adata[:, adata.var.space_variable_features]
+adata_sp=adata.copy()
+adata_sp
+
+
+# ## Tangram model
+# 
+# Tangram is a Python package, written in PyTorch and based on scanpy, for mapping single-cell (or single-nucleus) gene expression data onto spatial gene expression data. The single-cell dataset and the spatial dataset should be collected from the same anatomical region/tissue type, ideally from a biological replicate, and need to share a set of genes. 
+# 
+# We can use `omicverse.space.Tangram` to apply the Tangram model.
+
+# In[7]:
+
+
+tg=ov.space.Tangram(adata_sc,adata_sp,clusters='Subset')
+
+
+# The function maps iteratively as specified by num_epochs. We typically interrupt mapping after the score plateaus. 
+# - The score measures the similarity between the gene expression of the mapped cells vs spatial data on the training genes. 
+# - The default mapping mode is mode=`cells`, which is recommended to run on a GPU. 
+# - Alternatively, one can specify mode=`clusters` which averages the single cells beloning to the same cluster (pass annotations via cluster_label). This is faster, and is our chioce when scRNAseq and spatial data come from different specimens. 
+# - If you wish to run Tangram with a GPU, set device=`cuda:0` otherwise use the set device=`cpu`. 
+# - density_prior specifies the cell density within each spatial voxel. Use uniform if the spatial voxels are at single cell resolution (ie MERFISH). The default value, rna_count_based, assumes that cell density is proportional to the number of RNA molecules
+
+# In[8]:
+
+
+tg.train(mode="clusters",num_epochs=500,device="cuda:0")
+
+
+# We can use `tg.cell2location()` to get the cell location in spatial spots.
+
+# In[9]:
+
+
+adata_plot=tg.cell2location()
+adata_plot.obs.columns
+
+
+# In[10]:
+
+
+annotation_list=['B_Cycling', 'B_GC_LZ', 'T_CD4+_TfH_GC', 'FDC',
+                         'B_naive', 'T_CD4+_naive', 'B_plasma', 'Endo']
+
+sc.pl.spatial(adata_plot, cmap='magma',
+                  # show first 8 cell types
+                  color=annotation_list,
+                  ncols=4, size=1.3,
+                  img_key='hires',
+                  # limit color scale at 99.2% quantile of cell abundance
+                  #vmin=0, vmax='p99.2'
+                 )
+
+
+# In[11]:
+
+
+color_dict=dict(zip(adata_sc.obs['Subset'].cat.categories,
+                   adata_sc.uns['Subset_colors']))
+
+
+# In[21]:
+
+
+import matplotlib as mpl
+clust_labels = annotation_list[:5]
+clust_col = ['' + str(i) for i in clust_labels] # in case column names differ from labels
+
+with mpl.rc_context({'figure.figsize': (8, 8),'axes.grid': False}):
+    fig = ov.pl.plot_spatial(
+        adata=adata_plot,
+        # labels to show on a plot
+        color=clust_col, labels=clust_labels,
+        show_img=True,
+        # 'fast' (white background) or 'dark_background'
+        style='fast',
+        # limit color scale at 99.2% quantile of cell abundance
+        max_color_quantile=0.992,
+        # size of locations (adjust depending on figure size)
+        circle_diameter=3,
+        reorder_cmap = [#0,
+            1,2,3,4,6], #['yellow', 'orange', 'blue', 'green', 'purple', 'grey', 'white'],
+        colorbar_position='right',
+        palette=color_dict
+    )
+    
+
+
+# In[ ]:
+
+
+
+

ovrawm/t_metacells.txt

@@ -0,0 +1,249 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# # Inference of MetaCell from Single-Cell RNA-seq
+# 
+# Metacells are cell groupings derived from single-cell sequencing data that represent highly granular, distinct cell states. Here, we present single-cell aggregation of cell-states (SEACells), an algorithm for identifying metacells; overcoming the sparsity of single-cell data, while retaining heterogeneity obscured by traditional cell clustering.
+# 
+# Paper: [SEACells: Inference of transcriptional and epigenomic cellular states from single-cell genomics data](https://www.nature.com/articles/s41587-023-01716-9)
+# 
+# Code: https://github.com/dpeerlab/SEACells
+# 
+
+# In[1]:
+
+
+import omicverse as ov
+import scanpy as sc
+import scvelo as scv
+
+ov.plot_set()
+
+
+# ## Data preprocessed
+# 
+# We need to normalized and scale the data at first.
+
+# In[3]:
+
+
+adata = scv.datasets.pancreas()
+adata
+
+
+# In[4]:
+
+
+#quantity control
+adata=ov.pp.qc(adata,
+              tresh={'mito_perc': 0.20, 'nUMIs': 500, 'detected_genes': 250},
+              mt_startswith='mt-')
+#normalize and high variable genes (HVGs) calculated
+adata=ov.pp.preprocess(adata,mode='shiftlog|pearson',n_HVGs=2000,)
+
+#save the whole genes and filter the non-HVGs
+adata.raw = adata
+adata = adata[:, adata.var.highly_variable_features]
+
+#scale the adata.X
+ov.pp.scale(adata)
+
+#Dimensionality Reduction
+ov.pp.pca(adata,layer='scaled',n_pcs=50)
+
+
+# ## Constructing a metacellular object
+# 
+# We can use `ov.single.MetaCell` to construct a metacellular object to train the SEACells model, the arguments can be found in below.
+# 
+# - :param ad: (AnnData) annotated data matrix
+# - :param build_kernel_on: (str) key corresponding to matrix in ad.obsm which is used to compute kernel for metacells
+#                         Typically 'X_pca' for scRNA or 'X_svd' for scATAC
+# - :param n_SEACells: (int) number of SEACells to compute
+# - :param use_gpu: (bool) whether to use GPU for computation
+# - :param verbose: (bool) whether to suppress verbose program logging
+# - :param n_waypoint_eigs: (int) number of eigenvectors to use for waypoint initialization
+# - :param n_neighbors: (int) number of nearest neighbors to use for graph construction
+# - :param convergence_epsilon: (float) convergence threshold for Franke-Wolfe algorithm
+# - :param l2_penalty: (float) L2 penalty for Franke-Wolfe algorithm
+# - :param max_franke_wolfe_iters: (int) maximum number of iterations for Franke-Wolfe algorithm
+# - :param use_sparse: (bool) whether to use sparse matrix operations. Currently only supported for CPU implementation.
+
+# In[5]:
+
+
+meta_obj=ov.single.MetaCell(adata,use_rep='scaled|original|X_pca',
+                            n_metacells=None,
+                           use_gpu='cuda:0')
+
+
+# In[6]:
+
+
+get_ipython().run_cell_magic('time', '', 'meta_obj.initialize_archetypes()\n')
+
+
+# ## Train and save the model
+
+# In[7]:
+
+
+get_ipython().run_cell_magic('time', '', 'meta_obj.train(min_iter=10, max_iter=50)\n')
+
+
+# In[9]:
+
+
+meta_obj.save('seacells/model.pkl')
+
+
+# In[6]:
+
+
+meta_obj.load('seacells/model.pkl')
+
+
+# ## Predicted the metacells
+# 
+# we can use `predicted` to predicted the metacells of raw scRNA-seq data. There are two method can be selected, one is `soft`, the other is `hard`. 
+# 
+# In the `soft` method, Aggregates cells within each SEACell, summing over all raw data x assignment weight for all cells belonging to a SEACell. Data is un-normalized and pseudo-raw aggregated counts are stored in .layers['raw']. Attributes associated with variables (.var) are copied over, but relevant per SEACell attributes must be manually copied, since certain attributes may need to be summed, or averaged etc, depending on the attribute.
+# 
+# In the `hard` method, Aggregates cells within each SEACell, summing over all raw data for all cells belonging to a SEACell. Data is unnormalized and raw aggregated counts are stored .layers['raw']. Attributes associated with variables (.var) are copied over, but relevant per SEACell attributes must be manually copied, since certain attributes may need to be summed, or averaged etc, depending on the attribute.
+
+# In[10]:
+
+
+ad=meta_obj.predicted(method='soft',celltype_label='clusters',
+                     summarize_layer='lognorm')
+
+
+# ## Benchmarking
+# 
+# Benchmarking metrics were computed for each metacell for all combinations of data modality, dataset and method. Cell type purity was used to assess the quality of PBMC metacells. Methods were compared using the Wilcoxon rank-sum test. We note that this test might possibly inflate significance due to the dependency between metacells, but it nonetheless provides an estimate of the direction of difference. Top-performing metacell approaches should have scores that are low on compactness, high on separation and high on cell type purity 
+
+# In[11]:
+
+
+SEACell_purity = meta_obj.compute_celltype_purity('clusters')
+separation = meta_obj.separation(use_rep='scaled|original|X_pca',nth_nbr=1)
+compactness = meta_obj.compactness(use_rep='scaled|original|X_pca')
+
+
+# In[12]:
+
+
+import seaborn as sns
+import matplotlib.pyplot as plt
+ov.plot_set()
+fig, axes = plt.subplots(1,3,figsize=(4,4))
+sns.boxplot(data=SEACell_purity, y='clusters_purity',ax=axes[0],
+           color=ov.utils.blue_color[3])
+sns.boxplot(data=compactness, y='compactness',ax=axes[1],
+           color=ov.utils.blue_color[4])
+sns.boxplot(data=separation, y='separation',ax=axes[2],
+           color=ov.utils.blue_color[4])
+plt.tight_layout()
+plt.suptitle('Evaluate of MetaCells',fontsize=13,y=1.05)
+for ax in axes:
+    ax.grid(False)
+    ax.spines['top'].set_visible(False)
+    ax.spines['right'].set_visible(False)
+    ax.spines['bottom'].set_visible(True)
+    ax.spines['left'].set_visible(True)
+
+
+# In[13]:
+
+
+import matplotlib.pyplot as plt
+fig, ax = plt.subplots(figsize=(4,4))
+ov.pl.embedding(
+    meta_obj.adata,
+    basis="X_umap",
+    color=['clusters'],
+    frameon='small',
+    title="Meta cells",
+    #legend_loc='on data',
+    legend_fontsize=14,
+    legend_fontoutline=2,
+    size=10,
+    ax=ax,
+    alpha=0.2,
+    #legend_loc='', 
+    add_outline=False, 
+    #add_outline=True,
+    outline_color='black',
+    outline_width=1,
+    show=False,
+    #palette=ov.utils.blue_color[:],
+    #legend_fontweight='normal'
+)
+ov.single.plot_metacells(ax,meta_obj.adata,color='#CB3E35',
+                                  )
+
+
+# ## Get the raw obs value from adata
+# 
+# There are times when we compute some floating point type data such as pseudotime on the raw single cell data. We want to get the result of the original data on the metacell, in this case, we can use the `ov.single` function to get it.
+# 
+# Note that the type parameter supports `str`,`max`,`min`,`mean`.
+
+# In[14]:
+
+
+ov.single.get_obs_value(ad,adata,groupby='S_score',
+                       type='mean')
+ad.obs.head()
+
+
+# ## Visualize the MetaCells
+
+# In[15]:
+
+
+import scanpy as sc
+ad.raw=ad.copy()
+sc.pp.highly_variable_genes(ad, n_top_genes=2000, inplace=True)
+ad=ad[:,ad.var.highly_variable]
+
+
+# In[16]:
+
+
+ov.pp.scale(ad)
+ov.pp.pca(ad,layer='scaled',n_pcs=30)
+ov.pp.neighbors(ad, n_neighbors=15, n_pcs=20,
+               use_rep='scaled|original|X_pca')
+
+
+# In[17]:
+
+
+ov.pp.umap(ad)
+
+
+# We want the metacells to take on the same colours as the original data, a noteworthy fact is that the colours of the original data are stored in the `adata.uns['_colors']`
+
+# In[18]:
+
+
+ad.obs['celltype']=ad.obs['celltype'].astype('category')
+ad.obs['celltype']=ad.obs['celltype'].cat.reorder_categories(adata.obs['clusters'].cat.categories)
+ad.uns['celltype_colors']=adata.uns['clusters_colors']
+
+
+# In[19]:
+
+
+ov.pl.embedding(ad, basis='X_umap',
+                color=["celltype","S_score"],
+                frameon='small',cmap='RdBu_r',
+               wspace=0.5)
+
+
+# In[ ]:
+
+
+
+

ovrawm/t_metatime.txt

@@ -0,0 +1,177 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# # Celltype auto annotation with MetaTiME
+# 
+# MetaTiME learns data-driven, interpretable, and reproducible gene programs by integrating millions of single cells from hundreds of tumor scRNA-seq data. The idea is to learn a map of single-cell space with biologically meaningful directions from large-scale data, which helps understand functional cell states and transfers knowledge to new data analysis. MetaTiME provides pretrained meta-components (MeCs) to automatically annotate fine-grained cell states and plot signature continuum for new single-cells of tumor microenvironment.
+# 
+# Here, we integrate MetaTiME in omicverse. This tutorial demonstrates how to use [MetaTiME (original code)](https://github.com/yi-zhang/MetaTiME/blob/main/docs/notebooks/metatime_annotator.ipynb) to annotate celltype in TME
+# 
+# Paper: [MetaTiME integrates single-cell gene expression to characterize the meta-components of the tumor immune microenvironment](https://www.nature.com/articles/s41467-023-38333-8)
+# 
+# Code: https://github.com/yi-zhang/MetaTiME
+# 
+# Colab_Reproducibility：https://colab.research.google.com/drive/1isvjTfSFM2cy6GzHWAwbuvSjveEJijzP?usp=sharing
+# 
+# ![metatime](https://media.springernature.com/full/springer-static/image/art%3A10.1038%2Fs41467-023-38333-8/MediaObjects/41467_2023_38333_Fig1_HTML.png)
+
+# In[1]:
+
+
+import omicverse as ov
+ov.utils.ov_plot_set()
+
+
+# ## Data normalize and Batch remove
+# 
+# The sample data has multiple patients , and we can use batch correction on patients. Here, we using [scVI](https://docs.scvi-tools.org/en/stable/) to remove batch.
+# 
+# <div class="admonition warning">
+#   <p class="admonition-title">Note</p>
+#   <p>
+#     If your data contains count matrix, we provide a wrapped function for pre-processing the data. Otherwise, if the data is already depth-normalized, log-transformed, and cells are filtered, we can skip this step.
+#   </p>
+# </div>
+
+# In[ ]:
+
+
+'''
+import scvi
+scvi.model.SCVI.setup_anndata(adata, layer="counts", batch_key="patient")
+vae = scvi.model.SCVI(adata, n_layers=2, n_latent=30, gene_likelihood="nb")
+vae.train()
+adata.obsm["X_scVI"] = vae.get_latent_representation()
+'''
+
+
+# Example data can be obtained from figshare: https://figshare.com/ndownloader/files/41440050
+
+# In[2]:
+
+
+import scanpy as sc
+adata=sc.read('TiME_adata_scvi.h5ad')
+adata
+
+
+# It is recommended that malignant cells are identified first and removed for best practice in cell state annotation.
+# 
+# In the BCC data, the cluster of malignant cells are identified with `inferCNV`. We can use the pre-saved column 'isTME' to keep Tumor Microenvironment cells.
+# 
+# These are the authors' exact words, but tests have found that the difference in annotation effect is not that great even without removing the malignant cells
+# 
+# But I think this step is not necessary
+
+# In[3]:
+
+
+#adata = adata[adata.obs['isTME']]
+
+
+# ## Neighborhood graph calculated
+# 
+# We note that scVI was used earlier to remove the batch effect from the data, so we need to recalculate the neighbourhood map based on what is stored in `adata.obsm['X_scVI']`. Note that if you are not using scVI but using another method to calculate the neighbourhood map, such as `X_pca`, then you need to change `X_scVI` to `X_pca` to complete the calculation
+# 
+# ```
+# #Example
+# #sc.tl.pca(adata)
+# #sc.pp.neighbors(adata, use_rep="X_pca")
+# ```
+
+# In[4]:
+
+
+sc.pp.neighbors(adata, use_rep="X_scVI")
+
+
+# To visualize the PCA’s embeddings, we use the `pymde` package wrapper in omicverse. This is an alternative to UMAP that is GPU-accelerated.
+
+# In[5]:
+
+
+adata.obsm["X_mde"] = ov.utils.mde(adata.obsm["X_scVI"])
+
+
+# In[6]:
+
+
+sc.pl.embedding(
+    adata,
+    basis="X_mde",
+    color=["patient"],
+    frameon=False,
+    ncols=1,
+)
+
+
+# In[7]:
+
+
+#adata.write_h5ad('adata_mde.h5ad',compression='gzip')
+#adata=sc.read('adata_mde.h5ad')
+
+
+# ## MeteTiME model init
+# 
+# Next, let's load the pre-computed MetaTiME MetaComponents (MeCs), and their functional annotation.
+
+# In[8]:
+
+
+TiME_object=ov.single.MetaTiME(adata,mode='table')
+
+
+# We can over-cluster the cells which is useful for fine-grained cell state annotation.
+# 
+# As the resolution gets larger, the number of clusters gets larger
+
+# In[9]:
+
+
+TiME_object.overcluster(resolution=8,clustercol = 'overcluster',)
+
+
+# ## TME celltype predicted
+# 
+# We using `TiME_object.predictTiME()` to predicted the latent celltype in TME. 
+# 
+# - The minor celltype will be stored in `adata.obs['MetaTiME']`
+# - The major celltype will be stored in `adata.obs['Major_MetaTiME']`
+
+# In[10]:
+
+
+TiME_object.predictTiME(save_obs_name='MetaTiME')
+
+
+# ## Visualize
+# 
+# The original author provides a drawing function that effectively avoids overlapping labels. Here I have expanded its parameters so that it can be visualised using parameters other than X_umap
+
+# In[13]:
+
+
+fig,ax=TiME_object.plot(cluster_key='MetaTiME',basis='X_mde',dpi=80)
+#fig.save
+
+
+# We can also use `sc.pl.embedding` to visualize the celltype
+
+# In[15]:
+
+
+sc.pl.embedding(
+    adata,
+    basis="X_mde",
+    color=["Major_MetaTiME"],
+    frameon=False,
+    ncols=1,
+)
+
+
+# In[ ]:
+
+
+
+

ovrawm/t_mofa.txt

@@ -0,0 +1,184 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# # Multi omics analysis by MOFA
+# MOFA is a factor analysis model that provides a general framework for the integration of multi-omic data sets in an unsupervised fashion.
+# 
+# This tutorial focuses on how to perform mofa in multi-omics like scRNA-seq and scATAC-seq
+# 
+# Paper: [MOFA+: a statistical framework for comprehensive integration of multi-modal single-cell data](https://genomebiology.biomedcentral.com/articles/10.1186/s13059-020-02015-1)
+# 
+# Code: https://github.com/bioFAM/mofapy2
+# 
+# Colab_Reproducibility：https://colab.research.google.com/drive/1UPGQA3BenrC-eLIGVtdKVftSnOKIwNrP?usp=sharing
+
+# ## Part.1 MOFA Model
+# In this part, we construct a model of mofa by scRNA-seq and scATAC-seq
+
+# In[1]:
+
+
+import omicverse as ov
+rna=ov.utils.read('data/sample/rna_p_n_raw.h5ad')
+atac=ov.utils.read('data/sample/atac_p_n_raw.h5ad')
+
+
+# In[2]:
+
+
+rna,atac
+
+
+# We only need to add anndata to `ov.single.mofa` to construct the base model
+
+# In[4]:
+
+
+test_mofa=ov.single.pyMOFA(omics=[rna,atac],
+                             omics_name=['RNA','ATAC'])
+
+
+# In[ ]:
+
+
+test_mofa.mofa_preprocess()
+test_mofa.mofa_run(outfile='models/brac_rna_atac.hdf5')
+
+
+# ## Part.2 MOFA Analysis
+# After get the model by mofa, we need to analysis the factor about different omics, we provide some method to do this
+
+# ### load data
+
+# In[1]:
+
+
+import omicverse as ov
+ov.utils.ov_plot_set()
+
+
+# In[2]:
+
+
+rna=ov.utils.read('data/sample/rna_test.h5ad')
+
+
+# ### add value of factor to anndata
+
+# In[3]:
+
+
+rna=ov.single.factor_exact(rna,hdf5_path='data/sample/MOFA_POS.hdf5')
+rna
+
+
+# ### analysis of the correlation between factor and cell type
+
+# In[4]:
+
+
+ov.single.factor_correlation(adata=rna,cluster='cell_type',factor_list=[1,2,3,4,5])
+
+
+# ### Get the gene/feature weights of different factor
+
+# In[5]:
+
+
+ov.single.get_weights(hdf5_path='data/sample/MOFA_POS.hdf5',view='RNA',factor=1)
+
+
+# ## Part.3 MOFA Visualize
+# 
+# To visualize the result of mofa, we provide a series of function to do this.
+
+# In[6]:
+
+
+pymofa_obj=ov.single.pyMOFAART(model_path='data/sample/MOFA_POS.hdf5')
+
+
+# We get the factor of each cell at first
+
+# In[7]:
+
+
+pymofa_obj.get_factors(rna)
+rna
+
+
+# We can also plot the varience in each View
+
+# In[8]:
+
+
+pymofa_obj.plot_r2()
+
+
+# In[9]:
+
+
+pymofa_obj.get_r2()
+
+
+# ### Visualize the correlation between factor and celltype
+
+# In[10]:
+
+
+pymofa_obj.plot_cor(rna,'cell_type')
+
+
+# We found that factor6 is correlated to Epithelial
+
+# In[11]:
+
+
+pymofa_obj.plot_factor(rna,'cell_type','Epi',figsize=(3,3),
+                    factor1=6,factor2=10,)
+
+
+# In[24]:
+
+
+import scanpy as sc
+sc.pp.neighbors(rna)
+sc.tl.umap(rna)
+sc.pl.embedding(
+    rna,
+    basis="X_umap",
+    color=["factor6","cell_type"],
+    frameon=False,
+    ncols=2,
+    #palette=ov.utils.pyomic_palette(),
+    show=False,
+    cmap='Greens',
+    vmin=0,
+)
+#plt.savefig("figures/umap_factor6.png",dpi=300,bbox_inches = 'tight')
+
+
+# In[12]:
+
+
+pymofa_obj.plot_weight_gene_d1(view='RNA',factor1=6,factor2=10,)
+
+
+# In[18]:
+
+
+pymofa_obj.plot_weights(view='RNA',factor=6,color='#5de25d',
+                        ascending=True)
+
+
+# In[14]:
+
+
+pymofa_obj.plot_top_feature_heatmap(view='RNA')
+
+
+# In[ ]:
+
+
+
+

ovrawm/t_mofa_glue.txt

@@ -0,0 +1,255 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# # Multi omics analysis by MOFA and GLUE
+# MOFA is a factor analysis model that provides a general framework for the integration of multi-omic data sets in an unsupervised fashion.
+# 
+# Most of the time, however, we did not get paired cells in the multi-omics analysis. Here, we can pair cells using GLUE, a dimensionality reduction algorithm that can integrate different histological layers, and it can efficiently merge data from different histological layers.
+# 
+# This tutorial focuses on how to perform mofa in multi-omics using GLUE.
+# 
+# Paper: [MOFA+: a statistical framework for comprehensive integration of multi-modal single-cell data](https://genomebiology.biomedcentral.com/articles/10.1186/s13059-020-02015-1) and [Multi-omics single-cell data integration and regulatory inference with graph-linked embedding](https://www.nature.com/articles/s41587-022-01284-4)
+# 
+# Code: https://github.com/bioFAM/mofapy2 and https://github.com/gao-lab/GLUE
+# 
+# Colab_Reproducibility：https://colab.research.google.com/drive/1zlakFf20IoBdyuOQDocwFQHu8XOVizRL?usp=sharing
+# 
+# We used the result anndata object `rna-emb.h5ad` and `atac.emb.h5ad` from [GLUE'tutorial](https://scglue.readthedocs.io/en/latest/training.html)
+
+# In[1]:
+
+
+import omicverse as ov
+ov.utils.ov_plot_set()
+
+
+# ## Load the data
+# 
+# We use `ov.utils.read` to read the `h5ad` files
+
+# In[2]:
+
+
+rna=ov.utils.read("chen_rna-emb.h5ad")
+atac=ov.utils.read("chen_atac-emb.h5ad")
+
+
+# ## Pair the omics 
+# 
+# Each cell in our rna and atac data has a feature vector, X_glue, based on which we can calculate the Pearson correlation coefficient to perform cell matching.
+
+# In[3]:
+
+
+pair_obj=ov.single.GLUE_pair(rna,atac)
+pair_obj.correlation()
+
+
+# We counted the top 50 highly correlated cells in another histology layer for each cell in one of the histology layers to avoid missing data due to one cell being highly correlated with multiple cells. The default minimum threshold for high correlation is 0.9. We can obtain more paired cells by increasing the depth, but note that increasing the depth may lead to higher errors in cell matching
+
+# In[4]:
+
+
+res_pair=pair_obj.find_neighbor_cell(depth=20)
+res_pair.to_csv('models/chen_pair_res.csv')
+
+
+# We filter to get paired cells
+
+# In[14]:
+
+
+rna1=rna[res_pair['omic_1']]
+atac1=atac[res_pair['omic_2']]
+rna1.obs.index=res_pair.index
+atac1.obs.index=res_pair.index
+rna1,atac1
+
+
+# We can use mudata to store the multi-omics 
+
+# In[6]:
+
+
+from mudata import MuData
+
+mdata = MuData({'rna': rna1, 'atac': atac1})
+mdata
+
+
+# In[7]:
+
+
+mdata.write("chen_mu.h5mu",compression='gzip')
+
+
+# ## MOFA prepare
+# 
+# In the MOFA analysis, we only need to use highly variable genes, for which we perform one filter
+
+# In[22]:
+
+
+rna1=mdata['rna']
+rna1=rna1[:,rna1.var['highly_variable']==True]
+atac1=mdata['atac']
+atac1=atac1[:,atac1.var['highly_variable']==True]
+rna1.obs.index=res_pair.index
+atac1.obs.index=res_pair.index
+
+
+# In[23]:
+
+
+import random
+random_obs_index=random.sample(list(rna1.obs.index),5000)
+
+
+# In[25]:
+
+
+from sklearn.metrics import adjusted_rand_score as ari
+ari_random=ari(rna1[random_obs_index].obs['cell_type'], atac1[random_obs_index].obs['cell_type'])
+ari_raw=ari(rna1.obs['cell_type'], atac1.obs['cell_type'])
+print('raw ari:{}, random ari:{}'.format(ari_raw,ari_random))
+
+
+# In[26]:
+
+
+#rna1=rna1[random_obs_index]
+#atac1=atac1[random_obs_index]
+
+
+# ## MOFA analysis
+# 
+# In this part, we construct a model of mofa by scRNA-seq and scATAC-seq
+
+# In[28]:
+
+
+test_mofa=ov.single.pyMOFA(omics=[rna1,atac1],
+                             omics_name=['RNA','ATAC'])
+
+
+# In[29]:
+
+
+test_mofa.mofa_preprocess()
+test_mofa.mofa_run(outfile='models/chen_rna_atac.hdf5')
+
+
+# ## MOFA Visualization
+# 
+# In this part, we provide a series of function to visualize the result of mofa.
+
+# In[30]:
+
+
+pymofa_obj=ov.single.pyMOFAART(model_path='models/chen_rna_atac.hdf5')
+
+
+# In[31]:
+
+
+pymofa_obj.get_factors(rna1)
+rna1
+
+
+# ### Visualize the varience of each view
+
+# In[32]:
+
+
+pymofa_obj.plot_r2()
+
+
+# In[33]:
+
+
+pymofa_obj.get_r2()
+
+
+# ### Visualize the correlation between factor and celltype
+
+# In[37]:
+
+
+pymofa_obj.plot_cor(rna1,'cell_type',figsize=(4,6))
+
+
+# In[38]:
+
+
+pymofa_obj.get_cor(rna1,'cell_type')
+
+
+# In[46]:
+
+
+pymofa_obj.plot_factor(rna1,'cell_type','Ast',figsize=(3,3),
+                    factor1=1,factor2=3,)
+
+
+# ### Visualize the factor in UMAP
+# 
+# To visualize the GLUE’s learned embeddings, we use the pymde package wrapperin scvi-tools. This is an alternative to UMAP that is GPU-accelerated.
+# 
+# You can use `sc.tl.umap` insteaded.
+
+# In[41]:
+
+
+from scvi.model.utils import mde
+import scanpy as sc
+sc.pp.neighbors(rna1, use_rep="X_glue", metric="cosine")
+rna1.obsm["X_mde"] = mde(rna1.obsm["X_glue"])
+
+
+# In[47]:
+
+
+sc.pl.embedding(
+    rna1,
+    basis="X_mde",
+    color=["factor1","factor3","cell_type"],
+    frameon=False,
+    ncols=3,
+    #palette=ov.utils.pyomic_palette(),
+    show=False,
+    cmap='Greens',
+    vmin=0,
+)
+
+
+# ### Weights ranked
+# A visualization of factor weights familiar to MOFA and MOFA+ users is implemented with some modifications in `plot_weight_gene_d1`, `plot_weight_gene_d2`, and `plot_weights`.
+
+# In[48]:
+
+
+pymofa_obj.plot_weight_gene_d1(view='RNA',factor1=1,factor2=3,)
+
+
+# In[50]:
+
+
+pymofa_obj.plot_weights(view='RNA',factor=1,
+                        ascending=False)
+
+
+# ### Weights heatmap
+# 
+# While trying to annotate factors, a global overview of top features defining them could be helpful.
+
+# In[51]:
+
+
+pymofa_obj.plot_top_feature_heatmap(view='RNA')
+
+
+# In[ ]:
+
+
+
+

ovrawm/t_network.txt

@@ -0,0 +1,88 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# # Protein-Protein interaction (PPI) analysis by String-db
+# STRING is a database of known and predicted protein-protein interactions. The interactions include direct (physical) and indirect (functional) associations; they stem from computational prediction, from knowledge transfer between organisms, and from interactions aggregated from other (primary) databases.
+# 
+# Here we produce a tutorial that use python to construct protein-protein interaction network
+# 
+# Colab_Reproducibility：https://colab.research.google.com/drive/1ReLCFA5cNNcem_WaMXYN9da7W0GN4gzl?usp=sharing
+
+# In[1]:
+
+
+import omicverse as ov
+ov.utils.ov_plot_set()
+
+
+# ## Prepare data
+# 
+# Here we use the example data of string-db to perform the analysis 
+# 
+# FAA4 and its ten most confident interactors.
+# FAA4 in yeast is a long chain fatty acyl-CoA synthetase; see it connected to other synthetases as well as regulators.
+# 
+# Saccharomyces cerevisiae
+# NCBI taxonomy Id: 4932
+# Other names: ATCC 18824, Candida robusta, NRRL Y-12632, S. cerevisiae, Saccharomyces capensis, Saccharomyces italicus, Saccharomyces oviformis, Saccharomyces uvarum var. melibiosus, lager beer yeast, yeast
+
+# In[2]:
+
+
+gene_list=['FAA4','POX1','FAT1','FAS2','FAS1','FAA1','OLE1','YJU3','TGL3','INA1','TGL5']
+
+
+# Besides, we also need to set the gene's type and color. Here, we randomly set the top 5 genes named Type1, other named Type2
+
+# In[3]:
+
+
+gene_type_dict=dict(zip(gene_list,['Type1']*5+['Type2']*6))
+gene_color_dict=dict(zip(gene_list,['#F7828A']*5+['#9CCCA4']*6))
+
+
+# ## STRING interaction analysis
+# 
+# The network API method also allows you to retrieve your STRING interaction network for one or multiple proteins in various text formats. It will tell you the combined score and all the channel specific scores for the set of proteins. You can also extend the network neighborhood by setting "add_nodes", which will add, to your network, new interaction partners in order of their confidence.
+
+# In[7]:
+
+
+G_res=ov.bulk.string_interaction(gene_list,4932)
+G_res.head()
+
+
+# ## STRING PPI network 
+# 
+# We also can use `ov.bulk.pyPPI` to get the PPI network of `gene_list`, we init it at first
+
+# In[5]:
+
+
+ppi=ov.bulk.pyPPI(gene=gene_list,
+                      gene_type_dict=gene_type_dict,
+                      gene_color_dict=gene_color_dict,
+                      species=4932)
+
+
+# Then we connect to string-db to calculate the protein-protein interaction
+
+# In[8]:
+
+
+ppi.interaction_analysis()
+
+
+# We provided a very simple function to plot the network, you can refer the `ov.utils.plot_network` to find out the parameter
+
+# In[9]:
+
+
+ppi.plot_network()
+
+
+# In[ ]:
+
+
+
+

ovrawm/t_nocd.txt

@@ -0,0 +1,112 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# # Overlapping Celltype annotation with GNN
+# Droplet based single cell transcriptomics has recently enabled parallel screening of tens of thousands of single cells. Clustering methods that scale for such high dimensional data without compromising accuracy are scarce.
+# 
+# This tutorial focuses on how to cluster the cell with overlapping and identify the cell with multi-fate
+# 
+# Colab_Reproducibility：https://colab.research.google.com/drive/1l7iHdVmTQcv9YmpIjhK_UzLuHbJ1Jv9E?usp=sharing
+# 
+# <div class="admonition warning">
+#   <p class="admonition-title">Warning</p>
+#   <p>
+#     NOCD's development is still in progress. The current version may not fully reproduce the original implementation’s results.
+#   </p>
+# </div>
+
+# ## Part.1 Data preprocess
+# 
+# In this part, we perform preliminary processing of the data, such as normalization and logarithmization, in order to make the data more interpretable
+
+# In[1]:
+
+
+import omicverse as ov
+import anndata
+import scanpy as sc
+import matplotlib.pyplot as plt
+import numpy as np
+import pandas as pd
+get_ipython().run_line_magic('matplotlib', 'inline')
+
+
+# In[2]:
+
+
+#param for visualization
+sc.settings.verbosity = 3             # verbosity: errors (0), warnings (1), info (2), hints (3)
+sc.settings.set_figure_params(dpi=80, facecolor='white')
+
+
+# In[3]:
+
+
+from matplotlib.colors import LinearSegmentedColormap
+sc_color=['#7CBB5F','#368650','#A499CC','#5E4D9A','#78C2ED','#866017', '#9F987F','#E0DFED',
+ '#EF7B77', '#279AD7','#F0EEF0', '#1F577B', '#A56BA7', '#E0A7C8', '#E069A6', '#941456', '#FCBC10',
+ '#EAEFC5', '#01A0A7', '#75C8CC', '#F0D7BC', '#D5B26C', '#D5DA48', '#B6B812', '#9DC3C3', '#A89C92', '#FEE00C', '#FEF2A1']
+sc_color_cmap = LinearSegmentedColormap.from_list('Custom', sc_color, len(sc_color))
+
+
+# In[4]:
+
+
+adata = anndata.read('sample/rna.h5ad')
+adata
+
+
+# In[5]:
+
+
+adata=ov.single.scanpy_lazy(adata)
+
+
+# ## Part.2 Overlapping Community Detection
+# In this part, we perform a graph neural network (GNN) basedmodel for overlapping community detection in scRNA-seq.
+# 
+# ![https://www.cs.cit.tum.de/fileadmin/w00cfj/daml/nocd/nocd.png](https://www.cs.cit.tum.de/fileadmin/w00cfj/daml/nocd/nocd.png)
+
+# In[6]:
+
+
+scbrca=ov.single.scnocd(adata)
+scbrca.matrix_transform()
+scbrca.matrix_normalize()
+scbrca.GNN_configure()
+scbrca.GNN_preprocess()
+scbrca.GNN_model()
+scbrca.GNN_result()
+scbrca.GNN_plot()
+#scbrca.calculate_nocd()
+scbrca.cal_nocd()
+
+
+# In[8]:
+
+
+scbrca.calculate_nocd()
+
+
+# ## Part.3 Visualization
+# In this part, we visualized the overlapping and non-overlapping cell.
+
+# In[9]:
+
+
+sc.pl.umap(scbrca.adata, color=['leiden','nocd'],wspace=0.4,palette=sc_color)
+
+
+# zero means the cell related to overlap
+
+# In[10]:
+
+
+sc.pl.umap(scbrca.adata, color=['leiden','nocd_n'],wspace=0.4,palette=sc_color)
+
+
+# In[ ]:
+
+
+
+

ovrawm/t_preprocess.txt

@@ -0,0 +1,421 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# # Preprocessing the data of scRNA-seq with omicverse
+# 
+# The count table, a numeric matrix of genes × cells, is the basic input data structure in the analysis of single-cell RNA-sequencing data. A common preprocessing step is to adjust the counts for variable sampling efficiency and to transform them so that the variance is similar across the dynamic range. 
+# 
+# Suitable methods to preprocess the scRNA-seq is important. Here, we introduce some preprocessing step to help researchers can perform downstream analysis easyier.
+# 
+# User can compare our tutorial with [scanpy'tutorial](https://scanpy-tutorials.readthedocs.io/en/latest/pbmc3k.html) to learn how to use omicverse well
+# 
+# Colab_Reproducibility：https://colab.research.google.com/drive/1DXLSls_ppgJmAaZTUvqazNC_E7EDCxUe?usp=sharing
+
+# In[1]:
+
+
+import omicverse as ov
+import scanpy as sc
+ov.ov_plot_set()
+
+
+# The data consist of 3k PBMCs from a Healthy Donor and are freely available from 10x Genomics ([here](http://cf.10xgenomics.com/samples/cell-exp/1.1.0/pbmc3k/pbmc3k_filtered_gene_bc_matrices.tar.gz) from this [webpage](https://support.10xgenomics.com/single-cell-gene-expression/datasets/1.1.0/pbmc3k)). On a unix system, you can uncomment and run the following to download and unpack the data. The last line creates a directory for writing processed data.
+
+# In[2]:
+
+
+# !mkdir data
+# !wget http://cf.10xgenomics.com/samples/cell-exp/1.1.0/pbmc3k/pbmc3k_filtered_gene_bc_matrices.tar.gz -O data/pbmc3k_filtered_gene_bc_matrices.tar.gz
+# !cd data; tar -xzf pbmc3k_filtered_gene_bc_matrices.tar.gz
+# !mkdir write
+
+
+# In[3]:
+
+
+adata = sc.read_10x_mtx(
+    'data/filtered_gene_bc_matrices/hg19/',  # the directory with the `.mtx` file
+    var_names='gene_symbols',                # use gene symbols for the variable names (variables-axis index)
+    cache=True)                              # write a cache file for faster subsequent reading
+adata
+
+
+# In[4]:
+
+
+adata.var_names_make_unique()
+adata.obs_names_make_unique()
+
+
+# ## Preprocessing
+# 
+# ### Quantity control
+# 
+# For single-cell data, we require quality control prior to analysis, including the removal of cells containing double cells, low-expressing cells, and low-expressing genes. In addition to this, we need to filter based on mitochondrial gene ratios, number of transcripts, number of genes expressed per cell, cellular Complexity, etc. For a detailed description of the different QCs please see the document: https://hbctraining.github.io/scRNA-seq/lessons/04_SC_quality_control.html
+
+# In[5]:
+
+
+adata=ov.pp.qc(adata,
+              tresh={'mito_perc': 0.05, 'nUMIs': 500, 'detected_genes': 250})
+adata
+
+
+# ### High variable Gene Detection
+# 
+# Here we try to use Pearson's method to calculate highly variable genes. This is the method that is proposed to be superior to ordinary normalisation. See [Article](https://www.nature.com/articles/s41592-023-01814-1#MOESM3) in *Nature Method* for details.
+# 
+
+# Sometimes we need to recover the original counts for some single-cell calculations, but storing them in the layer layer may result in missing data, so we provide two functions here, a store function and a release function, to save the original data.
+# 
+# We set `layers=counts`, the counts will be stored in `adata.uns['layers_counts']`
+
+# In[6]:
+
+
+ov.utils.store_layers(adata,layers='counts')
+adata
+
+
+# normalize|HVGs：We use | to control the preprocessing step, | before for the normalisation step, either `shiftlog` or `pearson`, and | after for the highly variable gene calculation step, either `pearson` or `seurat`. Our default is `shiftlog|pearson`.
+# 
+# - if you use `mode`=`shiftlog|pearson` you need to set `target_sum=50*1e4`, more people like to se `target_sum=1e4`, we test the result think 50*1e4 will be better
+# - if you use `mode`=`pearson|pearson`, you don't need to set `target_sum`
+# 
+# <div class="admonition warning">
+#   <p class="admonition-title">Note</p>
+#   <p>
+#     if the version of `omicverse` lower than `1.4.13`, the mode can only be set between `scanpy` and `pearson`.
+#   </p>
+# </div>
+# 
+
+# In[7]:
+
+
+adata=ov.pp.preprocess(adata,mode='shiftlog|pearson',n_HVGs=2000,)
+adata
+
+
+# Set the .raw attribute of the AnnData object to the normalized and logarithmized raw gene expression for later use in differential testing and visualizations of gene expression. This simply freezes the state of the AnnData object.
+
+# In[8]:
+
+
+adata.raw = adata
+adata = adata[:, adata.var.highly_variable_features]
+adata
+
+
+# We find that the adata.X matrix is normalized at this point, including the data in raw, but we want to get the unnormalized data, so we can use the retrieve function `ov.utils.retrieve_layers`
+
+# In[9]:
+
+
+adata_counts=adata.copy()
+ov.utils.retrieve_layers(adata_counts,layers='counts')
+print('normalize adata:',adata.X.max())
+print('raw count adata:',adata_counts.X.max())
+
+
+# In[10]:
+
+
+adata_counts
+
+
+# If we wish to recover the original count matrix at the whole gene level, we can try the following code
+
+# In[11]:
+
+
+adata_counts=adata.raw.to_adata().copy()
+ov.utils.retrieve_layers(adata_counts,layers='counts')
+print('normalize adata:',adata.X.max())
+print('raw count adata:',adata_counts.X.max())
+adata_counts
+
+
+# ## Principal component analysis
+# 
+# In contrast to scanpy, we do not directly scale the variance of the original expression matrix, but store the results of the variance scaling in the layer, due to the fact that scale may cause changes in the data distribution, and we have not found scale to be meaningful in any scenario other than a principal component analysis
+
+# In[12]:
+
+
+ov.pp.scale(adata)
+adata
+
+
+# If you want to perform pca in normlog layer, you can set `layer`=`normlog`, but we think scaled is necessary in PCA.
+
+# In[13]:
+
+
+ov.pp.pca(adata,layer='scaled',n_pcs=50)
+adata
+
+
+# In[14]:
+
+
+adata.obsm['X_pca']=adata.obsm['scaled|original|X_pca']
+ov.utils.embedding(adata,
+                  basis='X_pca',
+                  color='CST3',
+                  frameon='small')
+
+
+# ## Embedding the neighborhood graph
+# 
+# We suggest embedding the graph in two dimensions using UMAP (McInnes et al., 2018), see below. It is potentially more faithful to the global connectivity of the manifold than tSNE, i.e., it better preserves trajectories. In some ocassions, you might still observe disconnected clusters and similar connectivity violations. They can usually be remedied by running:
+
+# In[15]:
+
+
+sc.pp.neighbors(adata, n_neighbors=15, n_pcs=50,
+               use_rep='scaled|original|X_pca')
+
+
+# To visualize the PCA’s embeddings, we use the `pymde` package wrapper in omicverse. This is an alternative to UMAP that is GPU-accelerated.
+
+# In[16]:
+
+
+adata.obsm["X_mde"] = ov.utils.mde(adata.obsm["scaled|original|X_pca"])
+adata
+
+
+# In[17]:
+
+
+ov.utils.embedding(adata,
+                basis='X_mde',
+                color='CST3',
+                frameon='small')
+
+
+# You also can use `umap` to visualize the neighborhood graph
+
+# In[18]:
+
+
+sc.tl.umap(adata)
+
+
+# In[19]:
+
+
+ov.utils.embedding(adata,
+                basis='X_umap',
+                color='CST3',
+                frameon='small')
+
+
+# ## Clustering the neighborhood graph
+# 
+# As with Seurat and many other frameworks, we recommend the Leiden graph-clustering method (community detection based on optimizing modularity) by Traag *et al.* (2018). Note that Leiden clustering directly clusters the neighborhood graph of cells, which we already computed in the previous section.
+
+# In[20]:
+
+
+sc.tl.leiden(adata)
+
+
+# We redesigned the visualisation of embedding to distinguish it from scanpy's embedding by adding the parameter `fraemon='small'`, which causes the axes to be scaled with the colourbar
+
+# In[21]:
+
+
+ov.utils.embedding(adata,
+                basis='X_mde',
+                color=['leiden', 'CST3', 'NKG7'],
+                frameon='small')
+
+
+# We also provide a boundary visualisation function `ov.utils.plot_ConvexHull` to visualise specific clusters.
+# 
+# Arguments: 
+# - color: if None will use the color of clusters
+# - alpha: default is 0.2
+
+# In[23]:
+
+
+import matplotlib.pyplot as plt
+fig,ax=plt.subplots( figsize = (4,4))
+
+ov.utils.embedding(adata,
+                basis='X_mde',
+                color=['leiden'],
+                frameon='small',
+                show=False,
+                ax=ax)
+
+ov.utils.plot_ConvexHull(adata,
+                basis='X_mde',
+                cluster_key='leiden',
+                hull_cluster='0',
+                ax=ax)
+
+
+# If you have too many labels, e.g. too many cell types, and you are concerned about cell overlap, then consider trying the `ov.utils.gen_mpl_labels` function, which improves text overlap.
+# In addition, we make use of the `patheffects` function, which makes our text have outlines
+# 
+# - adjust_kwargs: it could be found in package `adjusttext`
+# - text_kwargs: it could be found in class `plt.texts`
+
+# In[67]:
+
+
+from matplotlib import patheffects
+import matplotlib.pyplot as plt
+fig, ax = plt.subplots(figsize=(4,4))
+
+ov.utils.embedding(adata,
+                  basis='X_mde',
+                  color=['leiden'],
+                   show=False, legend_loc=None, add_outline=False, 
+                   frameon='small',legend_fontoutline=2,ax=ax
+                 )
+
+ov.utils.gen_mpl_labels(
+    adata,
+    'leiden',
+    exclude=("None",),  
+    basis='X_mde',
+    ax=ax,
+    adjust_kwargs=dict(arrowprops=dict(arrowstyle='-', color='black')),
+    text_kwargs=dict(fontsize= 12 ,weight='bold',
+                     path_effects=[patheffects.withStroke(linewidth=2, foreground='w')] ),
+)
+
+
+# In[47]:
+
+
+marker_genes = ['IL7R', 'CD79A', 'MS4A1', 'CD8A', 'CD8B', 'LYZ', 'CD14',
+                'LGALS3', 'S100A8', 'GNLY', 'NKG7', 'KLRB1',
+                'FCGR3A', 'MS4A7', 'FCER1A', 'CST3', 'PPBP']
+
+
+# In[48]:
+
+
+sc.pl.dotplot(adata, marker_genes, groupby='leiden',
+             standard_scale='var');
+
+
+# ## Finding marker genes
+# 
+# Let us compute a ranking for the highly differential genes in each cluster. For this, by default, the .raw attribute of AnnData is used in case it has been initialized before. The simplest and fastest method to do so is the t-test.
+
+# In[49]:
+
+
+sc.tl.dendrogram(adata,'leiden',use_rep='scaled|original|X_pca')
+sc.tl.rank_genes_groups(adata, 'leiden', use_rep='scaled|original|X_pca',
+                        method='t-test',use_raw=False,key_added='leiden_ttest')
+sc.pl.rank_genes_groups_dotplot(adata,groupby='leiden',
+                                cmap='Spectral_r',key='leiden_ttest',
+                                standard_scale='var',n_genes=3)
+
+
+# cosg is also considered to be a better algorithm for finding marker genes. Here, omicverse provides the calculation of cosg
+# 
+# Paper: [Accurate and fast cell marker gene identification with COSG](https://academic.oup.com/bib/advance-article-abstract/doi/10.1093/bib/bbab579/6511197?redirectedFrom=fulltext)
+# 
+# Code: https://github.com/genecell/COSG
+# 
+
+# In[50]:
+
+
+sc.tl.rank_genes_groups(adata, groupby='leiden', 
+                        method='t-test',use_rep='scaled|original|X_pca',)
+ov.single.cosg(adata, key_added='leiden_cosg', groupby='leiden')
+sc.pl.rank_genes_groups_dotplot(adata,groupby='leiden',
+                                cmap='Spectral_r',key='leiden_cosg',
+                                standard_scale='var',n_genes=3)
+
+
+# ## Other plotting
+# 
+# Next, let's try another chart, which we call the Stacked Volcano Chart. We need to prepare two dictionaries, a `data_dict` and a `color_dict`, both of which have the same key requirements.
+# 
+# For `data_dict`. we require the contents within each key to be a DataFrame containing ['names','logfoldchanges','pvals_adj'], where names stands for gene names, logfoldchanges stands for differential expression multiplicity, pvals_adj stands for significance p-value
+# 
+
+# In[51]:
+
+
+data_dict={}
+for i in adata.obs['leiden'].cat.categories:
+    data_dict[i]=sc.get.rank_genes_groups_df(adata, group=i, key='leiden_ttest',
+                                            pval_cutoff=None,log2fc_min=None)
+
+
+# In[65]:
+
+
+data_dict.keys()
+
+
+# In[64]:
+
+
+data_dict[i].head()
+
+
+# For `color_dict`, we require that the colour to be displayed for the current key is stored within each key.`
+
+# In[63]:
+
+
+type_color_dict=dict(zip(adata.obs['leiden'].cat.categories,
+                         adata.uns['leiden_colors']))
+type_color_dict
+
+
+# There are a number of parameters available here for us to customise the settings. Note that when drawing stacking_vol with omicverse version less than 1.4.13, there is a bug that the vertical coordinate is constant at [-15,15], so we have added some code in this tutorial for visualisation.
+# 
+# - data_dict: dict, in each key, there is a dataframe with columns of ['logfoldchanges','pvals_adj','names']
+# - color_dict: dict, in each key, there is a color for each omic
+# - pval_threshold: float, pvalue threshold for significant genes
+# - log2fc_threshold: float, log2fc threshold for significant genes
+# - figsize: tuple, figure size
+# - sig_color: str, color for significant genes
+# - normal_color: str, color for non-significant genes
+# - plot_genes_num: int, number of genes to plot
+# - plot_genes_fontsize: int, fontsize for gene names
+# - plot_genes_weight: str, weight for gene names
+
+# In[62]:
+
+
+fig,axes=ov.utils.stacking_vol(data_dict,type_color_dict,
+            pval_threshold=0.01,
+            log2fc_threshold=2,
+            figsize=(8,4),
+            sig_color='#a51616',
+            normal_color='#c7c7c7',
+            plot_genes_num=2,
+            plot_genes_fontsize=6,
+            plot_genes_weight='bold',
+            )
+
+#The following code will be removed in future
+y_min,y_max=0,0
+for i in data_dict.keys():
+    y_min=min(y_min,data_dict[i]['logfoldchanges'].min())
+    y_max=max(y_max,data_dict[i]['logfoldchanges'].max())
+for i in adata.obs['leiden'].cat.categories:
+    axes[i].set_ylim(y_min,y_max)
+plt.suptitle('Stacking_vol',fontsize=12)   
+
+
+# In[ ]:
+
+
+
+

ovrawm/t_preprocess_cpu.txt

@@ -0,0 +1,404 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# # Preprocessing the data of scRNA-seq with omicverse
+# 
+# The count table, a numeric matrix of genes × cells, is the basic input data structure in the analysis of single-cell RNA-sequencing data. A common preprocessing step is to adjust the counts for variable sampling efficiency and to transform them so that the variance is similar across the dynamic range. 
+# 
+# Suitable methods to preprocess the scRNA-seq is important. Here, we introduce some preprocessing step to help researchers can perform downstream analysis easyier.
+# 
+# User can compare our tutorial with [scanpy'tutorial](https://scanpy-tutorials.readthedocs.io/en/latest/pbmc3k.html) to learn how to use omicverse well
+# 
+# Colab_Reproducibility：https://colab.research.google.com/drive/1DXLSls_ppgJmAaZTUvqazNC_E7EDCxUe?usp=sharing
+
+# In[1]:
+
+
+import scanpy as sc
+import omicverse as ov
+ov.plot_set()
+
+
+# The data consist of 3k PBMCs from a Healthy Donor and are freely available from 10x Genomics ([here](http://cf.10xgenomics.com/samples/cell-exp/1.1.0/pbmc3k/pbmc3k_filtered_gene_bc_matrices.tar.gz) from this [webpage](https://support.10xgenomics.com/single-cell-gene-expression/datasets/1.1.0/pbmc3k)). On a unix system, you can uncomment and run the following to download and unpack the data. The last line creates a directory for writing processed data.
+
+# In[ ]:
+
+
+# !mkdir data
+get_ipython().system('wget http://cf.10xgenomics.com/samples/cell-exp/1.1.0/pbmc3k/pbmc3k_filtered_gene_bc_matrices.tar.gz -O data/pbmc3k_filtered_gene_bc_matrices.tar.gz')
+get_ipython().system('cd data; tar -xzf pbmc3k_filtered_gene_bc_matrices.tar.gz')
+# !mkdir write
+
+
+# In[2]:
+
+
+adata = sc.read_10x_mtx(
+    'data/filtered_gene_bc_matrices/hg19/',  # the directory with the `.mtx` file
+    var_names='gene_symbols',                # use gene symbols for the variable names (variables-axis index)
+    cache=True)                              # write a cache file for faster subsequent reading
+adata
+
+
+# In[3]:
+
+
+adata.var_names_make_unique()
+adata.obs_names_make_unique()
+
+
+# ## Preprocessing
+# 
+# ### Quantity control
+# 
+# For single-cell data, we require quality control prior to analysis, including the removal of cells containing double cells, low-expressing cells, and low-expressing genes. In addition to this, we need to filter based on mitochondrial gene ratios, number of transcripts, number of genes expressed per cell, cellular Complexity, etc. For a detailed description of the different QCs please see the document: https://hbctraining.github.io/scRNA-seq/lessons/04_SC_quality_control.html
+# 
+# <div class="admonition warning">
+#   <p class="admonition-title">Note</p>
+#   <p>
+#     if the version of `omicverse` larger than `1.6.4`, the `doublets_method` can be set between `scrublet` and `sccomposite`.
+#   </p>
+# </div>
+# 
+# COMPOSITE (COMpound POiSson multIplet deTEction model) is a computational tool for multiplet detection in both single-cell single-omics and multiomics settings. It has been implemented as an automated pipeline and is available as both a cloud-based application with a user-friendly interface and a Python package.
+# 
+# Hu, H., Wang, X., Feng, S. et al. A unified model-based framework for doublet or multiplet detection in single-cell multiomics data. Nat Commun 15, 5562 (2024). https://doi.org/10.1038/s41467-024-49448-x
+
+# In[4]:
+
+
+get_ipython().run_cell_magic('time', '', "adata=ov.pp.qc(adata,\n              tresh={'mito_perc': 0.2, 'nUMIs': 500, 'detected_genes': 250},\n               doublets_method='sccomposite',\n              batch_key=None)\nadata\n")
+
+
+# ### High variable Gene Detection
+# 
+# Here we try to use Pearson's method to calculate highly variable genes. This is the method that is proposed to be superior to ordinary normalisation. See [Article](https://www.nature.com/articles/s41592-023-01814-1#MOESM3) in *Nature Method* for details.
+# 
+
+# normalize|HVGs：We use | to control the preprocessing step, | before for the normalisation step, either `shiftlog` or `pearson`, and | after for the highly variable gene calculation step, either `pearson` or `seurat`. Our default is `shiftlog|pearson`.
+# 
+# - if you use `mode`=`shiftlog|pearson` you need to set `target_sum=50*1e4`, more people like to se `target_sum=1e4`, we test the result think 50*1e4 will be better
+# - if you use `mode`=`pearson|pearson`, you don't need to set `target_sum`
+# 
+# <div class="admonition warning">
+#   <p class="admonition-title">Note</p>
+#   <p>
+#     if the version of `omicverse` lower than `1.4.13`, the mode can only be set between `scanpy` and `pearson`.
+#   </p>
+# </div>
+# 
+
+# In[5]:
+
+
+get_ipython().run_cell_magic('time', '', "adata=ov.pp.preprocess(adata,mode='shiftlog|pearson',n_HVGs=2000,)\nadata\n")
+
+
+# Set the .raw attribute of the AnnData object to the normalized and logarithmized raw gene expression for later use in differential testing and visualizations of gene expression. This simply freezes the state of the AnnData object.
+
+# In[6]:
+
+
+get_ipython().run_cell_magic('time', '', 'adata.raw = adata\nadata = adata[:, adata.var.highly_variable_features]\nadata\n')
+
+
+# ## Principal component analysis
+# 
+# In contrast to scanpy, we do not directly scale the variance of the original expression matrix, but store the results of the variance scaling in the layer, due to the fact that scale may cause changes in the data distribution, and we have not found scale to be meaningful in any scenario other than a principal component analysis
+
+# In[7]:
+
+
+get_ipython().run_cell_magic('time', '', 'ov.pp.scale(adata)\nadata\n')
+
+
+# If you want to perform pca in normlog layer, you can set `layer`=`normlog`, but we think scaled is necessary in PCA.
+
+# In[8]:
+
+
+get_ipython().run_cell_magic('time', '', "ov.pp.pca(adata,layer='scaled',n_pcs=50)\nadata\n")
+
+
+# In[9]:
+
+
+adata.obsm['X_pca']=adata.obsm['scaled|original|X_pca']
+ov.pl.embedding(adata,
+                  basis='X_pca',
+                  color='CST3',
+                  frameon='small')
+
+
+# ## Embedding the neighborhood graph
+# 
+# We suggest embedding the graph in two dimensions using UMAP (McInnes et al., 2018), see below. It is potentially more faithful to the global connectivity of the manifold than tSNE, i.e., it better preserves trajectories. In some ocassions, you might still observe disconnected clusters and similar connectivity violations. They can usually be remedied by running:
+
+# In[10]:
+
+
+get_ipython().run_cell_magic('time', '', "ov.pp.neighbors(adata, n_neighbors=15, n_pcs=50,\n               use_rep='scaled|original|X_pca')\n")
+
+
+# You also can use `umap` to visualize the neighborhood graph
+
+# In[11]:
+
+
+get_ipython().run_cell_magic('time', '', 'ov.pp.umap(adata)\n')
+
+
+# In[12]:
+
+
+ov.pl.embedding(adata,
+                basis='X_umap',
+                color='CST3',
+                frameon='small')
+
+
+# To visualize the PCA’s embeddings, we use the `pymde` package wrapper in omicverse. This is an alternative to UMAP that is GPU-accelerated.
+
+# In[13]:
+
+
+ov.pp.mde(adata,embedding_dim=2,n_neighbors=15, basis='X_mde',
+          n_pcs=50, use_rep='scaled|original|X_pca',)
+
+
+# In[14]:
+
+
+ov.pl.embedding(adata,
+                basis='X_mde',
+                color='CST3',
+                frameon='small')
+
+
+# ## Score cell cyle
+# 
+# In OmicVerse, we store both G1M/S and G2M genes into the function (both human and mouse), so you can run cell cycle analysis without having to manually enter cycle genes!
+
+# In[19]:
+
+
+adata_raw=adata.raw.to_adata()
+ov.pp.score_genes_cell_cycle(adata_raw,species='human')
+
+
+# In[21]:
+
+
+ov.pl.embedding(adata_raw,
+                basis='X_mde',
+                color='phase',
+                frameon='small')
+
+
+# ## Clustering the neighborhood graph
+# 
+# As with Seurat and many other frameworks, we recommend the Leiden graph-clustering method (community detection based on optimizing modularity) by Traag *et al.* (2018). Note that Leiden clustering directly clusters the neighborhood graph of cells, which we already computed in the previous section.
+
+# In[22]:
+
+
+ov.pp.leiden(adata,resolution=1)
+
+
+# We redesigned the visualisation of embedding to distinguish it from scanpy's embedding by adding the parameter `fraemon='small'`, which causes the axes to be scaled with the colourbar
+
+# In[23]:
+
+
+ov.pl.embedding(adata,
+                basis='X_mde',
+                color=['leiden', 'CST3', 'NKG7'],
+                frameon='small')
+
+
+# We also provide a boundary visualisation function `ov.utils.plot_ConvexHull` to visualise specific clusters.
+# 
+# Arguments: 
+# - color: if None will use the color of clusters
+# - alpha: default is 0.2
+
+# In[24]:
+
+
+import matplotlib.pyplot as plt
+fig,ax=plt.subplots( figsize = (4,4))
+
+ov.pl.embedding(adata,
+                basis='X_mde',
+                color=['leiden'],
+                frameon='small',
+                show=False,
+                ax=ax)
+
+ov.pl.ConvexHull(adata,
+                basis='X_mde',
+                cluster_key='leiden',
+                hull_cluster='0',
+                ax=ax)
+
+
+# If you have too many labels, e.g. too many cell types, and you are concerned about cell overlap, then consider trying the `ov.utils.gen_mpl_labels` function, which improves text overlap.
+# In addition, we make use of the `patheffects` function, which makes our text have outlines
+# 
+# - adjust_kwargs: it could be found in package `adjusttext`
+# - text_kwargs: it could be found in class `plt.texts`
+
+# In[25]:
+
+
+from matplotlib import patheffects
+import matplotlib.pyplot as plt
+fig, ax = plt.subplots(figsize=(4,4))
+
+ov.pl.embedding(adata,
+                  basis='X_mde',
+                  color=['leiden'],
+                   show=False, legend_loc=None, add_outline=False, 
+                   frameon='small',legend_fontoutline=2,ax=ax
+                 )
+
+ov.utils.gen_mpl_labels(
+    adata,
+    'leiden',
+    exclude=("None",),  
+    basis='X_mde',
+    ax=ax,
+    adjust_kwargs=dict(arrowprops=dict(arrowstyle='-', color='black')),
+    text_kwargs=dict(fontsize= 12 ,weight='bold',
+                     path_effects=[patheffects.withStroke(linewidth=2, foreground='w')] ),
+)
+
+
+# In[26]:
+
+
+marker_genes = ['IL7R', 'CD79A', 'MS4A1', 'CD8A', 'CD8B', 'LYZ', 'CD14',
+                'LGALS3', 'S100A8', 'GNLY', 'NKG7', 'KLRB1',
+                'FCGR3A', 'MS4A7', 'FCER1A', 'CST3', 'PPBP']
+
+
+# In[27]:
+
+
+sc.pl.dotplot(adata, marker_genes, groupby='leiden',
+             standard_scale='var');
+
+
+# ## Finding marker genes
+# 
+# Let us compute a ranking for the highly differential genes in each cluster. For this, by default, the .raw attribute of AnnData is used in case it has been initialized before. The simplest and fastest method to do so is the t-test.
+
+# In[28]:
+
+
+sc.tl.dendrogram(adata,'leiden',use_rep='scaled|original|X_pca')
+sc.tl.rank_genes_groups(adata, 'leiden', use_rep='scaled|original|X_pca',
+                        method='t-test',use_raw=False,key_added='leiden_ttest')
+sc.pl.rank_genes_groups_dotplot(adata,groupby='leiden',
+                                cmap='Spectral_r',key='leiden_ttest',
+                                standard_scale='var',n_genes=3)
+
+
+# cosg is also considered to be a better algorithm for finding marker genes. Here, omicverse provides the calculation of cosg
+# 
+# Paper: [Accurate and fast cell marker gene identification with COSG](https://academic.oup.com/bib/advance-article-abstract/doi/10.1093/bib/bbab579/6511197?redirectedFrom=fulltext)
+# 
+# Code: https://github.com/genecell/COSG
+# 
+
+# In[29]:
+
+
+sc.tl.rank_genes_groups(adata, groupby='leiden', 
+                        method='t-test',use_rep='scaled|original|X_pca',)
+ov.single.cosg(adata, key_added='leiden_cosg', groupby='leiden')
+sc.pl.rank_genes_groups_dotplot(adata,groupby='leiden',
+                                cmap='Spectral_r',key='leiden_cosg',
+                                standard_scale='var',n_genes=3)
+
+
+# ## Other plotting
+# 
+# Next, let's try another chart, which we call the Stacked Volcano Chart. We need to prepare two dictionaries, a `data_dict` and a `color_dict`, both of which have the same key requirements.
+# 
+# For `data_dict`. we require the contents within each key to be a DataFrame containing ['names','logfoldchanges','pvals_adj'], where names stands for gene names, logfoldchanges stands for differential expression multiplicity, pvals_adj stands for significance p-value
+# 
+
+# In[51]:
+
+
+data_dict={}
+for i in adata.obs['leiden'].cat.categories:
+    data_dict[i]=sc.get.rank_genes_groups_df(adata, group=i, key='leiden_ttest',
+                                            pval_cutoff=None,log2fc_min=None)
+
+
+# In[65]:
+
+
+data_dict.keys()
+
+
+# In[64]:
+
+
+data_dict[i].head()
+
+
+# For `color_dict`, we require that the colour to be displayed for the current key is stored within each key.`
+
+# In[63]:
+
+
+type_color_dict=dict(zip(adata.obs['leiden'].cat.categories,
+                         adata.uns['leiden_colors']))
+type_color_dict
+
+
+# There are a number of parameters available here for us to customise the settings. Note that when drawing stacking_vol with omicverse version less than 1.4.13, there is a bug that the vertical coordinate is constant at [-15,15], so we have added some code in this tutorial for visualisation.
+# 
+# - data_dict: dict, in each key, there is a dataframe with columns of ['logfoldchanges','pvals_adj','names']
+# - color_dict: dict, in each key, there is a color for each omic
+# - pval_threshold: float, pvalue threshold for significant genes
+# - log2fc_threshold: float, log2fc threshold for significant genes
+# - figsize: tuple, figure size
+# - sig_color: str, color for significant genes
+# - normal_color: str, color for non-significant genes
+# - plot_genes_num: int, number of genes to plot
+# - plot_genes_fontsize: int, fontsize for gene names
+# - plot_genes_weight: str, weight for gene names
+
+# In[62]:
+
+
+fig,axes=ov.utils.stacking_vol(data_dict,type_color_dict,
+            pval_threshold=0.01,
+            log2fc_threshold=2,
+            figsize=(8,4),
+            sig_color='#a51616',
+            normal_color='#c7c7c7',
+            plot_genes_num=2,
+            plot_genes_fontsize=6,
+            plot_genes_weight='bold',
+            )
+
+#The following code will be removed in future
+y_min,y_max=0,0
+for i in data_dict.keys():
+    y_min=min(y_min,data_dict[i]['logfoldchanges'].min())
+    y_max=max(y_max,data_dict[i]['logfoldchanges'].max())
+for i in adata.obs['leiden'].cat.categories:
+    axes[i].set_ylim(y_min,y_max)
+plt.suptitle('Stacking_vol',fontsize=12)   
+
+
+# In[ ]:
+
+
+
+

ovrawm/t_preprocess_gpu.txt

@@ -0,0 +1,416 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# # Preprocessing the data of scRNA-seq with omicverse[GPU]
+# 
+# The count table, a numeric matrix of genes × cells, is the basic input data structure in the analysis of single-cell RNA-sequencing data. A common preprocessing step is to adjust the counts for variable sampling efficiency and to transform them so that the variance is similar across the dynamic range. 
+# 
+# Suitable methods to preprocess the scRNA-seq is important. Here, we introduce some preprocessing step to help researchers can perform downstream analysis easyier.
+# 
+# User can compare our tutorial with [scanpy'tutorial](https://scanpy-tutorials.readthedocs.io/en/latest/pbmc3k.html) to learn how to use omicverse well
+# 
+# Colab_Reproducibility：https://colab.research.google.com/drive/1DXLSls_ppgJmAaZTUvqazNC_E7EDCxUe?usp=sharing
+
+# ## Installation
+# 
+# Note that the GPU module is not directly present and needs to be installed separately, for a detailed [tutorial](https://rapids-singlecell.readthedocs.io/en/latest/index.html) see [https://rapids-singlecell.readthedocs.io/en/latest/index.html](https://rapids-singlecell.readthedocs.io/en/latest/index.html)
+# 
+# ### pip
+# ```shell
+# pip install rapids-singlecell
+# #rapids
+# pip install \
+#     --extra-index-url=https://pypi.nvidia.com \
+#     cudf-cu12==24.4.* dask-cudf-cu12==24.4.* cuml-cu12==24.4.* \
+#     cugraph-cu12==24.4.* cuspatial-cu12==24.4.* cuproj-cu12==24.4.* \
+#     cuxfilter-cu12==24.4.* cucim-cu12==24.4.* pylibraft-cu12==24.4.* \
+#     raft-dask-cu12==24.4.* cuvs-cu12==24.4.*
+# #cupy
+# pip install cupy-cuda12x
+# ```
+# 
+# ### conda-env
+# Note that in order to avoid conflicts, we will consider installing rapid_singlecell first before installing omicverse.
+# 
+# The easiest way to install rapids-singlecell is to use one of the yaml file provided in the [conda](https://github.com/Starlitnightly/omicverse/tree/master/conda) folder. These yaml files install everything needed to run the example notebooks and get you started.
+# ```shell
+# conda env create -f conda/omicverse_gpu.yml
+# # or
+# mamba env create -f conda/omicverse_gpu.yml
+# ```
+
+# In[1]:
+
+
+import omicverse as ov
+import scanpy as sc
+ov.plot_set()
+ov.settings.gpu_init()
+
+
+# The data consist of 3k PBMCs from a Healthy Donor and are freely available from 10x Genomics ([here](http://cf.10xgenomics.com/samples/cell-exp/1.1.0/pbmc3k/pbmc3k_filtered_gene_bc_matrices.tar.gz) from this [webpage](https://support.10xgenomics.com/single-cell-gene-expression/datasets/1.1.0/pbmc3k)). On a unix system, you can uncomment and run the following to download and unpack the data. The last line creates a directory for writing processed data.
+
+# In[2]:
+
+
+# !mkdir data
+#!wget http://cf.10xgenomics.com/samples/cell-exp/1.1.0/pbmc3k/pbmc3k_filtered_gene_bc_matrices.tar.gz -O data/pbmc3k_filtered_gene_bc_matrices.tar.gz
+#!cd data; tar -xzf pbmc3k_filtered_gene_bc_matrices.tar.gz
+# !mkdir write
+
+
+# In[3]:
+
+
+adata = sc.read_10x_mtx(
+    'data/filtered_gene_bc_matrices/hg19/',  # the directory with the `.mtx` file
+    var_names='gene_symbols',                # use gene symbols for the variable names (variables-axis index)
+    cache=True)                              # write a cache file for faster subsequent reading
+adata
+
+
+# In[4]:
+
+
+adata.var_names_make_unique()
+adata.obs_names_make_unique()
+
+
+# ## Preprocessing
+# 
+# ### Quantity control
+# 
+# For single-cell data, we require quality control prior to analysis, including the removal of cells containing double cells, low-expressing cells, and low-expressing genes. In addition to this, we need to filter based on mitochondrial gene ratios, number of transcripts, number of genes expressed per cell, cellular Complexity, etc. For a detailed description of the different QCs please see the document: https://hbctraining.github.io/scRNA-seq/lessons/04_SC_quality_control.html
+
+# In[5]:
+
+
+ov.pp.anndata_to_GPU(adata)
+
+
+# In[6]:
+
+
+get_ipython().run_cell_magic('time', '', "adata=ov.pp.qc(adata,\n              tresh={'mito_perc': 0.2, 'nUMIs': 500, 'detected_genes': 250},\n              batch_key=None)\nadata\n")
+
+
+# ### High variable Gene Detection
+# 
+# Here we try to use Pearson's method to calculate highly variable genes. This is the method that is proposed to be superior to ordinary normalisation. See [Article](https://www.nature.com/articles/s41592-023-01814-1#MOESM3) in *Nature Method* for details.
+# 
+
+# normalize|HVGs：We use | to control the preprocessing step, | before for the normalisation step, either `shiftlog` or `pearson`, and | after for the highly variable gene calculation step, either `pearson` or `seurat`. Our default is `shiftlog|pearson`.
+# 
+# - if you use `mode`=`shiftlog|pearson` you need to set `target_sum=50*1e4`, more people like to se `target_sum=1e4`, we test the result think 50*1e4 will be better
+# - if you use `mode`=`pearson|pearson`, you don't need to set `target_sum`
+# 
+# <div class="admonition warning">
+#   <p class="admonition-title">Note</p>
+#   <p>
+#     if the version of `omicverse` lower than `1.4.13`, the mode can only be set between `scanpy` and `pearson`.
+#   </p>
+# </div>
+# 
+
+# In[7]:
+
+
+get_ipython().run_cell_magic('time', '', "adata=ov.pp.preprocess(adata,mode='shiftlog|pearson',n_HVGs=2000,)\nadata\n")
+
+
+# Set the .raw attribute of the AnnData object to the normalized and logarithmized raw gene expression for later use in differential testing and visualizations of gene expression. This simply freezes the state of the AnnData object.
+
+# In[8]:
+
+
+adata.raw = adata
+adata = adata[:, adata.var.highly_variable_features]
+adata
+
+
+# ## Principal component analysis
+# 
+# In contrast to scanpy, we do not directly scale the variance of the original expression matrix, but store the results of the variance scaling in the layer, due to the fact that scale may cause changes in the data distribution, and we have not found scale to be meaningful in any scenario other than a principal component analysis
+
+# In[9]:
+
+
+get_ipython().run_cell_magic('time', '', 'ov.pp.scale(adata)\nadata\n')
+
+
+# If you want to perform pca in normlog layer, you can set `layer`=`normlog`, but we think scaled is necessary in PCA.
+
+# In[10]:
+
+
+get_ipython().run_cell_magic('time', '', "ov.pp.pca(adata,layer='scaled',n_pcs=50)\nadata\n")
+
+
+# In[11]:
+
+
+adata.obsm['X_pca']=adata.obsm['scaled|original|X_pca']
+ov.utils.embedding(adata,
+                  basis='X_pca',
+                  color='CST3',
+                  frameon='small')
+
+
+# ## Embedding the neighborhood graph
+# 
+# We suggest embedding the graph in two dimensions using UMAP (McInnes et al., 2018), see below. It is potentially more faithful to the global connectivity of the manifold than tSNE, i.e., it better preserves trajectories. In some ocassions, you might still observe disconnected clusters and similar connectivity violations. They can usually be remedied by running:
+
+# In[23]:
+
+
+get_ipython().run_cell_magic('time', '', "ov.pp.neighbors(adata, n_neighbors=15, n_pcs=50,\n               use_rep='scaled|original|X_pca',method='cagra')\n")
+
+
+# To visualize the PCA’s embeddings, we use the `pymde` package wrapper in omicverse. This is an alternative to UMAP that is GPU-accelerated.
+
+# In[19]:
+
+
+adata.obsm["X_mde"] = ov.utils.mde(adata.obsm["scaled|original|X_pca"])
+adata
+
+
+# In[20]:
+
+
+ov.pl.embedding(adata,
+                basis='X_mde',
+                color='CST3',
+                frameon='small')
+
+
+# You also can use `umap` to visualize the neighborhood graph
+
+# In[21]:
+
+
+ov.pp.umap(adata)
+
+
+# In[22]:
+
+
+ov.pl.embedding(adata,
+                basis='X_umap',
+                color='CST3',
+                frameon='small')
+
+
+# ## Clustering the neighborhood graph
+# 
+# As with Seurat and many other frameworks, we recommend the Leiden graph-clustering method (community detection based on optimizing modularity) by Traag *et al.* (2018). Note that Leiden clustering directly clusters the neighborhood graph of cells, which we already computed in the previous section.
+
+# In[24]:
+
+
+ov.pp.leiden(adata)
+
+
+# In[30]:
+
+
+ov.pp.anndata_to_CPU(adata)
+
+
+# We redesigned the visualisation of embedding to distinguish it from scanpy's embedding by adding the parameter `fraemon='small'`, which causes the axes to be scaled with the colourbar
+
+# In[25]:
+
+
+ov.pl.embedding(adata,
+                basis='X_mde',
+                color=['leiden', 'CST3', 'NKG7'],
+                frameon='small')
+
+
+# We also provide a boundary visualisation function `ov.utils.plot_ConvexHull` to visualise specific clusters.
+# 
+# Arguments: 
+# - color: if None will use the color of clusters
+# - alpha: default is 0.2
+
+# In[26]:
+
+
+import matplotlib.pyplot as plt
+fig,ax=plt.subplots( figsize = (4,4))
+
+ov.pl.embedding(adata,
+                basis='X_mde',
+                color=['leiden'],
+                frameon='small',
+                show=False,
+                ax=ax)
+
+ov.pl.ConvexHull(adata,
+                basis='X_mde',
+                cluster_key='leiden',
+                hull_cluster='0',
+                ax=ax)
+
+
+# If you have too many labels, e.g. too many cell types, and you are concerned about cell overlap, then consider trying the `ov.utils.gen_mpl_labels` function, which improves text overlap.
+# In addition, we make use of the `patheffects` function, which makes our text have outlines
+# 
+# - adjust_kwargs: it could be found in package `adjusttext`
+# - text_kwargs: it could be found in class `plt.texts`
+
+# In[27]:
+
+
+from matplotlib import patheffects
+import matplotlib.pyplot as plt
+fig, ax = plt.subplots(figsize=(4,4))
+
+ov.pl.embedding(adata,
+                  basis='X_mde',
+                  color=['leiden'],
+                   show=False, legend_loc=None, add_outline=False, 
+                   frameon='small',legend_fontoutline=2,ax=ax
+                 )
+
+ov.utils.gen_mpl_labels(
+    adata,
+    'leiden',
+    exclude=("None",),  
+    basis='X_mde',
+    ax=ax,
+    adjust_kwargs=dict(arrowprops=dict(arrowstyle='-', color='black')),
+    text_kwargs=dict(fontsize= 12 ,weight='bold',
+                     path_effects=[patheffects.withStroke(linewidth=2, foreground='w')] ),
+)
+
+
+# In[28]:
+
+
+marker_genes = ['IL7R', 'CD79A', 'MS4A1', 'CD8A', 'CD8B', 'LYZ', 'CD14',
+                'LGALS3', 'S100A8', 'GNLY', 'NKG7', 'KLRB1',
+                'FCGR3A', 'MS4A7', 'FCER1A', 'CST3', 'PPBP']
+
+
+# In[29]:
+
+
+sc.pl.dotplot(adata, marker_genes, groupby='leiden',
+             standard_scale='var');
+
+
+# ## Finding marker genes
+# 
+# Let us compute a ranking for the highly differential genes in each cluster. For this, by default, the .raw attribute of AnnData is used in case it has been initialized before. The simplest and fastest method to do so is the t-test.
+
+# In[31]:
+
+
+sc.tl.dendrogram(adata,'leiden',use_rep='scaled|original|X_pca')
+sc.tl.rank_genes_groups(adata, 'leiden', use_rep='scaled|original|X_pca',
+                        method='t-test',use_raw=False,key_added='leiden_ttest')
+sc.pl.rank_genes_groups_dotplot(adata,groupby='leiden',
+                                cmap='Spectral_r',key='leiden_ttest',
+                                standard_scale='var',n_genes=3)
+
+
+# cosg is also considered to be a better algorithm for finding marker genes. Here, omicverse provides the calculation of cosg
+# 
+# Paper: [Accurate and fast cell marker gene identification with COSG](https://academic.oup.com/bib/advance-article-abstract/doi/10.1093/bib/bbab579/6511197?redirectedFrom=fulltext)
+# 
+# Code: https://github.com/genecell/COSG
+# 
+
+# In[32]:
+
+
+sc.tl.rank_genes_groups(adata, groupby='leiden', 
+                        method='t-test',use_rep='scaled|original|X_pca',)
+ov.single.cosg(adata, key_added='leiden_cosg', groupby='leiden')
+sc.pl.rank_genes_groups_dotplot(adata,groupby='leiden',
+                                cmap='Spectral_r',key='leiden_cosg',
+                                standard_scale='var',n_genes=3)
+
+
+# ## Other plotting
+# 
+# Next, let's try another chart, which we call the Stacked Volcano Chart. We need to prepare two dictionaries, a `data_dict` and a `color_dict`, both of which have the same key requirements.
+# 
+# For `data_dict`. we require the contents within each key to be a DataFrame containing ['names','logfoldchanges','pvals_adj'], where names stands for gene names, logfoldchanges stands for differential expression multiplicity, pvals_adj stands for significance p-value
+# 
+
+# In[33]:
+
+
+data_dict={}
+for i in adata.obs['leiden'].cat.categories:
+    data_dict[i]=sc.get.rank_genes_groups_df(adata, group=i, key='leiden_ttest',
+                                            pval_cutoff=None,log2fc_min=None)
+
+
+# In[34]:
+
+
+data_dict.keys()
+
+
+# In[35]:
+
+
+data_dict[i].head()
+
+
+# For `color_dict`, we require that the colour to be displayed for the current key is stored within each key.`
+
+# In[36]:
+
+
+type_color_dict=dict(zip(adata.obs['leiden'].cat.categories,
+                         adata.uns['leiden_colors']))
+type_color_dict
+
+
+# There are a number of parameters available here for us to customise the settings. Note that when drawing stacking_vol with omicverse version less than 1.4.13, there is a bug that the vertical coordinate is constant at [-15,15], so we have added some code in this tutorial for visualisation.
+# 
+# - data_dict: dict, in each key, there is a dataframe with columns of ['logfoldchanges','pvals_adj','names']
+# - color_dict: dict, in each key, there is a color for each omic
+# - pval_threshold: float, pvalue threshold for significant genes
+# - log2fc_threshold: float, log2fc threshold for significant genes
+# - figsize: tuple, figure size
+# - sig_color: str, color for significant genes
+# - normal_color: str, color for non-significant genes
+# - plot_genes_num: int, number of genes to plot
+# - plot_genes_fontsize: int, fontsize for gene names
+# - plot_genes_weight: str, weight for gene names
+
+# In[37]:
+
+
+fig,axes=ov.utils.stacking_vol(data_dict,type_color_dict,
+            pval_threshold=0.01,
+            log2fc_threshold=2,
+            figsize=(8,4),
+            sig_color='#a51616',
+            normal_color='#c7c7c7',
+            plot_genes_num=2,
+            plot_genes_fontsize=6,
+            plot_genes_weight='bold',
+            )
+
+#The following code will be removed in future
+y_min,y_max=0,0
+for i in data_dict.keys():
+    y_min=min(y_min,data_dict[i]['logfoldchanges'].min())
+    y_max=max(y_max,data_dict[i]['logfoldchanges'].max())
+for i in adata.obs['leiden'].cat.categories:
+    axes[i].set_ylim(y_min,y_max)
+plt.suptitle('Stacking_vol',fontsize=12)   
+
+
+# In[ ]:
+
+
+
+

ovrawm/t_scdeg.txt

@@ -0,0 +1,316 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# # Differential expression analysis in single cell
+# 
+# Sometimes we need to compare differentially expressed genes or differentially expressed features between two cell types on single cell data, but existing methods focus more on cell-specific gene analysis. Researchers need to transfer bulk RNA-seq analysis to single-cell analysis, which involves interaction between different programming languages or programming tools, adding significantly to the workload of the researcher.
+# 
+# Here, we use omicverse's bulk RNA-seq pyDEG method to complete differential expression analysis at the single cell level. We will present two different perspectives, one from the perspective of all cells and one from the perspective of the metacellular.
+# 
+# Colab_Reproducibility：https://colab.research.google.com/drive/12faBRh0xT7v6KSy8NCSRqbegF_AEoDXr?usp=sharing
+
+# In[1]:
+
+
+import omicverse as ov
+import scanpy as sc
+import scvelo as scv
+
+ov.utils.ov_plot_set()
+
+
+# ## Data preprocessed 
+# 
+# We need to normalized and scale the data at first.
+
+# In[2]:
+
+
+adata = scv.datasets.pancreas()
+adata
+
+
+# In[3]:
+
+
+adata.X.max()
+
+
+# We found that the max value of anndata object larger than 10 and type is int. We need to normalize and log1p it
+
+# In[4]:
+
+
+#quantity control
+adata=ov.pp.qc(adata,
+              tresh={'mito_perc': 0.05, 'nUMIs': 500, 'detected_genes': 250})
+#normalize and high variable genes (HVGs) calculated
+adata=ov.pp.preprocess(adata,mode='shiftlog|pearson',n_HVGs=2000,)
+
+#save the whole genes and filter the non-HVGs
+adata.raw = adata
+adata = adata[:, adata.var.highly_variable_features]
+
+#scale the adata.X
+ov.pp.scale(adata)
+
+#Dimensionality Reduction
+ov.pp.pca(adata,layer='scaled',n_pcs=50)
+
+
+# In[5]:
+
+
+adata.X.max()
+
+
+# ## Different expression in total level
+# 
+# We then select the target cells to be analysed, including `Alpha` and `Beta`, derive the expression matrix using `to_df()` and build the differential expression analysis module using `pyDEG`
+
+# In[6]:
+
+
+test_adata=adata[adata.obs['clusters'].isin(['Alpha','Beta'])]
+test_adata
+
+
+# In[7]:
+
+
+dds=ov.bulk.pyDEG(test_adata.to_df(layer='lognorm').T)
+
+
+# In[8]:
+
+
+dds.drop_duplicates_index()
+print('... drop_duplicates_index success')
+
+
+# We also need to set up an experimental group and a control group, i.e. the two types of cells to be compared and analysed
+
+# In[9]:
+
+
+treatment_groups=test_adata.obs[test_adata.obs['clusters']=='Alpha'].index.tolist()
+control_groups=test_adata.obs[test_adata.obs['clusters']=='Beta'].index.tolist()
+result=dds.deg_analysis(treatment_groups,control_groups,method='ttest')
+
+
+# In[10]:
+
+
+result.sort_values('qvalue').head()
+
+
+# In[11]:
+
+
+# -1 means automatically calculates
+dds.foldchange_set(fc_threshold=-1,
+                   pval_threshold=0.05,
+                   logp_max=10)
+
+
+# In[12]:
+
+
+dds.plot_volcano(title='DEG Analysis',figsize=(4,4),
+                 plot_genes_num=8,plot_genes_fontsize=12,)
+
+
+# In[13]:
+
+
+dds.plot_boxplot(genes=['Irx1','Adra2a'],treatment_groups=treatment_groups,
+                control_groups=control_groups,figsize=(2,3),fontsize=12,
+                 legend_bbox=(2,0.55))
+
+
+# In[14]:
+
+
+ov.utils.embedding(adata,
+                   basis='X_umap',
+                    frameon='small',
+                   color=['clusters','Irx1','Adra2a'])
+
+
+# ## Different expression in Metacells level
+# 
+# Here, we calculated the metacells from the whole scRNA-seq datasets using SEACells, and the same analyze with total level.
+
+# ### Constructing a metacellular object
+# 
+# We can use `ov.single.MetaCell` to construct a metacellular object to train the SEACells model, the arguments can be found in below.
+# 
+# - :param ad: (AnnData) annotated data matrix
+# - :param build_kernel_on: (str) key corresponding to matrix in ad.obsm which is used to compute kernel for metacells
+#                         Typically 'X_pca' for scRNA or 'X_svd' for scATAC
+# - :param n_SEACells: (int) number of SEACells to compute
+# - :param use_gpu: (bool) whether to use GPU for computation
+# - :param verbose: (bool) whether to suppress verbose program logging
+# - :param n_waypoint_eigs: (int) number of eigenvectors to use for waypoint initialization
+# - :param n_neighbors: (int) number of nearest neighbors to use for graph construction
+# - :param convergence_epsilon: (float) convergence threshold for Franke-Wolfe algorithm
+# - :param l2_penalty: (float) L2 penalty for Franke-Wolfe algorithm
+# - :param max_franke_wolfe_iters: (int) maximum number of iterations for Franke-Wolfe algorithm
+# - :param use_sparse: (bool) whether to use sparse matrix operations. Currently only supported for CPU implementation.
+
+# In[15]:
+
+
+meta_obj=ov.single.MetaCell(adata,use_rep='scaled|original|X_pca',n_metacells=150,
+                           use_gpu=True)
+
+
+# In[16]:
+
+
+meta_obj.initialize_archetypes()
+
+
+# ## Train and save the model
+
+# In[17]:
+
+
+meta_obj.train(min_iter=10, max_iter=50)
+
+
+# In[34]:
+
+
+meta_obj.save('seacells/model.pkl')
+
+
+# In[ ]:
+
+
+meta_obj.load('seacells/model.pkl')
+
+
+# ## Predicted the metacells
+# 
+# we can use `predicted` to predicted the metacells of raw scRNA-seq data. There are two method can be selected, one is `soft`, the other is `hard`. 
+# 
+# In the `soft` method, Aggregates cells within each SEACell, summing over all raw data x assignment weight for all cells belonging to a SEACell. Data is un-normalized and pseudo-raw aggregated counts are stored in .layers['raw']. Attributes associated with variables (.var) are copied over, but relevant per SEACell attributes must be manually copied, since certain attributes may need to be summed, or averaged etc, depending on the attribute.
+# 
+# In the `hard` method, Aggregates cells within each SEACell, summing over all raw data for all cells belonging to a SEACell. Data is unnormalized and raw aggregated counts are stored .layers['raw']. Attributes associated with variables (.var) are copied over, but relevant per SEACell attributes must be manually copied, since certain attributes may need to be summed, or averaged etc, depending on the attribute.
+
+# In[19]:
+
+
+ad=meta_obj.predicted(method='soft',celltype_label='clusters',
+                     summarize_layer='lognorm')
+
+
+# In[20]:
+
+
+ad.X.min(),ad.X.max()
+
+
+# In[21]:
+
+
+import matplotlib.pyplot as plt
+fig, ax = plt.subplots(figsize=(4,4))
+ov.utils.embedding(
+    meta_obj.adata,
+    basis="X_umap",
+    color=['clusters'],
+    frameon='small',
+    title="Meta cells",
+    #legend_loc='on data',
+    legend_fontsize=14,
+    legend_fontoutline=2,
+    size=10,
+    ax=ax,
+    alpha=0.2,
+    #legend_loc='', 
+    add_outline=False, 
+    #add_outline=True,
+    outline_color='black',
+    outline_width=1,
+    show=False,
+    #palette=ov.utils.blue_color[:],
+    #legend_fontweight='normal'
+)
+ov.single._metacell.plot_metacells(ax,meta_obj.adata,color='#CB3E35',
+                                  )
+
+
+# ### Differentially expressed analysis
+# 
+# Similar to total cells for differential expression analysis, we used metacells for differential expression in the same way.
+
+# In[23]:
+
+
+test_adata=ad[ad.obs['celltype'].isin(['Alpha','Beta'])]
+test_adata
+
+
+# In[24]:
+
+
+dds_meta=ov.bulk.pyDEG(test_adata.to_df().T)
+
+
+# In[25]:
+
+
+dds_meta.drop_duplicates_index()
+print('... drop_duplicates_index success')
+
+
+# We also need to set up an experimental group and a control group, i.e. the two types of cells to be compared and analysed
+
+# In[27]:
+
+
+treatment_groups=test_adata.obs[test_adata.obs['celltype']=='Alpha'].index.tolist()
+control_groups=test_adata.obs[test_adata.obs['celltype']=='Beta'].index.tolist()
+result=dds_meta.deg_analysis(treatment_groups,control_groups,method='ttest')
+
+
+# In[28]:
+
+
+result.sort_values('qvalue').head()
+
+
+# In[29]:
+
+
+# -1 means automatically calculates
+dds_meta.foldchange_set(fc_threshold=-1,
+                   pval_threshold=0.05,
+                   logp_max=10)
+
+
+# In[30]:
+
+
+dds_meta.plot_volcano(title='DEG Analysis',figsize=(4,4),
+                 plot_genes_num=8,plot_genes_fontsize=12,)
+
+
+# In[31]:
+
+
+dds_meta.plot_boxplot(genes=['Ctxn2','Mnx1'],treatment_groups=treatment_groups,
+                control_groups=control_groups,figsize=(2,3),fontsize=12,
+                 legend_bbox=(2,0.55))
+
+
+# In[32]:
+
+
+ov.utils.embedding(adata,
+                   basis='X_umap',
+                    frameon='small',
+                   color=['clusters','Ctxn2','Mnx1'])
+

ovrawm/t_scdrug.txt

@@ -0,0 +1,225 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# # Drug response predict with scDrug
+# 
+# scDrug is a database that can be used to predict the drug sensitivity of single cells based on an existing database of drug responses. In the downstream tasks of single cell analysis, especially in tumours, we are fully interested in potential drugs and combination therapies. To this end, we have integrated scDrug's IC50 prediction and inferCNV to infer the function of tumour cells to build a drug screening pipeline.
+# 
+# Paper: [scDrug: From single-cell RNA-seq to drug response prediction](https://www.sciencedirect.com/science/article/pii/S2001037022005505)
+# 
+# Code: https://github.com/ailabstw/scDrug
+# 
+# Colab_Reproducibility：https://colab.research.google.com/drive/1mayoMO7I7qjYIRjrZEi8r5zuERcxAEcF?usp=sharing
+
+# In[1]:
+
+
+import omicverse as ov
+import scanpy as sc
+import infercnvpy as cnv
+import matplotlib.pyplot as plt
+import os
+
+sc.settings.verbosity = 3             # verbosity: errors (0), warnings (1), info (2), hints (3)
+sc.settings.set_figure_params(dpi=80, facecolor='white')
+
+
+# ## Infer the Tumor from scRNA-seq
+# 
+# Here we use Infercnvpy's example data to complete the tumour analysis, you can also refer to the official tutorial for this step: https://infercnvpy.readthedocs.io/en/latest/notebooks/tutorial_3k.html
+# 
+# So, we provide a utility function ov.utils.get_gene_annotation to supplement the coordinate information from GTF files. The following usage assumes that the adata.var_names correspond to “gene_name” attribute in the GTF file. For other cases, please check the function documentation.
+# 
+# The GTF file used here can be downloaded from [GENCODE](http://ftp.ebi.ac.uk/pub/databases/gencode/Gencode_mouse/release_M25/).
+# 
+# T2T-CHM13 gtf file can be download from [figshare](https://figshare.com/ndownloader/files/40628072)
+
+# In[3]:
+
+
+adata = cnv.datasets.maynard2020_3k()
+
+ov.utils.get_gene_annotation(
+    adata, gtf="gencode.v43.basic.annotation.gtf.gz",
+    gtf_by="gene_name"
+)
+
+
+# In[ ]:
+
+
+adata=adata[:,~adata.var['chrom'].isnull()]
+adata.var['chromosome']=adata.var['chrom']
+adata.var['start']=adata.var['chromStart']
+adata.var['end']=adata.var['chromEnd']
+adata.var['ensg']=adata.var['gene_id']
+adata.var.loc[:, ["ensg", "chromosome", "start", "end"]].head()
+
+
+# We noted that infercnvpy need to normalize and log the matrix at first
+
+# In[4]:
+
+
+adata
+
+
+# We use the immune cells as reference and infer the cnv score of each cells in scRNA-seq
+
+# In[5]:
+
+
+# We provide all immune cell types as "normal cells".
+cnv.tl.infercnv(
+    adata,
+    reference_key="cell_type",
+    reference_cat=[
+        "B cell",
+        "Macrophage",
+        "Mast cell",
+        "Monocyte",
+        "NK cell",
+        "Plasma cell",
+        "T cell CD4",
+        "T cell CD8",
+        "T cell regulatory",
+        "mDC",
+        "pDC",
+    ],
+    window_size=250,
+)
+cnv.tl.pca(adata)
+cnv.pp.neighbors(adata)
+cnv.tl.leiden(adata)
+cnv.tl.umap(adata)
+cnv.tl.cnv_score(adata)
+
+
+# In[6]:
+
+
+sc.pl.umap(adata, color="cnv_score", show=False)
+
+
+# We set an appropriate threshold for the cnv_score, here we set it to 0.03 and identify cells greater than 0.03 as tumour cells
+
+# In[7]:
+
+
+adata.obs["cnv_status"] = "normal"
+adata.obs.loc[
+    adata.obs["cnv_score"]>0.03, "cnv_status"
+] = "tumor"
+
+
+# In[8]:
+
+
+sc.pl.umap(adata, color="cnv_status", show=False)
+
+
+# We extracted tumour cells separately for drug prediction response
+
+# In[11]:
+
+
+tumor=adata[adata.obs['cnv_status']=='tumor']
+tumor.X.max()
+
+
+# ## Tumor preprocessing
+# 
+# We need to extract the highly variable genes in the tumour for further analysis, and found out the sub-cluster in tumor
+
+# In[12]:
+
+
+adata=tumor
+print('Preprocessing...')
+sc.pp.filter_cells(adata, min_genes=200)
+sc.pp.filter_genes(adata, min_cells=3)
+adata.var['mt'] = adata.var_names.str.startswith('MT-')
+sc.pp.calculate_qc_metrics(adata, qc_vars=['mt'], percent_top=None, log1p=False, inplace=True)
+if not (adata.obs.pct_counts_mt == 0).all():
+    adata = adata[adata.obs.pct_counts_mt < 30, :]
+
+adata.raw = adata.copy()
+
+sc.pp.highly_variable_genes(adata)
+adata = adata[:, adata.var.highly_variable]
+sc.pp.scale(adata)
+sc.tl.pca(adata, svd_solver='arpack')
+
+
+# In[13]:
+
+
+sc.pp.neighbors(adata, n_pcs=20)
+sc.tl.umap(adata)
+
+
+# Here, we need to download the scDrug database and mods so that the subsequent predictions can be made properly
+
+# In[27]:
+
+
+ov.utils.download_GDSC_data()
+ov.utils.download_CaDRReS_model()
+
+
+# Then, we apply Single-Cell Data Analysis once again to carry out sub-clustering on the tumor clusters at automatically determined resolution.
+
+# In[18]:
+
+
+adata, res,plot_df = ov.single.autoResolution(adata,cpus=4)
+
+
+# Don't forget to save your data
+
+# In[20]:
+
+
+results_file = os.path.join('./', 'scanpyobj.h5ad')
+adata.write(results_file)
+
+
+# In[21]:
+
+
+results_file = os.path.join('./', 'scanpyobj.h5ad')
+adata=sc.read(results_file)
+
+
+# ## IC50 predicted
+# 
+# Drug Response Prediction examined scanpyobj.h5ad generated in Single-Cell Data Analysis, reported clusterwise IC50 and cell death percentages to drugs in the GDSC database via CaDRReS-Sc (a recommender system framework for in silico drug response prediction), or drug sensitivity AUC in the PRISM database from [DepMap Portal PRISM-19Q4](https://doi.org/10.1038/s43018-019-0018-6). 
+# 
+# Note we need to download the CaDRReS-Sc from github by `git clone https://github.com/CSB5/CaDRReS-Sc`
+
+# In[24]:
+
+
+get_ipython().system('git clone https://github.com/CSB5/CaDRReS-Sc')
+
+
+# To run drug response predicted, we need to set:
+# 
+# - scriptpath: the CaDRReS-Sc path we downloaded just now
+# - modelpath: the model path we downloaded just now
+# - output: the save path of drug response predicted result 
+
+# In[25]:
+
+
+import ov
+job=ov.single.Drug_Response(adata,scriptpath='CaDRReS-Sc',
+                                modelpath='models/',
+                                output='result')
+
+
+# In[ ]:
+
+
+
+

ovrawm/t_scmulan.txt

@@ -0,0 +1,199 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# ## Using scMulan to annotate cell types in Heart, Lung, Liver, Bone marrow, Blood, Brain, and Thymus
+
+# In this study, the authors enrich the pre-training paradigm by integrating an abundance of metadata and a multiplicity of pre-training tasks, and obtain scMulan, a multitask generative pre-trained language model tailored for single-cell analysis. They represent a cell as a structured cell sentence (c-sentence) by encoding its gene expression, metadata terms, and target tasks as words of tuples, each consisting of entities and their corresponding values. They construct a unified generative framework to model the cell language on c-sentence and design three pretraining tasks to bridge the microscopic and macroscopic information within the c-sentences. They pre-train scMulan on 10 million single-cell transcriptomic data and their corresponding metadata, with 368 million parameters. As a single model, scMulan can accomplish tasks zero-shot for cell type annotation, batch integration, and conditional cell generation, guided by different task prompts.
+
+# #### we provide a liver dataset sampled (percentage of 20%) from Suo C, 2022 (doi/10.1126/science.abo0510)
+# **Paper:** [scMulan: a multitask generative pre-trained language model for single-cell analysis](https://www.biorxiv.org/content/10.1101/2024.01.25.577152v1)    
+# **Data download:** https://cloud.tsinghua.edu.cn/f/45a7fd2a27e543539f59/?dl=1   
+# **Pre-train model download:** https://cloud.tsinghua.edu.cn/f/2250c5df51034b2e9a85/?dl=1
+# 
+# If you found this tutorial helpful, please cite scMulan and OmicVerse:         
+# Bian H, Chen Y, Dong X, et al. scMulan: a multitask generative pre-trained language model for single-cell analysis[C]//International Conference on Research in Computational Molecular Biology. Cham: Springer Nature Switzerland, 2024: 479-482.
+
+# In[36]:
+
+
+import os
+#os.environ["CUDA_VISIBLE_DEVICES"] = "-1" # if use CPU only
+import scanpy as sc
+import omicverse as ov
+ov.plot_set()
+#import scMulan
+#from scMulan import GeneSymbolUniform
+
+
+# ## 1. load h5ad
+# You can download the liver dataset from the following link: https://cloud.tsinghua.edu.cn/f/45a7fd2a27e543539f59/?dl=1 
+# 
+# It's recommended that you use h5ad here with raw count (and after your QC)
+# 
+
+# In[4]:
+
+
+adata = sc.read('./data/liver_test.h5ad')
+
+
+# In[5]:
+
+
+adata
+
+
+# In[6]:
+
+
+from scipy.sparse import csc_matrix
+adata.X = csc_matrix(adata.X)
+
+
+# ## 2. transform original h5ad with uniformed genes (42117 genes)
+
+# This step transform the genes in input adata to 42117 gene symbols and reserves the corresponding gene expression values. The gene symbols are the same as the pre-trained scMulan model.
+
+# In[7]:
+
+
+adata_GS_uniformed = ov.externel.scMulan.GeneSymbolUniform(input_adata=adata,
+                                 output_dir="./data",
+                                 output_prefix='liver')
+
+
+# ## 3. process uniformed data (simply norm and log1p)
+
+# In[8]:
+
+
+## you can read the saved uniformed adata
+
+adata_GS_uniformed=sc.read_h5ad('./data/liver_uniformed.h5ad')
+
+
+# In[9]:
+
+
+adata_GS_uniformed
+
+
+# In[10]:
+
+
+# norm and log1p count matrix
+# in some case, the count matrix is not normalized, and log1p is not applied.
+# So we need to normalize the count matrix
+if adata_GS_uniformed.X.max() > 10:
+    sc.pp.normalize_total(adata_GS_uniformed, target_sum=1e4) 
+    sc.pp.log1p(adata_GS_uniformed)
+
+
+# ## 4. load scMulan
+
+# In[11]:
+
+
+# you should first download ckpt from https://cloud.tsinghua.edu.cn/f/2250c5df51034b2e9a85/?dl=1
+# put it under .ckpt/ckpt_scMulan.pt
+# by: wget https://cloud.tsinghua.edu.cn/f/2250c5df51034b2e9a85/?dl=1  -O ckpt/ckpt_scMulan.pt
+
+ckp_path = './ckpt/ckpt_scMulan.pt'
+
+
+# In[12]:
+
+
+scml = ov.externel.scMulan.model_inference(ckp_path, adata_GS_uniformed)
+base_process = scml.cuda_count()
+
+
+# In[13]:
+
+
+scml.get_cell_types_and_embds_for_adata(parallel=True, n_process = 1)
+# scml.get_cell_types_and_embds_for_adata(parallel=False) # for only using CPU, but it is really slow.
+
+
+# The predicted cell types are stored in scml.adata.obs['cell_type_from_scMulan'], besides the cell embeddings (for multibatch integration) in scml.adata.obsm['X_scMulan'] (not used in this tutorial).
+
+# ## 5. visualization
+# 
+# Here, we visualize the cell types predicted by scMulan. And we also visualize the original cell types in the dataset.
+
+# In[14]:
+
+
+adata_mulan = scml.adata.copy()
+
+
+# In[15]:
+
+
+# calculated the 2-D embedding of the adata using pyMDE
+ov.pp.scale(adata_mulan)
+ov.pp.pca(adata_mulan)
+
+#sc.pl.pca_variance_ratio(adata_mulan)
+ov.pp.mde(adata_mulan,embedding_dim=2,n_neighbors=15, basis='X_mde',
+          n_pcs=10, use_rep='scaled|original|X_pca',)
+
+
+# In[26]:
+
+
+# Here, we can see the cell type annotation from scMulan
+ov.pl.embedding(adata_mulan,basis='X_mde',
+                color=["cell_type_from_scMulan",],
+                ncols=1,frameon='small')
+
+
+# In[29]:
+
+
+adata_mulan.obsm['X_umap']=adata_mulan.obsm['X_mde']
+
+
+# In[30]:
+
+
+# you can run smoothing function to filter the false positives
+ov.externel.scMulan.cell_type_smoothing(adata_mulan, threshold=0.1)
+
+
+# In[31]:
+
+
+# cell_type_from_mulan_smoothing: pred+smoothing
+# cell_type: original annotations by the authors
+ov.pl.embedding(adata_mulan,basis='X_mde',
+                color=["cell_type_from_mulan_smoothing","cell_type"],
+                ncols=1,frameon='small')
+
+
+# In[32]:
+
+
+adata_mulan
+
+
+# In[33]:
+
+
+top_celltypes = adata_mulan.obs.cell_type_from_scMulan.value_counts().index[:20]
+
+
+# In[34]:
+
+
+# you can select some cell types of interest (from scMulan's prediction) for visulization
+# selected_cell_types = ["NK cell", "Kupffer cell", "Conventional dendritic cell 2"] # as example
+selected_cell_types = top_celltypes
+ov.externel.scMulan.visualize_selected_cell_types(adata_mulan,selected_cell_types,smoothing=True)
+
+
+# In[ ]:
+
+
+
+

ovrawm/t_simba.txt

@@ -0,0 +1,146 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# # Data integration and batch correction with SIMBA
+# 
+# Here we will use three scRNA-seq human pancreas datasets of different studies as an example to illustrate how SIMBA performs scRNA-seq batch correction for multiple batches
+# 
+# We follow the corresponding tutorial at [SIMBA](https://simba-bio.readthedocs.io/en/latest/rna_human_pancreas.html). We do not provide much explanation, and instead refer to the original tutorial.
+# 
+# Paper: [SIMBA: single-cell embedding along with features](https://www.nature.com/articles/s41592-023-01899-8)
+# 
+# Code: https://github.com/huidongchen/simba
+
+# In[1]:
+
+
+import omicverse as ov
+from omicverse.utils import mde
+workdir = 'result_human_pancreas'
+ov.utils.ov_plot_set()
+
+
+# We need to install simba at first
+# 
+# ```
+# conda install -c bioconda simba
+# ```
+# 
+# or
+# 
+# ```
+# pip install git+https://github.com/huidongchen/simba
+# pip install git+https://github.com/pinellolab/simba_pbg
+# ```
+
+# ## Read data
+# 
+# The anndata object was concat from three anndata in simba: `simba.datasets.rna_baron2016()`, `simba.datasets.rna_segerstolpe2016()`, and `simba.datasets.rna_muraro2016()`
+# 
+# It can be downloaded from figshare: https://figshare.com/ndownloader/files/41418600
+
+# In[2]:
+
+
+adata=ov.utils.read('simba_adata_raw.h5ad')
+
+
+# We need to set workdir to initiate the pySIMBA object
+
+# In[3]:
+
+
+simba_object=ov.single.pySIMBA(adata,workdir)
+
+
+# ## Preprocess
+# 
+# Follow the raw tutorial, we set the paragument as default.
+
+# In[4]:
+
+
+simba_object.preprocess(batch_key='batch',min_n_cells=3,
+                    method='lib_size',n_top_genes=3000,n_bins=5)
+
+
+# ## Generate a graph for training
+# 
+# Observations and variables within each Anndata object are both represented as nodes (entities).
+# 
+# the data store in `simba_object.uns['simba_batch_edge_dict']`
+
+# In[5]:
+
+
+simba_object.gen_graph()
+
+
+# ## PBG training
+# 
+# Before training, let’s take a look at the current parameters:
+# 
+# - dict_config['workers'] = 12 #The number of CPUs.
+
+# In[10]:
+
+
+simba_object.train(num_workers=6)
+
+
+# In[6]:
+
+
+simba_object.load('result_human_pancreas/pbg/graph0')
+
+
+# ## Batch correction
+# 
+# Here, we use `simba_object.batch_correction()` to perform the batch correction
+# 
+# <div class="admonition note">
+#   <p class="admonition-title">Note</p>
+#   <p>
+#     If the batch is greater than 10, then the batch correction is less effective
+#   </p>
+# </div>
+
+# In[7]:
+
+
+adata=simba_object.batch_correction()
+adata
+
+
+# ## Visualize
+# 
+# We also use `mde` instead `umap` to visualize the result
+
+# In[8]:
+
+
+adata.obsm["X_mde"] = mde(adata.obsm["X_simba"])
+
+
+# In[11]:
+
+
+sc.pl.embedding(adata,basis='X_mde',color=['cell_type1','batch'])
+
+
+# Certainly, umap can also be used to visualize
+
+# In[10]:
+
+
+import scanpy as sc
+sc.pp.neighbors(adata, use_rep="X_simba")
+sc.tl.umap(adata)
+sc.pl.umap(adata,color=['cell_type1','batch'])
+
+
+# In[ ]:
+
+
+
+

ovrawm/t_single_batch.txt

@@ -0,0 +1,333 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# # Data integration and batch correction
+# 
+# An important task of single-cell analysis is the integration of several samples, which we can perform with omicverse. 
+# 
+# Here we demonstrate how to merge data using omicverse and perform a corrective analysis for batch effects. We provide a total of 4 methods for batch effect correction in omicverse, including harmony, scanorama and combat which do not require GPU, and SIMBA which requires GPU. if available, we recommend using GPU-based scVI and scANVI to get the best batch effect correction results.
+# 
+# 
+
+# In[1]:
+
+
+import omicverse as ov
+#print(f"omicverse version: {ov.__version__}")
+import scanpy as sc
+#print(f"scanpy version: {sc.__version__}")
+ov.utils.ov_plot_set()
+
+
+# ## Data integration
+# 
+# First, we need to concat the data of scRNA-seq from different batch. We can use `sc.concat` to perform it。
+# 
+# The dataset we will use to demonstrate data integration contains several samples of bone marrow mononuclear cells. These samples were originally created for the Open Problems in Single-Cell Analysis NeurIPS Competition 2021.
+# 
+# We selected sample of `s1d3`, `s2d1` and `s3d7` to perform integrate. The individual data can be downloaded from figshare.
+# 
+# - s1d3:
+# - s2d1:
+# - s3d7:
+
+# In[2]:
+
+
+adata1=ov.read('neurips2021_s1d3.h5ad')
+adata1.obs['batch']='s1d3'
+adata2=ov.read('neurips2021_s2d1.h5ad')
+adata2.obs['batch']='s2d1'
+adata3=ov.read('neurips2021_s3d7.h5ad')
+adata3.obs['batch']='s3d7'
+
+
+# In[3]:
+
+
+adata=sc.concat([adata1,adata2,adata3],merge='same')
+adata
+
+
+# We can see that there are now three elements in the batch
+
+# In[4]:
+
+
+adata.obs['batch'].unique()
+
+
+# In[7]:
+
+
+import numpy as np
+adata.X=adata.X.astype(np.int64)
+
+
+# ## Data preprocess and Batch visualize
+# 
+# We first performed quality control of the data and normalisation with screening for highly variable genes. Then visualise potential batch effects in the data.
+# 
+# Here, we can set `batch_key=batch` to correct the doublet detectation and Highly variable genes identifcation.
+
+# In[8]:
+
+
+adata=ov.pp.qc(adata,
+              tresh={'mito_perc': 0.2, 'nUMIs': 500, 'detected_genes': 250},
+              batch_key='batch')
+adata
+
+
+# We can store the raw counts if we need the raw counts after filtered the HVGs.
+
+# In[10]:
+
+
+adata=ov.pp.preprocess(adata,mode='shiftlog|pearson',
+                       n_HVGs=3000,batch_key=None)
+adata
+
+
+# In[11]:
+
+
+adata.raw = adata
+adata = adata[:, adata.var.highly_variable_features]
+adata
+
+
+# We can save the pre-processed data.
+
+# In[12]:
+
+
+adata.write_h5ad('neurips2021_batch_normlog.h5ad',compression='gzip')
+
+
+# Similarly, we calculated PCA for HVGs and visualised potential batch effects in the data using pymde. pymde is GPU-accelerated UMAP.
+
+# In[13]:
+
+
+ov.pp.scale(adata)
+ov.pp.pca(adata,layer='scaled',n_pcs=50,mask_var='highly_variable_features')
+
+adata.obsm["X_mde_pca"] = ov.utils.mde(adata.obsm["scaled|original|X_pca"])
+
+
+# There is a very clear batch effect in the data
+
+# In[14]:
+
+
+ov.utils.embedding(adata,
+                basis='X_mde_pca',frameon='small',
+                color=['batch','cell_type'],show=False)
+
+
+# ## Harmony
+# 
+# Harmony is an algorithm for performing integration of single cell genomics datasets. Please check out manuscript on [Nature Methods](https://www.nature.com/articles/s41592-019-0619-0).
+# 
+# ![harmony](https://portals.broadinstitute.org/harmony/articles/main.jpg)
+
+# The function `ov.single.batch_correction` can be set in three methods: `harmony`,`combat` and `scanorama`
+
+# In[40]:
+
+
+adata_harmony=ov.single.batch_correction(adata,batch_key='batch',
+                                        methods='harmony',n_pcs=50)
+adata
+
+
+# In[41]:
+
+
+adata.obsm["X_mde_harmony"] = ov.utils.mde(adata.obsm["X_harmony"])
+
+
+# In[42]:
+
+
+ov.utils.embedding(adata,
+                basis='X_mde_harmony',frameon='small',
+                color=['batch','cell_type'],show=False)
+
+
+# ## Combat
+# 
+# combat is a batch effect correction method that is very widely used in bulk RNA-seq, and it works just as well on single-cell sequencing data.
+# 
+# 
+
+# In[43]:
+
+
+adata_combat=ov.single.batch_correction(adata,batch_key='batch',
+                                        methods='combat',n_pcs=50)
+adata
+
+
+# In[44]:
+
+
+adata.obsm["X_mde_combat"] = ov.utils.mde(adata.obsm["X_combat"])
+
+
+# In[45]:
+
+
+ov.utils.embedding(adata,
+                basis='X_mde_combat',frameon='small',
+                color=['batch','cell_type'],show=False)
+
+
+# ## scanorama
+# 
+# Integration of single-cell RNA sequencing (scRNA-seq) data from multiple experiments, laboratories and technologies can uncover biological insights, but current methods for scRNA-seq data integration are limited by a requirement for datasets to derive from functionally similar cells. We present Scanorama, an algorithm that identifies and merges the shared cell types among all pairs of datasets and accurately integrates heterogeneous collections of scRNA-seq data. 
+# 
+# ![scanorama](https://media.springernature.com/full/springer-static/image/art%3A10.1038%2Fs41587-019-0113-3/MediaObjects/41587_2019_113_Fig1_HTML.png?as=webp)
+
+# In[46]:
+
+
+adata_scanorama=ov.single.batch_correction(adata,batch_key='batch',
+                                        methods='scanorama',n_pcs=50)
+adata
+
+
+# In[47]:
+
+
+adata.obsm["X_mde_scanorama"] = ov.utils.mde(adata.obsm["X_scanorama"])
+
+
+# In[48]:
+
+
+ov.utils.embedding(adata,
+                basis='X_mde_scanorama',frameon='small',
+                color=['batch','cell_type'],show=False)
+
+
+# ## scVI
+# 
+# An important task of single-cell analysis is the integration of several samples, which we can perform with scVI. For integration, scVI treats the data as unlabelled. When our dataset is fully labelled (perhaps in independent studies, or independent analysis pipelines), we can obtain an integration that better preserves biology using scANVI, which incorporates cell type annotation information. Here we demonstrate this functionality with an integrated analysis of cells from the lung atlas integration task from the scIB manuscript. The same pipeline would generally be used to analyze any collection of scRNA-seq datasets.
+
+# In[3]:
+
+
+adata_scvi=ov.single.batch_correction(adata,batch_key='batch',
+                           methods='scVI',n_layers=2, n_latent=30, gene_likelihood="nb")
+adata
+
+
+# In[4]:
+
+
+adata.obsm["X_mde_scVI"] = ov.utils.mde(adata.obsm["X_scVI"])
+
+
+# In[5]:
+
+
+ov.utils.embedding(adata,
+                basis='X_mde_scVI',frameon='small',
+                color=['batch','cell_type'],show=False)
+
+
+# ## MIRA+CODAL
+# 
+# Topic modeling of batched single-cell data is challenging because these models cannot typically distinguish between biological and technical effects of the assay. CODAL (COvariate Disentangling Augmented Loss) uses a novel mutual information regularization technique to explicitly disentangle these two sources of variation.
+
+# In[15]:
+
+
+LDA_obj=ov.utils.LDA_topic(adata,feature_type='expression',
+                  highly_variable_key='highly_variable_features',
+                 layers='counts',batch_key='batch',learning_rate=1e-3)
+
+
+# In[16]:
+
+
+LDA_obj.plot_topic_contributions(6)
+
+
+# In[17]:
+
+
+LDA_obj.predicted(15)
+
+
+# In[37]:
+
+
+adata.obsm["X_mde_mira_topic"] = ov.utils.mde(adata.obsm["X_topic_compositions"])
+adata.obsm["X_mde_mira_feature"] = ov.utils.mde(adata.obsm["X_umap_features"])
+
+
+# In[38]:
+
+
+ov.utils.embedding(adata,
+                basis='X_mde_mira_topic',frameon='small',
+                color=['batch','cell_type'],show=False)
+
+
+# In[39]:
+
+
+ov.utils.embedding(adata,
+                basis='X_mde_mira_feature',frameon='small',
+                color=['batch','cell_type'],show=False)
+
+
+# ## Benchmarking test
+# 
+# The methods demonstrated here are selected based on results from benchmarking experiments including the single-cell integration benchmarking project [Luecken et al., 2021]. This project also produced a software package called [scib](https://www.github.com/theislab/scib) that can be used to run a range of integration methods as well as the metrics that were used for evaluation. In this section, we show how to use this package to evaluate the quality of an integration.
+
+# In[6]:
+
+
+adata.write_h5ad('neurips2021_batch_all.h5ad',compression='gzip')
+
+
+# In[2]:
+
+
+adata=sc.read('neurips2021_batch_all.h5ad')
+
+
+# In[7]:
+
+
+adata.obsm['X_pca']=adata.obsm['scaled|original|X_pca'].copy()
+adata.obsm['X_mira_topic']=adata.obsm['X_topic_compositions'].copy()
+adata.obsm['X_mira_feature']=adata.obsm['X_umap_features'].copy()
+
+
+# In[ ]:
+
+
+from scib_metrics.benchmark import Benchmarker
+bm = Benchmarker(
+    adata,
+    batch_key="batch",
+    label_key="cell_type",
+    embedding_obsm_keys=["X_pca", "X_combat", "X_harmony",
+                         'X_scanorama','X_mira_topic','X_mira_feature','X_scVI'],
+    n_jobs=8,
+)
+bm.benchmark()
+
+
+# In[9]:
+
+
+bm.plot_results_table(min_max_scale=False)
+
+
+# We can find that harmony removes the batch effect the best of the three methods that do not use the GPU, scVI is method to remove batch effect using GPU.

ovrawm/t_slat.txt

@@ -0,0 +1,365 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# # Single cell spatial alignment tools
+# 
+# SLAT (Spatially-Linked Alignment Tool), a graph-based algorithm for efficient and effective alignment of spatial slices. Adopting a graph adversarial matching strategy, SLAT is the first algorithm capable of aligning heterogenous spatial data across distinct technologies and modalities. 
+# 
+# We made two improvements in integrating the STT algorithm in OmicVerse:
+# 
+# - **Fix the running error in alignment**: We fixed some issues with the scSLAT package on pypi.
+# - **Added more downstream analysis**: We have expanded on the original tutorial by combining the tutorial and reproduce code given by the authors for downstream analysis.
+# 
+# If you found this tutorial helpful, please cite SLAT and OmicVerse: 
+# 
+# - Xia, CR., Cao, ZJ., Tu, XM. et al. Spatial-linked alignment tool (SLAT) for aligning heterogenous slices. Nat Commun 14, 7236 (2023). https://doi.org/10.1038/s41467-023-43105-5
+
+# In[1]:
+
+
+import omicverse as ov
+import os
+
+import scanpy as sc
+import numpy as np
+import pandas as pd
+import torch
+ov.plot_set()
+
+
+# In[2]:
+
+
+#import scSLAT
+from omicverse.externel.scSLAT.model import load_anndatas, Cal_Spatial_Net, run_SLAT, scanpy_workflow, spatial_match
+from omicverse.externel.scSLAT.viz import match_3D_multi, hist, Sankey, match_3D_celltype, Sankey,Sankey_multi,build_3D
+from omicverse.externel.scSLAT.metrics import region_statistics
+
+
+# ## Preprocess Data
+# 
+# adata1.h5ad: E11.5 mouse embryo dataset, download from [here](https://drive.google.com/uc?export=download&id=1KkuJt6aSlKS1AJzFZjE_odypY-GINRuD)
+# 
+# adata2.h5ad: E12.5 mouse embryo dataset, download from [here](https://drive.google.com/uc?export=download&id=1YIiEmjGfHxcDbGn4nv2kzmTHUo3_q5hJ)
+
+# In[3]:
+
+
+adata1 = sc.read_h5ad('data/E115_Stereo.h5ad')
+adata2 = sc.read_h5ad('data/E125_Stereo.h5ad')
+
+
+# In[4]:
+
+
+adata1.obs['week']='E11.5'
+adata2.obs['week']='E12.5'
+
+
+# In[5]:
+
+
+sc.pl.spatial(adata1, color='annotation', spot_size=3)
+sc.pl.spatial(adata2, color='annotation', spot_size=3)
+
+
+# ## Run SLAT
+# 
+# Then we run SLAT as usual
+
+# In[6]:
+
+
+Cal_Spatial_Net(adata1, k_cutoff=20, model='KNN')
+Cal_Spatial_Net(adata2, k_cutoff=20, model='KNN')
+edges, features = load_anndatas([adata1, adata2], feature='DPCA', check_order=False)
+
+
+# In[7]:
+
+
+embd0, embd1, time = run_SLAT(features, edges, LGCN_layer=5)
+
+
+# In[8]:
+
+
+best, index, distance = spatial_match([embd0, embd1], reorder=False, adatas=[adata1,adata2])
+
+
+# In[9]:
+
+
+matching = np.array([range(index.shape[0]), best])
+best_match = distance[:,0]
+region_statistics(best_match, start=0.5, number_of_interval=10)
+
+
+# ## Visualization of alignment
+
+# In[10]:
+
+
+import matplotlib.pyplot as plt
+matching_list=[matching]
+model = build_3D([adata1,adata2], matching_list,subsample_size=300, )
+ax=model.draw_3D(hide_axis=True, line_color='#c2c2c2', height=1, size=[6,6], line_width=1)
+
+
+# Then we check the alignment quality of the whole slide
+
+# In[11]:
+
+
+adata2.obs['low_quality_index']= best_match
+adata2.obs['low_quality_index'] = adata2.obs['low_quality_index'].astype(float)
+
+
+# In[12]:
+
+
+adata2.obsm['spatial']
+
+
+# In[13]:
+
+
+sc.pl.spatial(adata2, color='low_quality_index', spot_size=3, title='Quality')
+
+
+# We use a Sankey diagram to show the correspondence between cell types at different stages of development
+
+# In[33]:
+
+
+fig=Sankey_multi(adata_li=[adata1,adata2],
+             prefix_li=['E11.5','E12.5'],
+             matching_li=[matching],
+                clusters='annotation',filter_num=10,
+             node_opacity = 0.8,
+             link_opacity = 0.2,
+                layout=[800,500],
+           font_size=12,
+           font_color='Black',
+           save_name=None,
+           format='png',
+           width=1200,
+           height=1000,
+           return_fig=True)
+fig.show()
+
+
+# In[34]:
+
+
+fig.write_html("slat_sankey.html")
+
+
+# ## Focus on developing Kidney
+# 
+# We highlighted the “Kidney” cells in E12.5 and their aligned precursor cells in E11.5 in alignment results. Consistent with our biological priors, the precursors of the kidney are the mesonephros and the metanephros
+# 
+# Then we focus on another organ: ‘Ovary’, and found ovary only has single spatial origin. It is interesting that precursors of ovary are spatially close to the mesonephros (see Kidney part), because mammalian ovary originates from the regressed mesonephros.
+
+# In[27]:
+
+
+color_dict1=dict(zip(adata1.obs['annotation'].cat.categories,
+                    adata1.uns['annotation_colors'].tolist()))
+adata1_df = pd.DataFrame({'index':range(embd0.shape[0]),
+                          'x': adata1.obsm['spatial'][:,0],
+                          'y': adata1.obsm['spatial'][:,1],
+                          'celltype':adata1.obs['annotation'],
+                         'color':adata1.obs['annotation'].map(color_dict1)
+                         }
+                        )
+color_dict2=dict(zip(adata2.obs['annotation'].cat.categories,
+                    adata2.uns['annotation_colors'].tolist()))
+adata2_df = pd.DataFrame({'index':range(embd1.shape[0]),
+                          'x': adata2.obsm['spatial'][:,0],
+                          'y': adata2.obsm['spatial'][:,1],
+                          'celltype':adata2.obs['annotation'],
+                         'color':adata2.obs['annotation'].map(color_dict2)
+                         }
+                        )
+
+
+# In[28]:
+
+
+kidney_align = match_3D_celltype(adata1_df, adata2_df, matching, meta='celltype', 
+                                 highlight_celltype = [['Urogenital ridge'],['Kidney','Ovary']],
+                                 subsample_size=10000, highlight_line = ['blue'], scale_coordinate = True )
+kidney_align.draw_3D(size= [6, 6], line_width =0.8, point_size=[0.6,0.6], hide_axis=True)
+
+
+# We can get the lineage of the query's cells and mappings using the following function
+
+# In[15]:
+
+
+def cal_matching_cell(target_adata,query_adata,matching,query_cell,clusters='annotation',):
+    adata1_df = pd.DataFrame({'index':range(target_adata.shape[0]),
+                          'x': target_adata.obsm['spatial'][:,0],
+                          'y': target_adata.obsm['spatial'][:,1],
+                          'celltype':target_adata.obs[clusters]})
+    adata2_df = pd.DataFrame({'index':range(query_adata.shape[0]),
+                              'x': query_adata.obsm['spatial'][:,0],
+                              'y': query_adata.obsm['spatial'][:,1],
+                              'celltype':query_adata.obs[clusters]})
+    query_adata = target_adata[matching[1,adata2_df.loc[adata2_df.celltype==query_cell,'index'].values],:]
+    #adata2_df['target_celltype'] = adata1_df.iloc[matching[1,:],:]['celltype'].to_list()
+    #adata2_df['target_obs_names'] = adata1_df.iloc[matching[1,:],:].index.to_list()
+    
+    #query_obs=adata2_df.loc[adata2_df['celltype']==query_cell,'target_obs_names'].tolist()
+    return query_adata
+    
+
+
+# We find that maps mapped on 3D also show up well on 2D
+
+# In[21]:
+
+
+query_adata=cal_matching_cell(target_adata=adata1,
+                              query_adata=adata2,
+                              matching=matching,
+                              query_cell='Kidney',clusters='annotation')
+query_adata
+
+
+# In[17]:
+
+
+adata1.obs['kidney_anno']=''
+adata1.obs.loc[query_adata.obs.index,'kidney_anno']=query_adata.obs['annotation']
+
+
+# In[18]:
+
+
+sc.pl.spatial(adata1, color='kidney_anno', spot_size=3,
+             palette=['#F5F5F5','#ff7f0e', 'green',])
+
+
+# We are concerned with Kidney lineage development, so we integrated the cells corresponding to the Kidney lineage on the two sections of E11 and E12, and then we could use the method of difference analysis to study the dynamic process of Kidney lineage development.
+
+# In[22]:
+
+
+kidney_lineage_ad=sc.concat([query_adata,adata2[adata2.obs['annotation']=='Kidney']],merge='same')
+kidney_lineage_ad=ov.pp.preprocess(kidney_lineage_ad,mode='shiftlog|pearson',n_HVGs=3000,target_sum=1e4)
+kidney_lineage_ad.raw = kidney_lineage_ad
+kidney_lineage_ad = kidney_lineage_ad[:, kidney_lineage_ad.var.highly_variable_features]
+ov.pp.scale(kidney_lineage_ad)
+ov.pp.pca(kidney_lineage_ad)
+ov.pp.neighbors(kidney_lineage_ad,use_rep='scaled|original|X_pca',metric="cosine")
+ov.utils.cluster(kidney_lineage_ad,method='leiden',resolution=1)
+ov.pp.umap(kidney_lineage_ad)
+
+
+# In[23]:
+
+
+ov.pl.embedding(kidney_lineage_ad,basis='X_umap',
+               color=['annotation','week','leiden'],
+               frameon='small')
+
+
+# In[25]:
+
+
+# Nphs1 https://www.nature.com/articles/s41467-021-22266-1
+sc.pl.dotplot(kidney_lineage_ad,{'nephron progenitors':['Wnt9b','Osr1','Nphs1','Lhx1','Pax2','Pax8'],
+                         'metanephric':['Eya1','Shisa3','Foxc1'], 
+                         'kidney':['Wt1','Wnt4','Nr2f2','Dach1','Cd44']} ,
+              'leiden',dendrogram=False,colorbar_title='Expression')
+
+
+# In[26]:
+
+
+kidney_lineage_ad.obs['re_anno'] = 'Unknown'
+kidney_lineage_ad.obs.loc[kidney_lineage_ad.obs.leiden.isin(['4']),'re_anno'] = 'Nephron progenitors (E11.5)'
+kidney_lineage_ad.obs.loc[kidney_lineage_ad.obs.leiden.isin(['2','3','1','5']),'re_anno'] = 'Metanephron progenitors (E11.5)'
+kidney_lineage_ad.obs.loc[kidney_lineage_ad.obs.leiden=='0','re_anno'] = 'Kidney (E12.5)'
+
+
+# In[28]:
+
+
+# kidney_all = kidney_all[kidney_all.obs.leiden!='3',:]
+kidney_lineage_ad.obs.leiden = list(kidney_lineage_ad.obs.leiden)
+ov.pl.embedding(kidney_lineage_ad,basis='X_umap',
+               color=['annotation','re_anno'],
+               frameon='small')
+
+
+# In[29]:
+
+
+adata1.obs['kidney_anno']=''
+adata1.obs.loc[kidney_lineage_ad[kidney_lineage_ad.obs['week']=='E11.5'].obs.index,'kidney_anno']=kidney_lineage_ad[kidney_lineage_ad.obs['week']=='E11.5'].obs['re_anno']
+
+
+# In[41]:
+
+
+import matplotlib.pyplot as plt
+fig, ax = plt.subplots(1, 1, figsize=(8, 8))
+sc.pl.spatial(adata1, color='kidney_anno', spot_size=1.5,
+             palette=['#F5F5F5','#ff7f0e', 'green',],show=False,ax=ax)
+
+
+# We can also differentially analyse Kidney's developmental pedigree to find different marker genes, and we can analyse transcription factors and thus find the regulatory units involved.
+
+# In[42]:
+
+
+test_adata=kidney_lineage_ad
+dds=ov.bulk.pyDEG(test_adata.to_df(layer='lognorm').T)
+dds.drop_duplicates_index()
+print('... drop_duplicates_index success')
+treatment_groups=test_adata.obs[test_adata.obs['week']=='E12.5'].index.tolist()
+control_groups=test_adata.obs[test_adata.obs['week']=='E11.5'].index.tolist()
+result=dds.deg_analysis(treatment_groups,control_groups,method='ttest')
+# -1 means automatically calculates
+dds.foldchange_set(fc_threshold=-1,
+                   pval_threshold=0.05,
+                   logp_max=10)
+
+
+# In[43]:
+
+
+dds.plot_volcano(title='DEG Analysis',figsize=(4,4),
+                 plot_genes_num=8,plot_genes_fontsize=12,)
+
+
+# In[52]:
+
+
+up_gene=dds.result.loc[dds.result['sig']=='up'].sort_values('qvalue')[:3].index.tolist()
+down_gene=dds.result.loc[dds.result['sig']=='down'].sort_values('qvalue')[:3].index.tolist()
+deg_gene=up_gene+down_gene
+
+
+# In[53]:
+
+
+sc.pl.dotplot(kidney_lineage_ad,deg_gene,
+             groupby='re_anno')
+
+
+# In addition to analysing directly using differential expression, we can also look for weekly marker genes by considering weeks as categories.
+
+# In[55]:
+
+
+sc.tl.dendrogram(kidney_lineage_ad,'re_anno',use_rep='scaled|original|X_pca')
+sc.tl.rank_genes_groups(kidney_lineage_ad, 're_anno', use_rep='scaled|original|X_pca',
+                        method='t-test',use_raw=False,key_added='re_anno_ttest')
+sc.pl.rank_genes_groups_dotplot(kidney_lineage_ad,groupby='re_anno',
+                                cmap='RdBu_r',key='re_anno_ttest',
+                                standard_scale='var',n_genes=3)
+

ovrawm/t_spaceflow.txt

@@ -0,0 +1,160 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# # Identifying Pseudo-Spatial Map
+# 
+# SpaceFlow is Python package for identifying spatiotemporal patterns and spatial domains from Spatial Transcriptomic (ST) Data. Based on deep graph network, SpaceFlow provides the following functions:  
+# 1. Encodes the ST data into **low-dimensional embeddings** that reflecting both expression similarity and the spatial proximity of cells in ST data.
+# 2. Incorporates **spatiotemporal** relationships of cells or spots in ST data through a **pseudo-Spatiotemporal Map (pSM)** derived from the embeddings.
+# 3. Identifies **spatial domains** with spatially-coherent expression patterns.
+# 
+# Check out [(Ren et al., Nature Communications, 2022)](https://www.nature.com/articles/s41467-022-31739-w) for the detailed methods and applications.
+# 
+# 
+# ![fig](https://media.springernature.com/full/springer-static/image/art%3A10.1038%2Fs41467-022-31739-w/MediaObjects/41467_2022_31739_Fig1_HTML.png)
+# 
+
+# In[1]:
+
+
+import omicverse as ov
+#print(f"omicverse version: {ov.__version__}")
+import scanpy as sc
+#print(f"scanpy version: {sc.__version__}")
+ov.utils.ov_plot_set()
+
+
+# ## Preprocess data
+# 
+# Here we present our re-analysis of 151676 sample of the dorsolateral prefrontal cortex (DLPFC) dataset. Maynard et al. has manually annotated DLPFC layers and white matter (WM) based on the morphological features and gene markers.
+# 
+# This tutorial demonstrates how to identify spatial domains on 10x Visium data using STAGATE. The processed data are available at https://github.com/LieberInstitute/spatialLIBD. We downloaded the manual annotation from the spatialLIBD package and provided at https://drive.google.com/drive/folders/10lhz5VY7YfvHrtV40MwaqLmWz56U9eBP?usp=sharing.
+
+# In[2]:
+
+
+adata = sc.read_visium(path='data', count_file='151676_filtered_feature_bc_matrix.h5')
+adata.var_names_make_unique()
+
+
+# <div class="admonition warning">
+#   <p class="admonition-title">Note</p>
+#   <p>
+#     We introduced the spatial special svg calculation module prost in omicverse versions greater than `1.6.0` to replace scanpy's HVGs, if you want to use scanpy's HVGs you can set mode=`scanpy` in `ov.space.svg` or use the following code.
+#   </p>
+# </div>
+# 
+# ```python
+# #adata=ov.pp.preprocess(adata,mode='shiftlog|pearson',n_HVGs=3000,target_sum=1e4)
+# #adata.raw = adata
+# #adata = adata[:, adata.var.highly_variable_features]
+# ```
+
+# In[3]:
+
+
+sc.pp.calculate_qc_metrics(adata, inplace=True)
+adata = adata[:,adata.var['total_counts']>100]
+adata=ov.space.svg(adata,mode='prost',n_svgs=3000,target_sum=1e4,platform="visium",)
+adata.raw = adata
+adata = adata[:, adata.var.space_variable_features]
+adata
+
+
+# We read the ground truth area of our spatial data
+
+# In[4]:
+
+
+# read the annotation
+import pandas as pd
+import os
+Ann_df = pd.read_csv(os.path.join('data', '151676_truth.txt'), sep='\t', header=None, index_col=0)
+Ann_df.columns = ['Ground Truth']
+adata.obs['Ground Truth'] = Ann_df.loc[adata.obs_names, 'Ground Truth']
+sc.pl.spatial(adata, img_key="hires", color=["Ground Truth"])
+
+
+# ## Training the SpaceFlow Model
+# 
+# Here, we used `ov.space.pySpaceFlow` to construct a SpaceFlow Object and train the model.
+# 
+# We need to store the space location info in `adata.obsm['spatial']`
+
+# In[5]:
+
+
+sf_obj=ov.space.pySpaceFlow(adata)
+
+
+# We then train a spatially regularized deep graph network model to learn a low-dimensional embedding that reflecting both expression similarity and the spatial proximity of cells in ST data.
+# 
+# Parameters:
+# - `spatial_regularization_strength`: the strength of spatial regularization, the larger the more of the spatial coherence in the identified spatial domains and spatiotemporal patterns. (default: 0.1)
+# - `z_dim`: the target size of the learned embedding. (default: 50)
+# - `lr`: learning rate for optimizing the model. (default: 1e-3)
+# - `epochs`: the max number of the epochs for model training. (default: 1000)
+# - `max_patience`: the max number of the epoch for waiting the loss decreasing. If loss does not decrease for epochs larger than this threshold, the learning will stop, and the model with the parameters that shows the minimal loss are kept as the best model. (default: 50) 
+# - `min_stop`: the earliest epoch the learning can stop if no decrease in loss for epochs larger than the `max_patience`. (default: 100) 
+# - `random_seed`: the random seed set to the random generators of the `random`, `numpy`, `torch` packages. (default: 42)
+# -  `gpu`: the index of the Nvidia GPU, if no GPU, the model will be trained via CPU, which is slower than the GPU training time. (default: 0) 
+# -  `regularization_acceleration`: whether or not accelerate the calculation of regularization loss using edge subsetting strategy (default: True)
+# -  `edge_subset_sz`: the edge subset size for regularization acceleration (default: 1000000)
+# 
+
+# In[6]:
+
+
+sf_obj.train(spatial_regularization_strength=0.1, 
+             z_dim=50, lr=1e-3, epochs=1000, 
+             max_patience=50, min_stop=100, 
+             random_seed=42, gpu=0, 
+             regularization_acceleration=True, edge_subset_sz=1000000)
+
+
+# ## Calculated the Pseudo-Spatial Map
+# 
+# Unlike the original SpaceFlow, we only need to use the `cal_PSM` function when calling SpaceFlow in omicverse to compute the pSM.
+
+# In[7]:
+
+
+sf_obj.cal_pSM(n_neighbors=20,resolution=1,
+                max_cell_for_subsampling=5000,psm_key='pSM_spaceflow')
+
+
+# In[8]:
+
+
+sc.pl.spatial(adata, color=['pSM_spaceflow','Ground Truth'],cmap='RdBu_r')
+
+
+# ## Clustering the space
+# 
+# We can use `GMM`, `leiden` or `louvain` to cluster the space.
+# 
+# ```python
+# sc.pp.neighbors(adata, n_neighbors=15, n_pcs=50,
+#                use_rep='spaceflow')
+# ov.utils.cluster(adata,use_rep='spaceflow',method='louvain',resolution=1)
+# ov.utils.cluster(adata,use_rep='spaceflow',method='leiden',resolution=1)
+# ```
+
+# In[9]:
+
+
+ov.utils.cluster(adata,use_rep='spaceflow',method='GMM',n_components=7,covariance_type='full',
+                      tol=1e-9, max_iter=1000, random_state=3607)
+
+
+# In[10]:
+
+
+sc.pl.spatial(adata, color=['gmm_cluster',"Ground Truth"])
+
+
+# In[ ]:
+
+
+
+

ovrawm/t_stagate.txt

@@ -0,0 +1,296 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# # Spatial clustering and denoising expressions
+# 
+# Spatial clustering, which shares an analogy with single-cell clustering, has expanded the scope of tissue physiology studies from cell-centroid to structure-centroid with spatially resolved transcriptomics (SRT) data.
+# 
+# Here, we presented two spatial clustering methods in OmicVerse.
+# 
+# We made three improvements in integrating the `GraphST` and `STAGATE` algorithm in OmicVerse:
+# - We removed the preprocessing that comes with `GraphST` and used the preprocessing consistent with all SRTs in OmicVerse
+# - We optimised the dimensional display of `GraphST`, and PCA is considered a self-contained computational step.
+# - We implemented `mclust` using Python, removing the R language dependency.
+# 
+# If you found this tutorial helpful, please cite `GraphST`, `STAGATE` and OmicVerse:
+# 
+# - Long, Y., Ang, K.S., Li, M. et al. Spatially informed clustering, integration, and deconvolution of spatial transcriptomics with GraphST. Nat Commun 14, 1155 (2023). https://doi.org/10.1038/s41467-023-36796-3
+# - Dong, K., Zhang, S. Deciphering spatial domains from spatially resolved transcriptomics with an adaptive graph attention auto-encoder. Nat Commun 13, 1739 (2022). https://doi.org/10.1038/s41467-022-29439-6
+# 
+# 
+
+# In[1]:
+
+
+import omicverse as ov
+#print(f"omicverse version: {ov.__version__}")
+import scanpy as sc
+#print(f"scanpy version: {sc.__version__}")
+ov.plot_set()
+
+
+# ## Preprocess data
+# 
+# Here we present our re-analysis of 151676 sample of the dorsolateral prefrontal cortex (DLPFC) dataset. Maynard et al. has manually annotated DLPFC layers and white matter (WM) based on the morphological features and gene markers.
+# 
+# This tutorial demonstrates how to identify spatial domains on 10x Visium data using STAGATE. The processed data are available at https://github.com/LieberInstitute/spatialLIBD. We downloaded the manual annotation from the spatialLIBD package and provided at https://drive.google.com/drive/folders/10lhz5VY7YfvHrtV40MwaqLmWz56U9eBP?usp=sharing.
+
+# In[2]:
+
+
+adata = sc.read_visium(path='data', count_file='151676_filtered_feature_bc_matrix.h5')
+adata.var_names_make_unique()
+
+
+# <div class="admonition warning">
+#   <p class="admonition-title">Note</p>
+#   <p>
+#     We introduced the spatial special svg calculation module prost in omicverse versions greater than `1.6.0` to replace scanpy's HVGs, if you want to use scanpy's HVGs you can set mode=`scanpy` in `ov.space.svg` or use the following code.
+#   </p>
+# </div>
+# 
+# ```python
+# #adata=ov.pp.preprocess(adata,mode='shiftlog|pearson',n_HVGs=3000,target_sum=1e4)
+# #adata.raw = adata
+# #adata = adata[:, adata.var.highly_variable_features]
+# ```
+
+# In[4]:
+
+
+sc.pp.calculate_qc_metrics(adata, inplace=True)
+adata = adata[:,adata.var['total_counts']>100]
+adata=ov.space.svg(adata,mode='prost',n_svgs=3000,target_sum=1e4,platform="visium",)
+adata
+
+
+# In[5]:
+
+
+adata.write('data/cluster_svg.h5ad',compression='gzip')
+
+
+# In[2]:
+
+
+#adata=ov.read('data/cluster_svg.h5ad',compression='gzip')
+
+
+# (Optional) We read the ground truth area of our spatial data
+# 
+# This step is not mandatory to run, in the tutorial, it's just to demonstrate the accuracy of our clustering effect, and in your own tasks, there is often no Ground_truth
+
+# In[3]:
+
+
+# read the annotation
+import pandas as pd
+import os
+Ann_df = pd.read_csv(os.path.join('data', '151676_truth.txt'), sep='\t', header=None, index_col=0)
+Ann_df.columns = ['Ground Truth']
+adata.obs['Ground Truth'] = Ann_df.loc[adata.obs_names, 'Ground Truth']
+sc.pl.spatial(adata, img_key="hires", color=["Ground Truth"])
+
+
+# ## GraphST model
+# 
+# GraphST was rated as one of the best spatial clustering algorithms on Nature Method 2024.04, so we first tried to call GraphST for spatial domain identification in OmicVerse.
+
+# In[64]:
+
+
+# define model
+model = ov.externel.GraphST.GraphST(adata, device='cuda:0')
+
+# train model
+adata = model.train(n_pcs=30)
+
+
+# ### Clustering the space
+# 
+# We can use `mclust`, `leiden` or `louvain` to cluster the space. 
+# 
+# Note that we also add optimal transport to optimise the distribution of labels, using `refine_label` for that processing.
+# 
+
+# In[84]:
+
+
+ov.utils.cluster(adata,use_rep='graphst|original|X_pca',method='mclust',n_components=10,
+                 modelNames='EEV', random_state=112,
+                )
+adata.obs['mclust_GraphST'] = ov.utils.refine_label(adata, radius=50, key='mclust') 
+
+
+# In[87]:
+
+
+sc.pp.neighbors(adata, n_neighbors=15, n_pcs=20,
+               use_rep='graphst|original|X_pca')
+ov.utils.cluster(adata,use_rep='graphst|original|X_pca',method='louvain',resolution=0.7)
+ov.utils.cluster(adata,use_rep='graphst|original|X_pca',method='leiden',resolution=0.7)
+adata.obs['louvain_GraphST'] = ov.utils.refine_label(adata, radius=50, key='louvain') 
+adata.obs['leiden_GraphST'] = ov.utils.refine_label(adata, radius=50, key='leiden') 
+
+
+# In[88]:
+
+
+sc.pl.spatial(adata, color=['mclust_GraphST','leiden_GraphST',
+                            'louvain_GraphST',"Ground Truth"])
+
+
+# <div class="admonition warning">
+#   <p class="admonition-title">Note</p>
+#   <p>
+#     If you find that the clustering is mediocre, you might consider re-running `model.train()`. Why the results improve
+#   </p>
+# </div>
+# 
+# 
+
+# ## STAGATE model
+# 
+# STAGATE is designed for spatial clustering and denoising expressions of spatial resolved transcriptomics (ST) data.
+# 
+# STAGATE learns low-dimensional latent embeddings with both spatial information and gene expressions via a graph attention auto-encoder. The method adopts an attention mechanism in the middle layer of the encoder and decoder, which adaptively learns the edge weights of spatial neighbor networks, and further uses them to update the spot representation by collectively aggregating information from its neighbors. The latent embeddings and the reconstructed expression profiles can be used to downstream tasks such as spatial domain identification, visualization, spatial trajectory inference, data denoising and 3D expression domain extraction.
+# 
+# Dong, Kangning, and Shihua Zhang. “Deciphering spatial domains from spatially resolved transcriptomics with an adaptive graph attention auto-encoder.” Nature Communications 13.1 (2022): 1-12.
+# 
+# 
+# Here, we used `ov.space.pySTAGATE` to construct a STAGATE object to train the model. 
+# 
+
+# In[14]:
+
+
+#This step sometimes needs to be run twice
+#and you need to check that adata.obs['X'] is correctly assigned instead of NA
+adata.obs['X'] = adata.obsm['spatial'][:,0]
+adata.obs['Y'] = adata.obsm['spatial'][:,1]
+adata.obs['X'][0]
+
+
+# In[15]:
+
+
+STA_obj=ov.space.pySTAGATE(adata,num_batch_x=3,num_batch_y=2,
+                 spatial_key=['X','Y'],rad_cutoff=200,num_epoch = 1000,lr=0.001,
+                weight_decay=1e-4,hidden_dims = [512, 30],
+                device='cuda:0')
+
+
+# In[16]:
+
+
+STA_obj.train()
+
+
+# We stored the latent embedding in `adata.obsm['STAGATE']`, and denoising expression in `adata.layers['STAGATE_ReX']`
+
+# In[17]:
+
+
+STA_obj.predicted()
+adata
+
+
+# ### Clustering the space
+# 
+
+# In[18]:
+
+
+ov.utils.cluster(adata,use_rep='STAGATE',method='mclust',n_components=8,
+                 modelNames='EEV', random_state=112,
+                )
+adata.obs['mclust_STAGATE'] = ov.utils.refine_label(adata, radius=50, key='mclust') 
+
+
+# In[21]:
+
+
+sc.pp.neighbors(adata, n_neighbors=15, n_pcs=20,
+               use_rep='STAGATE')
+ov.utils.cluster(adata,use_rep='STAGATE',method='louvain',resolution=0.5)
+ov.utils.cluster(adata,use_rep='STAGATE',method='leiden',resolution=0.5)
+adata.obs['louvain_STAGATE'] = ov.utils.refine_label(adata, radius=50, key='louvain') 
+adata.obs['leiden_STAGATE'] = ov.utils.refine_label(adata, radius=50, key='leiden') 
+
+
+# In[22]:
+
+
+sc.pl.spatial(adata, color=['mclust_STAGATE','leiden_STAGATE',
+                            'louvain_STAGATE',"Ground Truth"])
+
+
+# ### Denoising
+
+# In[23]:
+
+
+adata.var.sort_values('PI',ascending=False).head(10)
+
+
+# In[24]:
+
+
+plot_gene = 'MBP'
+import matplotlib.pyplot as plt
+fig, axs = plt.subplots(1, 2, figsize=(8, 4))
+sc.pl.spatial(adata, img_key="hires", color=plot_gene, show=False, ax=axs[0], title='RAW_'+plot_gene, vmax='p99')
+sc.pl.spatial(adata, img_key="hires", color=plot_gene, show=False, ax=axs[1], title='STAGATE_'+plot_gene, layer='STAGATE_ReX', vmax='p99')
+
+
+# ### Calculated the Pseudo-Spatial Map
+# 
+# We compared the model results from `SpaceFlow` and `STAGATE`, and to our surprise, STAGATE can also be applied to predict pSM.
+
+# In[25]:
+
+
+STA_obj.cal_pSM(n_neighbors=20,resolution=1,
+                       max_cell_for_subsampling=5000)
+adata
+
+
+# In[26]:
+
+
+sc.pl.spatial(adata, color=['Ground Truth','pSM_STAGATE'],
+             cmap='RdBu_r')
+
+
+# ## Evaluate cluster
+# 
+# We use ARI to evaluate the scoring of our clusters against the true values
+# 
+
+# In[86]:
+
+
+from sklearn.metrics.cluster import adjusted_rand_score
+
+obs_df = adata.obs.dropna()
+#GraphST
+ARI = adjusted_rand_score(obs_df['mclust_GraphST'], obs_df['Ground Truth'])
+print('mclust_GraphST: Adjusted rand index = %.2f' %ARI)
+
+ARI = adjusted_rand_score(obs_df['leiden_GraphST'], obs_df['Ground Truth'])
+print('leiden_GraphST: Adjusted rand index = %.2f' %ARI)
+
+ARI = adjusted_rand_score(obs_df['louvain_GraphST'], obs_df['Ground Truth'])
+print('louvain_GraphST: Adjusted rand index = %.2f' %ARI)
+
+ARI = adjusted_rand_score(obs_df['mclust_STAGATE'], obs_df['Ground Truth'])
+print('mclust_STAGATE: Adjusted rand index = %.2f' %ARI)
+
+ARI = adjusted_rand_score(obs_df['leiden_STAGATE'], obs_df['Ground Truth'])
+print('leiden_STAGATE: Adjusted rand index = %.2f' %ARI)
+
+ARI = adjusted_rand_score(obs_df['louvain_STAGATE'], obs_df['Ground Truth'])
+print('louvain_STAGATE: Adjusted rand index = %.2f' %ARI)
+
+
+# It seems that STAGATE outperforms GraphST on this task, but OmicVerse only provides a unified implementation of the algorithm and does not do a full benchmark of the algorithm.

ovrawm/t_staligner.txt

@@ -0,0 +1,155 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# # Spatial integration and clustering
+# 
+# STAligner is designed for alignment and integration of spatially resolved transcriptomics data.
+# 
+# STAligner first normalizes the expression profiles for all spots and constructs a spatial neighbor network using the spatial coordinates. STAligner further employs a graph attention auto-encoder neural network to extract spatially aware embedding, and constructs the spot triplets based on current embeddings to guide the alignment process by attracting similar spots and discriminating dissimilar spots across slices. STAligner introduces the triplet loss to update the spot embedding to reduce the distance from the anchor to positive spot, and increase the distance from the anchor to negative spot. The triplet construction and auto-encoder training are optimized iteratively until batch-corrected embeddings are generated. STAligner can be applied to integrate ST datasets to achieve alignment and simultaneous identification of spatial domains from different biological samples in (a), technological platforms (I), developmental (embryonic) stages (II), disease conditions (III) and consecutive slices of a tissue for 3D slice alignment (IV
+# 
+# Zhou, X., Dong, K. & Zhang, S. Integrating spatial transcriptomics data across different conditions, technologies and developmental stages. Nat Comput Sci 3, 894–906 (2023). https://doi.org/10.1038/s43588-023-00528-w
+# 
+# ![image.png](attachment:00790548-59f9-4fad-a1e3-a52c3ae98d44.png)).
+
+# In[1]:
+
+
+from scipy.sparse import csr_matrix
+import omicverse as ov
+import scanpy as sc
+import anndata as ad
+import pandas as pd
+import os
+
+ov.utils.ov_plot_set()
+
+
+# # Preprocess data
+# 
+# Here, We use the mouse olfactory bulb data generated by Stereo-seq and Slide-seqV2. The processed Stereo-seq and Slide-seqV2 data can be downloaded from https://drive.google.com/drive/folders/1Omte1adVFzyRDw7VloOAQYwtv_NjdWcG?usp=share_link. and the original tutorals can be finded from https://staligner.readthedocs.io/en/latest
+# 
+# Here is a critical point that must be clarified: for STAligner, it first calculates highly variable genes before concating annadata samples. Therefore, the number of highly variable genes should not be selected too low. Otherwise, in the case of a large number of samples, the downstream features for STAligner training would be insufficient, impacting the model's performance.
+# 
+# When using STAligner, it is necessary to adjust the **rad_cutoff** parameter according to different data to ensure that each spot has an **average of 5-10 adjacent spots** connected to it. Such as: "11.3356 neighbors per cell on average."
+# 
+
+# In[2]:
+
+
+Batch_list = []
+adj_list = []
+section_ids = ['Slide-seqV2_MoB', 'Stereo-seq_MoB']
+print(section_ids)
+pathway = '/storage/zengjianyangLab/hulei/scRNA-seq/scripts/STAligner'
+
+for section_id in section_ids:
+    print(section_id)
+    adata = sc.read_h5ad(os.path.join(pathway,section_id+".h5ad"))
+
+    # check whether the adata.X is sparse matrix
+    if isinstance(adata.X, pd.DataFrame):
+        adata.X = csr_matrix(adata.X)
+    else:
+        pass
+
+    adata.var_names_make_unique(join="++")
+
+    # make spot name unique
+    adata.obs_names = [x+'_'+section_id for x in adata.obs_names]
+
+    # Constructing the spatial network
+    ov.space.Cal_Spatial_Net(adata, rad_cutoff=50) # the spatial network are saved in adata.uns[‘adj’]
+
+    # Normalization
+    sc.pp.highly_variable_genes(adata, flavor="seurat_v3", n_top_genes=10000)
+    sc.pp.normalize_total(adata, target_sum=1e4)
+    sc.pp.log1p(adata)
+
+    adata = adata[:, adata.var['highly_variable']]
+    adj_list.append(adata.uns['adj'])
+    Batch_list.append(adata)
+
+
+# In[3]:
+
+
+Batch_list
+
+
+# In[4]:
+
+
+adata_concat = ad.concat(Batch_list, label="slice_name", keys=section_ids)
+adata_concat.obs["batch_name"] = adata_concat.obs["slice_name"].astype('category')
+print('adata_concat.shape: ', adata_concat.shape)
+
+
+# # Training STAligner model
+# 
+# Here, we used `ov.space.pySTAligner` to construct a STAGATE object to train the model.
+# 
+# We are using the `train_STAligner_subgraph` function from STAligner to reduce GPU memory usage, each slice is considered as a subgraph for training.
+
+# In[5]:
+
+
+get_ipython().run_cell_magic('time', '', "# iter_comb is used to specify the order of integration. For example, (0, 1) means slice 0 will be algined with slice 1 as reference.\niter_comb = [(i, i + 1) for i in range(len(section_ids) - 1)]\n\n# Here, to reduce GPU memory usage, each slice is considered as a subgraph for training.\nSTAligner_obj = ov.space.pySTAligner(adata_concat, verbose=True, knn_neigh = 100, n_epochs = 600, iter_comb = iter_comb,\n                                     batch_key = 'batch_name',  key_added='STAligner', Batch_list = Batch_list)\n")
+
+
+# In[6]:
+
+
+STAligner_obj.train()
+
+
+# We stored the latent embedding in `adata.obsm['STAligner']`.
+
+# In[7]:
+
+
+adata = STAligner_obj.predicted()
+
+
+# # Clustering the space
+# 
+# We can use `GMM`, `leiden` or `louvain` to cluster the space.
+# 
+# `ov.utils.cluster(adata,use_rep='STAligner',method='GMM',n_components=7,covariance_type='full', tol=1e-9, max_iter=1000, random_state=3607`
+# 
+# or `sc.pp.neighbors(adata, use_rep='STAligner', random_state=666)`            
+# `ov.utils.cluster(adata,use_rSTAlignerGATE',method='leiden',resolution=1)`
+
+# In[8]:
+
+
+sc.pp.neighbors(adata, use_rep='STAligner', random_state=666)
+ov.utils.cluster(adata,use_rep='STAligner',method='leiden',resolution=0.4)
+sc.tl.umap(adata, random_state=666)
+sc.pl.umap(adata, color=['batch_name',"leiden"],wspace=0.5)
+
+
+# We can also map the clustering results back to the original spatial coordinates to obtain spatially specific clustering results.
+
+# In[9]:
+
+
+import matplotlib.pyplot as plt
+spot_size = 50
+title_size = 15
+fig, ax = plt.subplots(1, 2, figsize=(6, 3), gridspec_kw={'wspace': 0.05, 'hspace': 0.2})
+_sc_0 = sc.pl.spatial(adata[adata.obs['batch_name'] == 'Slide-seqV2_MoB'], img_key=None, color=['leiden'], title=['Slide-seqV2'],
+                      legend_fontsize=10, show=False, ax=ax[0], frameon=False, spot_size=spot_size, legend_loc=None)
+_sc_0[0].set_title('Slide-seqV2', size=title_size)
+
+_sc_1 = sc.pl.spatial(adata[adata.obs['batch_name'] == 'Stereo-seq_MoB'], img_key=None, color=['leiden'], title=['Stereo-seq'],
+                      legend_fontsize=10, show=False, ax=ax[1], frameon=False, spot_size=spot_size)
+_sc_1[0].set_title('Stereo-seq',size=title_size)
+_sc_1[0].invert_yaxis()
+plt.show()
+
+
+# In[ ]:
+
+
+
+

ovrawm/t_starfysh.txt

@@ -0,0 +1,519 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# # Deconvolution spatial transcriptomic without scRNA-seq
+# 
+# This is a tutorial on an example real Spatial Transcriptomics (ST) data (CID44971_TNBC) from Wu et al., 2021. Raw tutorial could be found in https://starfysh.readthedocs.io/en/latest/notebooks/Starfysh_tutorial_real.html
+# 
+# 
+# Starfysh performs cell-type deconvolution followed by various downstream analyses to discover spatial interactions in tumor microenvironment. Specifically, Starfysh looks for anchor spots (presumably with the highest compositions of one given cell type) informed by user-provided gene signatures ([see example](https://drive.google.com/file/d/1AXWQy_mwzFEKNjAdrJjXuegB3onxJoOM/view?usp=share_link)) as priors to guide the deconvolution inference, which further enables downstream analyses such as sample integration, spatial hub characterization, cell-cell interactions, etc. This tutorial focuses on the deconvolution task. Overall, Starfysh provides the following options:
+# 
+# At omicverse, we have made the following improvements:
+# - Easier visualization, you can use omicverse unified visualization for scientific mapping
+# - Reduce installation dependency errors, we optimized the automatic selection of different packages, you don't need to install too many extra packages and cause conflicts.
+# 
+# **Base feature**:
+# 
+# - Spot-level deconvolution with expected cell types and corresponding annotated signature gene sets (default)
+# 
+# **Optional**:
+# 
+# - Archetypal Analysis (AA):
+# 
+#     *If gene signatures are not provided*
+#     
+#     - Unsupervised cell type annotation
+# 
+#     *If gene signatures are provided but require refinement*:
+#     
+#     - Novel cell type / cell state discovery (complementary to known cell types from the *signatures*)
+#     - Refine known marker genes by appending archetype-specific differentially expressed genes, and update anchor spots accordingly
+#     
+# - Product-of-Experts (PoE) integration
+# 
+#     Multi-modal integrative predictions with expression & histology image by leverging additional side information (e.g. cell density) from H&E image.
+# 
+# He, S., Jin, Y., Nazaret, A. et al.
+# Starfysh integrates spatial transcriptomic and histologic data to reveal heterogeneous tumor–immune hubs.
+# Nat Biotechnol (2024).
+# https://doi.org/10.1038/s41587-024-02173-8
+
+# In[1]:
+
+
+import scanpy as sc
+import omicverse as ov
+ov.plot_set()
+
+
+# In[2]:
+
+
+from omicverse.externel.starfysh import (AA, utils, plot_utils, post_analysis)
+from omicverse.externel.starfysh import _starfysh as sf_model
+
+
+# ### (1). load data and marker genes
+# 
+# File Input:
+# - Spatial transcriptomics
+#     - Count matrix: `adata`
+#     - (Optional): Paired histology & spot coordinates: `img`, `map_info`
+# 
+# - Annotated signatures (marker genes) for potential cell types: `gene_sig`
+# 
+# Starfysh is built upon scanpy and Anndata. The common ST/Visium data sample folder consists a expression count file (usually `filtered_featyur_bc_matrix.h5`), and a subdirectory with corresponding H&E image and spatial information, as provided by Visium platform.
+# 
+# For example, our example real ST data has the following structure:
+# ```
+# ├── data_folder
+#     signature.csv
+# 
+#     ├── CID44971:
+#         \__ filtered_feature_bc_mactrix.h5
+# 
+#         ├── spatial:
+#             \__ aligned_fiducials.jpg
+#                 detected_tissue_image.jpg
+#                 scalefactors_json.json
+#                 tissue_hires_image.png
+#                 tissue_lowres_image.png
+#                 tissue_positions_list.csv
+# ```
+# 
+# For data that doesn't follow the common visium data structure (e.g. missing `filtered_featyur_bc_matrix.h5` or the given `.h5ad` count matrix file lacks spatial metadata), please construct the data as Anndata synthesizing information as the example simulated data shows:
+
+# [Note]: If you’re running this tutorial locally, please download the sample [data](https://drive.google.com/drive/folders/1RIp0Z2eF1m8Ortx0sgB4z5g5ISsRFzJ4?usp=share_link) and [signature gene sets](https://drive.google.com/file/d/1AXWQy_mwzFEKNjAdrJjXuegB3onxJoOM/view?usp=share_link), and save it in the relative path `data/star_data` (otherwise please modify the data_path defined in the cell below):
+
+# In[3]:
+
+
+# Specify data paths
+data_path = 'data/star_data'
+sample_id = 'CID44971_TNBC'
+sig_name = 'bc_signatures_version_1013.csv'
+
+
+# In[4]:
+
+
+# Load expression counts and signature gene sets
+adata, adata_normed = utils.load_adata(data_folder=data_path,
+                                       sample_id=sample_id, # sample id
+                                       n_genes=2000  # number of highly variable genes to keep
+                                       )
+
+
+# In[5]:
+
+
+import pandas as pd
+import os
+gene_sig = pd.read_csv(os.path.join(data_path, sig_name))
+gene_sig = utils.filter_gene_sig(gene_sig, adata.to_df())
+gene_sig.head()
+
+
+# **If there's no input signature gene sets, Starfysh defines "archetypal marker genes" as *signatures*. Please refer to the following code snippet and see details in section (3).**
+# 
+# ```Python
+# aa_model = AA.ArchetypalAnalysis(adata_orig=adata_normed)
+# archetype, arche_dict, major_idx, evs = aa_model.compute_archetypes(r=40)
+# gene_sig = aa_model.find_markers(n_markers=30, display=False)
+# gene_sig = utils.filter_gene_sig(gene_sig, adata.to_df())
+# gene_sig.head()
+# ```
+
+# In[6]:
+
+
+# Load spatial information
+img_metadata = utils.preprocess_img(data_path,
+                                    sample_id,
+                                    adata_index=adata.obs.index,
+                                    #hchannel=False
+                                    )
+img, map_info, scalefactor = img_metadata['img'], img_metadata['map_info'], img_metadata['scalefactor']
+umap_df = utils.get_umap(adata, display=True)
+
+
+# In[7]:
+
+
+import matplotlib.pyplot as plt
+plt.figure(figsize=(6, 6), dpi=80)
+plt.imshow(img)
+
+
+# In[8]:
+
+
+map_info.head()
+
+
+# ### (2). Preprocessing (finding anchor spots)
+# - Identify anchor spot locations.
+# 
+# Instantiate parameters for Starfysh model training:
+# - Raw & normalized counts after taking highly variable genes
+# - filtered signature genes
+# - library size & spatial smoothed library size (log-transformed)
+# - Anchor spot indices (`anchors_df`) for each cell type & their signature means (`sig_means`)
+# 
+
+# In[ ]:
+
+
+# Parameters for training
+visium_args = utils.VisiumArguments(adata,
+                                    adata_normed,
+                                    gene_sig,
+                                    img_metadata,
+                                    n_anchors=60,
+                                    window_size=3,
+                                    sample_id=sample_id
+                                   )
+
+adata, adata_normed = visium_args.get_adata()
+anchors_df = visium_args.get_anchors()
+
+
+# In[10]:
+
+
+adata.obs['log library size']=visium_args.log_lib
+adata.obs['windowed log library size']=visium_args.win_loglib
+
+
+# In[11]:
+
+
+sc.pl.spatial(adata, cmap='magma',
+                  # show first 8 cell types
+                  color='log library size',
+                  ncols=4, size=1.3,
+                  img_key='hires',
+                  #palette=Layer_color
+                  # limit color scale at 99.2% quantile of cell abundance
+                  #vmin=0, vmax='p99.2'
+                 )
+
+
+# In[12]:
+
+
+sc.pl.spatial(adata, cmap='magma',
+                  # show first 8 cell types
+                  color='windowed log library size',
+                  ncols=4, size=1.3,
+                  img_key='hires',
+                  #palette=Layer_color
+                  # limit color scale at 99.2% quantile of cell abundance
+                  #vmin=0, vmax='p99.2'
+                 )
+
+
+# plot raw gene expression:
+
+# In[13]:
+
+
+sc.pl.spatial(adata, cmap='magma',
+                  # show first 8 cell types
+                  color='IL7R',
+                  ncols=4, size=1.3,
+                  img_key='hires',
+                  #palette=Layer_color
+                  # limit color scale at 99.2% quantile of cell abundance
+                  #vmin=0, vmax='p99.2'
+                 )
+
+
+# In[14]:
+
+
+plot_utils.plot_anchor_spots(umap_df,
+                             visium_args.pure_spots,
+                             visium_args.sig_mean,
+                             bbox_x=2
+                            )
+
+
+# ### (3). Optional: Archetypal Analysis
+# Overview:
+# If users don't provide annotated gene signature sets with cell types, Starfysh identifies candidates for cell types via archetypal analysis (AA). The underlying assumption is that the geometric "extremes" are identified as the purest cell types, whereas all other spots are mixture of the "archetypes". If the users provide the gene signature sets, they can still optionally apply AA to refine marker genes and update anchor spots for known cell types. In addition, AA can identify & assign potential novel cell types / states. Here are the features provided by the optional archetypal analysis:
+# - Finding archetypal spots & assign 1-1 mapping to their closest anchor spot neighbors
+# - Finding archetypal marker genes & append them to marker genes of annotated cell types
+# - Assigning novel cell type / cell states as the most distant archetypes
+# 
+# Overall, Starfysh provides the archetypal analysis as a complementary toolkit for automatic cell-type annotation & signature gene completion:<br><br>
+# 
+# 1. *If signature genes aren't provided:* <br><br>Archetypal analysis defines the geometric extrema of the data as major cell types with corresponding marker genes.<br><br>
+# 
+# 2. *If complete signature genes are known*: <br><br>Users can skip this section and use only the signature priors<br><br>
+# 
+# 3. *If signature genes are incomplete or need refinement*: <br><br>Archetypal analysis can be applied to
+#     a. Refine signatures of certain cell types
+#     b. Find novel cell types / states that haven't been provided from the input signature
+
+# #### If signature genes aren't provided
+# 
+# Note: <br>
+# - Intrinsic Dimension (ID) estimator is implemented to estimate the lower-bound for the number of archetypes $k$, followed by elbow method with iterations to identify the optimal $k$. By default, a [conditional number](https://scikit-dimension.readthedocs.io/en/latest/skdim.id.FisherS.html) is set as 30; if you believe there are potentially more / fewer cell types, please increase / decrease `cn` accordingly.
+
+# Major cell types & corresponding markers are represented by the inferred archetypes:<br><br>
+# 
+# 
+# 
+# ```Python
+# aa_model = AA.ArchetypalAnalysis(adata_orig=adata_normed)
+# archetype, arche_dict, major_idx, evs = aa_model.compute_archetypes(r=40)
+# 
+# # (1). Find archetypal spots & archetypal clusters
+# arche_df = aa_model.find_archetypal_spots(major=True)
+# 
+# # (2). Define "signature genes" as marker genes associated with each archetypal cluster
+# gene_sig = aa_model.find_markers(n_markers=30, display=False)
+# gene_sig.head()
+# ```
+
+# #### If complete signature genes are known
+# 
+# Users can skip th section & run Starfysh
+# 
+# #### If signature genes are incomplete or require refinement
+# 
+# **In this tutorial, we'll show an example of applying best-aligned archetypes to existing `anchors` of given cell type(s) to append signature genes.**
+
+# In[ ]:
+
+
+aa_model = AA.ArchetypalAnalysis(adata_orig=adata_normed)
+archetype, arche_dict, major_idx, evs = aa_model.compute_archetypes(cn=40)
+# (1). Find archetypal spots & archetypal clusters
+arche_df = aa_model.find_archetypal_spots(major=True)
+
+# (2). Find marker genes associated with each archetypal cluster
+markers_df = aa_model.find_markers(n_markers=30, display=False)
+
+# (3). Map archetypes to closest anchors (1-1 per cell type)
+map_df, map_dict = aa_model.assign_archetypes(anchors_df)
+
+# (4). Optional: Find the most distant archetypes that are not assigned to any annotated cell types
+distant_arches = aa_model.find_distant_archetypes(anchors_df, n=3)
+
+
+# In[16]:
+
+
+plot_utils.plot_evs(evs, kmin=aa_model.kmin)
+
+
+# - Visualize archetypes
+
+# In[17]:
+
+
+aa_model.plot_archetypes(do_3d=False, major=True, disp_cluster=False)
+
+
+# - Visualize archetypal - cell type mapping:
+
+# In[18]:
+
+
+aa_model.plot_mapping(map_df)
+
+
+# - Application: appending marker genes Append archetypal marker genes with the best-aligned anchors:
+
+# In[ ]:
+
+
+visium_args = utils.refine_anchors(
+    visium_args,
+    aa_model,
+    #thld=0.7,  # alignment threshold
+    n_genes=5,
+    #n_iters=1
+)
+
+# Get updated adata & signatures
+adata, adata_normed = visium_args.get_adata()
+gene_sig = visium_args.gene_sig
+cell_types = gene_sig.columns
+
+
+# ## Run starfysh without histology integration
+# 
+# 
+# 
+# We perform `n_repeat` random restarts and select the best model with lowest loss for parameter `c` (inferred cell-type proportions):
+# 
+# ### (1). Model parameters
+
+# In[20]:
+
+
+import torch
+n_repeats = 3
+epochs = 200
+patience = 50
+device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+
+
+# ### (2). Model training
+# 
+# Users can choose to run the one of the following `Starfysh` model without/with histology integration:
+# 
+# Without histology integration: setting `utils.run_starfysh(poe=False)` (default)
+# 
+# With histology integration: setting `utils.run_starfysh(poe=True)`
+
+# In[21]:
+
+
+# Run models
+model, loss = utils.run_starfysh(visium_args,
+                                 n_repeats=n_repeats,
+                                 epochs=epochs,
+                                 #patience=patience,
+                                 device=device
+                                )
+
+
+# ### Downstream analysis
+# 
+# ### Parse Starfysh inference output
+
+# In[22]:
+
+
+adata, adata_normed = visium_args.get_adata()
+inference_outputs, generative_outputs,adata_ = sf_model.model_eval(model,
+                                                            adata,
+                                                            visium_args,
+                                                            poe=False,
+                                                            device=device)
+
+
+# ### Visualize starfysh deconvolution results
+# 
+# **Gene sig mean vs. inferred prop**
+
+# In[31]:
+
+
+import numpy as np
+n_cell_types = gene_sig.shape[1]
+idx = np.random.randint(0, n_cell_types)
+post_analysis.gene_mean_vs_inferred_prop(inference_outputs,
+                                         visium_args,
+                                         idx=idx,
+                                         figsize=(4,4)
+                                        )
+
+
+# ### Spatial visualizations:
+# 
+# Inferred density on Spatial map:
+
+# In[24]:
+
+
+plot_utils.pl_spatial_inf_feature(adata_, feature='ql_m', cmap='Blues')
+
+
+# **Inferred cell-type proportions (spatial map):**
+# 
+
+# In[25]:
+
+
+def cell2proportion(adata):
+    adata_plot=sc.AnnData(adata.X)
+    adata_plot.obs=utils.extract_feature(adata_, 'qc_m').obs.copy()
+    adata_plot.var=adata.var.copy()
+    adata_plot.obsm=adata.obsm.copy()
+    adata_plot.obsp=adata.obsp.copy()
+    adata_plot.uns=adata.uns.copy()
+    return adata_plot
+adata_plot=cell2proportion(adata_)
+
+
+# In[26]:
+
+
+adata_plot
+
+
+# In[27]:
+
+
+sc.pl.spatial(adata_plot, cmap='Spectral_r',
+                  # show first 8 cell types
+                  color=['Basal','LumA','LumB'],
+                  ncols=4, size=1.3,
+                  img_key='hires',
+                  vmin=0, vmax='p90'
+                 )
+
+
+# In[28]:
+
+
+ov.pl.embedding(adata_plot,
+               basis='z_umap',
+                color=['Basal', 'LumA', 'MBC', 'Normal epithelial'],
+               frameon='small',
+                vmin=0, vmax='p90',
+                cmap='Spectral_r',
+               )
+
+
+# In[29]:
+
+
+pred_exprs = sf_model.model_ct_exp(model,
+                                   adata,
+                                   visium_args,
+                                   device=device)
+
+
+# Plot spot-level expression (e.g. `IL7R` from *Effector Memory T cells (Tem)*):
+# 
+
+# In[30]:
+
+
+gene='IL7R'
+gene_celltype='Tem'
+adata_.layers[f'infer_{gene_celltype}']=pred_exprs[gene_celltype]
+
+sc.pl.spatial(adata_, cmap='Spectral_r',
+                  # show first 8 cell types
+                  color=gene,
+                  title=f'{gene} (Predicted expression)\n{gene_celltype}',
+                  layer=f'infer_{gene_celltype}',
+                  ncols=4, size=1.3,
+                  img_key='hires',
+                  #vmin=0, vmax='p90'
+                 )
+
+
+# ### Save model & inferred parameters
+
+# In[ ]:
+
+
+# Specify output directory
+outdir = './results/'
+if not os.path.exists(outdir):
+    os.mkdir(outdir)
+
+# save the model
+torch.save(model.state_dict(), os.path.join(outdir, 'starfysh_model.pt'))
+
+# save `adata` object with inferred parameters
+adata.write(os.path.join(outdir, 'st.h5ad'))
+

ovrawm/t_stt.txt

@@ -0,0 +1,274 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# # Spatial transition tensor of single cells
+# 
+# spatial transition tensor (STT), a method that uses messenger RNA splicing and spatial transcriptomes through a multiscale dynamical model to characterize multistability in space. By learning a four-dimensional transition tensor and spatial-constrained random walk, STT reconstructs cell-state-specific dynamics and spatial state transitions via both short-time local tensor streamlines between cells and long-time transition paths among attractors.
+# 
+# We made three improvements in integrating the STT algorithm in OmicVerse:
+# 
+# - **More user-friendly function implementation**: we refreshed the unnecessary extra assignments in the original documentation and automated their encapsulation into the `omicverse.space.STT` class.
+# - **Removed version dependencies**: We removed all the strong dependencies such as ``CellRank==1.3.1`` in the original `requierment.txt`, so that users only need to install the OmicVerse package and the latest version of CellRank to make it work perfectly.
+# - **Added clearer function notes**: We have reorganised the unclear areas described in the original tutorial, where you need to go back to the paper to read, in this document.
+# 
+# If you found this tutorial helpful, please cite STT and OmicVerse: 
+# 
+# - Zhou, P., Bocci, F., Li, T. et al. Spatial transition tensor of single cells. Nat Methods (2024). https://doi.org/10.1038/s41592-024-02266-x
+
+# In[1]:
+
+
+import omicverse as ov
+#import omicverse.STT as st
+import scvelo as scv
+import scanpy as sc
+ov.plot_set()
+
+
+# ## Preprocess data
+# 
+# In this tutorial, we focus on demonstrating and reproducing the original author's data, which has been completed with calculations such as `adata.layers[‘Ms’]`, `adata.layers[‘Mu’]`, and so on. And when analysing our own data, we need the following functions to preprocess the raw data
+# 
+# ```
+# scv.pp.filter_and_normalise(adata, min_shared_counts=20, n_top_genes=3000)
+# scv.pp.moments(adata, n_pcs=30, n_neighbors=30)
+# ```
+# 
+# The `mouse_brain.h5ad` could be found in the [Github:STT](https://github.com/cliffzhou92/STT/tree/release/data)
+
+# In[2]:
+
+
+adata = sc.read_h5ad('mouse_brain.h5ad')
+adata
+
+
+# ## Training STT model
+# 
+# Here, we used ov.space.STT to construct a STAGATE object to train the model. We need to set the following parameters during initialisation:
+# 
+# - `spatial_loc`: The nulling coordinates for each spot, in 10x genomic data, are typically `adata.obsm[‘spatial’]`, so this parameter is typically set to `spatial`, but here we store it in `xy_loc`.
+# - `region`: This parameter is considered to be the region of the attractor, which we would normally define using spatial annotations or cellular annotation information.
+
+# In[3]:
+
+
+STT_obj=ov.space.STT(adata,spatial_loc='xy_loc',region='Region')
+
+
+# Note that we need to specify the number of potential attractors first when predicting attractors. In the author's original tutorial and original paper, there is no clear definition for the specification of this parameter. After referring to the author's tutorial, we use the calculated number of leiden of `adata_aggr` as a prediction of the number of potential attractors.
+
+# In[4]:
+
+
+STT_obj.stage_estimate()
+
+
+# The authors noted in the original tutorial that a key parameter called ‘spa_weight’ controls the relative weight of the spatial location similarity kernel.
+# 
+# Other parameters are further described in the api documentation. Typically `n_stage` is the parameter we are interested in modifying
+
+# In[ ]:
+
+
+STT_obj.train(n_states = 9, n_iter = 15, weight_connectivities = 0.5, 
+            n_neighbors = 50,thresh_ms_gene = 0.2, spa_weight =0.3)
+
+
+# After the prediction is complete, the attractor is stored in `adata.obs[‘attractor’]`. We can use `ov.pl.embedding` to visualize it.
+
+# In[12]:
+
+
+ov.pl.embedding(adata, basis="xy_loc", 
+                color=["attractor"],frameon='small',
+               palette=ov.pl.sc_color[11:])
+
+
+# In[7]:
+
+
+ov.pl.embedding(adata, basis="xy_loc", 
+                color=["Region"],frameon='small',
+               )
+
+
+# ## Pathway analysis
+# 
+# In the original tutorial, the author encapsulated the `gseapy==1.0.4` version for access enrichment. Note that the use of this function requires networking, which we have modified so that we can enrich using the local pathway dataset
+# 
+# We can download good access data directly in [enrichr](https://maayanlab.cloud/Enrichr/#libraries), such as the `KEGG_2019_mouse` used in this study.
+# 
+# https://maayanlab.cloud/Enrichr/geneSetLibrary?mode=text&libraryName=KEGG_2019_Mouse
+
+# In[8]:
+
+
+pathway_dict=ov.utils.geneset_prepare('genesets/KEGG_2019_Mouse.txt',organism='Mouse')
+
+
+# In[ ]:
+
+
+STT_obj.compute_pathway(pathway_dict)
+
+
+# After running the function, we can use the `plot_pathway` function to visualize the similairty between pathway dynamics in the low dimensional embeddings.
+
+# In[11]:
+
+
+fig = STT_obj.plot_pathway(figsize = (10,8),size = 100,fontsize = 12)
+for ax in fig.axes:
+    ax.set_xlabel('Embedding 1', fontsize=20)  # Adjust font size as needed
+    ax.set_ylabel('Embedding 2', fontsize=20)  # Adjust font size as needed
+fig.show()
+
+
+# If we are interested in the specific pathways, we can use the `plot_tensor_pathway` function to visualize the streamlines.
+
+# In[13]:
+
+
+import matplotlib.pyplot as plt
+fig, ax = plt.subplots(1, 1, figsize=(4, 4))
+STT_obj.plot_tensor_pathway(pathway_name = 'Wnt signaling pathway',basis = 'xy_loc',
+                           ax=ax)
+
+
+# In[14]:
+
+
+fig, ax = plt.subplots(1, 1, figsize=(4, 4))
+STT_obj.plot_tensor_pathway( 'TGF-beta signaling pathway',basis = 'xy_loc',
+                           ax=ax)
+
+
+# ## Tensor analysis
+# 
+# In the author's original paper, a very interesting concept is mentioned, attractor-averaged and attractor-specific tensors.
+# 
+# We can analyse the Joint Tensor and thus study the steady state processes of different attractors. If the streamlines are passing through the attractor then the attractor is in a steady state, if the streamlines are emanating/converging from the attractor then the attractor is in a dynamic state.
+# 
+# In addition to this, the Unspliced Tensor also reflects the strength as well as the size of the attraction.
+
+# In[4]:
+
+
+STT_obj.plot_tensor(list_attractor = [1,3,5,6],
+                filter_cells = True, member_thresh = 0.1, density = 1)
+
+
+# ## Landscape analysis
+# 
+# Each attractor corresponds to a spatial steady state, then we can use contour plots to visualise this steady state and use CellRank's correlation function to infer state transitions between different attractors.
+
+# In[17]:
+
+
+STT_obj.construct_landscape(coord_key = 'X_xy_loc')
+
+
+# In[14]:
+
+
+sc.pl.embedding(adata, color = ['attractor', 'Region'],basis= 'trans_coord')
+
+
+# Method to infer the lineage, either ‘MPFT’(maxium probability flow tree, global) or ‘MPPT’(most probable path tree, local)
+
+# In[15]:
+
+
+STT_obj.infer_lineage(si=3,sf=4, method = 'MPPT',flux_fraction=0.8,color_palette_name = 'tab10',size_point = 8,
+                   size_text=12)
+
+
+# The Sankey plot displaying the relation between STT attractors (left) and spatial region annotations (right). The width of links indicates the number of cells that share the connected attractor label and region annotation label simultaneously
+
+# In[16]:
+
+
+fig = STT_obj.plot_sankey(adata.obs['attractor'].tolist(),adata.obs['Region'].tolist())
+
+
+# ## Saving and Loading Data
+# 
+# We need to save the data after the calculation is complete, and we provide the load function to load it directly in the next analysis without having to re-analyse it.
+
+# In[24]:
+
+
+#del adata.uns['r2_keep_train']
+#del adata.uns['r2_keep_test']
+#del adata.uns['kernel']
+#del adata.uns['kernel_connectivities']
+
+STT_obj.adata.write('data/mouse_brain_adata.h5ad')
+STT_obj.adata_aggr.write('data/mouse_brain_adata_aggr.h5ad')
+
+
+# In[2]:
+
+
+adata=ov.read('data/mouse_brain_adata.h5ad')
+adata_aggr=ov.read('data/mouse_brain_adata_aggr.h5ad')
+
+
+# In[3]:
+
+
+STT_obj=ov.space.STT(adata,spatial_loc='xy_loc',region='Region')
+STT_obj.load(adata,adata_aggr)
+
+
+# ## Gene Dynamic
+# 
+# The genes with high multistability scores possess varying expression levels in both unspliced and spliced counts within various attractors, and show a gradual change during stage transitions. These gene were stored in `adata.var['r2_test']`
+# 
+# 
+
+# In[18]:
+
+
+adata.var['r2_test'].sort_values(ascending=False)
+
+
+# In[11]:
+
+
+STT_obj.plot_top_genes(top_genes = 6, ncols = 2, figsize = (8,8),)
+
+
+# We analysed the attractor 1-related gene Sim1, and we found that in the unspliced Mu matrix (smooth unspliced), Sim1 is expressed low at attractor 1; in the spliced matrix Ms, Sim1 is expressed high at attractor 1. It indicates that there is a dynamic tendency of Sim1 gene at attractor 1, i.e., the direction of Sim1 expression is flowing towards attractor 1.
+# 
+# Velo analyses can also illustrate this, although Sim1 expression is not highest at attractor 1.
+
+# In[26]:
+
+
+import matplotlib.pyplot as plt
+fig, axes = plt.subplots(1, 4, figsize=(12, 3))
+ov.pl.embedding(adata, basis="xy_loc", 
+                color=["Sim1"],frameon='small',
+                title='Sim1:Ms',show=False,
+                layer='Ms',cmap='RdBu_r',ax=axes[0]
+               )
+ov.pl.embedding(adata, basis="xy_loc", 
+                color=["Sim1"],frameon='small',
+                title='Sim1:Mu',show=False,
+                layer='Mu',cmap='RdBu_r',ax=axes[1]
+               )
+ov.pl.embedding(adata, basis="xy_loc", 
+                color=["Sim1"],frameon='small',
+                title='Sim1:Velo',show=False,
+                layer='velo',cmap='RdBu_r',ax=axes[2]
+               )
+ov.pl.embedding(adata, basis="xy_loc", 
+                color=["Sim1"],frameon='small',
+                title='Sim1:exp',show=False,
+                #layer='Mu',
+                cmap='RdBu_r',ax=axes[3]
+               )
+plt.tight_layout()
+

ovrawm/t_tcga.txt

@@ -0,0 +1,96 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# # TCGA database preprocess
+# 
+# We often download patient survival data from the TCGA database for analysis in order to verify the importance of genes in cancer. However, the pre-processing of the TCGA database is often a headache. Here, we have introduced the TCGA module in ov, a way to quickly process the file formats we download from the TCGA database. We need to prepare 3 files as input:
+# 
+# - gdc_sample_sheet (`.tsv`): The `Sample Sheet` button of TCGA, and we can get `tsv` file from it
+# - gdc_download_files (`folder`): The `Download/Cart` button of TCGA, and we get `tar.gz` included all file you selected/
+# - clinical_cart (`folder`): The `Clinical` button of TCGA, and we can get `tar.gz` included all clinical of your files
+
+# In[1]:
+
+
+import omicverse as ov
+import scanpy as sc
+ov.plot_set()
+
+
+# ## TCGA counts read
+# 
+# Here, we use `ov.bulk.TCGA` to perform the `gdc_sample_sheet`, `gdc_download_files` and `clinical_cart` you download before. The raw count, fpkm and tpm matrix will be stored in anndata object
+
+# In[2]:
+
+
+get_ipython().run_cell_magic('time', '', "gdc_sample_sheep='data/TCGA_OV/gdc_sample_sheet.2024-07-05.tsv'\ngdc_download_files='data/TCGA_OV/gdc_download_20240705_180129.081531'\nclinical_cart='data/TCGA_OV/clinical.cart.2024-07-05'\naml_tcga=ov.bulk.pyTCGA(gdc_sample_sheep,gdc_download_files,clinical_cart)\naml_tcga.adata_init()\n")
+
+
+# We can save the anndata object for the next use
+
+# In[3]:
+
+
+aml_tcga.adata.write_h5ad('data/TCGA_OV/ov_tcga_raw.h5ad',compression='gzip')
+
+
+# Note: Each time we read the anndata file, we need to initialize the TCGA object using three paths so that the subsequent TCGA functions such as survival analysis can be used properly
+# 
+# If you wish to create your own TCGA data, we have provided [sample data](https://figshare.com/ndownloader/files/47461946) here for download:
+# 
+# TCGA OV: https://figshare.com/ndownloader/files/47461946
+
+# In[2]:
+
+
+gdc_sample_sheep='data/TCGA_OV/gdc_sample_sheet.2024-07-05.tsv'
+gdc_download_files='data/TCGA_OV/gdc_download_20240705_180129.081531'
+clinical_cart='data/TCGA_OV/clinical.cart.2024-07-05'
+aml_tcga=ov.bulk.pyTCGA(gdc_sample_sheep,gdc_download_files,clinical_cart)
+aml_tcga.adata_read('data/TCGA_OV/ov_tcga_raw.h5ad')
+
+
+# ## Meta init
+# As the TCGA reads the gene_id, we need to convert it to gene_name as well as adding basic information about the patient. Therefore we need to initialise the patient's meta information.
+
+# In[3]:
+
+
+aml_tcga.adata_meta_init()
+
+
+# ## Survial init
+# We set up the path for Clinical earlier, but in fact we did not import the patient information in the previous process, we only initially determined the id of the patient's TCGA, so we attracted to initialize the clinical information
+
+# In[4]:
+
+
+aml_tcga.survial_init()
+aml_tcga.adata
+
+
+# To visualize the gene you interested, we can use `survival_analysis` to finish it. 
+
+# In[5]:
+
+
+aml_tcga.survival_analysis('MYC',layer='deseq_normalize',plot=True)
+
+
+# If you want to calculate the survival of all genes, you can also use the `survial_analysis_all` to finish it. It may calculate a lot of times.
+
+# In[ ]:
+
+
+aml_tcga.survial_analysis_all()
+aml_tcga.adata
+
+
+# Don't forget to save your result.
+
+# In[ ]:
+
+
+aml_tcga.adata.write_h5ad('data/TCGA_OV/ov_tcga_survial_all.h5ad',compression='gzip')
+

ovrawm/t_tosica.txt

@@ -0,0 +1,317 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# # Celltype annotation migration(mapping) with TOSICA
+# 
+# We know that when all samples cannot be obtained at the same time, it would be desirable to classify the cell types on the first batch of data and use them to annotate the data obtained later or to be obtained in the future with the same standard, without the need to processing and mapping them together again.
+# 
+# So migration(mapping) the reference cell annotation is necessary. This tutorial focuses on how to migration(mapping) the cell annotation from reference scRNA-seq atlas to new scRNA-seq data.
+# 
+# Paper: [Transformer for one stop interpretable cell type annotation](https://www.nature.com/articles/s41467-023-35923-4)
+# 
+# Code: https://github.com/JackieHanLab/TOSICA
+# 
+# Colab_Reproducibility：https://colab.research.google.com/drive/1BjPEG-kLAgicP8iQvtVtpzzbIOmk1X23?usp=sharing
+# 
+# ![tosica](https://raw.githubusercontent.com/JackieHanLab/TOSICA/main/figure.png)
+# 
+
+# In[1]:
+
+
+import omicverse as ov
+import scanpy as sc
+ov.utils.ov_plot_set()
+
+
+# ## Loading data
+# 
+# + `demo_train.h5ad` :  Braon(GSE84133) and Muraro(GSE85241)
+# 
+# + `demo_test.h5ad` :  xin(GSE81608), segerstolpe(E-MTAB-5061) and Lawlor(GSE86473)
+# 
+# They can be downloaded at https://figshare.com/projects/TOSICA_demo/158489.
+
+# In[2]:
+
+
+ref_adata = sc.read('demo_train.h5ad')
+ref_adata = ref_adata[:,ref_adata.var_names]
+print(ref_adata)
+print(ref_adata.obs.Celltype.value_counts())
+
+
+# In[4]:
+
+
+query_adata = sc.read('demo_test.h5ad')
+query_adata = query_adata[:,ref_adata.var_names]
+print(query_adata)
+print(query_adata.obs.Celltype.value_counts())
+
+
+# We need to select the same gene training and predicting the celltype
+
+# In[ ]:
+
+
+ref_adata.var_names_make_unique()
+query_adata.var_names_make_unique()
+ret_gene=list(set(query_adata.var_names) & set(ref_adata.var_names))
+len(ret_gene)
+
+
+# In[ ]:
+
+
+query_adata=query_adata[:,ret_gene]
+ref_adata=ref_adata[:,ret_gene]
+
+
+# In[5]:
+
+
+print(f"The max of ref_adata is {ref_adata.X.max()}, query_data is {query_adata.X.max()}",)
+
+
+# By comparing the maximum values of the two data, we can see that the data has been normalised `sc.pp.normalize_total` and logarithmised `sc.pp.log1p`. The same treatment is applied to the data when we use our own data for analysis.
+# 
+# ## Download Genesets
+# 
+# Here, we need to download the genesets as pathway at first. You can use `ov.utils.download_tosica_gmt()` to download automatically or manual download from:
+# 
+# - 'GO_bp':'https://figshare.com/ndownloader/files/41460072',
+# - 'TF':'https://figshare.com/ndownloader/files/41460066',
+# - 'reactome':'https://figshare.com/ndownloader/files/41460051',
+# - 'm_GO_bp':'https://figshare.com/ndownloader/files/41460060',
+# - 'm_TF':'https://figshare.com/ndownloader/files/41460057',
+# - 'm_reactome':'https://figshare.com/ndownloader/files/41460054',
+# - 'immune':'https://figshare.com/ndownloader/files/41460063',
+# 
+
+# In[ ]:
+
+
+ov.utils.download_tosica_gmt()
+
+
+# ## Initialisation the TOSICA model
+# 
+# We first need to train the TOSICA model on the REFERENCE dataset, where omicverse provides a simple class `pyTOSICA`, and all subsequent operations can be done with `pyTOSICA`. We need to set the parameters for model initialisation.
+# 
+# - `adata`: the reference adata object
+# - `gmt_path`: default pre-prepared mask or path to .gmt files. you can use `ov.utils.download_tosica_gmt()` to obtain the genesets
+# - `depth`: the depth of transformer model, When it is set to 2, a memory leak may occur
+# - `label_name`: the reference key of celltype in `adata.obs`
+# - `project_path`: the save path of TOSICA model
+# - `batch_size`: indicates the number of cells passed to the programme for training in a single pass
+
+# In[6]:
+
+
+tosica_obj=ov.single.pyTOSICA(adata=ref_adata,
+                              gmt_path='genesets/GO_bp.gmt', depth=1,
+                              label_name='Celltype',
+                              project_path='hGOBP_demo',
+                              batch_size=8)
+
+
+# ## Training the TOSICA model
+# 
+# There're 4 arguments to set when training the TOSICA model.
+# 
+# - pre_weights: The path of the pre-trained weights.
+# - lr: The learning rate.
+# - epochs: The number of epochs.
+# - lrf: The learning rate of the last layer.
+
+# In[5]:
+
+
+tosica_obj.train(epochs=5)
+
+
+# We can use `.save` to store the `TOSICA` model in `project_path`
+
+# In[6]:
+
+
+tosica_obj.save()
+
+
+# The model can be loaded from `project_path` automatically.
+
+# In[7]:
+
+
+tosica_obj.load()
+
+
+# ## Update with query
+# 
+
+# In[8]:
+
+
+new_adata=tosica_obj.predicted(pre_adata=query_adata)
+
+
+# ## Visualize the reference and mapping
+# 
+# We first compute the lower dimensional space of query_data, where we use omicverse's preprocessing method as well as the mde method for dimensionality reduction
+# 
+# To visualize the PCA’s embeddings, we use the `pymde` package wrapper in omicverse. This is an alternative to UMAP that is GPU-accelerated.
+
+# In[15]:
+
+
+ov.pp.scale(query_adata)
+ov.pp.pca(query_adata,layer='scaled',n_pcs=50)
+sc.pp.neighbors(query_adata, n_neighbors=15, n_pcs=50,
+               use_rep='scaled|original|X_pca')
+query_adata.obsm["X_mde"] = ov.utils.mde(query_adata.obsm["scaled|original|X_pca"])
+query_adata
+
+
+# Since new_adata and query_adata have the same cells, their low-dimensional spaces are also the same. So we proceed directly to the assignment operation.
+
+# In[16]:
+
+
+new_adata.obsm=query_adata[new_adata.obs.index].obsm.copy()
+new_adata.obsp=query_adata[new_adata.obs.index].obsp.copy()
+new_adata
+
+
+# Since the predicted cell types are not exactly the same as the original cell types, the colours are not exactly the same. For the visualisation effect, we manually set the colour of the predicted cell type with the original cell type.
+
+# In[18]:
+
+
+import numpy as np
+col = np.array([
+"#98DF8A","#E41A1C" ,"#377EB8", "#4DAF4A" ,"#984EA3" ,"#FF7F00" ,"#FFFF33" ,"#A65628" ,"#F781BF" ,"#999999","#1F77B4","#FF7F0E","#279E68","#FF9896"
+]).astype('<U7')
+
+celltype = ("alpha","beta","ductal","acinar","delta","PP","PSC","endothelial","epsilon","mast","macrophage","schwann",'t_cell')
+new_adata.obs['Prediction'] = new_adata.obs['Prediction'].astype('category')
+new_adata.obs['Prediction'] = new_adata.obs['Prediction'].cat.reorder_categories(list(celltype))
+new_adata.uns['Prediction_colors'] = col[1:]
+
+celltype = ("MHC class II","alpha","beta","ductal","acinar","delta","PP","PSC","endothelial","epsilon","mast")
+new_adata.obs['Celltype'] = new_adata.obs['Celltype'].astype('category')
+new_adata.obs['Celltype'] = new_adata.obs['Celltype'].cat.reorder_categories(list(celltype))
+new_adata.uns['Celltype_colors'] = col[:11]
+
+
+# In[24]:
+
+
+sc.pl.embedding(
+    new_adata,
+    basis="X_mde",
+    color=['Celltype', 'Prediction'],
+    frameon=False,
+    #ncols=1,
+    wspace=0.5,
+    #palette=ov.utils.pyomic_palette()[11:],
+    show=False,
+)
+
+
+# ## Pathway attention
+# 
+# TOSICA has another special feature, which is the ability to computationally use self-attention mechanisms to find pathways associated with cell types. Here we demonstrate the approach of this downstream analysis.
+# 
+# We first need to filter out the predicted types of cells with cell counts less than 5.
+
+# In[30]:
+
+
+cell_idx=new_adata.obs['Prediction'].value_counts()[new_adata.obs['Prediction'].value_counts()<5].index
+new_adata=new_adata[~new_adata.obs['Prediction'].isin(cell_idx)]
+
+
+# We then used `sc.tl.rank_genes_groups` to calculate the differential pathways with the highest attention across cell types. This differential pathway is derived from the gmt genesets used for the previous calculation.
+
+# In[31]:
+
+
+sc.tl.rank_genes_groups(new_adata, 'Prediction', method='wilcoxon')
+
+
+# In[34]:
+
+
+sc.pl.rank_genes_groups_dotplot(new_adata,
+                                n_genes=3,standard_scale='var',)
+
+
+# If you want to get the cell-specific pathway, you can use `sc.get.rank_genes_groups_df` to get.
+# 
+# For example, we would like to obtain the pathway with the highest attention for the cell type `PP`
+
+# In[35]:
+
+
+degs = sc.get.rank_genes_groups_df(new_adata, group='PP', key='rank_genes_groups',
+                                            pval_cutoff=0.05)
+degs.head()
+
+
+# In[36]:
+
+
+sc.pl.embedding(
+    new_adata,
+    basis="X_mde",
+    color=['Prediction','GOBP_REGULATION_OF_MUSCLE_SYSTEM_PROCESS'],
+    frameon=False,
+    #ncols=1,
+    wspace=0.5,
+    #palette=ov.utils.pyomic_palette()[11:],
+    show=False,
+)
+
+
+# If you call omciverse to complete a TOSICA analysis, don't forget to cite the following literature:
+# 
+# ```
+# @article{pmid:36641532,
+# journal = {Nature communications},
+# doi = {10.1038/s41467-023-35923-4},
+# issn = {2041-1723},
+# number = {1},
+# pmid = {36641532},
+# pmcid = {PMC9840170},
+# address = {England},
+# title = {Transformer for one stop interpretable cell type annotation},
+# volume = {14},
+# author = {Chen, Jiawei and Xu, Hao and Tao, Wanyu and Chen, Zhaoxiong and Zhao, Yuxuan and Han, Jing-Dong J},
+# note = {[Online; accessed 2023-07-18]},
+# pages = {223},
+# date = {2023-01-14},
+# year = {2023},
+# month = {1},
+# day = {14},
+# }
+# 
+# 
+# @misc{doi:10.1101/2023.06.06.543913,
+# doi = {10.1101/2023.06.06.543913},
+# publisher = {Cold Spring Harbor Laboratory},
+# title = {OmicVerse: A single pipeline for exploring the entire transcriptome universe},
+# author = {Zeng, Zehua and Ma, Yuqing and Hu, Lei and Xiong, Yuanyan and Du, Hongwu},
+# note = {[Online; accessed 2023-07-18]},
+# date = {2023-06-08},
+# year = {2023},
+# month = {6},
+# day = {8},
+# }
+# ```
+
+# In[ ]:
+
+
+
+

ovrawm/t_traj.txt

@@ -0,0 +1,238 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# # Trajectory Inference with PAGA or Palantir
+# 
+# Diffusion maps were introduced by Ronald Coifman and Stephane Lafon, and the underlying idea is to assume that the data are samples from a diffusion process.
+# 
+# Palantir is an algorithm to align cells along differentiation trajectories. Palantir models differentiation as a stochastic process where stem cells differentiate to terminally differentiated cells by a series of steps through a low dimensional phenotypic manifold. Palantir effectively captures the continuity in cell states and the stochasticity in cell fate determination.
+# 
+# Note that both methods require the input of cells in their initial state, and we will introduce other methods that do not require the input of artificial information, such as pyVIA, in subsequent analyses.
+# 
+# 
+# ## Preprocess data
+# 
+# As an example, we apply differential kinetic analysis to dentate gyrus neurogenesis, which comprises multiple heterogeneous subpopulations.
+
+# In[1]:
+
+
+import scanpy as sc
+import scvelo as scv
+import matplotlib.pyplot as plt
+import omicverse as ov
+ov.plot_set()
+
+
+# In[2]:
+
+
+import scvelo as scv
+adata=scv.datasets.dentategyrus()
+adata
+
+
+# In[3]:
+
+
+adata=ov.pp.preprocess(adata,mode='shiftlog|pearson',n_HVGs=3000,)
+adata.raw = adata
+adata = adata[:, adata.var.highly_variable_features]
+ov.pp.scale(adata)
+ov.pp.pca(adata,layer='scaled',n_pcs=50)
+
+
+# Let us inspect the contribution of single PCs to the total variance in the data. This gives us information about how many PCs we should consider in order to compute the neighborhood relations of cells. In our experience, often a rough estimate of the number of PCs does fine.
+
+# In[4]:
+
+
+ov.utils.plot_pca_variance_ratio(adata)
+
+
+# ## Trajectory inference with diffusion map
+# 
+# Here, we used `ov.single.TrajInfer` to construct a Trajectory Inference object.
+
+# In[5]:
+
+
+Traj=ov.single.TrajInfer(adata,basis='X_umap',groupby='clusters',
+                         use_rep='scaled|original|X_pca',n_comps=50,)
+Traj.set_origin_cells('nIPC')
+
+
+# In[6]:
+
+
+Traj.inference(method='diffusion_map')
+
+
+# In[7]:
+
+
+ov.utils.embedding(adata,basis='X_umap',
+                   color=['clusters','dpt_pseudotime'],
+                   frameon='small',cmap='Reds')
+
+
+# PAGA graph abstraction has benchmarked as top-performing method for trajectory inference. It provides a graph-like map of the data topology with weighted edges corresponding to the connectivity between two clusters. 
+# 
+# Here, PAGA is extended by neighbor directionality.
+
+# In[8]:
+
+
+ov.utils.cal_paga(adata,use_time_prior='dpt_pseudotime',vkey='paga',
+                 groups='clusters')
+
+
+# In[9]:
+
+
+ov.utils.plot_paga(adata,basis='umap', size=50, alpha=.1,title='PAGA LTNN-graph',
+            min_edge_width=2, node_size_scale=1.5,show=False,legend_loc=False)
+
+
+# ## Trajectory inference with Slingshot
+# 
+# Provides functions for inferring continuous, branching lineage structures in low-dimensional data. Slingshot was designed to model developmental trajectories in single-cell RNA sequencing data and serve as a component in an analysis pipeline after dimensionality reduction and clustering. It is flexible enough to handle arbitrarily many branching events and allows for the incorporation of prior knowledge through supervised graph construction.
+
+# In[10]:
+
+
+Traj=ov.single.TrajInfer(adata,basis='X_umap',groupby='clusters',
+                         use_rep='scaled|original|X_pca',n_comps=50)
+Traj.set_origin_cells('nIPC')
+#Traj.set_terminal_cells(["Granule mature","OL","Astrocytes"])
+
+
+# If you only need the proposed timing and not the lineage of the process, then you can leave the debug_axes parameter unset.
+
+# In[ ]:
+
+
+Traj.inference(method='slingshot',num_epochs=1)
+
+
+# else, you can set `debug_axes` to visualize the lineage
+
+# In[13]:
+
+
+fig, axes = plt.subplots(nrows=2, ncols=2, figsize=(8, 8))
+Traj.inference(method='slingshot',num_epochs=1,debug_axes=axes)
+
+
+# In[14]:
+
+
+ov.utils.embedding(adata,basis='X_umap',
+                   color=['clusters','slingshot_pseudotime'],
+                   frameon='small',cmap='Reds')
+
+
+# In[15]:
+
+
+sc.pp.neighbors(adata,use_rep='scaled|original|X_pca')
+ov.utils.cal_paga(adata,use_time_prior='slingshot_pseudotime',vkey='paga',
+                 groups='clusters')
+
+
+# In[16]:
+
+
+ov.utils.plot_paga(adata,basis='umap', size=50, alpha=.1,title='PAGA Slingshot-graph',
+            min_edge_width=2, node_size_scale=1.5,show=False,legend_loc=False)
+
+
+# ## Trajectory inference with Palantir
+# 
+# Palantir can be run by specifying an approxiate early cell.
+# 
+# Palantir can automatically determine the terminal states as well. In this dataset, we know the terminal states and we will set them using the terminal_states parameter
+# 
+# Here, we used `ov.single.TrajInfer` to construct a Trajectory Inference object.
+
+# In[17]:
+
+
+Traj=ov.single.TrajInfer(adata,basis='X_umap',groupby='clusters',
+                         use_rep='scaled|original|X_pca',n_comps=50)
+Traj.set_origin_cells('nIPC')
+Traj.set_terminal_cells(["Granule mature","OL","Astrocytes"])
+
+
+# In[18]:
+
+
+Traj.inference(method='palantir',num_waypoints=500)
+
+
+# Palantir results can be visualized on the tSNE or UMAP using the plot_palantir_results function
+
+# In[19]:
+
+
+Traj.palantir_plot_pseudotime(embedding_basis='X_umap',cmap='RdBu_r',s=3)
+
+
+# Once the cells are selected, it's often helpful to visualize the selection on the pseudotime trajectory to ensure we've isolated the correct cells for our specific trend. We can do this using the plot_branch_selection function:
+
+# In[20]:
+
+
+Traj.palantir_cal_branch(eps=0)
+
+
+# In[22]:
+
+
+ov.externel.palantir.plot.plot_trajectory(adata, "Granule mature",
+                                cell_color="palantir_entropy",
+                                n_arrows=10,
+                                color="red",
+                                scanpy_kwargs=dict(cmap="RdBu_r"),
+                                )
+
+
+# Palantir uses Mellon Function Estimator to determine the gene expression trends along different lineages. The marker trends can be determined using the following snippet. This computes the trends for all lineages. A subset of lineages can be used using the lineages parameter.
+
+# In[23]:
+
+
+gene_trends = Traj.palantir_cal_gene_trends(
+    layers="MAGIC_imputed_data",
+)
+
+
+# In[24]:
+
+
+genes = ['Cdca3','Rasl10a','Mog','Aqp4']
+Traj.palantir_plot_gene_trends(genes)
+plt.show()
+
+
+# We can also use paga to visualize the cell stages
+
+# In[25]:
+
+
+ov.utils.cal_paga(adata,use_time_prior='palantir_pseudotime',vkey='paga',
+                 groups='clusters')
+
+
+# In[26]:
+
+
+ov.utils.plot_paga(adata,basis='umap', size=50, alpha=.1,title='PAGA LTNN-graph',
+            min_edge_width=2, node_size_scale=1.5,show=False,legend_loc=False)
+
+
+# In[ ]:
+
+
+
+

ovrawm/t_via.txt

@@ -0,0 +1,193 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# # Trajectory Inference with VIA
+# 
+# VIA is a single-cell Trajectory Inference method that offers topology construction, pseudotimes, automated terminal state prediction and automated plotting of temporal gene dynamics along lineages. Here, we have improved the original author's colouring logic and user habits so that users can use the anndata object directly for analysis。 
+# 
+# We have completed this tutorial using the analysis provided by the original VIA authors.
+# 
+# Paper: [Generalized and scalable trajectory inference in single-cell omics data with VIA](https://www.nature.com/articles/s41467-021-25773-3)
+# 
+# Code: https://github.com/ShobiStassen/VIA
+# 
+# Colab_Reproducibility：https://colab.research.google.com/drive/1A2X23z_RLJaYLbXaiCbZa-fjNbuGACrD?usp=sharing
+
+# In[1]:
+
+
+import omicverse as ov
+import scanpy as sc
+import matplotlib.pyplot as plt
+ov.utils.ov_plot_set()
+
+
+# ## Data loading and preprocessing
+# 
+# We have used the dataset scRNA_hematopoiesis provided by the authors for this analysis, noting that the data have been normalized and logarithmicized but not scaled.
+
+# In[2]:
+
+
+adata = ov.single.scRNA_hematopoiesis()
+sc.tl.pca(adata, svd_solver='arpack', n_comps=200)
+adata
+
+
+# ## Model construct and run
+# 
+# We need to specify the cell feature vector `adata_key` used for VIA inference, which can be X_pca, X_scVI or X_glue, depending on the purpose of your analysis, here we use X_pca directly. We also need to specify how many components to be used, the components should not larger than the length of vector.
+# 
+# Besides, we need to specify the `clusters` to be colored and calculate for VIA. If the `root_user` is None, it will be calculated the root cell automatically.
+# 
+# We need to set `basis` argument stored in `adata.obsm`. An example setting `tsne` because it stored in `obsm: 'tsne', 'MAGIC_imputed_data', 'palantir_branch_probs', 'X_pca'`
+# 
+# We also need to set `clusters` argument stored in `adata.obs`. It means the celltype key.
+# 
+# Other explaination of argument and attributes could be found at https://pyvia.readthedocs.io/en/latest/Parameters%20and%20Attributes.html
+
+# In[3]:
+
+
+v0 = ov.single.pyVIA(adata=adata,adata_key='X_pca',adata_ncomps=80, basis='tsne',
+                         clusters='label',knn=30,random_seed=4,root_user=[4823],)
+
+v0.run()
+
+
+# ## Visualize and analysis
+# 
+# Before the subsequent analysis, we need to specify the colour of each cluster. Here we use sc.pl.embedding to automatically colour each cluster, if you need to specify your own colours, specify the palette parameter
+
+# In[4]:
+
+
+fig, ax = plt.subplots(1,1,figsize=(4,4))
+sc.pl.embedding(
+    adata,
+    basis="tsne",
+    color=['label'],
+    frameon=False,
+    ncols=1,
+    wspace=0.5,
+    show=False,
+    ax=ax
+)
+fig.savefig('figures/via_fig1.png',dpi=300,bbox_inches = 'tight')
+
+
+# ## VIA graph
+# 
+# To visualize the results of the Trajectory inference in various ways. Via offers various plotting functions.We first show the cluster-graph level trajectory abstraction consisting of two subplots colored by annotated (true_label) composition and by pseudotime
+
+# In[5]:
+
+
+fig, ax, ax1 = v0.plot_piechart_graph(clusters='label',cmap='Reds',dpi=80,
+                                   show_legend=False,ax_text=False,fontsize=4)
+fig.savefig('figures/via_fig2.png',dpi=300,bbox_inches = 'tight')
+
+
+# In[ ]:
+
+
+#you can use `v0.model.single_cell_pt_markov` to extract the pseudotime
+v0.get_pseudotime(v0.adata)
+v0.adata
+
+
+# ## Visualise gene/feature graph
+# 
+# View the gene expression along the VIA graph. We use the computed HNSW small world graph in VIA to accelerate the gene imputation calculations (using similar approach to MAGIC) as follows: 
+# 
+
+# In[6]:
+
+
+gene_list_magic = ['IL3RA', 'IRF8', 'GATA1', 'GATA2', 'ITGA2B', 'MPO', 'CD79B', 'SPI1', 'CD34', 'CSF1R', 'ITGAX']
+fig,axs=v0.plot_clustergraph(gene_list=gene_list_magic[:4],figsize=(12,3),)
+fig.savefig('figures/via_fig2_1.png',dpi=300,bbox_inches = 'tight')
+
+
+# ## Trajectory projection
+# 
+# Visualize the overall VIA trajectory projected onto a 2D embedding (UMAP, Phate, TSNE etc) in different ways.
+# 
+# - Draw the high-level clustergraph abstraction onto the embedding;
+# - Draws a vector field plot of the more fine-grained directionality of cells along the trajectory projected onto an embedding.
+# - Draw high-edge resolution directed graph
+
+# In[7]:
+
+
+fig,ax1,ax2=v0.plot_trajectory_gams(basis='tsne',clusters='label',draw_all_curves=False)
+fig.savefig('figures/via_fig3.png',dpi=300,bbox_inches = 'tight')
+
+
+# In[8]:
+
+
+fig,ax=v0.plot_stream(basis='tsne',clusters='label',
+               density_grid=0.8, scatter_size=30, scatter_alpha=0.3, linewidth=0.5)
+fig.savefig('figures/via_fig4.png',dpi=300,bbox_inches = 'tight')
+
+
+# In[9]:
+
+
+fig,ax=v0.plot_stream(basis='tsne',density_grid=0.8, scatter_size=30, color_scheme='time', linewidth=0.5,
+                             min_mass = 1, cutoff_perc = 5, scatter_alpha=0.3, marker_edgewidth=0.1,
+                             density_stream = 2, smooth_transition=1, smooth_grid=0.5)
+fig.savefig('figures/via_fig5.png',dpi=300,bbox_inches = 'tight')
+
+
+# ## Probabilistic pathways
+# 
+# Visualize the probabilistic pathways from root to terminal state as indicated by the lineage likelihood. The higher the linage likelihood, the greater the potential of that particular cell to differentiate towards the terminal state of interest.
+
+# In[10]:
+
+
+fig,axs=v0.plot_lineage_probability(figsize=(8,4),)
+fig.savefig('figures/via_fig6.png',dpi=300,bbox_inches = 'tight')
+
+
+# We can specify a specific linkage for visualisation
+
+# In[11]:
+
+
+fig,axs=v0.plot_lineage_probability(figsize=(6,3),marker_lineages = [2,3])
+fig.savefig('figures/via_fig7.png',dpi=300,bbox_inches = 'tight')
+
+
+# ## Gene Dynamics
+# 
+# The gene dynamics along pseudotime for all detected lineages are automatically inferred by VIA. These can be interpreted as the change in gene expression along any given lineage. 
+
+# In[12]:
+
+
+fig,axs=v0.plot_gene_trend(gene_list=gene_list_magic,figsize=(8,6),)
+fig.savefig('figures/via_fig8.png',dpi=300,bbox_inches = 'tight')
+
+
+# In[14]:
+
+
+fig,ax=v0.plot_gene_trend_heatmap(gene_list=gene_list_magic,figsize=(4,4),
+                          marker_lineages=[2])
+fig.savefig('figures/via_fig9.png',dpi=300,bbox_inches = 'tight')
+
+
+# In[ ]:
+
+
+
+
+
+# In[ ]:
+
+
+
+

ovrawm/t_via_velo.txt

@@ -0,0 +1,102 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# # Trajectory Inference with VIA and scVelo
+# 
+# When scRNA-velocity is available, it can be used to guide the trajectory inference and automate initial state prediction. However, because RNA velocitycan be misguided by(Bergen 2021) boosts in expression, variable transcription rates and data capture scope limited to steady-state populations only, users might find it useful to adjust the level of influence the RNA-velocity data should exercise on the inferred TI.
+# 
+# Paper: [Generalized and scalable trajectory inference in single-cell omics data with VIA](https://www.nature.com/articles/s41467-021-25773-3)
+# 
+# Code: https://github.com/ShobiStassen/VIA
+# 
+# Colab_Reproducibility：https://colab.research.google.com/drive/1MtGr3e9uUb_BWOzKlcbOTiCYsZpljEyF?usp=sharing
+
+# In[7]:
+
+
+import omicverse as ov
+import scanpy as sc
+import scvelo as scv
+import cellrank as cr
+ov.utils.ov_plot_set()
+
+
+# ## Data loading and preprocessing
+# 
+# We use a familiar endocrine-genesis dataset (Bastidas-Ponce et al. (2019).) to demonstrate initial state prediction at the EP Ngn3 low cells and automatic captures of the 4 differentiated islets (alpha, beta, delta and epsilon). As mentioned, it us useful to control the level of influence of RNA-velocity relative to gene-gene distance and this is done using the velo_weight parameter.
+
+# In[2]:
+
+
+adata = cr.datasets.pancreas()
+adata
+
+
+# In[3]:
+
+
+n_pcs = 30
+scv.pp.filter_and_normalize(adata, min_shared_counts=20, n_top_genes=5000)
+sc.tl.pca(adata, n_comps = n_pcs)
+scv.pp.moments(adata, n_pcs=None, n_neighbors=None)
+scv.tl.velocity(adata, mode='stochastic') # good results acheived with mode = 'stochastic' too
+
+
+# ## Initialize and run VIA
+# 
+# 
+
+# In[4]:
+
+
+v0 = ov.single.pyVIA(adata=adata,adata_key='X_pca',adata_ncomps=n_pcs, basis='X_umap',
+                         clusters='clusters',knn=20, root_user=None,
+                         dataset='', random_seed=42,is_coarse=True, preserve_disconnected=True, pseudotime_threshold_TS=50,
+                         piegraph_arrow_head_width=0.15,piegraph_edgeweight_scalingfactor=2.5,
+                         velocity_matrix=adata.layers['velocity'],gene_matrix=adata.X.todense(),velo_weight=0.5,
+                         edgebundle_pruning_twice=False, edgebundle_pruning=0.15, pca_loadings = adata.varm['PCs']
+                        )
+
+v0.run()
+
+
+# In[ ]:
+
+
+
+
+
+# In[5]:
+
+
+fig, ax, ax1 = v0.plot_piechart_graph(clusters='clusters',cmap='Reds',dpi=80,
+                                   show_legend=False,ax_text=False,fontsize=4)
+fig.set_size_inches(8,4)
+
+
+# ## Visualize trajectory and cell progression
+# 
+# Fine grained vector fields
+
+# In[8]:
+
+
+v0.plot_trajectory_gams(basis='X_umap',clusters='clusters',draw_all_curves=False)
+
+
+# In[9]:
+
+
+v0.plot_stream(basis='X_umap',clusters='clusters',
+               density_grid=0.8, scatter_size=30, scatter_alpha=0.3, linewidth=0.5)
+
+
+# ## Draw lineage likelihoods
+# 
+# These indicate potential pathways corresponding to the 4 islets (two types of Beta islets Lineage 5 and 12)
+
+# In[10]:
+
+
+v0.plot_lineage_probability()
+

ovrawm/t_visualize_bulk.txt

@@ -0,0 +1,170 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# # Visualization of Bulk RNA-seq
+# 
+# In this part, we will introduce the tutorial of special plot of `omicverse`.
+
+# In[1]:
+
+
+import omicverse as ov
+import scanpy as sc
+import matplotlib.pyplot as plt
+ov.plot_set()
+
+
+# ## Venn plot
+# 
+# In transcriptome analyses, we often have to study differential genes that are common to different groups. Here, we provide `ov.pl.venn` to draw venn plots to visualise differential genes.
+# 
+# **Function**: `ov.pl.venn`: 
+# - sets: Subgroups requiring venn plots, Dictionary format, keys no more than 4
+# - palette: You can also re-specify the colour bar that needs to be drawn, just set `palette=['#FFFFFF','#000000']`, we have prepared `ov.pl.red_color`,`ov.pl.blue_color`,`ov.pl.green_color`,`ov.pl.orange_color`, by default.
+# - fontsize: the fontsize and linewidth to visualize, fontsize will be multiplied by 2
+
+# In[20]:
+
+
+fig,ax=plt.subplots(figsize = (4,4))
+#dict of sets
+sets = {
+    'Set1:name': {1,2,3},
+    'Set2': {1,2,3,4},
+    'Set3': {3,4},
+    'Set4': {5,6}
+}
+#plot venn   
+ov.pl.venn(sets=sets,palette=ov.pl.sc_color,
+           fontsize=5.5,ax=ax,
+           )
+
+
+#If we need to annotate genes, we can use plt.annotate for this purpose, 
+#we need to modify the text content, xy and xytext parameters.
+plt.annotate('gene1,gene2', xy=(50,30), xytext=(0,-100),
+             ha='center', textcoords='offset points', 
+             bbox=dict(boxstyle='round,pad=0.5', fc='gray', alpha=0.1),
+             arrowprops=dict(arrowstyle='->', color='gray'),size=12)
+
+#Set the title
+plt.title('Venn4',fontsize=13)
+
+#save figure
+fig.savefig("figures/bulk_venn4.png",dpi=300,bbox_inches = 'tight')
+
+
+# In[9]:
+
+
+fig,ax=plt.subplots(figsize = (4,4))
+#dict of sets
+sets = {
+    'Set1:name': {1,2,3},
+    'Set2': {1,2,3,4},
+    'Set3': {3,4},
+}
+    
+ov.pl.venn(sets=sets,ax=ax,fontsize=5.5,
+           palette=ov.pl.red_color)
+
+plt.title('Venn3',fontsize=13)
+
+
+# ## Volcano plot
+# 
+# For differentially expressed genes, we tend to visualise them only with volcano plots. Here, we present a method for mapping volcanoes using Python `ov.pl.volcano`.
+# 
+# **Function**: `ov.pl.venn`: 
+# 
+# main argument
+# - result: the DEGs result
+# - pval_name: the names of the columns whose vertical coordinates need to be plotted, stored in result.columns. In Bulk RNA-seq experiments, we usually set this to qvalue.
+# - fc_name: The names of the columns for which you need to plot the horizontal coordinates, stored in result.columns. In Bulk RNA-seq experiments, we typically set this to log2FC.
+# - fc_max: We need to set the threshold for the difference foldchange
+# - fc_min: We need to set the threshold for the difference foldchange
+# - pval_threshold: We need to set the threshold for the qvalue
+# - pval_max: We also need to set boundary values so that the data is not too large to affect the visualisation
+# - FC_max: We also need to set boundary values so that the data is not too large to affect the visualisation
+# 
+# plot argument
+# - figsize: The size of the generated figure, by default (4,4).
+# - title: The title of the plot, by default ''.
+# - titlefont: A dictionary of font properties for the plot title, by default {'weight':'normal','size':14,}.
+# - up_color: The color of the up-regulated genes in the plot, by default '#e25d5d'.
+# - down_color: The color of the down-regulated genes in the plot, by default '#7388c1'.
+# - normal_color: The color of the non-significant genes in the plot, by default '#d7d7d7'.
+# - legend_bbox: A tuple containing the coordinates of the legend's bounding box, by default (0.8, -0.2).
+# - legend_ncol: The number of columns in the legend, by default 2.
+# - legend_fontsize: The font size of the legend, by default 12.
+# - plot_genes: A list of genes to be plotted on the volcano plot, by default None.
+# - plot_genes_num: The number of genes to be plotted on the volcano plot, by default 10.
+# - plot_genes_fontsize: The font size of the genes to be plotted on the volcano plot, by default 10.
+# - ticks_fontsize: The font size of the ticks, by default 12.
+
+# In[3]:
+
+
+result=ov.read('data/dds_result.csv',index_col=0)
+result.head()
+
+
+# In[4]:
+
+
+ov.pl.volcano(result,pval_name='qvalue',fc_name='log2FoldChange',
+                     pval_threshold=0.05,fc_max=1.5,fc_min=-1.5,
+                      pval_max=10,FC_max=10,
+                    figsize=(4,4),title='DEGs in Bulk',titlefont={'weight':'normal','size':14,},
+                     up_color='#e25d5d',down_color='#7388c1',normal_color='#d7d7d7',
+                     up_fontcolor='#e25d5d',down_fontcolor='#7388c1',normal_fontcolor='#d7d7d7',
+                     legend_bbox=(0.8, -0.2),legend_ncol=2,legend_fontsize=12,
+                     plot_genes=None,plot_genes_num=10,plot_genes_fontsize=11,
+                     ticks_fontsize=12,)
+
+
+# ## Box plot
+# 
+# For differentially expressed genes in different groups, we sometimes need to compare the differences between different groups, and this is when we need to use box-and-line plots to do the comparison
+# 
+# **Function**: `ov.pl.boxplot`: 
+# 
+# - data: the data to visualize the boxplt example could be found in `seaborn.load_dataset("tips")`
+# - x_value, y_value, hue: Inputs for plotting long-form data. See examples for interpretation.
+# - figsize: The size of the generated figure, by default (4,4).
+# - fontsize: The font size of the tick and labels, by default 12.
+# - title: The title of the plot, by default ''.
+# 
+# 
+# **Function**: `ov.pl.add_palue`: 
+# - ax: the axes of bardotplot
+# - line_x1: The left side of the p-value line to be plotted
+# - line_x2: The right side of the p-value line to be plotted|
+# - line_y: The height of the p-value line to be plotted
+# - text_y: How much above the p-value line is plotted text
+# - text: the text of p-value, you can set `***` to instead `p<0.001`
+# - fontsize: the fontsize of text
+# - fontcolor: the color of text
+# - horizontalalignment: the location of text
+
+# In[3]:
+
+
+import seaborn as sns
+data = sns.load_dataset("tips")
+data.head()
+
+
+# In[7]:
+
+
+fig,ax=ov.pl.boxplot(data,hue='sex',x_value='day',y_value='total_bill',
+              palette=ov.pl.red_color,
+              figsize=(4,2),fontsize=12,title='Tips',)
+
+ov.pl.add_palue(ax,line_x1=-0.5,line_x2=0.5,line_y=40,
+          text_y=0.2,
+          text='$p={}$'.format(round(0.001,3)),
+          fontsize=11,fontcolor='#000000',
+          horizontalalignment='center',)
+

ovrawm/t_visualize_colorsystem.txt

@@ -0,0 +1,223 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# # Color system
+# 
+# In OmicVerse, we offer a color system based on Eastern aesthetics, featuring 384 representative colors derived from the Forbidden City. We will utilize these colors in combination for future visualizations.
+# 
+# All color come from the book: "中国传统色：故宫里的色彩美学" (ISBN: 9787521716054)
+
+# In[ ]:
+
+
+import omicverse as ov
+import scanpy as sc
+#import scvelo as scv
+ov.plot_set()
+
+
+# We utilized single-cell RNA-seq data (GEO accession: GSE95753) obtained from the dentate gyrus of the hippocampus in mouse.
+
+# In[2]:
+
+
+adata = ov.read('data/DentateGyrus/10X43_1.h5ad')
+adata
+
+
+# ## Understanding the Color System
+# 
+# In OmicVerse, we offer a color system based on Eastern aesthetics, featuring 384 representative colors derived from the Forbidden City. We will utilize these colors in combination for future visualizations.
+
+# In[3]:
+
+
+fb=ov.pl.ForbiddenCity()
+
+
+# In[7]:
+
+
+from IPython.display import HTML
+HTML(fb.visual_color(loc_range=(0,384),
+                    num_per_row=24))
+
+
+# we can get a color using `get_color`
+
+# In[14]:
+
+
+fb.get_color(name='凝夜紫')
+
+
+# ## Default Colormap
+# 
+# We have provided a range of default colors including `green`, `red`, `pink`, `purple`, `yellow`, `brown`, `blue`, and `grey`. Each of these colors comes with its own set of sub-colormaps, providing a more granular level of color differentiation.
+# 
+# Here's a breakdown of the sub-colormaps available for each default color:
+# 
+# - **Green**: 
+#   - `green1`: `Forbidden_Cmap(range(1, 19))`
+#   - `green2`: `Forbidden_Cmap(range(19, 41))`
+#   - `green3`: `Forbidden_Cmap(range(41, 62))`
+# 
+# - **Red**: 
+#   - `red1`: `Forbidden_Cmap(range(62, 77))`
+#   - `red2`: `Forbidden_Cmap(range(77, 104))`
+# 
+# - **Pink**: 
+#   - `pink1`: `Forbidden_Cmap(range(104, 134))`
+#   - `pink2`: `Forbidden_Cmap(range(134, 148))`
+# 
+# - **Purple**: 
+#   - `purple1`: `Forbidden_Cmap(range(148, 162))`
+#   - `purple2`: `Forbidden_Cmap(range(162, 176))`
+# 
+# - **Yellow**: 
+#   - `yellow1`: `Forbidden_Cmap(range(176, 196))`
+#   - `yellow2`: `Forbidden_Cmap(range(196, 207))`
+#   - `yellow3`: `Forbidden_Cmap(range(255, 276))`
+# 
+# - **Brown**: 
+#   - `brown1`: `Forbidden_Cmap(range(207, 228))`
+#   - `brown2`: `Forbidden_Cmap(range(228, 246))`
+#   - `brown3`: `Forbidden_Cmap(range(246, 255))`
+#   - `brown4`: `Forbidden_Cmap(range(276, 293))`
+# 
+# - **Blue**: 
+#   - `blue1`: `Forbidden_Cmap(range(293, 312))`
+#   - `blue2`: `Forbidden_Cmap(range(312, 321))`
+#   - `blue3`: `Forbidden_Cmap(range(321, 333))`
+#   - `blue4`: `Forbidden_Cmap(range(333, 339))`
+# 
+# - **Grey**: 
+#   - `grey1`: `Forbidden_Cmap(range(339, 356))`
+#   - `grey2`: `Forbidden_Cmap(range(356, 385))`
+# 
+# Each main color can be represented as a combination of its sub-colormaps:
+# 
+# - `green = green1 + green2 + green3`
+# - `red = red1 + red2`
+# - `pink = pink1 + pink2`
+# - `purple = purple1 + purple2`
+# - `yellow = yellow1 + yellow2 + yellow3`
+# - `brown = brown1 + brown2 + brown3 + brown4`
+# - `blue = blue1 + blue2 + blue3 + blue4`
+# - `grey = grey1 + grey2`
+# 
+# These colormaps can be utilized in various applications where color differentiation is necessary, providing flexibility in visual representation.
+# 
+# `palette` is the argument we need to revise
+
+# In[13]:
+
+
+import matplotlib.pyplot as plt
+fig, axes = plt.subplots(1,3,figsize=(9,3)) 
+ov.pl.embedding(adata,
+                   basis='X_umap',
+                    frameon='small',
+                   color=["clusters"],
+                   palette=fb.red[:],
+                   ncols=3,
+                show=False,
+                legend_loc=None,
+                    ax=axes[0])
+
+ov.pl.embedding(adata,
+                   basis='X_umap',
+                    frameon='small',
+                   color=["clusters"],
+                   palette=fb.pink1[:],
+                   ncols=3,show=False,
+                legend_loc=None,
+                    ax=axes[1])
+
+ov.pl.embedding(adata,
+                   basis='X_umap',
+                    frameon='small',
+                   color=["clusters"],
+                   palette=fb.red1[:4]+fb.blue1,
+                   ncols=3,show=False,
+                    ax=axes[2])
+
+
+
+
+# In[31]:
+
+
+color_dict={'Astrocytes': '#e40414',
+ 'Cajal Retzius': '#ec5414',
+ 'Cck-Tox': '#ec4c2c',
+ 'Endothelial': '#d42c24',
+ 'GABA': '#2c5ca4',
+ 'Granule immature': '#acd4ec',
+ 'Granule mature': '#a4bcdc',
+ 'Microglia': '#8caccc',
+ 'Mossy': '#8cacdc',
+ 'Neuroblast': '#6c9cc4',
+ 'OL': '#6c94cc',
+ 'OPC': '#5c74bc',
+ 'Radial Glia-like': '#4c94c4',
+ 'nIPC': '#3474ac'}
+
+ov.pl.embedding(adata,
+                   basis='X_umap',
+                    frameon='small',
+                   color=["clusters"],
+                   palette=color_dict,
+                   ncols=3,show=False,
+                    )
+
+
+
+# ## Segmented Colormap
+# 
+# When we need to create a continuous color gradient, we will use another function: `get_cmap_seg`, and we can combine the colors we need for visualization.
+
+# In[22]:
+
+
+colors=[
+    fb.get_color_rgb('群青'),
+    fb.get_color_rgb('半见'),
+    fb.get_color_rgb('丹罽'),
+]
+fb.get_cmap_seg(colors)
+
+
+# In[24]:
+
+
+colors=[
+    fb.get_color_rgb('群青'),
+    fb.get_color_rgb('山矾'),
+    fb.get_color_rgb('丹罽'),
+]
+fb.get_cmap_seg(colors)
+
+
+# In[25]:
+
+
+colors=[
+    fb.get_color_rgb('山矾'),
+    fb.get_color_rgb('丹罽'),
+]
+fb.get_cmap_seg(colors)
+
+
+# In[27]:
+
+
+ov.pl.embedding(adata,
+                basis='X_umap',
+                frameon='small',
+                color=["Sox7"],
+                cmap=fb.get_cmap_seg(colors),
+                ncols=3,show=False,
+                #vmin=-1,vmax=1
+                )
+

ovrawm/t_visualize_single.txt

@@ -0,0 +1,534 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# # Visualization of single cell RNA-seq
+# 
+# In this part, we will introduce the tutorial of special plot of `omicverse`.
+
+# In[1]:
+
+
+import omicverse as ov
+import scanpy as sc
+#import scvelo as scv
+ov.plot_set()
+
+
+# We utilized single-cell RNA-seq data (GEO accession: GSE95753) obtained from the dentate gyrus of the hippocampus in mouse.
+
+# In[2]:
+
+
+adata = ov.read('data/DentateGyrus/10X43_1.h5ad')
+adata
+
+
+# ## Optimizing color mapping
+# 
+# Visualizing spatially resolved biological data with appropriate color mapping can significantly facilitate the exploration of underlying patterns and heterogeneity. Spaco (spatial colorization) provides a spatially constrained approach that generates discriminate color assignments for visualizing single-cell spatial data in various scenarios.
+# 
+# Jing Z, Zhu Q, Li L, Xie Y, Wu X, Fang Q, et al. [Spaco: A comprehensive tool for coloring spatial data at single-cell resolution.](https://doi.org/10.1016/j.patter.2023.100915) Patterns. 2024;100915
+# 
+# 
+# **Function**: `ov.pl.optim_palette`: 
+# - adata: the datasets of scRNA-seq
+# - basis: he position on the plane should be set to `X_spatial` in spatial RNA-seq, `X_umap`,`X_tsne`,`X_mde` in scRNA-seq and should not be set to `X_pca`
+# - colors: Specify the colour to be optimised, which should be for one of the columns in adata.obs, noting that it should have the colour first, and that we can use ov.pl.embedding to colour the cell types. If there is no colour then colour blind optimisation colour will be used.
+# - palette: You can also re-specify the colour bar that needs to be drawn, just set `palette=['#FFFFFF','#000000']`, we have prepared `ov.pl.red_color`,`ov.pl.blue_color`,`ov.pl.green_color`,`ov.pl.orange_color`, by default.
+
+# In[ ]:
+
+
+optim_palette=ov.pl.optim_palette(adata,basis='X_umap',colors='clusters')
+
+
+# In[4]:
+
+
+import matplotlib.pyplot as plt
+fig,ax=plt.subplots(figsize = (4,4))
+ov.pl.embedding(adata,
+                basis='X_umap',
+               color='clusters',
+               frameon='small',
+               show=False,
+               palette=optim_palette,
+               ax=ax,)
+plt.title('Cell Type of DentateGyrus',fontsize=15)
+
+
+# In[5]:
+
+
+ov.pl.embedding(adata,
+                basis='X_umap',
+               color='age(days)',
+               frameon='small',
+               show=False,)
+
+
+# ## Stacked histogram of cell proportions
+# 
+# This is a graph that appears widely in various CNS-level journals, and is limited to the fact that `scanpy` does not have a proper way of plotting it, and we provide `ov.pl.cellproportion` for plotting it here.
+# 
+# **Function**: `ov.pl.cellproportion`: 
+# - adata: the datasets of scRNA-seq
+# - celltype_clusters: Specify the colour to plot, which should be for one of the columns in adata.obs, noting that it should have the colour first, and that we can use ov.pl.embedding to colour the cell types. If there is no colour then colour blind optimisation colour will be used.
+# - groupby: The group variable for the different groups of cell types we need to display, in this case we are displaying different ages, so we set it to `age(days)`
+# - groupby_li: If there are too many groups, we can also select the ones we are interested in plotting, here we use groupby_li to plot the groups
+# - figsize: If we specify axes, then this variable can be left empty
+# - legend: Whether to show a legend
+
+# In[6]:
+
+
+import matplotlib.pyplot as plt
+fig,ax=plt.subplots(figsize = (1,4))
+ov.pl.cellproportion(adata=adata,celltype_clusters='clusters',
+                    groupby='age(days)',legend=True,ax=ax)
+
+
+# In[7]:
+
+
+fig,ax=plt.subplots(figsize = (2,2))
+ov.pl.cellproportion(adata=adata,celltype_clusters='age(days)',
+                    groupby='clusters',groupby_li=['nIPC','Granule immature','Granule mature'],
+                     legend=True,ax=ax)
+
+
+# If you are interested in the changes in cell types in different groups, we recommend using a stacked area graph.
+
+# In[8]:
+
+
+fig,ax=plt.subplots(figsize = (2,2))
+ov.pl.cellstackarea(adata=adata,celltype_clusters='age(days)',
+                    groupby='clusters',groupby_li=['nIPC','Granule immature','Granule mature'],
+                     legend=True,ax=ax)
+
+
+# ## A collection of some interesting embedded plot
+# 
+# Our first presentation is an embedding map with the number and proportion of cell types. This graph visualises the low-dimensional representation of cells in addition to the number of cell proportions, etc. It should be noted that the cell proportions plotted on the left side may be distorted when there are too many cell types, and we would be grateful if anyone would be interested in fixing this bug.
+# 
+# **Function**: `ov.pl.embedding_celltype`: 
+# - adata: the datasets of scRNA-seq
+# - figsize: Note that we don't usually provide the ax parameter for combinatorial graphs, this is due to the fact that combinatorial graphs are made up of multiple axes, so the figsize parameter is more important, here we set it to `figsize=(7,4)`.
+# - basis: he position on the plane should be set to `X_spatial` in spatial RNA-seq, `X_umap`,`X_tsne`,`X_mde` in scRNA-seq and should not be set to `X_pca`
+# - celltype_key: Specify the colour to be optimised, which should be for one of the columns in adata.obs, noting that it should have the colour first, and that we can use ov.pl.embedding to colour the cell types. If there is no colour then colour blind optimisation colour will be used.
+# - title: Note that the space entered in title is used to control the position.
+# - celltype_range: Since our number of cell types is different in each data, we want to have the flexibility to control where the cell scale plot is drawn, here we set it to `(1,10)`. You can also tweak the observations yourself to find the parameter that best suits your data.
+# - embedding_range: As with the positional parameters of the cell types, they need to be adjusted several times on their own for optimal results.
+
+# In[9]:
+
+
+ov.pl.embedding_celltype(adata,figsize=(7,4),basis='X_umap',
+                            celltype_key='clusters',
+                            title='            Cell type',
+                            celltype_range=(1,10),
+                            embedding_range=(4,10),)
+
+
+# Sometimes we want to be able to circle a certain type of cell that we are interested in, and here we use convex polygons to achieve this, while the shape of the convex polygons may be optimised in future versions.
+# 
+# **Function**: `ov.pl.ConvexHull`: 
+# - adata: the datasets of scRNA-seq
+# - basis: he position on the plane should be set to `X_spatial` in spatial RNA-seq, `X_umap`,`X_tsne`,`X_mde` in scRNA-seq and should not be set to `X_pca`
+# - cluster_key: Specify the celltype to be optimised, which should be for one of the columns in adata.obs, noting that it should have the colour first, and that we can use ov.pl.embedding to colour the cell types. If there is no colour then colour blind optimisation colour will be used.
+# - hull_cluster: the target celltype to be circled.
+
+# In[10]:
+
+
+import matplotlib.pyplot as plt
+fig,ax=plt.subplots(figsize = (4,4))
+
+ov.pl.embedding(adata,
+                basis='X_umap',
+                color=['clusters'],
+                frameon='small',
+                show=False,
+                ax=ax)
+
+ov.pl.ConvexHull(adata,
+                basis='X_umap',
+                cluster_key='clusters',
+                hull_cluster='Granule mature',
+                ax=ax)
+
+
+# Besides, if you don't want to plot convexhull, you can plot the contour instead. 
+# 
+# **Function**: `ov.pl.contour`: 
+# - adata: the datasets of scRNA-seq
+# - basis: he position on the plane should be set to `X_spatial` in spatial RNA-seq, `X_umap`,`X_tsne`,`X_mde` in scRNA-seq and should not be set to `X_pca`
+# - groupby: Specify the celltype to be optimised, which should be for one of the columns in adata.obs, noting that it should have the colour first, and that we can use ov.pl.embedding to colour the cell types. If there is no colour then colour blind optimisation colour will be used.
+# - clusters: the target celltype to be circled. 
+# - colors: the color of the contour
+# - linestyles: the linestyles of the contour
+# - **kwargs: more kwargs could be found from `plt.contour`
+
+# In[11]:
+
+
+import matplotlib.pyplot as plt
+fig,ax=plt.subplots(figsize = (4,4))
+
+ov.pl.embedding(adata,
+                basis='X_umap',
+                color=['clusters'],
+                frameon='small',
+                show=False,
+                ax=ax)
+
+ov.pl.contour(ax=ax,adata=adata,groupby='clusters',clusters=['Granule immature','Granule mature'],
+       basis='X_umap',contour_threshold=0.1,colors='#000000',
+        linestyles='dashed',)
+
+
+# In scanpy's default `embedding` plotting function, when we set legend=True, legend masking may occur. To solve this problem, we introduced `ov.pl.embedding_adjust` in omicverse to automatically adjust the position of the legend.
+# 
+# **Function**: `ov.pl.embedding_adjust`: 
+# - adata: the datasets of scRNA-seq
+# - basis: he position on the plane should be set to `X_spatial` in spatial RNA-seq, `X_umap`,`X_tsne`,`X_mde` in scRNA-seq and should not be set to `X_pca`
+# - groupby: Specify the celltype to be optimised, which should be for one of the columns in adata.obs, noting that it should have the colour first, and that we can use ov.pl.embedding to colour the cell types. If there is no colour then colour blind optimisation colour will be used.
+# - exclude: We can specify which cell types are not to be plotted, in this case we set it to `OL`
+# - adjust_kwargs: We can manually specify the parameters of [adjustText](https://adjusttext.readthedocs.io/en/latest/), the specific parameters see the documentation of adjustText, it should be noted that we have to use dict to specify the parameters here.
+# - text_kwargs: We can also specify the font colour manually by specifying the [text_kwargs](https://matplotlib.org/stable/api/_as_gen/matplotlib.pyplot.text.html) parameter
+
+# In[12]:
+
+
+from matplotlib import patheffects
+import matplotlib.pyplot as plt
+fig, ax = plt.subplots(figsize=(4,4))
+
+ov.pl.embedding(adata,
+                  basis='X_umap',
+                  color=['clusters'],
+                   show=False, legend_loc=None, add_outline=False, 
+                   frameon='small',legend_fontoutline=2,ax=ax
+                 )
+
+ov.pl.embedding_adjust(
+    adata,
+    groupby='clusters',
+    exclude=("OL",),  
+    basis='X_umap',
+    ax=ax,
+    adjust_kwargs=dict(arrowprops=dict(arrowstyle='-', color='black')),
+    text_kwargs=dict(fontsize=12 ,weight='bold',
+                     path_effects=[patheffects.withStroke(linewidth=2, foreground='w')] ),
+)
+
+
+# Sometimes we are interested in the distribution density of a certain class of cell types in a categorical variable, which is cumbersome to plot in the `scanpy` implementation, so we have simplified the implementation in omicverse and ensured the same plotting.
+# 
+# **Function**: `ov.pl.embedding_density`: 
+# - adata: the datasets of scRNA-seq
+# - basis: he position on the plane should be set to `X_spatial` in spatial RNA-seq, `X_umap`,`X_tsne`,`X_mde` in scRNA-seq and should not be set to `X_pca`
+# - groupby: Specify the celltype to be optimised, which should be for one of the columns in adata.obs, noting that it should have the colour first, and that we can use ov.pl.embedding to colour the cell types. If there is no colour then colour blind optimisation colour will be used.
+# - target_clusters: We can specify which cell types are to be plotted, in this case we set it to `Granule mature`
+# - kwargs: other parameter can be found in `scanpy.pl.embedding`
+
+# In[13]:
+
+
+ov.pl.embedding_density(adata,
+                 basis='X_umap',
+                 groupby='clusters',
+                 target_clusters='Granule mature',
+                 frameon='small',
+                show=False,cmap='RdBu_r',alpha=0.8)
+
+
+# ## Bar graph with overlapping dots (Bar-dot) plot
+# 
+# In biological research, bardotplot plots are the most common class of graphs we use, but unfortunately, there is no direct implementation of plotting functions in either matplotlib, seaborn or scanpy. To compensate for this, we implement bardotplot plotting in omicverse and provide manual addition of p-values (it should be noted that manual addition refers to the manual addition of p-values for model fitting rather than making up p-values yourself).
+
+# In[14]:
+
+
+ov.single.geneset_aucell(adata,
+                            geneset_name='Sox',
+                            geneset=['Sox17', 'Sox4', 'Sox7', 'Sox18', 'Sox5'])
+
+
+# In[15]:
+
+
+ov.pl.embedding(adata,
+                basis='X_umap',
+                color=['Sox4'],
+                frameon='small',
+                show=False,)
+
+
+# In[18]:
+
+
+ov.pl.violin(adata,keys='Sox4',groupby='clusters',figsize=(6,3))
+
+
+# **Function**: `ov.pl.embedding_density`: 
+# - adata: the datasets of scRNA-seq
+# - groupby: Specify the celltype to be optimised, which should be for one of the columns in adata.obs, noting that it should have the colour first, and that we can use ov.pl.embedding to colour the cell types. If there is no colour then colour blind optimisation colour will be used.
+# - color: The size of the variable to be plotted, which can be a gene, stored in adata.var, or a cell value, stored in adata.obs.
+# - bar_kwargs: We provide the parameters of the barplot for input, see the matplotlib documentation for more [details](https://matplotlib.org/stable/api/_as_gen/matplotlib.pyplot.bar.html)
+# - scatter_kwargs: We also provide the parameters of the scatter for input, see the matplotlib documentation for more [details](https://matplotlib.org/stable/api/_as_gen/matplotlib.pyplot.scatter.html)
+# 
+# **Function**: `ov.pl.add_palue`: 
+# - ax: the axes of bardotplot
+# - line_x1: The left side of the p-value line to be plotted
+# - line_x2: The right side of the p-value line to be plotted|
+# - line_y: The height of the p-value line to be plotted
+# - text_y: How much above the p-value line is plotted text
+# - text: the text of p-value, you can set `***` to instead `p<0.001`
+# - fontsize: the fontsize of text
+# - fontcolor: the color of text
+# - horizontalalignment: the location of text
+
+# In[19]:
+
+
+fig, ax = plt.subplots(figsize=(6,2))
+ov.pl.bardotplot(adata,groupby='clusters',color='Sox_aucell',figsize=(6,2),
+           ax=ax,
+          ylabel='Expression',
+           bar_kwargs={'alpha':0.5,'linewidth':2,'width':0.6,'capsize':4},
+           scatter_kwargs={'alpha':0.8,'s':10,'marker':'o'})
+
+ov.pl.add_palue(ax,line_x1=3,line_x2=4,line_y=0.1,
+          text_y=0.02,
+          text='$p={}$'.format(round(0.001,3)),
+          fontsize=11,fontcolor='#000000',
+             horizontalalignment='center',)
+
+
+# In[20]:
+
+
+fig, ax = plt.subplots(figsize=(6,2))
+ov.pl.bardotplot(adata,groupby='clusters',color='Sox17',figsize=(6,2),
+           ax=ax,
+          ylabel='Expression',xlabel='Cell Type',
+           bar_kwargs={'alpha':0.5,'linewidth':2,'width':0.6,'capsize':4},
+           scatter_kwargs={'alpha':0.8,'s':10,'marker':'o'})
+
+ov.pl.add_palue(ax,line_x1=3,line_x2=4,line_y=2,
+          text_y=0.2,
+          text='$p={}$'.format(round(0.001,3)),
+          fontsize=11,fontcolor='#000000',
+             horizontalalignment='center',)
+
+
+# ## Boxplot with jitter points 
+# A box plot, also known as a box-and-whisker plot, is a graphical representation used to display the distribution and summary statistics of a dataset. It provides a concise and visual way to understand the central tendency, spread, and potential outliers in the data.
+
+# **Function**: `ov.pl.single_group_boxplot`: 
+#  
+# - adata (AnnData object): The data object containing the information for plotting. 
+# - groupby (str): The variable used for grouping the data
+# - color (str): The variable used for coloring the data points.
+# - type_color_dict (dict): A dictionary mapping group categories to specific colors.  
+# - scatter_kwargs (dict): Additional keyword arguments for customizing the scatter plot.  
+# - ax (matplotlib.axes.Axes): A pre-existing axes object for plotting (optional).  (optional).(optional).
+#     
+
+# In[21]:
+
+
+import pandas as pd
+import seaborn as sns
+#sns.set_style('white')
+
+ov.pl.single_group_boxplot(adata,groupby='clusters',
+             color='Sox_aucell',
+             type_color_dict=dict(zip(pd.Categorical(adata.obs['clusters']).categories, adata.uns['clusters_colors'])),
+             x_ticks_plot=True,
+             figsize=(5,2),
+             kruskal_test=True,
+             ylabel='Sox_aucell',
+             legend_plot=False,
+             bbox_to_anchor=(1,1),
+             title='Expression',
+             scatter_kwargs={'alpha':0.8,'s':10,'marker':'o'},
+             point_number=15,
+             sort=False,
+             save=False,
+             )
+plt.grid(False)
+plt.xticks(rotation=90,fontsize=12)
+
+
+# ## Complexheatmap
+# 
+# A complex heatmap, also known as a clustered heatmap, is a data visualization technique used to represent complex relationships and patterns in multivariate data. It combines several elements, including clustering, color mapping, and hierarchical organization, to provide a comprehensive view of data across multiple dimensions. 
+
+# **Function**: `ov.pl.single_group_boxplot`: 
+#  
+# - adata (AnnData): Annotated data object containing single-cell RNA-seq data.
+# - groupby (str, optional): Grouping variable for the heatmap. Default is ''.
+# - figsize (tuple, optional): Figure size. Default is (6, 10).
+# - layer (str, optional): Data layer to use. Default is None.
+# - use_raw (bool, optional): Whether to use the raw data. Default is False.
+# - var_names (list or None, optional): List of genes to include in the heatmap. Default is None.
+# - gene_symbols (None, optional): Not used in the function.
+# - standard_scale (str, optional): Method for standardizing values. Options: 'obs', 'var', None. Default is None.
+# - col_color_bars (dict, optional): Dictionary mapping columns types to colors.
+# - col_color_labels (dict, optional): Dictionary mapping column labels to colors.
+# - left_color_bars (dict, optional): Dictionary mapping left types to colors.
+# - left_color_labels (dict, optional): Dictionary mapping left labels to colors.
+# - right_color_bars (dict, optional): Dictionary mapping right types to colors.
+# - right_color_labels (dict, optional): Dictionary mapping right labels to colors.
+# - marker_genes_dict (dict, optional): Dictionary mapping cell types to marker genes.
+# - index_name (str, optional): Name for the index column in the melted DataFrame. Default is ''.
+# - value_name (str, optional): Name for the value column in the melted DataFrame. Default is ''.
+# - cmap (str, optional): Colormap for the heatmap. Default is 'parula'.
+# - xlabel (str, optional): X-axis label. Default is ''.
+# - ylabel (str, optional): Y-axis label. Default is ''.
+# - label (str, optional): Label for the plot. Default is ''.
+# - save (bool, optional): Whether to save the plot. Default is False.
+# - save_pathway (str, optional): File path for saving the plot. Default is ''.
+# - legend_gap (int, optional): Gap between legend items. Default is 7.
+# - legend_hpad (int, optional): Horizontal space between the heatmap and legend, default is 2 [mm].
+# - show (bool, optional): Whether to display the plot. Default is False.
+# 
+#     
+
+# In[22]:
+
+
+import pandas as pd
+marker_genes_dict = {
+    'Sox':['Sox4', 'Sox7', 'Sox18', 'Sox5'],
+}
+
+color_dict = {'Sox':'#EFF3D8',}
+
+gene_color_dict = {}
+gene_color_dict_black = {}
+for cell_type, genes in marker_genes_dict.items():
+    cell_type_color = color_dict.get(cell_type)
+    for gene in genes:
+        gene_color_dict[gene] = cell_type_color
+        gene_color_dict_black[gene] = '#000000'
+
+cm = ov.pl.complexheatmap(adata,
+                       groupby ='clusters',
+                       figsize =(5,2),
+                       layer = None,
+                       use_raw = False,
+                       standard_scale = 'var',
+                       col_color_bars = dict(zip(pd.Categorical(adata.obs['clusters']).categories, adata.uns['clusters_colors'])),
+                       col_color_labels = dict(zip(pd.Categorical(adata.obs['clusters']).categories, adata.uns['clusters_colors'])),
+                       left_color_bars = color_dict,
+                       left_color_labels = None,
+                       right_color_bars = color_dict,
+                       right_color_labels = gene_color_dict_black,
+                       marker_genes_dict = marker_genes_dict,
+                       cmap = 'coolwarm', #parula,jet
+                       legend_gap = 15,
+                       legend_hpad = 0,
+                       left_add_text = True,
+                       col_split_gap = 2,
+                       row_split_gap = 1,
+                       col_height = 6,
+                       left_height = 4,
+                       right_height = 6,
+                       col_split = None,
+                       row_cluster = False,
+                       col_cluster = False,
+                       value_name='Gene',
+                       xlabel = "Expression of selected genes",
+                       label = 'Gene Expression',
+                       save = True,
+                       show = False,
+                       legend = False,
+                       plot_legend = False,
+                      #save_pathway = "complexheatmap.png",
+                            )
+
+
+# ## Marker gene plot
+# 
+# In single-cell analysis, a marker gene heatmap is a powerful visualization tool that helps researchers to understand the expression patterns of specific marker genes across different cell populations. Here we provide `ov.pl.marker_heatmap` for visualizing the patterns of marker genes.
+
+# We first preprocess the data and define the dictionary of cell type and marker gene.
+# **Please ensure that each gene in the dictionary appears only once** (i.e. different cells cannot have the same marker gene, otherwise an error will be reported).
+
+# In[23]:
+
+
+adata=ov.pp.preprocess(adata,mode='shiftlog|pearson',n_HVGs=2000,)
+
+marker_genes_dict = {'Granule immature': ['Sepw1', 'Camk2b', 'Cnih2'],
+ 'Radial Glia-like': ['Dbi', 'Fabp7', 'Aldoc'],
+ 'Granule mature': ['Malat1', 'Rasl10a', 'Ppp3ca'],
+ 'Neuroblast': ['Igfbpl1', 'Tubb2b', 'Tubb5'],
+ 'Microglia': ['Lgmn', 'C1qa', 'C1qb'],
+ 'Cajal Retzius': ['Diablo', 'Ramp1', 'Stmn1'],
+ 'OPC': ['Olig1', 'C1ql1', 'Pllp'],
+ 'Cck-Tox': ['Tshz2', 'Cck', 'Nap1l5'],
+ 'GABA': ['Gad2', 'Gad1', 'Snhg11'],
+ 'Endothelial': ['Sparc', 'Myl12a', 'Itm2a'],
+ 'Astrocytes': ['Apoe',  'Atp1a2'],
+ 'OL': ['Plp1', 'Mog', 'Mag'],
+ 'Mossy': ['Arhgdig', 'Camk4'],
+ 'nIPC': ['Hmgn2', 'Ptma', 'H2afz']}
+
+
+# **Function**: `ov.pl.marker_heatmap`: 
+#  
+# - adata: AnnData object
+#         Annotated data matrix.
+# - marker_genes_dict: dict
+#         A dictionary containing the marker genes for each cell type.
+# - groupby: str
+#         The key in adata.obs that will be used for grouping the cells.
+# - color_map: str
+#         The color map to use for the value of heatmap.
+# - use_raw: bool
+#         Whether to use the raw data of AnnDta object for plotting.
+# - standard_scale: str
+#         The standard scale for the heatmap.
+# - expression_cutoff: float
+#         The cutoff value for the expression of genes.
+# - bbox_to_anchor: tuple
+#         The position of the legend bbox (x, y) in axes coordinates.
+# - figsize: tuple
+#         The size of the plot figure in inches (width, height).
+# - spines: bool
+#         Whether to show the spines of the plot.
+# - fontsize: int
+#         The font size of the text in the plot.
+# - show_rownames: bool
+#         Whether to show the row names in the heatmap.
+# - show_colnames: bool
+#         Whether to show the column names in the heatmap.
+# - save_pathway: str 
+#         The file path for saving the plot.
+# - ax: matplotlib.axes.Axes
+#         A pre-existing axes object for plotting (optional).
+
+# In[24]:
+
+
+ov.pl.marker_heatmap(
+    adata,
+    marker_genes_dict,
+    groupby='clusters',
+    color_map="RdBu_r",
+    use_raw=False,
+    standard_scale="var",
+    expression_cutoff=0.0,
+    fontsize=12,
+    bbox_to_anchor=(7, -2),
+    figsize=(8.5,4),
+    spines=False,
+    show_rownames=False,
+    show_colnames=True,
+)
+

ovrawm/t_wgcna.txt

@@ -0,0 +1,252 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+# # WGCNA (Weighted gene co-expression network analysis) analysis
+# Weighted gene co-expression network analysis (WGCNA) is a systems biology approach to characterize gene association patterns between different samples and can be used to identify highly synergistic gene sets and identify candidate biomarker genes or therapeutic targets based on the endogeneity of the gene sets and the association between the gene sets and the phenotype.
+# 
+# Paper: [WGCNA: an R package for weighted correlation network analysis](https://bmcbioinformatics.biomedcentral.com/articles/10.1186/1471-2105-9-559#Sec21)
+# 
+# Narges Rezaie, Farilie Reese, Ali Mortazavi, PyWGCNA: a Python package for weighted gene co-expression network analysis, Bioinformatics, Volume 39, Issue 7, July 2023, btad415, https://doi.org/10.1093/bioinformatics/btad415
+# 
+# Code: Reproduce by Python. Raw is http://www.genetics.ucla.edu/labs/horvath/CoexpressionNetwork/Rpackages/WGCNA
+# 
+# Colab_Reproducibility：https://colab.research.google.com/drive/1EbP-Tq1IwYO9y1_-zzw23XlPbzrxP0og?usp=sharing
+# 
+# Here, you will be briefly guided through the basics of how to use omicverse to perform wgcna anlysis. Once you are set
+
+# In[1]:
+
+
+import scanpy as sc
+import omicverse as ov
+import matplotlib.pyplot as plt
+ov.plot_set()
+
+
+# ## Load the data
+# The analysis is based on the in-built WGCNA tutorial data. All the data can be download from https://github.com/mortazavilab/PyWGCNA/tree/main/tutorials/5xFAD_paper
+
+# In[2]:
+
+
+import pandas as pd
+data=ov.utils.read('data/5xFAD_paper/expressionList.csv',
+                           index_col=0)
+data.head()
+
+
+# In[3]:
+
+
+from statsmodels import robust #import package
+gene_mad=data.apply(robust.mad) #use function to calculate MAD
+data=data.T
+data=data.loc[gene_mad.sort_values(ascending=False).index[:2000]]
+data.head()
+
+
+# In[5]:
+
+
+#import PyWGCNA
+pyWGCNA_5xFAD = ov.bulk.pyWGCNA(name='5xFAD_2k', 
+                              species='mus musculus', 
+                              geneExp=data.T, 
+                              outputPath='',
+                              save=True)
+pyWGCNA_5xFAD.geneExpr.to_df().head(5)
+
+
+# ## Pre-processing workflow
+# 
+# PyWGCNA allows you to easily preproces the data including removing genes with too many missing values or lowly-expressed genes across samples (by default we suggest to remove genes without that are expressed below 1 TPM) and removing samples with too many missing values. Keep in your mind that these options can be adjusted by changing `TPMcutoff` and `cut`
+
+# In[6]:
+
+
+pyWGCNA_5xFAD.preprocess()
+
+
+# ## Construction of the gene network and identification of modules
+# 
+# PyWGCNA compresses all the steps of network construction and module detection in one function called `findModules` which performs the following steps:
+# 1. Choosing the soft-thresholding power: analysis of network topology
+# 2. Co-expression similarity and adjacency
+# 3. Topological Overlap Matrix (TOM)
+# 4. Clustering using TOM
+# 5. Merging of modules whose expression profiles are very similar
+# 
+# In this tutorial, we will perform the analysis step by step.
+
+# In[7]:
+
+
+pyWGCNA_5xFAD.calculate_soft_threshold()
+
+
+# In[8]:
+
+
+pyWGCNA_5xFAD.calculating_adjacency_matrix()
+
+
+# In[9]:
+
+
+pyWGCNA_5xFAD.calculating_TOM_similarity_matrix()
+
+
+# ## Building a network of co-expressions
+# 
+# We use the dynamicTree to build the co-expressions module basing TOM matrix
+
+# In[10]:
+
+
+pyWGCNA_5xFAD.calculate_geneTree()
+pyWGCNA_5xFAD.calculate_dynamicMods(kwargs_function={'cutreeHybrid': {'deepSplit': 2, 'pamRespectsDendro': False}})
+pyWGCNA_5xFAD.calculate_gene_module(kwargs_function={'moduleEigengenes': {'softPower': 8}})
+
+
+# In[11]:
+
+
+pyWGCNA_5xFAD.plot_matrix(save=False)
+
+
+# ## Saving and loading your PyWGCNA
+# You can save or load your PyWGCNA object with the `saveWGCNA()` or `readWGCNA()` functions respectively.
+
+# In[12]:
+
+
+pyWGCNA_5xFAD.saveWGCNA()
+
+
+# In[2]:
+
+
+pyWGCNA_5xFAD=ov.bulk.readWGCNA('5xFAD_2k.p')
+
+
+# In[14]:
+
+
+pyWGCNA_5xFAD.mol.head()
+
+
+# In[15]:
+
+
+pyWGCNA_5xFAD.datExpr.var.head()
+
+
+# ## Sub co-expression module
+# 
+# Sometimes we are interested in a gene, or a module of a pathway, and we need to extract the sub-modules of the gene for analysis and mapping. For example, we have selected two modules, 6 and 12, as sub-modules for analysis
+
+# In[13]:
+
+
+sub_mol=pyWGCNA_5xFAD.get_sub_module(['gold','lightgreen'],
+                            mod_type='module_color')
+sub_mol.head(),sub_mol.shape
+
+
+# We found a total of 151 genes for 'gold' and 'lightgreen'. Next, we used the scale-free network constructed earlier, with the threshold set to 0.95, to construct a gene correlation network graph for modules 'gold' and 'lightgreen'
+
+# In[17]:
+
+
+G_sub=pyWGCNA_5xFAD.get_sub_network(mod_list=['lightgreen'],
+                            mod_type='module_color',correlation_threshold=0.2)
+G_sub
+
+
+# In[18]:
+
+
+len(G_sub.edges())
+
+
+# pyWGCNA provides a simple visualisation function `plot_sub_network` to visualise the gene-free network of our interest.
+
+# In[19]:
+
+
+pyWGCNA_5xFAD.plot_sub_network(['gold','lightgreen'],pos_type='kamada_kawai',pos_scale=10,pos_dim=2,
+                         figsize=(8,8),node_size=10,label_fontsize=8,correlation_threshold=0.2,
+                        label_bbox={"ec": "white", "fc": "white", "alpha": 0.6})
+
+
+# We also can merge two previous steps by calling `runWGCNA()` function.
+# 
+# ## Updating sample information and assiging color to them for dowstream analysis
+
+# In[3]:
+
+
+pyWGCNA_5xFAD.updateSampleInfo(path='data/5xFAD_paper/sampleInfo.csv', sep=',')
+
+# add color for metadata
+pyWGCNA_5xFAD.setMetadataColor('Sex', {'Female': 'green',
+                                       'Male': 'yellow'})
+pyWGCNA_5xFAD.setMetadataColor('Genotype', {'5xFADWT': 'darkviolet',
+                                            '5xFADHEMI': 'deeppink'})
+pyWGCNA_5xFAD.setMetadataColor('Age', {'4mon': 'thistle',
+                                       '8mon': 'plum',
+                                       '12mon': 'violet',
+                                       '18mon': 'purple'})
+pyWGCNA_5xFAD.setMetadataColor('Tissue', {'Hippocampus': 'red',
+                                          'Cortex': 'blue'})
+
+
+# **note**: For doing downstream analysis, we keep aside the Gray modules which is the collection of genes that could not be assigned to any other module.
+# 
+# ## Relating modules to external information and identifying important genes
+# PyWGCNA gather some important analysis after identifying modules in `analyseWGCNA()` function including:
+# 
+# 1. Quantifying module–trait relationship 
+# 2. Gene relationship to trait and modules
+# 
+# Keep in your mind before you start analysis to add any sample or gene information.
+# 
+# For showing module relationship heatmap, PyWGCNA needs user to choose and set colors from [Matplotlib colors](https://matplotlib.org/stable/gallery/color/named_colors.html) for metadata by using `setMetadataColor()` function.
+# 
+# You also can select which data trait in which order you wish to show in module eigengene heatmap
+
+# In[4]:
+
+
+pyWGCNA_5xFAD.analyseWGCNA()
+
+
+# In[5]:
+
+
+metadata = pyWGCNA_5xFAD.datExpr.obs.columns.tolist()
+
+
+# In[10]:
+
+
+pyWGCNA_5xFAD.plotModuleEigenGene('lightgreen', metadata, show=True)
+
+
+# In[11]:
+
+
+pyWGCNA_5xFAD.barplotModuleEigenGene('lightgreen', metadata, show=True)
+
+
+# ## Finding hub genes for each modules
+# 
+# you can also ask about hub genes in each modules based on their connectivity by using `top_n_hub_genes()` function.
+# 
+# It will give you dataframe sorted by connectivity with additional gene information you have in your expression data.
+
+# In[12]:
+
+
+pyWGCNA_5xFAD.top_n_hub_genes(moduleName="lightgreen", n=10)
+