harmony_all_analysis

1 Parameters

2 Functions

library(stringi)
library(reticulate)
source_from_github(repositoy = "DEG_functions",version = "0.2.24")
source_from_github(repositoy = "cNMF_functions",version = "0.3.77",script_name = "cnmf_function_Harmony.R")

no_neg <- function(x) {
  x = x + abs(min(x))
  x
}

sum_2_one <- function(x) {
  x =x/sum(x)
  x
}

# import python functions:
import types

get_norm_counts  = r.get_norm_counts
code_obj = compile(get_norm_counts, '<string>', 'exec')
get_norm_counts = types.FunctionType(code_obj.co_consts[0], globals())

get_usage_from_score  = r.get_usage_from_score
code_obj = compile(get_usage_from_score, '<string>', 'exec')
get_usage_from_score = types.FunctionType(code_obj.co_consts[0], globals())

3 Data

xeno = readRDS("./Data/10x_xeno_1000.Rds")
lung = readRDS("./Data/lung_cancercells_withTP_onlyPatients.rds")
lung_patients = lung$patient.ident %>% unique() %>% as.character()
lung_patients_filtered = lung_patients[!(lung_patients %in% c("X1055new","X1099"))] # remove patients with less than 100 malignant cells
lung = subset(x = lung,subset = patient.ident %in% lung_patients_filtered)

4 Models 2K vargenes

suffix = r.suffix
import pickle
from cnmf import cNMF
f = open('./Data/cnmf/cnmf_objects/models_2Kvargenes_cnmf_obj.pckl', 'rb')
cnmf_obj = pickle.load(f)
f.close()

gep_scores = readRDS("/sci/labs/yotamd/lab_share/avishai.wizel/R_projects/EGFR/Data/cnmf/harmony_models_gep_scores.rds")

selected_k = 3
density_threshold = 0.1
# cnmf_obj.consensus(k=selected_k, density_threshold=density_threshold)
usage_norm, gep_scores, gep_tpm, topgenes = cnmf_obj.load_results(K=selected_k, density_threshold=density_threshold)

4.1 programs enrichment

gep_scores = py$gep_scores
gep_tpm = py$gep_tpm
all_metagenes= py$usage_norm

for (program  in names (gep_scores)) {
  print(
    ggplot(gep_scores, aes(gep_scores[,program])) + stat_ecdf(geom = "step")

  )
}

ntop = 200
plt_list = list()
for (i in 1:ncol(gep_scores)) {
  top_genes = gep_scores  %>%  arrange(desc(gep_scores[i])) #sort by score a
  top = head(rownames(top_genes),ntop) #take top top_genes_num
  res = genes_vec_enrichment(genes = top,background = rownames(gep_scores),homer = T,title = 
                    i,silent = T,return_all = T,custom_pathways  = NULL)
   
  plt_list[[i]] = res$plt
}
gridExtra::grid.arrange(grobs = plt_list)

xeno = FindVariableFeatures(object = xeno,nfeatures = 2000)

Calculating gene variances
0%   10   20   30   40   50   60   70   80   90   100%
[----|----|----|----|----|----|----|----|----|----|
**************************************************|
Calculating feature variances of standardized and clipped values
0%   10   20   30   40   50   60   70   80   90   100%
[----|----|----|----|----|----|----|----|----|----|
**************************************************|

vargenes = VariableFeatures(object = xeno)
xeno = DietSeurat(xeno)
gc()

            used   (Mb) gc trigger    (Mb)   max used    (Mb)
Ncells  12013833  641.7   20862907  1114.2   20862907  1114.2
Vcells 513038452 3914.2 1751618077 13363.8 1524897220 11634.1

xeno_expression = t(as.matrix(GetAssayData(xeno,slot='counts'))) 
xeno_expression = xeno_expression[,vargenes] %>% as.data.frame()

all_0_genes = colnames(xeno_expression)[colSums(xeno_expression==0, na.rm=TRUE)==nrow(xeno_expression)] #delete rows that have all 0
vargenes = vargenes[!vargenes %in% all_0_genes]
gc()

            used   (Mb) gc trigger    (Mb)   max used    (Mb)
Ncells  12013886  641.7   20862907  1114.2   20862907  1114.2
Vcells 513040458 3914.2 2309359557 17619.1 2798942453 21354.3

def get_usage_from_score(counts,tpm, genes,cnmf_obj,k):
      import anndata as ad
      import scanpy as sc
      import numpy as np
      from sklearn.decomposition import non_negative_factorization
      import pandas as pd
      counts_adata = ad.AnnData(counts)
      tpm_adata = ad.AnnData(tpm)
      norm_counts = get_norm_counts(counts=counts_adata,tpm=tpm_adata,high_variance_genes_filter=np.array(genes)) #norm counts as cnmf
      spectra = cnmf_obj.get_median_spectra(k=k) #get score 
      spectra = spectra[spectra.columns.intersection(genes)] #remove genes not in @genes
      spectra = spectra.T.reindex(norm_counts.to_df().columns).T #reorder spectra genes like norm_counts
      
      usage_by_calc,_,_ = non_negative_factorization(X=norm_counts.X, H = spectra.values, update_H=False,n_components = k,max_iter=1000,init ='random')
      usage_by_calc = pd.DataFrame(usage_by_calc, index=counts.index, columns=spectra.index) #insert to df+add names
      usage_by_calc = usage_by_calc.div(usage_by_calc.sum(axis=1), axis=0) # sum rows to 1 
      return(usage_by_calc)

5 Calculate usage

import numpy as np
import scanpy as sc
xeno_expression = r.xeno_expression
vargenes = r.vargenes

ad = sc.read_h5ad('./Data/cnmf/xeno_Harmony_NoNeg_2Kvargenes.h5ad')
ad = ad.to_df()
def compute_tpm(input_counts):
    """
    Default TPM normalization
    """
    tpm = input_counts.copy()
    tpm = sc.pp.normalize_per_cell(tpm, counts_per_cell_after=1e6,copy = True)
    return(tpm)

tpm =  compute_tpm(xeno_expression)
cnmf_genes = ad.keys().to_list()
usage_by_calc = get_usage_from_score(counts=xeno_expression,tpm=tpm,genes=vargenes,cnmf_obj=cnmf_obj,k=3)

/sci/labs/yotamd/lab_share/avishai.wizel/python_envs/miniconda/envs/cnmf_env_6/bin/python3.7:7: FutureWarning: X.dtype being converted to np.float32 from float64. In the next version of anndata (0.9) conversion will not be automatic. Pass dtype explicitly to avoid this warning. Pass `AnnData(X, dtype=X.dtype, ...)` to get the future behavour.
/sci/labs/yotamd/lab_share/avishai.wizel/python_envs/miniconda/envs/cnmf_env_6/bin/python3.7:8: FutureWarning: X.dtype being converted to np.float32 from float64. In the next version of anndata (0.9) conversion will not be automatic. Pass dtype explicitly to avoid this warning. Pass `AnnData(X, dtype=X.dtype, ...)` to get the future behavour.

# cnmf_genes = py$cnmf_genes %>% as.data.frame()
# a = vargenes %>% as.data.frame()
usage_by_calc = py$usage_by_calc
usage_by_calc = usage_by_calc[,c(3,2,1)]
usage_norm = py$usage_norm
cor(usage_by_calc,usage_norm)

           1          2          3
3  0.7706987 -0.3294875 -0.4757492
2 -0.4583164  0.8211901 -0.4757045
1 -0.3128294 -0.4755204  0.9323277

all_metagenes = usage_by_calc

6 programs expression

names (all_metagenes) = c("Hypoxia","TNFa","Cell_cycle")
#add each metagene to metadata
for (i  in 1:ncol(all_metagenes)) {
  metage_metadata = all_metagenes %>% dplyr::select(i)
  xeno = AddMetaData(object = xeno,metadata = metage_metadata)
}

print_tab(plt = FeaturePlot(object = xeno,features = colnames(all_metagenes)),title = "umap expression")

umap expression

metagenes_mean_compare(dataset = xeno,time.point_var = "treatment",prefix = "model",patient.ident_var = "orig.ident",pre_on = c("NT","OSI"))

Hypoxia per patient

Hypoxia

TNFa per patient

TNFa

NA

6.1 program assignment

larger_by = 1.25
xeno = program_assignment(dataset = xeno,larger_by = larger_by,program_names = colnames(all_metagenes))

print_tab(plt = 
            DimPlot(xeno,group.by = "program.assignment",cols = c(Hypoxia = "red",TNFa = "green",Cell_cycle = "blue","NA" = "grey"))
          ,title = "program.assignment",subtitle_num = 3)

program.assignment

print_tab(plt = 
              DimPlot(xeno,group.by = "orig.ident")
          ,title = "orig.ident",subtitle_num = 3)

orig.ident

print_tab(plt = 
            DimPlot(xeno,group.by = "treatment")
          ,title = "treatment",subtitle_num = 3)

treatment

p = cell_percentage(dataset = xeno,time.point_var = "treatment",by_program = T,x_order = c("NT","OSI","res"))
print_tab(plt = p,title = "by program",subtitle_num = 3)

by program

p = cell_percentage(dataset = xeno,time.point_var = "treatment",by_tp  = T,x_order =c("Hypoxia","TNFa","Cell_cycle","NA"))
print_tab(plt = p,title = "by time point",subtitle_num = 3)

by time point

NA

7 Patients programs expression

lung = FindVariableFeatures(object = lung,nfeatures = 2000)
genes = rownames(lung)[rownames(lung) %in% VariableFeatures(object = xeno)[1:2000]]
lung_expression = t(as.matrix(GetAssayData(lung,slot='data'))) 
lung_expression = 2**lung_expression #convert from log2(tpm+1) to tpm
lung_expression = lung_expression-1
lung_expression = lung_expression[,genes] %>% as.data.frame()

all_0_genes = colnames(lung_expression)[colSums(lung_expression==0, na.rm=TRUE)==nrow(lung_expression)] #delete rows that have all 0
genes = genes[!genes %in% all_0_genes]
lung_expression = lung_expression[,!colnames(lung_expression) %in% all_0_genes]
gc()

usage_by_calc = get_usage_from_score(counts=lung_expression,tpm=lung_expression,genes=genes,cnmf_obj=cnmf_obj,k=3)

/sci/labs/yotamd/lab_share/avishai.wizel/python_envs/miniconda/envs/cnmf_env_6/bin/python3.7:7: FutureWarning: X.dtype being converted to np.float32 from float64. In the next version of anndata (0.9) conversion will not be automatic. Pass dtype explicitly to avoid this warning. Pass `AnnData(X, dtype=X.dtype, ...)` to get the future behavour.
/sci/labs/yotamd/lab_share/avishai.wizel/python_envs/miniconda/envs/cnmf_env_6/bin/python3.7:8: FutureWarning: X.dtype being converted to np.float32 from float64. In the next version of anndata (0.9) conversion will not be automatic. Pass dtype explicitly to avoid this warning. Pass `AnnData(X, dtype=X.dtype, ...)` to get the future behavour.

all_metagenes = py$usage_by_calc
all_metagenes = all_metagenes[,c(3,2,1)]

names (all_metagenes) = c("Hypoxia","TNFa","Cell_cycle")
#add each metagene to metadata
for (i  in 1:ncol(all_metagenes)) {
  metage_metadata = all_metagenes %>% dplyr::select(i)
  lung = AddMetaData(object = lung,metadata = metage_metadata)
}

FeaturePlot(object = lung,features = colnames(all_metagenes))

metagenes_mean_compare(dataset = lung,time.point_var = "time.point",prefix = "patient",patient.ident_var = "patient.ident",pre_on = c("pre-treatment","on-treatment"))

Hypoxia per patient

Hypoxia

TNFa per patient

TNFa

NA

7.1 lung program assignment

larger_by = 1.25
lung = program_assignment(dataset = lung,larger_by = larger_by,program_names = colnames(all_metagenes))

print_tab(plt = 
            DimPlot(lung,group.by = "program.assignment",cols = c(Hypoxia = "red",TNFa = "green",Cell_cycle = "blue","NA" = "grey"))
          ,title = "program.assignment",subtitle_num = 3)

program.assignment

print_tab(plt = 
              DimPlot(lung,group.by = "patient.ident")
          ,title = "patient.ident",subtitle_num = 3)

patient.ident

print_tab(plt = 
            DimPlot(lung,group.by = "time.point")
          ,title = "time.point",subtitle_num = 3)

time.point

p = cell_percentage(dataset = lung,time.point_var = "time.point",by_program = T,x_order = c("pre-treatment","on-treatment","resistant"))
print_tab(plt = p,title = "by program",subtitle_num = 3)

by program

p = cell_percentage(dataset = lung,time.point_var = "time.point",by_tp  = T,x_order =c("Hypoxia","TNFa","Cell_cycle","NA"))
print_tab(plt = p,title = "by time point",subtitle_num = 3)

by time point

NA

  top_genes = gep_scores  %>%  arrange(desc(gep_scores["Hypoxia"])) #sort by score a
  top = head(rownames(top_genes),200) #take top top_genes_num
  expr = xeno_expression[,colnames(xeno_expression) %in% top]
  expr_cor = cor(expr)

  pht1 = pheatmap(expr_cor,show_colnames = F,show_rownames = F, silent = T)
      
  
  num_of_clusters = 4
clustering_distance = "euclidean"
myannotation = as.data.frame(cutree(pht1[["tree_row"]], k = num_of_clusters)) #split into k clusters
 
names(myannotation)[1] = "cluster"
  myannotation$cluster = as.factor(myannotation$cluster)
  
  palette1 <-brewer.pal(num_of_clusters, "Paired")

  names(palette1) = unique(myannotation$cluster)
  ann_colors = list (cluster = palette1)
  annotation = list(ann_colors = ann_colors, myannotation = myannotation)
  
  colors <- c(seq(-1,1,by=0.01))
  my_palette <- c("blue",colorRampPalette(colors = c("blue", "white", "red"))
                                                   (n = length(colors)-3), "red")


  print_tab(plt = 
                pheatmap(mat = expr_cor,annotation_col =  annotation[["myannotation"]], annotation_colors = annotation[["ann_colors"]], clustering_distance_rows = clustering_distance,clustering_distance_cols = clustering_distance,color = my_palette,breaks = colors,show_rownames = F,show_colnames = F)
            ,title = "genes expression heatmap")

##   genes expression heatmap {.unnumbered }  

NA

NA

  top_genes = gep_scores  %>%  arrange(desc(gep_scores["TNFa"])) #sort by score a
  top = head(rownames(top_genes),200) #take top top_genes_num
  expr = xeno_expression[,colnames(xeno_expression) %in% top]
  expr_cor = cor(expr)

  pht1 = pheatmap(expr_cor,show_colnames = F,show_rownames = F, silent = T)
      
  
  num_of_clusters = 4
clustering_distance = "euclidean"
myannotation = as.data.frame(cutree(pht1[["tree_row"]], k = num_of_clusters)) #split into k clusters
 
names(myannotation)[1] = "cluster"
  myannotation$cluster = as.factor(myannotation$cluster)
  
  palette1 <-brewer.pal(num_of_clusters, "Paired")

  names(palette1) = unique(myannotation$cluster)
  ann_colors = list (cluster = palette1)
  annotation = list(ann_colors = ann_colors, myannotation = myannotation)
  
  colors <- c(seq(-1,1,by=0.01))
  my_palette <- c("blue",colorRampPalette(colors = c("blue", "white", "red"))
                                                   (n = length(colors)-3), "red")


  print_tab(plt = 
                pheatmap(mat = expr_cor,annotation_col =  annotation[["myannotation"]], annotation_colors = annotation[["ann_colors"]], clustering_distance_rows = clustering_distance,clustering_distance_cols = clustering_distance,color = my_palette,breaks = colors,show_rownames = F,show_colnames = F)
            ,title = "genes expression heatmap")

##   genes expression heatmap {.unnumbered }  

NA

NA

  top_genes = gep_scores  %>%  arrange(desc(gep_scores["Cell_cycle"])) #sort by score a
  top = head(rownames(top_genes),200) #take top top_genes_num
  expr = xeno_expression[,colnames(xeno_expression) %in% top]
  expr_cor = cor(expr)

  pht1 = pheatmap(expr_cor,show_colnames = F,show_rownames = F, silent = T)
      
  
  num_of_clusters = 4
clustering_distance = "euclidean"
myannotation = as.data.frame(cutree(pht1[["tree_row"]], k = num_of_clusters)) #split into k clusters
 
names(myannotation)[1] = "cluster"
  myannotation$cluster = as.factor(myannotation$cluster)
  
  palette1 <-brewer.pal(num_of_clusters, "Paired")

  names(palette1) = unique(myannotation$cluster)
  ann_colors = list (cluster = palette1)
  annotation = list(ann_colors = ann_colors, myannotation = myannotation)
  
  colors <- c(seq(-1,1,by=0.01))
  my_palette <- c("blue",colorRampPalette(colors = c("blue", "white", "red"))
                                                   (n = length(colors)-3), "red")


  print_tab(plt = 
                pheatmap(mat = expr_cor,annotation_col =  annotation[["myannotation"]], annotation_colors = annotation[["ann_colors"]], clustering_distance_rows = clustering_distance,clustering_distance_cols = clustering_distance,color = my_palette,breaks = colors,show_rownames = F,show_colnames = F)
            ,title = "Cell_cycle")

##   Cell_cycle {.unnumbered }  

NA

NA

intersect(hif_targets,hypoxia_genes)

 [1] "ERO1A"   "ENO1"    "HILPDA"  "PGK1"    "ENO2"    "EGLN3"   "NDRG1"   "STC2"    "P4HA1"   "OSMR"    "SLC16A3"