mirror of
https://github.com/haniffalab/scRNA-seq_analysis.git
synced 2024-10-23 08:29:24 -07:00
scRNA-seq_analysis
This commit is contained in:
veghp
commit
82cc2d191e
188 changed files with 146184 additions and 0 deletions
130
pipelines/06_add_dr/add_dr.R
Executable file
130
pipelines/06_add_dr/add_dr.R
Executable file
|
|
@ -0,0 +1,130 @@
|
|||
args = commandArgs(trailingOnly=T)
|
||||
args = paste(args, collapse = "")
|
||||
args = unlist(strsplit(args, ";"))
|
||||
|
||||
arguments.list = "
|
||||
seurat.addr.arg = args[1]
|
||||
do.normalize.arg = args[2]
|
||||
add.PCA.arg = args[3]
|
||||
add.TSNE.arg = args[4]
|
||||
add.UMAP.arg = args[5]
|
||||
add.FDG.arg = args[6]
|
||||
save.dr.arg = args[7]
|
||||
"
|
||||
|
||||
expected_arguments = unlist(strsplit(arguments.list, "\n"))
|
||||
expected_arguments = expected_arguments[!(expected_arguments == "")]
|
||||
|
||||
if(length(args) != length(expected_arguments)){
|
||||
error.msg = sprintf('This pipeline requires %s parameters', as.character(length(expected_arguments)))
|
||||
expected_arguments = paste(unlist(lapply(strsplit(expected_arguments, ".arg"), "[", 1)), collapse = "\n")
|
||||
stop(sprintf('This pipeline requires %s parameters: '))
|
||||
}
|
||||
|
||||
eval(parse(text = arguments.list))
|
||||
|
||||
for(n in 1:length(expected_arguments)){
|
||||
argument = expected_arguments[n]
|
||||
argument = gsub(pattern=" ", replacement="", x=argument)
|
||||
argument.name = unlist(strsplit(argument, "="))[1]
|
||||
variable.name = gsub(pattern=".arg", replacement="", argument.name)
|
||||
argument.content = eval(parse(text = argument.name))
|
||||
eval(parse(text = argument.content))
|
||||
if (!exists(variable.name)){
|
||||
stop(sprintf("Argument %s not passed. Stopping ... ", variable.name))
|
||||
}
|
||||
}
|
||||
|
||||
# create required folders for output and work material
|
||||
output_folder = gsub(pattern="^\\d+_", replacement="", x=basename(getwd()))
|
||||
output_folder = paste(output_folder, seurat.addr, sep = "_")
|
||||
c.time = Sys.time()
|
||||
c.time = gsub(pattern=" BST", replacement="", x=c.time)
|
||||
c.time = gsub(pattern=":", replacement="", x=c.time)
|
||||
c.time = gsub(pattern=" ", replacement="", x=c.time)
|
||||
c.time = gsub(pattern="-", replacement="", x=c.time)
|
||||
c.time = substr(x=c.time, start=3, stop=nchar(c.time))
|
||||
output_folder = paste(output_folder, c.time, sep = "_")
|
||||
output_folder = file.path("../../output", output_folder)
|
||||
dir.create(output_folder)
|
||||
|
||||
seurat.addr = file.path("../../data", seurat.addr)
|
||||
|
||||
source("../../tools/bunddle_utils.R")
|
||||
|
||||
library(Seurat)
|
||||
library("sva")
|
||||
library(plyr)
|
||||
library(dplyr)
|
||||
library(reshape)
|
||||
|
||||
#######################################################################################################
|
||||
|
||||
# load data
|
||||
print("loading data ... ")
|
||||
seurat.obj = readRDS(seurat.addr)
|
||||
print("Data loaded.")
|
||||
|
||||
if(do.normalize){
|
||||
print("Normalizing data ... ")
|
||||
seurat.obj = NormalizeData(object = seurat.obj, normalization.method = "LogNormalize", scale.factor = 10000)
|
||||
|
||||
seurat.obj = FindVariableGenes(object = seurat.obj, mean.function = ExpMean,
|
||||
dispersion.function = LogVMR, x.low.cutoff = .0125,
|
||||
x.high.cutoff = 3, y.cutoff = .625)
|
||||
|
||||
print("Scaling data ... ")
|
||||
seurat.obj = ScaleData(object=seurat.obj)
|
||||
}
|
||||
|
||||
if(add.PCA){
|
||||
print("Performing PCA")
|
||||
seurat.obj = RunPCA(object = seurat.obj, pc.genes = seurat.obj@var.genes, do.print = FALSE)
|
||||
}
|
||||
|
||||
if (add.TSNE){
|
||||
print("Performing tSNE")
|
||||
seurat.obj = RunTSNE(object=seurat.obj, dims.use=1:20, seed.use=42, do.fast=TRUE)
|
||||
}
|
||||
|
||||
if (add.UMAP){
|
||||
# run umap
|
||||
print("running UMAP")
|
||||
umap.coordinates = RunUMAP(pca.df=seurat.obj@dr$pca@cell.embeddings, tool_addr=tool_addr, python.addr=python.addr)
|
||||
rownames(umap.coordinates) = names(seurat.obj@ident)
|
||||
seurat.obj = SetDimReduction(object=seurat.obj, reduction.type="umap", slot="cell.embeddings", new.data=as.matrix(umap.coordinates))
|
||||
seurat.obj = SetDimReduction(object=seurat.obj, reduction.type="umap", slot="key", new.data="umap")
|
||||
}
|
||||
|
||||
if (add.FDG){
|
||||
# run force-directed graph
|
||||
print("Running force directed graph")
|
||||
seurat.obj = BuildSNN(object=seurat.obj, reduction.type="pca", dims.use=1:20, plot.SNN=F, force.recalc=T)
|
||||
fdg_coordinates = runFDG(pca.df=seurat.obj@dr$pca@cell.embeddings, snn=seurat.obj@snn, iterations=2000, tool_addr=tool_addr, python.addr=python.addr)
|
||||
seurat.obj = SetDimReduction(object=seurat.obj, reduction.type="fdg", slot="cell.embeddings", new.data=as.matrix(fdg_coordinates))
|
||||
seurat.obj = SetDimReduction(object=seurat.obj, reduction.type="fdg", slot = "key", new.data = "fdg")
|
||||
}
|
||||
|
||||
print("Saving Seurat object")
|
||||
saveRDS(seurat.obj, seurat.addr)
|
||||
|
||||
if(save.dr){
|
||||
CellNames = as.vector(names(seurat.obj@ident))
|
||||
tSNEdata = seurat.obj@dr$tsne@cell.embeddings
|
||||
UMAPdata = seurat.obj@dr$umap@cell.embeddings
|
||||
FDGdata = seurat.obj@dr$fdg@cell.embeddings
|
||||
PCAdata = seurat.obj@dr$pca@cell.embeddings
|
||||
colnames(tSNEdata) = c("tSNEx", "tSNEy")
|
||||
colnames(UMAPdata) = c("UMAPx", "UMAPy")
|
||||
colnames(FDGdata) = c("FDGx", "FDGy")
|
||||
dr_md_df = data.frame(CellNames = CellNames)
|
||||
dr_md_df = cbind(dr_md_df, tSNEdata, UMAPdata, FDGdata, PCAdata, seurat.obj@meta.data)
|
||||
save.to = file.path(output_folder, "dr_and_metadata.csv")
|
||||
write.csv(dr_md_df, save.to)
|
||||
}else{
|
||||
unlink(output_folder, recursive=T, force=T)
|
||||
}
|
||||
|
||||
file.remove("Rplots.pdf")
|
||||
|
||||
print("Ended beautifully ... ")
|
||||
16
pipelines/06_add_dr/add_dr.sh
Executable file
16
pipelines/06_add_dr/add_dr.sh
Executable file
|
|
@ -0,0 +1,16 @@
|
|||
#!/bin/bash
|
||||
|
||||
#$ -cwd
|
||||
#$ -N add_dr
|
||||
#$ -V
|
||||
#$ -l h_rt=47:59:59
|
||||
#$ -l h_vmem=300G
|
||||
|
||||
if [ "$#" -ne 1 ]; then
|
||||
echo "Illegal number of parameters"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
Rscript add_dr.R $1
|
||||
|
||||
echo "End on `date`"
|
||||
159
pipelines/06_add_dr/add_dr_COMBAT.R
Executable file
159
pipelines/06_add_dr/add_dr_COMBAT.R
Executable file
|
|
@ -0,0 +1,159 @@
|
|||
args = commandArgs(trailingOnly=T)
|
||||
args = paste(args, collapse = "")
|
||||
args = unlist(strsplit(args, ";"))
|
||||
|
||||
arguments.list = "
|
||||
seurat.addr.arg = args[1]
|
||||
do.normalize.arg = args[2]
|
||||
add.PCA.arg = args[3]
|
||||
add.TSNE.arg = args[4]
|
||||
add.UMAP.arg = args[5]
|
||||
add.FDG.arg = args[6]
|
||||
save.dr.arg = args[7]
|
||||
"
|
||||
|
||||
expected_arguments = unlist(strsplit(arguments.list, "\n"))
|
||||
expected_arguments = expected_arguments[!(expected_arguments == "")]
|
||||
|
||||
if(length(args) != length(expected_arguments)){
|
||||
error.msg = sprintf('This pipeline requires %s parameters', as.character(length(expected_arguments)))
|
||||
expected_arguments = paste(unlist(lapply(strsplit(expected_arguments, ".arg"), "[", 1)), collapse = "\n")
|
||||
stop(sprintf('This pipeline requires %s parameters: '))
|
||||
}
|
||||
|
||||
eval(parse(text = arguments.list))
|
||||
|
||||
for(n in 1:length(expected_arguments)){
|
||||
argument = expected_arguments[n]
|
||||
argument = gsub(pattern=" ", replacement="", x=argument)
|
||||
argument.name = unlist(strsplit(argument, "="))[1]
|
||||
variable.name = gsub(pattern=".arg", replacement="", argument.name)
|
||||
argument.content = eval(parse(text = argument.name))
|
||||
eval(parse(text = argument.content))
|
||||
if (!exists(variable.name)){
|
||||
stop(sprintf("Argument %s not passed. Stopping ... ", variable.name))
|
||||
}
|
||||
}
|
||||
|
||||
# create required folders for output and work material
|
||||
output_folder = gsub(pattern="^\\d+_", replacement="", x=basename(getwd()))
|
||||
output_folder = paste(output_folder, seurat.addr, sep = "_")
|
||||
c.time = Sys.time()
|
||||
c.time = gsub(pattern=" BST", replacement="", x=c.time)
|
||||
c.time = gsub(pattern=":", replacement="", x=c.time)
|
||||
c.time = gsub(pattern=" ", replacement="", x=c.time)
|
||||
c.time = gsub(pattern="-", replacement="", x=c.time)
|
||||
c.time = substr(x=c.time, start=3, stop=nchar(c.time))
|
||||
output_folder = paste(output_folder, c.time, sep = "_")
|
||||
output_folder = file.path("../../output", output_folder)
|
||||
dir.create(output_folder)
|
||||
|
||||
seurat.addr = file.path("../../data", seurat.addr)
|
||||
|
||||
source("../../tools/bunddle_utils.R")
|
||||
|
||||
library(Seurat)
|
||||
library("sva")
|
||||
library(plyr)
|
||||
library(dplyr)
|
||||
library(reshape)
|
||||
|
||||
#######################################################################################################
|
||||
|
||||
# load data
|
||||
print("loading data ... ")
|
||||
seurat.obj = readRDS(seurat.addr)
|
||||
print("Data loaded.")
|
||||
|
||||
if(do.normalize){
|
||||
print("Normalizing data ... ")
|
||||
seurat.obj = NormalizeData(object = seurat.obj, normalization.method = "LogNormalize", scale.factor = 10000)
|
||||
|
||||
print("Applying COMBAT ...")
|
||||
expression.data = as.matrix(seurat.obj@data[seurat.obj@var.genes, ])
|
||||
pheno.data = data.frame(sample = names(seurat.obj@ident),
|
||||
batch = seurat.obj@meta.data$fetal.ids,
|
||||
stages = seurat.obj@meta.data$stages)
|
||||
batch = as.numeric(pheno.data$batch)
|
||||
pheno.data$batch = batch
|
||||
mod = model.matrix(~as.factor(stages), data=pheno.data)
|
||||
colnames(expression.data) = gsub(pattern="CD45[+]", replacement="CD45Pos", x=colnames(expression.data))
|
||||
colnames(expression.data) = gsub(pattern="CD45[-]", replacement="CD45Neg", x=colnames(expression.data))
|
||||
|
||||
write.csv(expression.data, file.path(output_folder, "data.csv"), row.names = T)
|
||||
batch = data.frame(Batch = batch)
|
||||
rownames(batch) = colnames(expression.data)
|
||||
write.csv(batch, file.path(output_folder, "batch.csv"))
|
||||
|
||||
command = sprintf("%s combat.py %s", python.addr, output_folder)
|
||||
system(command, wait = T)
|
||||
rm(mod, expression.data)
|
||||
|
||||
combat_data = read.csv(file.path(output_folder, "combat.csv"), sep = ",", row.names = 1)
|
||||
print("COMBAT data loaded.")
|
||||
|
||||
colnames(combat_data) = gsub(pattern="CD45Pos", replacement="CD45+", x=colnames(combat_data))
|
||||
colnames(combat_data) = gsub(pattern="CD45Neg", replacement="CD45-", x = colnames(combat_data))
|
||||
|
||||
combat_data = as(as.matrix(combat_data), "dgCMatrix")
|
||||
genes.not = rownames(seurat.obj@data)[!(rownames(seurat.obj@data) %in% rownames(combat_data))]
|
||||
all.expression = seurat.obj@data[genes.not, ]
|
||||
all.expression = rbind(all.expression, combat_data)
|
||||
all.expression = all.expression[rownames(seurat.obj@data), ]
|
||||
seurat.obj@data = combat_data
|
||||
|
||||
print("Scaling data ... ")
|
||||
seurat.obj = ScaleData(object=seurat.obj)
|
||||
}
|
||||
|
||||
if(add.PCA){
|
||||
print("Performing PCA")
|
||||
seurat.obj = RunPCA(object = seurat.obj, pc.genes = seurat.obj@var.genes, do.print = FALSE)
|
||||
}
|
||||
|
||||
if (add.TSNE){
|
||||
print("Performing tSNE")
|
||||
seurat.obj = RunTSNE(object=seurat.obj, dims.use=1:20, seed.use=42, do.fast=TRUE)
|
||||
}
|
||||
|
||||
if (add.UMAP){
|
||||
# run umap
|
||||
print("running UMAP")
|
||||
umap.coordinates = RunUMAP(pca.df=seurat.obj@dr$pca@cell.embeddings, tool_addr=tool_addr, python.addr=python.addr)
|
||||
rownames(umap.coordinates) = names(seurat.obj@ident)
|
||||
seurat.obj = SetDimReduction(object=seurat.obj, reduction.type="umap", slot="cell.embeddings", new.data=as.matrix(umap.coordinates))
|
||||
seurat.obj = SetDimReduction(object=seurat.obj, reduction.type="umap", slot="key", new.data="umap")
|
||||
}
|
||||
|
||||
if (add.FDG){
|
||||
# run force-directed graph
|
||||
print("Running force directed graph")
|
||||
seurat.obj = BuildSNN(object=seurat.obj, reduction.type="pca", dims.use=1:20, plot.SNN=F, force.recalc=T)
|
||||
fdg_coordinates = runFDG(pca.df=seurat.obj@dr$pca@cell.embeddings, snn=seurat.obj@snn, iterations=2000, tool_addr=tool_addr, python.addr=python.addr)
|
||||
seurat.obj = SetDimReduction(object=seurat.obj, reduction.type="fdg", slot="cell.embeddings", new.data=as.matrix(fdg_coordinates))
|
||||
seurat.obj = SetDimReduction(object=seurat.obj, reduction.type="fdg", slot = "key", new.data = "fdg")
|
||||
}
|
||||
|
||||
print("Saving Seurat object")
|
||||
saveRDS(seurat.obj, seurat.addr)
|
||||
|
||||
if(save.dr){
|
||||
CellNames = as.vector(names(seurat.obj@ident))
|
||||
tSNEdata = seurat.obj@dr$tsne@cell.embeddings
|
||||
UMAPdata = seurat.obj@dr$umap@cell.embeddings
|
||||
FDGdata = seurat.obj@dr$fdg@cell.embeddings
|
||||
PCAdata = seurat.obj@dr$pca@cell.embeddings
|
||||
colnames(tSNEdata) = c("tSNEx", "tSNEy")
|
||||
colnames(UMAPdata) = c("UMAPx", "UMAPy")
|
||||
colnames(FDGdata) = c("FDGx", "FDGy")
|
||||
dr_md_df = data.frame(CellNames = CellNames)
|
||||
dr_md_df = cbind(dr_md_df, tSNEdata, UMAPdata, FDGdata, PCAdata, seurat.obj@meta.data)
|
||||
save.to = file.path(output_folder, "dr_and_metadata.csv")
|
||||
write.csv(dr_md_df, save.to)
|
||||
}else{
|
||||
unlink(output_folder, recursive=T, force=T)
|
||||
}
|
||||
|
||||
file.remove("Rplots.pdf")
|
||||
|
||||
print("Ended beautifully ... ")
|
||||
16
pipelines/06_add_dr/add_dr_COMBAT.sh
Executable file
16
pipelines/06_add_dr/add_dr_COMBAT.sh
Executable file
|
|
@ -0,0 +1,16 @@
|
|||
#!/bin/bash
|
||||
|
||||
#$ -cwd
|
||||
#$ -N add_dr
|
||||
#$ -V
|
||||
#$ -l h_rt=47:59:59
|
||||
#$ -l h_vmem=300G
|
||||
|
||||
if [ "$#" -ne 1 ]; then
|
||||
echo "Illegal number of parameters"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
Rscript add_dr_COMBAT.R $1
|
||||
|
||||
echo "End on `date`"
|
||||
208
pipelines/06_add_dr/combat.py
Executable file
208
pipelines/06_add_dr/combat.py
Executable file
|
|
@ -0,0 +1,208 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
Created on Fri Mar 1 18:37:16 2019
|
||||
|
||||
@author: doru
|
||||
"""
|
||||
|
||||
import pandas as pd
|
||||
import patsy
|
||||
import sys
|
||||
import numpy.linalg as la
|
||||
import numpy as np
|
||||
|
||||
|
||||
def adjust_nums(numerical_covariates, drop_idxs):
|
||||
# if we dropped some values, have to adjust those with a larger index.
|
||||
if numerical_covariates is None: return drop_idxs
|
||||
return [nc - sum(nc < di for di in drop_idxs) for nc in numerical_covariates]
|
||||
|
||||
def design_mat(mod, numerical_covariates, batch_levels):
|
||||
# require levels to make sure they are in the same order as we use in the
|
||||
# rest of the script.
|
||||
design = patsy.dmatrix("~ 0 + C(batch, levels=%s)" % str(batch_levels),
|
||||
mod, return_type="dataframe")
|
||||
|
||||
mod = mod.drop(["batch"], axis=1)
|
||||
numerical_covariates = list(numerical_covariates)
|
||||
sys.stderr.write("found %i batches\n" % design.shape[1])
|
||||
other_cols = [c for i, c in enumerate(mod.columns)
|
||||
if not i in numerical_covariates]
|
||||
factor_matrix = mod[other_cols]
|
||||
design = pd.concat((design, factor_matrix), axis=1)
|
||||
if numerical_covariates is not None:
|
||||
sys.stderr.write("found %i numerical covariates...\n"
|
||||
% len(numerical_covariates))
|
||||
for i, nC in enumerate(numerical_covariates):
|
||||
cname = mod.columns[nC]
|
||||
sys.stderr.write("\t{0}\n".format(cname))
|
||||
design[cname] = mod[mod.columns[nC]]
|
||||
sys.stderr.write("found %i categorical variables:" % len(other_cols))
|
||||
sys.stderr.write("\t" + ", ".join(other_cols) + '\n')
|
||||
return design
|
||||
|
||||
|
||||
def combat(data, batch, model=None, numerical_covariates=None):
|
||||
"""Correct for batch effects in a dataset
|
||||
Parameters
|
||||
----------
|
||||
data : pandas.DataFrame
|
||||
A (n_features, n_samples) dataframe of the expression or methylation
|
||||
data to batch correct
|
||||
batch : pandas.Series
|
||||
A column corresponding to the batches in the data, with index same as
|
||||
the columns that appear in ``data``
|
||||
model : patsy.design_info.DesignMatrix, optional
|
||||
A model matrix describing metadata on the samples which could be
|
||||
causing batch effects. If not provided, then will attempt to coarsely
|
||||
correct just from the information provided in ``batch``
|
||||
numerical_covariates : list-like
|
||||
List of covariates in the model which are numerical, rather than
|
||||
categorical
|
||||
Returns
|
||||
-------
|
||||
corrected : pandas.DataFrame
|
||||
A (n_features, n_samples) dataframe of the batch-corrected data
|
||||
"""
|
||||
if isinstance(numerical_covariates, str):
|
||||
numerical_covariates = [numerical_covariates]
|
||||
if numerical_covariates is None:
|
||||
numerical_covariates = []
|
||||
|
||||
if model is not None and isinstance(model, pd.DataFrame):
|
||||
model["batch"] = list(batch)
|
||||
else:
|
||||
model = pd.DataFrame({'batch': batch})
|
||||
|
||||
batch_items = model.groupby("batch").groups.items()
|
||||
batch_levels = [k for k, v in batch_items]
|
||||
batch_info = [v for k, v in batch_items]
|
||||
n_batch = len(batch_info)
|
||||
n_batches = np.array([len(v) for v in batch_info])
|
||||
n_array = float(sum(n_batches))
|
||||
|
||||
# drop intercept
|
||||
drop_cols = [cname for cname, inter in ((model == 1).all()).iteritems() if inter == True]
|
||||
drop_idxs = [list(model.columns).index(cdrop) for cdrop in drop_cols]
|
||||
model = model[[c for c in model.columns if not c in drop_cols]]
|
||||
numerical_covariates = [list(model.columns).index(c) if isinstance(c, str) else c
|
||||
for c in numerical_covariates if not c in drop_cols]
|
||||
|
||||
design = design_mat(model, numerical_covariates, batch_levels)
|
||||
|
||||
sys.stderr.write("Standardizing Data across genes.\n")
|
||||
B_hat = np.dot(np.dot(la.inv(np.dot(design.T, design)), design.T), data.T)
|
||||
grand_mean = np.dot((n_batches / n_array).T, B_hat[:n_batch,:])
|
||||
var_pooled = np.dot(((data - np.dot(design, B_hat).T)**2), np.ones((int(n_array), 1)) / int(n_array))
|
||||
|
||||
stand_mean = np.dot(grand_mean.T.reshape((len(grand_mean), 1)), np.ones((1, int(n_array))))
|
||||
tmp = np.array(design.copy())
|
||||
tmp[:,:n_batch] = 0
|
||||
stand_mean += np.dot(tmp, B_hat).T
|
||||
|
||||
s_data = ((data - stand_mean) / np.dot(np.sqrt(var_pooled), np.ones((1, int(n_array)))))
|
||||
|
||||
sys.stderr.write("Fitting L/S model and finding priors\n")
|
||||
batch_design = design[design.columns[:n_batch]]
|
||||
gamma_hat = np.dot(np.dot(la.inv(np.dot(batch_design.T, batch_design)), batch_design.T), s_data.T)
|
||||
|
||||
delta_hat = []
|
||||
|
||||
for i, batch_idxs in enumerate(batch_info):
|
||||
#batches = [list(model.columns).index(b) for b in batches]
|
||||
delta_hat.append(s_data[batch_idxs].var(axis=1))
|
||||
|
||||
gamma_bar = gamma_hat.mean(axis=1)
|
||||
t2 = gamma_hat.var(axis=1)
|
||||
|
||||
|
||||
a_prior = list(map(aprior, delta_hat))
|
||||
b_prior = list(map(bprior, delta_hat))
|
||||
|
||||
sys.stderr.write("Finding parametric adjustments\n")
|
||||
gamma_star, delta_star = [], []
|
||||
for i, batch_idxs in enumerate(batch_info):
|
||||
#print '18 20 22 28 29 31 32 33 35 40 46'
|
||||
#print batch_info[batch_id]
|
||||
|
||||
temp = it_sol(s_data[batch_idxs], gamma_hat[i],
|
||||
delta_hat[i], gamma_bar[i], t2[i], a_prior[i], b_prior[i])
|
||||
|
||||
gamma_star.append(temp[0])
|
||||
delta_star.append(temp[1])
|
||||
|
||||
sys.stdout.write("Adjusting data\n")
|
||||
bayesdata = s_data
|
||||
gamma_star = np.array(gamma_star)
|
||||
delta_star = np.array(delta_star)
|
||||
|
||||
|
||||
for j, batch_idxs in enumerate(batch_info):
|
||||
|
||||
dsq = np.sqrt(delta_star[j,:])
|
||||
dsq = dsq.reshape((len(dsq), 1))
|
||||
denom = np.dot(dsq, np.ones((1, n_batches[j])))
|
||||
numer = np.array(bayesdata[batch_idxs] - np.dot(batch_design.loc[batch_idxs], gamma_star).T)
|
||||
|
||||
bayesdata[batch_idxs] = numer / denom
|
||||
|
||||
vpsq = np.sqrt(var_pooled).reshape((len(var_pooled), 1))
|
||||
bayesdata = bayesdata * np.dot(vpsq, np.ones((1, int(n_array)))) + stand_mean
|
||||
|
||||
return bayesdata
|
||||
|
||||
def it_sol(sdat, g_hat, d_hat, g_bar, t2, a, b, conv=0.0001):
|
||||
n = (1 - np.isnan(sdat)).sum(axis=1)
|
||||
g_old = g_hat.copy()
|
||||
d_old = d_hat.copy()
|
||||
|
||||
change = 1
|
||||
count = 0
|
||||
while change > conv:
|
||||
#print g_hat.shape, g_bar.shape, t2.shape
|
||||
g_new = postmean(g_hat, g_bar, n, d_old, t2)
|
||||
sum2 = ((sdat - np.dot(g_new.values.reshape((g_new.shape[0], 1)), np.ones((1, sdat.shape[1])))) ** 2).sum(axis=1)
|
||||
d_new = postvar(sum2, n, a, b)
|
||||
|
||||
change = max((abs(g_new - g_old) / g_old).max(), (abs(d_new - d_old) / d_old).max())
|
||||
g_old = g_new #.copy()
|
||||
d_old = d_new #.copy()
|
||||
count = count + 1
|
||||
adjust = (g_new, d_new)
|
||||
return adjust
|
||||
|
||||
|
||||
|
||||
def aprior(gamma_hat):
|
||||
m = gamma_hat.mean()
|
||||
s2 = gamma_hat.var()
|
||||
return (2 * s2 +m**2) / s2
|
||||
|
||||
def bprior(gamma_hat):
|
||||
m = gamma_hat.mean()
|
||||
s2 = gamma_hat.var()
|
||||
return (m*s2+m**3)/s2
|
||||
|
||||
def postmean(g_hat, g_bar, n, d_star, t2):
|
||||
return (t2*n*g_hat+d_star * g_bar) / (t2*n+d_star)
|
||||
|
||||
def postvar(sum2, n, a, b):
|
||||
return (0.5 * sum2 + b) / (n / 2.0 + a - 1.0)
|
||||
|
||||
# get arguments
|
||||
|
||||
output_folder = "COMBAT"
|
||||
output_folder = sys.argv[1]
|
||||
|
||||
from os.path import join
|
||||
|
||||
data_file = join(output_folder, "data.csv")
|
||||
batch_file = join(output_folder, "batch.csv")
|
||||
|
||||
data = pd.read_csv(data_file, index_col=0)
|
||||
batch = pd.read_csv(batch_file, index_col = 0)
|
||||
|
||||
combat_data = combat(data, batch['Batch'])
|
||||
combat_data.to_csv(join(output_folder, "combat.csv"), sep=",")
|
||||
|
||||
Loading…
Add table
Add a link
Reference in a new issue