Most efficient list to data.frame method?
Since a data.frame
is already a list and you know that each list element is the same length (X), the fastest thing would probably be to just update the class
and row.names
attributes:
set.seed(21)
n <- 1e6
x <- list(x=rnorm(n), y=rnorm(n), z=rnorm(n))
x <- c(x,x,x,x,x,x)
system.time(a <- as.data.frame(x))
system.time(b <- do.call(data.frame,x))
system.time({
d <- x # Skip 'c' so Joris doesn't down-vote me! ;-)
class(d) <- "data.frame"
rownames(d) <- 1:n
names(d) <- make.unique(names(d))
})
identical(a, b) # TRUE
identical(b, d) # TRUE
Update - this is ~2x faster than creating d
:
system.time({
e <- x
attr(e, "row.names") <- c(NA_integer_,n)
attr(e, "class") <- "data.frame"
attr(e, "names") <- make.names(names(e), unique=TRUE)
})
identical(d, e) # TRUE
Update 2 - I forgot about memory consumption. The last update makes two copies of e
. Using the attributes
function reduces that to only one copy.
set.seed(21)
f <- list(x=rnorm(n), y=rnorm(n), z=rnorm(n))
f <- c(f,f,f,f,f,f)
tracemem(f)
system.time({ # makes 2 copies
attr(f, "row.names") <- c(NA_integer_,n)
attr(f, "class") <- "data.frame"
attr(f, "names") <- make.names(names(f), unique=TRUE)
})
set.seed(21)
g <- list(x=rnorm(n), y=rnorm(n), z=rnorm(n))
g <- c(g,g,g,g,g,g)
tracemem(g)
system.time({ # only makes 1 copy
attributes(g) <- list(row.names=c(NA_integer_,n),
class="data.frame", names=make.names(names(g), unique=TRUE))
})
identical(f,g) # TRUE
This appears to need a data.table
suggestion given that efficiency for large datasets is required. Notably setattr
sets by reference and does not copy
library(data.table)
set.seed(21)
n <- 1e6
h <- list(x=rnorm(n), y=rnorm(n), z=rnorm(n))
h <- c(h,h,h,h,h,h)
tracemem(h)
system.time({h <- as.data.table(h)
setattr(h, 'names', make.names(names(h), unique=T))})
as.data.table
, however does make a copy.
Edit - no copying version
Using @MatthewDowle's suggestion setattr(h,'class','data.frame')
which will convert to data.frame by reference (no copies)
set.seed(21)
n <- 1e6
i <- list(x=rnorm(n), y=rnorm(n), z=rnorm(n))
i <- c(i,i,i,i,i,i)
tracemem(i)
system.time({
setattr(i, 'class', 'data.frame')
setattr(i, "row.names", c(NA_integer_,n))
setattr(i, "names", make.names(names(i), unique=TRUE))
})