有人要求我将此功能添加到stackoverflow R包中。鉴于它是一个tinyverse包(不能依赖于第三方包)，以下是我想到的:

#' Bulk import data files 
#' 
#' Read in each file at a path and then unnest them. Defaults to csv format.
#' 
#' @param path        a character vector of full path names
#' @param pattern     an optional \link[=regex]{regular expression}. Only file names which match the regular expression will be returned.
#' @param reader      a function that can read data from a file name.
#' @param ...         optional arguments to pass to the reader function (eg \code{stringsAsFactors}).
#' @param reducer     a function to unnest the individual data files. Use I to retain the nested structure. 
#' @param recursive     logical. Should the listing recurse into directories?
#'  
#' @author Neal Fultz
#' @references \url{https://stackoverflow.com/questions/11433432/how-to-import-multiple-csv-files-at-once}
#' 
#' @importFrom utils read.csv
#' @export
read.directory <- function(path='.', pattern=NULL, reader=read.csv, ..., 
                           reducer=function(dfs) do.call(rbind.data.frame, dfs), recursive=FALSE) {
  files <- list.files(path, pattern, full.names = TRUE, recursive = recursive)

  reducer(lapply(files, reader, ...))
}

通过参数化读卡器和减速器功能，人们可以使用数据。table或dplyr(如果他们选择的话)，或者只使用适用于较小数据集的基本R函数。

有人要求我将此功能添加到stackoverflow R包中。鉴于它是一个tinyverse包(不能依赖于第三方包)，以下是我想到的:

#' Bulk import data files 
#' 
#' Read in each file at a path and then unnest them. Defaults to csv format.
#' 
#' @param path        a character vector of full path names
#' @param pattern     an optional \link[=regex]{regular expression}. Only file names which match the regular expression will be returned.
#' @param reader      a function that can read data from a file name.
#' @param ...         optional arguments to pass to the reader function (eg \code{stringsAsFactors}).
#' @param reducer     a function to unnest the individual data files. Use I to retain the nested structure. 
#' @param recursive     logical. Should the listing recurse into directories?
#'  
#' @author Neal Fultz
#' @references \url{https://stackoverflow.com/questions/11433432/how-to-import-multiple-csv-files-at-once}
#' 
#' @importFrom utils read.csv
#' @export
read.directory <- function(path='.', pattern=NULL, reader=read.csv, ..., 
                           reducer=function(dfs) do.call(rbind.data.frame, dfs), recursive=FALSE) {
  files <- list.files(path, pattern, full.names = TRUE, recursive = recursive)

  reducer(lapply(files, reader, ...))
}

通过参数化读卡器和减速器功能，人们可以使用数据。table或dplyr(如果他们选择的话)，或者只使用适用于较小数据集的基本R函数。

一个快速而简洁的解决方案: (比Base R的read.csv快两倍多)

tbl <-
    list.files(pattern = "*.csv") %>% 
    map_df(~read_csv(.))

和数据。Table的fread()甚至可以再次将加载时间减少一半。(1/4底R乘以)

library(data.table)

tbl_fread <- 
    list.files(pattern = "*.csv") %>% 
    map_df(~fread(.))

stringsAsFactors = FALSE参数保持数据帧因子不受影响(正如marbel指出的那样，这是fread的默认设置)。

如果类型转换不够厚颜无耻，可以使用col_types参数强制所有列都是字符。

tbl <-
    list.files(pattern = "*.csv") %>% 
    map_df(~read_csv(., col_types = cols(.default = "c")))

如果您想要深入子目录来构造最终要绑定的文件列表，那么请确保包含路径名，并在列表中以文件的全名注册这些文件。这将允许绑定工作在当前目录之外进行。(将完整的路径名视为护照，允许跨目录“边界”返回。)

tbl <-
    list.files(path = "./subdirectory/",
               pattern = "*.csv", 
               full.names = T) %>% 
    map_df(~read_csv(., col_types = cols(.default = "c"))) 

正如Hadley在这里描述的(大约在中间):

Map_df (x, f)实际上与do相同。调用("rbind"， lapply(x, f))....

额外功能-添加文件名的记录每尼克功能请求在评论下面: *为每条记录添加原始文件名。

代码解释:创建一个函数，在初始读取表期间将文件名附加到每个记录。然后使用该函数而不是简单的read_csv()函数。

read_plus <- function(flnm) {
    read_csv(flnm) %>% 
        mutate(filename = flnm)
}

tbl_with_sources <-
    list.files(pattern = "*.csv", 
               full.names = T) %>% 
    map_df(~read_plus(.))

(类型强制转换和子目录处理方法也可以在read_plus()函数中以与上面建议的第二个和第三个变体相同的方式进行处理。)

### Benchmark Code & Results 
library(tidyverse)
library(data.table)
library(microbenchmark)

### Base R Approaches
#### Instead of a dataframe, this approach creates a list of lists
#### removed from analysis as this alone doubled analysis time reqd
# lapply_read.delim <- function(path, pattern = "*.csv") {
#     temp = list.files(path, pattern, full.names = TRUE)
#     myfiles = lapply(temp, read.delim)
# }

#### `read.csv()`
do.call_rbind_read.csv <- function(path, pattern = "*.csv") {
    files = list.files(path, pattern, full.names = TRUE)
    do.call(rbind, lapply(files, function(x) read.csv(x, stringsAsFactors = FALSE)))
}

map_df_read.csv <- function(path, pattern = "*.csv") {
    list.files(path, pattern, full.names = TRUE) %>% 
    map_df(~read.csv(., stringsAsFactors = FALSE))
}


### *dplyr()*
#### `read_csv()`
lapply_read_csv_bind_rows <- function(path, pattern = "*.csv") {
    files = list.files(path, pattern, full.names = TRUE)
    lapply(files, read_csv) %>% bind_rows()
}

map_df_read_csv <- function(path, pattern = "*.csv") {
    list.files(path, pattern, full.names = TRUE) %>% 
    map_df(~read_csv(., col_types = cols(.default = "c")))
}

### *data.table* / *purrr* hybrid
map_df_fread <- function(path, pattern = "*.csv") {
    list.files(path, pattern, full.names = TRUE) %>% 
    map_df(~fread(.))
}

### *data.table*
rbindlist_fread <- function(path, pattern = "*.csv") {
    files = list.files(path, pattern, full.names = TRUE)
    rbindlist(lapply(files, function(x) fread(x)))
}

do.call_rbind_fread <- function(path, pattern = "*.csv") {
    files = list.files(path, pattern, full.names = TRUE)
    do.call(rbind, lapply(files, function(x) fread(x, stringsAsFactors = FALSE)))
}


read_results <- function(dir_size){
    microbenchmark(
        # lapply_read.delim = lapply_read.delim(dir_size), # too slow to include in benchmarks
        do.call_rbind_read.csv = do.call_rbind_read.csv(dir_size),
        map_df_read.csv = map_df_read.csv(dir_size),
        lapply_read_csv_bind_rows = lapply_read_csv_bind_rows(dir_size),
        map_df_read_csv = map_df_read_csv(dir_size),
        rbindlist_fread = rbindlist_fread(dir_size),
        do.call_rbind_fread = do.call_rbind_fread(dir_size),
        map_df_fread = map_df_fread(dir_size),
        times = 10L) 
}

read_results_lrg_mid_mid <- read_results('./testFolder/500MB_12.5MB_40files')
print(read_results_lrg_mid_mid, digits = 3)

read_results_sml_mic_mny <- read_results('./testFolder/5MB_5KB_1000files/')
read_results_sml_tny_mod <- read_results('./testFolder/5MB_50KB_100files/')
read_results_sml_sml_few <- read_results('./testFolder/5MB_500KB_10files/')

read_results_med_sml_mny <- read_results('./testFolder/50MB_5OKB_1000files')
read_results_med_sml_mod <- read_results('./testFolder/50MB_5OOKB_100files')
read_results_med_med_few <- read_results('./testFolder/50MB_5MB_10files')

read_results_lrg_sml_mny <- read_results('./testFolder/500MB_500KB_1000files')
read_results_lrg_med_mod <- read_results('./testFolder/500MB_5MB_100files')
read_results_lrg_lrg_few <- read_results('./testFolder/500MB_50MB_10files')

read_results_xlg_lrg_mod <- read_results('./testFolder/5000MB_50MB_100files')


print(read_results_sml_mic_mny, digits = 3)
print(read_results_sml_tny_mod, digits = 3)
print(read_results_sml_sml_few, digits = 3)

print(read_results_med_sml_mny, digits = 3)
print(read_results_med_sml_mod, digits = 3)
print(read_results_med_med_few, digits = 3)

print(read_results_lrg_sml_mny, digits = 3)
print(read_results_lrg_med_mod, digits = 3)
print(read_results_lrg_lrg_few, digits = 3)

print(read_results_xlg_lrg_mod, digits = 3)

# display boxplot of my typical use case results & basic machine max load
par(oma = c(0,0,0,0)) # remove overall margins if present
par(mfcol = c(1,1)) # remove grid if present
par(mar = c(12,5,1,1) + 0.1) # to display just a single boxplot with its complete labels
boxplot(read_results_lrg_mid_mid, las = 2, xlab = "", ylab = "Duration (seconds)", main = "40 files @ 12.5MB (500MB)")
boxplot(read_results_xlg_lrg_mod, las = 2, xlab = "", ylab = "Duration (seconds)", main = "100 files @ 50MB (5GB)")

# generate 3x3 grid boxplots
par(oma = c(12,1,1,1)) # margins for the whole 3 x 3 grid plot
par(mfcol = c(3,3)) # create grid (filling down each column)
par(mar = c(1,4,2,1)) # margins for the individual plots in 3 x 3 grid
boxplot(read_results_sml_mic_mny, las = 2, xlab = "", ylab = "Duration (seconds)", main = "1000 files @ 5KB (5MB)", xaxt = 'n')
boxplot(read_results_sml_tny_mod, las = 2, xlab = "", ylab = "Duration (milliseconds)", main = "100 files @ 50KB (5MB)", xaxt = 'n')
boxplot(read_results_sml_sml_few, las = 2, xlab = "", ylab = "Duration (milliseconds)", main = "10 files @ 500KB (5MB)",)

boxplot(read_results_med_sml_mny, las = 2, xlab = "", ylab = "Duration (microseconds)        ", main = "1000 files @ 50KB (50MB)", xaxt = 'n')
boxplot(read_results_med_sml_mod, las = 2, xlab = "", ylab = "Duration (microseconds)", main = "100 files @ 500KB (50MB)", xaxt = 'n')
boxplot(read_results_med_med_few, las = 2, xlab = "", ylab = "Duration (seconds)", main = "10 files @ 5MB (50MB)")

boxplot(read_results_lrg_sml_mny, las = 2, xlab = "", ylab = "Duration (seconds)", main = "1000 files @ 500KB (500MB)", xaxt = 'n')
boxplot(read_results_lrg_med_mod, las = 2, xlab = "", ylab = "Duration (seconds)", main = "100 files @ 5MB (500MB)", xaxt = 'n')
boxplot(read_results_lrg_lrg_few, las = 2, xlab = "", ylab = "Duration (seconds)", main = "10 files @ 50MB (500MB)")

中等用例

更大的用例

用例的多样性

行:文件计数(1000,100,10) 列:最终数据帧大小(5MB, 50MB, 500MB) (点击图片查看原始尺寸)

在最小的用例中，使用purrr和dplyr的C库带来的开销超过了执行大规模处理任务时所观察到的性能收益，因此基本R结果更好。

如果您想运行自己的测试，您可能会发现这个bash脚本很有帮助。

for ((i=1; i<=$2; i++)); do 
  cp "$1" "${1:0:8}_${i}.csv";
done

bash what_you_name_this_script.sh " filename_you_want_copies " 100将创建100份按顺序编号的文件(在文件名的前8个字符和下划线之后)。

归因和欣赏

特别感谢:

Tyler Rinker和Akrun演示了微基准测试。 Jake Kaupp向我介绍了map_df()。 David McLaughlin提供了关于改进可视化和讨论/确认在小文件、小数据框架分析结果中观察到的性能反转的有用反馈。 Marbel指出了fread()的默认行为。(我需要研究一下数据表。)

这是我读取多个文件并将它们组合成1个数据帧的具体示例:

path<- file.path("C:/folder/subfolder")
files <- list.files(path=path, pattern="/*.csv",full.names = T)
library(data.table)
data = do.call(rbind, lapply(files, function(x) read.csv(x, stringsAsFactors = FALSE)))

除了使用lapply或R中的其他循环构造，您还可以将CSV文件合并到一个文件中。

在Unix中，如果文件没有头文件，那么很简单:

cat *.csv > all.csv

或者如果有标题，你可以找到一个字符串匹配标题，只有标题(即假设标题行都以“年龄”开头)，你会这样做:

cat *.csv | grep -v ^Age > all.csv

我认为在Windows中，你可以通过DOS命令框中的COPY和SEARCH(或FIND或其他什么)来做到这一点，但为什么不安装cygwin并获得Unix命令shell的强大功能呢?

使用许多文件和许多核心，fread xargs cat(如下所述)比前3个答案中最快的解决方案快大约50倍。

rbindlist lapply read.delim  500s <- 1st place & accepted answer
rbindlist lapply fread       250s <- 2nd & 3rd place answers
rbindlist mclapply fread      10s
fread xargs cat                5s

是时候把121401个csv读入一个data.table了。每次平均跑三次，然后四舍五入。每个csv有3列，一个标题行，平均有4.510行。Machine是一个96核的GCP VM。

@A5C1D2H2I1M1N2O1R2T1、@ leersej和@marbel给出的前三个答案本质上都是相同的:对每个文件应用fread(或read.delim)，然后rbind/rbindlist结果data.tables。对于小型数据集，我通常使用rbindlist(lapply(list.files("*.csv")，fread))表单。对于中等规模的数据集，我使用并行的mclapply而不是lapply，如果有多个核心，那么lapply速度要快得多。

这比其他r内部的替代方案更好，但对于大量的小型csv来说，在速度问题上不是最好的。在这种情况下，首先使用cat将所有csv连接到一个csv中会更快，就像@Spacedman的答案一样。我将在R中添加一些关于如何做到这一点的细节:

x = fread(cmd='cat *.csv', header=F)

但是，如果每个csv都有一个头呢?

x = fread(cmd="awk 'NR==1||FNR!=1' *.csv", header=T)

如果您有太多文件，以至于*.csv shell glob失败了怎么办?

x = fread(cmd='find . -name "*.csv" | xargs cat', header=F)

如果所有文件都有头文件，而且文件太多怎么办?

header = fread(cmd='find . -name "*.csv" | head -n1 | xargs head -n1', header=T)
x = fread(cmd='find . -name "*.csv" | xargs tail -q -n+2', header=F)
setnames(x,header)

如果结果连接的csv对于系统内存来说太大怎么办?(例如，/dev/shm out of space错误)

system('find . -name "*.csv" | xargs cat > combined.csv')
x = fread('combined.csv', header=F)

头吗?

system('find . -name "*.csv" | head -n1 | xargs head -n1 > combined.csv')
system('find . -name "*.csv" | xargs tail -q -n+2 >> combined.csv')
x = fread('combined.csv', header=T)

最后，如果您不希望将所有.csv文件放在一个目录中，而希望将其放在一组特定的文件中，该怎么办?(而且，它们都有头文件。)(这是我的用例。)

fread(text=paste0(system("xargs cat|awk 'NR==1||$1!=\"<column one name>\"'",input=paths,intern=T),collapse="\n"),header=T,sep="\t")

这和普通的fread xargs cat的速度差不多:)

注:用于数据。表pre-v1.11.6(2018年9月19日)，从fread中省略cmd= (cmd=。

总之，如果您对速度感兴趣，并且有很多文件和内核，那么fread xargs cat比前3个答案中最快的解决方案快大约50倍。

更新:这里是我写的一个函数，可以轻松地应用最快的解决方案。我在生产环境中使用了它，但是在信任它之前，您应该用自己的数据彻底测试它。

fread_many = function(files,header=T,...){
  if(length(files)==0) return()
  if(typeof(files)!='character') return()
  files = files[file.exists(files)]
  if(length(files)==0) return()
  tmp = tempfile(fileext = ".csv")
  # note 1: requires awk, not cat or tail because some files have no final newline
  # note 2: parallel --xargs is 40% slower
  # note 3: reading to var is 15% slower and crashes R if the string is too long
  # note 4: shorter paths -> more paths per awk -> fewer awks -> measurably faster
  #         so best cd to the csv dir and use relative paths
  if(header==T){
    system(paste0('head -n1 ',files[1],' > ',tmp))
    system(paste0("xargs awk 'FNR>1' >> ",tmp),input=files)
  } else {
    system(paste0("xargs awk '1' > ",tmp),input=files)
  }
  DT = fread(file=tmp,header=header,...)
  file.remove(tmp)
  DT
}

更新2:这里是fread_many函数的一个更复杂的版本，用于需要结果数据的情况。表中包含每个csv的inpath的列。在这种情况下，还必须使用sep参数显式地指定csv分隔符。

fread_many = function(files,header=T,keep_inpath=F,sep="auto",...){
  if(length(files)==0) return()
  if(typeof(files)!='character') return()
  files = files[file.exists(files)]
  if(length(files)==0) return()
  tmp = tempfile(fileext = ".csv")
  if(keep_inpath==T){
    stopifnot(sep!="auto")
    if(header==T){
      system(paste0('/usr/bin/echo -ne inpath"',sep,'" > ',tmp))
      system(paste0('head -n1 ',files[1],' >> ',tmp))
      system(paste0("xargs awk -vsep='",sep,"' 'BEGIN{OFS=sep}{if(FNR>1)print FILENAME,$0}' >> ",tmp),input=files)
    } else {
      system(paste0("xargs awk -vsep='",sep,"' 'BEGIN{OFS=sep}{print FILENAME,$0}' > ",tmp),input=files)
    }
  } else {
    if(header==T){
      system(paste0('head -n1 ',files[1],' > ',tmp))
      system(paste0("xargs awk 'FNR>1' >> ",tmp),input=files)
    } else {
      system(paste0("xargs awk '1' > ",tmp),input=files)
    }
  }
  DT = fread(file=tmp,header=header,sep=sep,...)
  file.remove(tmp)
  DT
}

注意:在读取csv之前，我的所有解决方案都假设它们都有相同的分隔符。如果不是所有的csv都使用相同的分隔符，可以分批使用rbindlist lapply fread、rbindlist mclapply fread或fread xargs cat，其中批处理中的所有csv都使用相同的分隔符。