我试图读取一个大文件一行在一次。我在Quora上找到了一个关于这个主题的问题,但我错过了一些联系,把整个事情联系在一起。

 var Lazy=require("lazy");
 new Lazy(process.stdin)
     .lines
     .forEach(
          function(line) { 
              console.log(line.toString()); 
          }
 );
 process.stdin.resume();

我想要弄清楚的是如何一次从文件中读取一行,而不是像本例中那样读取STDIN。

我试着:

 fs.open('./VeryBigFile.csv', 'r', '0666', Process);

 function Process(err, fd) {
    if (err) throw err;
    // DO lazy read 
 }

但这并不奏效。我知道在必要时我可以使用PHP之类的东西,但我想弄清楚这个问题。

我不认为其他答案会起作用,因为文件比我运行它的服务器的内存大得多。


当前回答

我已经看了以上所有的答案,他们都使用第三方库来解决它。它在Node的API中有一个简单的解决方案。如

const fs= require('fs')

let stream = fs.createReadStream('<filename>', { autoClose: true })

stream.on('data', chunk => {
    let row = chunk.toString('ascii')
}))

其他回答

在Node.js v18.11.0中添加了一个逐行读取文件的新函数

filehandle.readLines([选项])

这就是如何将此功能用于想要读取的文本文件

import { open } from 'node:fs/promises';
myFileReader();
async function myFileReader() {
    const file = await open('./TextFileName.txt');
    for await (const line of file.readLines()) {
        console.log(line)
    }
}

为了了解更多read Node.js文档,这里有文件系统readlines()的链接: https://nodejs.org/api/fs.html#filehandlereadlinesoptions

我对缺乏全面的解决方案感到沮丧,所以我把自己的尝试(git / npm)放在一起。复制粘贴功能列表:

Interactive line processing (callback-based, no loading the entire file into RAM) Optionally, return all lines in an array (detailed or raw mode) Interactively interrupt streaming, or perform map/filter like processing Detect any newline convention (PC/Mac/Linux) Correct eof / last line treatment Correct handling of multi-byte UTF-8 characters Retrieve byte offset and byte length information on per-line basis Random access, using line-based or byte-based offsets Automatically map line-offset information, to speed up random access Zero dependencies Tests

国家卫生研究院?你决定:-)

虽然您可能应该像上面的答案所建议的那样使用readline模块,但readline似乎面向命令行接口,而不是行读取。关于缓冲,它也有点不透明。(任何需要流式行阅读器的人可能都想调整缓冲区大小)。readline模块大约有1000行,而这个,加上统计和测试,是34行。

const EventEmitter = require('events').EventEmitter;
class LineReader extends EventEmitter{
    constructor(f, delim='\n'){
        super();
        this.totalChars = 0;
        this.totalLines = 0;
        this.leftover = '';

        f.on('data', (chunk)=>{
            this.totalChars += chunk.length;
            let lines = chunk.split(delim);
            if (lines.length === 1){
                this.leftover += chunk;
                return;
            }
            lines[0] = this.leftover + lines[0];
            this.leftover = lines[lines.length-1];
            if (this.leftover) lines.pop();
            this.totalLines += lines.length;
            for (let l of lines) this.onLine(l);
        });
        // f.on('error', ()=>{});
        f.on('end', ()=>{console.log('chars', this.totalChars, 'lines', this.totalLines)});
    }
    onLine(l){
        this.emit('line', l);
    }
}
//Command line test
const f = require('fs').createReadStream(process.argv[2], 'utf8');
const delim = process.argv[3];
const lineReader = new LineReader(f, delim);
lineReader.on('line', (line)=> console.log(line));

下面是一个更短的版本,没有统计数据,只有19行:

class LineReader extends require('events').EventEmitter{
    constructor(f, delim='\n'){
        super();
        this.leftover = '';
        f.on('data', (chunk)=>{
            let lines = chunk.split(delim);
            if (lines.length === 1){
                this.leftover += chunk;
                return;
            }
            lines[0] = this.leftover + lines[0];
            this.leftover = lines[lines.length-1];
            if (this.leftover) 
                lines.pop();
            for (let l of lines)
                this.emit('line', l);
        });
    }
}

2019年更新

Nodejs官方文档中已经发布了一个很棒的例子。在这里

这需要在您的机器上安装最新的Nodejs。> 11.4

const fs = require('fs');
const readline = require('readline');

async function processLineByLine() {
  const fileStream = fs.createReadStream('input.txt');

  const rl = readline.createInterface({
    input: fileStream,
    crlfDelay: Infinity
  });
  // Note: we use the crlfDelay option to recognize all instances of CR LF
  // ('\r\n') in input.txt as a single line break.

  for await (const line of rl) {
    // Each line in input.txt will be successively available here as `line`.
    console.log(`Line from file: ${line}`);
  }
}

processLineByLine();
function createLineReader(fileName){
    var EM = require("events").EventEmitter
    var ev = new EM()
    var stream = require("fs").createReadStream(fileName)
    var remainder = null;
    stream.on("data",function(data){
        if(remainder != null){//append newly received data chunk
            var tmp = new Buffer(remainder.length+data.length)
            remainder.copy(tmp)
            data.copy(tmp,remainder.length)
            data = tmp;
        }
        var start = 0;
        for(var i=0; i<data.length; i++){
            if(data[i] == 10){ //\n new line
                var line = data.slice(start,i)
                ev.emit("line", line)
                start = i+1;
            }
        }
        if(start<data.length){
            remainder = data.slice(start);
        }else{
            remainder = null;
        }
    })

    stream.on("end",function(){
        if(null!=remainder) ev.emit("line",remainder)
    })

    return ev
}


//---------main---------------
fileName = process.argv[2]

lineReader = createLineReader(fileName)
lineReader.on("line",function(line){
    console.log(line.toString())
    //console.log("++++++++++++++++++++")
})