[javascript] best way to get folder and file list in Javascript

I'm using node-webkit, and am trying to have a user select a folder, and I'll return the directory structure of that folder and recursively get its children.

I've got this working fairly simply with this code (in an Angular Controller).

var fs = require('fs');

$scope.explorer=[];
$scope.openFile = function(){
    $scope.explorer = [tree_entry($scope.path)];    
    get_folder($scope.path, $scope.explorer[0].children);
};

function get_folder(path, tree){
    fs.readdir(path, function(err,files){
        if (err) return console.log(err);

        files.forEach( function (file,idx){
            tree.push(tree_entry(file));
            fs.lstat(path+'/'+file,function(err,stats){
                if(err) return console.log(err);
                if(stats.isDirectory()){
                    get_folder(path+'/'+file,tree[idx].children);
                }
            });
        });
    });
    console.log($scope.explorer);

    return;
}

function tree_entry(entry){
    return { label : entry, children: []}
}

Taking a moderate sized folder with 22 child folders and about 4 levels deep, it is taking a few minutes to get the entire directory structure.

Is there something that I'm obviously doing wrong here? I can't believe it takes that long, seeing as I'm using the built in Node fs methods. Or is there a way to get the entire contents of a directory without touching each and every file?

I'm going to want to be able to use an Angular filter on the file names all the way down the tree, and possibly on the contents too, so delaying processing the entire tree isn't likely a solution that would work.

fs/promises and fs.Dirent

Here's an efficient, non-blocking ls program using Node's fast fs.Dirent objects and fs/promises module. This approach allows you to skip wasteful fs.exist or fs.stat calls on every path -

// main.js
import { readdir } from "fs/promises"
import { join } from "path"

async function* ls (path = ".")
{ yield path
  for (const dirent of await readdir(path, { withFileTypes: true }))
    if (dirent.isDirectory())
      yield* ls(join(path, dirent.name))
    else
      yield join(path, dirent.name)
}

async function* empty () {}

async function toArray (iter = empty())
{ let r = []
  for await (const x of iter)
    r.push(x)
  return r
}

toArray(ls(".")).then(console.log, console.error)

Let's get some sample files so we can see ls working -

$ yarn add immutable     # (just some example package)
$ node main.js

[
  '.',
  'main.js',
  'node_modules',
  'node_modules/.yarn-integrity',
  'node_modules/immutable',
  'node_modules/immutable/LICENSE',
  'node_modules/immutable/README.md',
  'node_modules/immutable/contrib',
  'node_modules/immutable/contrib/cursor',
  'node_modules/immutable/contrib/cursor/README.md',
  'node_modules/immutable/contrib/cursor/__tests__',
  'node_modules/immutable/contrib/cursor/__tests__/Cursor.ts.skip',
  'node_modules/immutable/contrib/cursor/index.d.ts',
  'node_modules/immutable/contrib/cursor/index.js',
  'node_modules/immutable/dist',
  'node_modules/immutable/dist/immutable-nonambient.d.ts',
  'node_modules/immutable/dist/immutable.d.ts',
  'node_modules/immutable/dist/immutable.es.js',
  'node_modules/immutable/dist/immutable.js',
  'node_modules/immutable/dist/immutable.js.flow',
  'node_modules/immutable/dist/immutable.min.js',
  'node_modules/immutable/package.json',
  'package.json',
  'yarn.lock'
]

For added explanation and other ways to leverage async generators, see this Q&A.

I don't like adding new package into my project just to handle this simple task.

And also, I try my best to avoid RECURSIVE algorithm.... since, for most cases it is slower compared to non Recursive one.

So I made a function to get all the folder content (and its sub folder).... NON-Recursively

var getDirectoryContent = function(dirPath) {
    /* 
        get list of files and directories from given dirPath and all it's sub directories
        NON RECURSIVE ALGORITHM
        By. Dreamsavior
    */
    var RESULT = {'files':[], 'dirs':[]};

    var fs = fs||require('fs');
    if (Boolean(dirPath) == false) {
        return RESULT;
    }
    if (fs.existsSync(dirPath) == false) {
        console.warn("Path does not exist : ", dirPath);
        return RESULT;
    }

    var directoryList = []
    var DIRECTORY_SEPARATOR = "\\";
    if (dirPath[dirPath.length -1] !== DIRECTORY_SEPARATOR) dirPath = dirPath+DIRECTORY_SEPARATOR;

    directoryList.push(dirPath); // initial

    while (directoryList.length > 0) {
        var thisDir  = directoryList.shift(); 
        if (Boolean(fs.existsSync(thisDir) && fs.lstatSync(thisDir).isDirectory()) == false) continue;

        var thisDirContent = fs.readdirSync(thisDir);
        while (thisDirContent.length > 0) { 
            var thisFile  = thisDirContent.shift(); 
            var objPath = thisDir+thisFile

            if (fs.existsSync(objPath) == false) continue;
            if (fs.lstatSync(objPath).isDirectory()) { // is a directory
                let thisDirPath = objPath+DIRECTORY_SEPARATOR; 
                directoryList.push(thisDirPath);
                RESULT['dirs'].push(thisDirPath);

            } else  { // is a file
                RESULT['files'].push(objPath); 

            } 
        } 

    }
    return RESULT;
}

the only drawback of this function is that this is Synchronous function... You have been warned ;)

Why to invent the wheel?

There is a very popular NPM package, that let you do things like that easy.

var recursive = require("recursive-readdir");

recursive("some/path", function (err, files) {
  // `files` is an array of file paths
  console.log(files);
});

Lear more:

• https://www.npmjs.com/package/recursive-readdir