Reading all files in a directory store them in objects and send the object

Question

I do not know if this is possible  but here goes  And working with callbacks makes it even more difficult   I have a directory with html files that I want to send back to the client in Object chunks with node js and socket io   All my files are in  tmpl  So socket needs to read all the files in  tmpl   for each file it has to store the data in an object with the filename as the key  and the content as the value     var data       this is wrong because it has to loop trough all files    fs readFile   dirname     tmpl filename html    utf8   function err  html         if err  throw err          filename must be without  html at the end       data  filename     html          socket emit  init    data  data      The final callback is also wrong  It has to be called when all the files in the directory are done   But I do not know how to create the code  anyone know if this is possibel

User · Answer

For all example below you need to import fs and path modules:

const fs = require('fs');
const path = require('path');

Read files asynchronously

function readFiles(dir, processFile) {
  // read directory
  fs.readdir(dir, (error, fileNames) => {
    if (error) throw error;

    fileNames.forEach(filename => {
      // get current file name
      const name = path.parse(filename).name;
      // get current file extension
      const ext = path.parse(filename).ext;
      // get current file path
      const filepath = path.resolve(dir, filename);

      // get information about the file
      fs.stat(filepath, function(error, stat) {
        if (error) throw error;

        // check if the current path is a file or a folder
        const isFile = stat.isFile();

        // exclude folders
        if (isFile) {
          // callback, do something with the file
          processFile(filepath, name, ext, stat);
        }
      });
    });
  });
}

Usage:

// use an absolute path to the folder where files are located
readFiles('absolute/path/to/directory/', (filepath, name, ext, stat) => {
  console.log('file path:', filepath);
  console.log('file name:', name);
  console.log('file extension:', ext);
  console.log('file information:', stat);
});

Read files synchronously, store in array, natural sorting

/**
 * @description Read files synchronously from a folder, with natural sorting
 * @param {String} dir Absolute path to directory
 * @returns {Object[]} List of object, each object represent a file
 * structured like so: `{ filepath, name, ext, stat }`
 */
function readFilesSync(dir) {
  const files = [];

  fs.readdirSync(dir).forEach(filename => {
    const name = path.parse(filename).name;
    const ext = path.parse(filename).ext;
    const filepath = path.resolve(dir, filename);
    const stat = fs.statSync(filepath);
    const isFile = stat.isFile();

    if (isFile) files.push({ filepath, name, ext, stat });
  });

  files.sort((a, b) => {
    // natural sort alphanumeric strings
    // https://stackoverflow.com/a/38641281
    return a.name.localeCompare(b.name, undefined, { numeric: true, sensitivity: 'base' });
  });

  return files;
}

Usage:

// return an array list of objects
// each object represent a file
const files = readFilesSync('absolute/path/to/directory/');

Read files async using promise

More info on promisify in this article.

const { promisify } = require('util');

const readdir_promise = promisify(fs.readdir);
const stat_promise = promisify(fs.stat);

function readFilesAsync(dir) {
  return readdir_promise(dir, { encoding: 'utf8' })
    .then(filenames => {
      const files = getFiles(dir, filenames);

      return Promise.all(files);
    })
    .catch(err => console.error(err));
}

function getFiles(dir, filenames) {
  return filenames.map(filename => {
    const name = path.parse(filename).name;
    const ext = path.parse(filename).ext;
    const filepath = path.resolve(dir, filename);

    return stat({ name, ext, filepath });
  });
}

function stat({ name, ext, filepath }) {
  return stat_promise(filepath)
    .then(stat => {
      const isFile = stat.isFile();

      if (isFile) return { name, ext, filepath, stat };
    })
    .catch(err => console.error(err));
}

Usage:

readFiles('absolute/path/to/directory/')
  // return an array list of objects
  // each object is a file
  // with those properties: { name, ext, filepath, stat }
  .then(files => console.log(files))
  .catch(err => console.log(err));

Note: return undefined for folders, if you want you can filter them out:

readFiles('absolute/path/to/directory/')
  .then(files => files.filter(file => file !== undefined))
  .catch(err => console.log(err));

User · Answer

Another version with Promise s modern method  It s shorter that the others responses based on Promise     const readFiles    dirname    gt       const readDirPr   new Promise   resolve  reject    gt        fs readdir dirname          err  filenames    gt   err    reject err    resolve filenames            return readDirPr then  filenames   gt  Promise all filenames map  filename    gt          return new Promise    resolve  reject    gt            fs readFile dirname   filename   utf-8              err  content    gt   err    reject err    resolve content                     catch  error   gt  Promise reject error        readFiles sourceFolder     then  allContents   gt            handle success treatment       error   gt  console log error

User · Answer

For the code to work smoothy in different enviroments  path resolve can be used in places where path is manipulated  Here is code which works better   Reading part   var fs   require  fs     function readFiles dirname  onFileContent  onError      fs readdir dirname  function err  filenames        if  err          onError err         return            filenames forEach function filename          fs readFile path resolve dirname  filename    utf-8   function err  content            if  err              onError err             return                    onFileContent filename  content                               Storing part   var data       readFiles path resolve   dirname   dirname     function filename  content      data filename    content     function error      throw err

User · Answer

Are you a lazy person like me and love npm module  D then check this out   npm install node-dir  example for reading files   var dir   require  node-dir     dir readFiles   dirname      function err  content  next            if  err  throw err          console log  content    content       get content of files         next               function err  files           if  err  throw err          console log  finished reading files    files      get filepath

User · Answer

If you have Node js 8 or later  you can use the new util promisify   I m marking as optional the parts of the code that have to do with reformatting as an object  which the original post requested      const fs   require  fs      const   promisify     require  util       let files     optional   promisify fs readdir  directory  then  filenames    gt        files   filenames     optional     return Promise all filenames map  filename    gt          return promisify fs readFile  directory   filename   encoding   utf8                   then  strArr    gt           optional      const data           strArr forEach  str  i    gt          data files i     str                 send data here      catch  err    gt        console log err

User · Answer

async await  const   promisify     require  util   const directory   path join   dirname    tmpl   const pathnames   promisify fs readdir  directory   try     async function emitData directory        let filenames   await pathnames     var ob          const data   filenames map async function filename  i          if  filename includes                 var storedFile   promisify fs readFile  directory        filename                encoding   utf8                      ob filename replace   js          await storedFile         socket emit  init     data  ob                  return ob               emitData directory    catch  err      console log err      Who wants to try with generators

User · Answer

This is a modern Promise version of the previous one  using a Promise all approach to resolve all promises when all files have been read          Promise all     author Loreto Parisi  loretoparisi at gmail dot com      function promiseAllP items  block        var promises           items forEach function item index            promises push  function item i                return new Promise function resolve  reject                    return block apply this  item index resolve reject                              item index               return Promise all promises       promiseAll         read files     param dirname string     return Promise     author Loreto Parisi  loretoparisi at gmail dot com      see http   stackoverflow com questions 10049557 reading-all-files-in-a-directory-store-them-in-objects-and-send-the-object     function readFiles dirname        return new Promise  resolve  reject    gt            fs readdir dirname  function err  filenames                if  err  return reject err               promiseAllP filenames               filename index resolve reject    gt                     fs readFile path resolve dirname  filename    utf-8   function err  content                        if  err  return reject err                       return resolve  filename  filename  contents  content                                                    then results   gt                    return resolve results                               catch error   gt                    return reject error                                         How to Use It   Just as simple as doing   readFiles  EMAIL ROOT         folder   then files   gt        console log   loaded    files length        files forEach   item  index    gt            console log   item  index   size    item contents length               catch  error   gt        console log  error          Supposed that you have another list of folders you can as well iterate over this list  since the internal promise all will resolve each of then asynchronously   var folders   spam   ham    folders forEach  folder   gt        readFiles  EMAIL ROOT         folder       then files   gt            console log   loaded    files length            files forEach   item  index    gt                console log   item  index   size    item contents length                           catch  error   gt            console log  error                  How it Works  The promiseAll does the magic  It takes a function block of signature function item index resolve reject   where item is the current item in the array  index its position in the array  and resolve and reject the Promise callback functions  Each promise will be pushed in a array at the current index and with the current item as arguments through a anonymous function call   promises push  function item i            return new Promise function resolve  reject                return block apply this  item index resolve reject                      item index     Then all promises will be resolved   return Promise all promises

User · Answer

I just wrote this and it looks more clean to me   x000D   x000D  const fs   require  fs    const util   require  util     const readdir   util promisify fs readdir   const readFile   util promisify fs readFile    const readFiles   async dirname   gt        try           const filenames   await readdir dirname           console log   filenames             const files promise   filenames map filename   gt                return readFile dirname   filename   utf-8                        const response   await Promise all files promise             console log   response              return response         return filenames reduce  accumlater  filename  currentIndex    gt                const content   response currentIndex               accumlater filename                      content                             return accumlater                        catch  error            console error error             const main   async      gt         const response   await readFiles             folder-name              console log   response        x000D   x000D   x000D   You can modify the response format according to your need  The response format from this code will look like        quot filename-01 quot           quot content quot   quot This is the sample content of the file quot            quot filename-02 quot           quot content quot   quot This is the sample content of the file quot

User · Answer

So  there are three parts  Reading  storing and sending   Here s the reading part   var fs   require  fs     function readFiles dirname  onFileContent  onError      fs readdir dirname  function err  filenames        if  err          onError err         return            filenames forEach function filename          fs readFile dirname   filename   utf-8   function err  content            if  err              onError err             return                    onFileContent filename  content                               Here s the storing part   var data       readFiles  dirname    function filename  content      data filename    content     function err      throw err        The sending part is up to you  You may want to send them one by one or after reading completion   If you want to send files after reading completion you should either use sync versions of fs functions or use promises  Async callbacks is not a good style   Additionally you asked about stripping an extension  You should proceed with questions one by one  Nobody will write a complete solution just for you

[node.js] Reading all files in a directory, store them in objects, and send the object

Read files asynchronously

Read files synchronously, store in array, natural sorting

Read files async using promise

Examples related to node.js