Remove duplicate values from JS array

TL;DR

Using the Set constructor and the spread syntax:

uniq = [...new Set(array)];

"Smart" but naïve way

uniqueArray = a.filter(function(item, pos) {
    return a.indexOf(item) == pos;
})

Basically, we iterate over the array and, for each element, check if the first position of this element in the array is equal to the current position. Obviously, these two positions are different for duplicate elements.

Using the 3rd ("this array") parameter of the filter callback we can avoid a closure of the array variable:

uniqueArray = a.filter(function(item, pos, self) {
    return self.indexOf(item) == pos;
})

Although concise, this algorithm is not particularly efficient for large arrays (quadratic time).

Hashtables to the rescue

function uniq(a) {
    var seen = {};
    return a.filter(function(item) {
        return seen.hasOwnProperty(item) ? false : (seen[item] = true);
    });
}

This is how it's usually done. The idea is to place each element in a hashtable and then check for its presence instantly. This gives us linear time, but has at least two drawbacks:

• since hash keys can only be strings or symbols in JavaScript, this code doesn't distinguish numbers and "numeric strings". That is, uniq([1,"1"]) will return just [1]
• for the same reason, all objects will be considered equal: uniq([{foo:1},{foo:2}]) will return just [{foo:1}].

That said, if your arrays contain only primitives and you don't care about types (e.g. it's always numbers), this solution is optimal.

The best from two worlds

A universal solution combines both approaches: it uses hash lookups for primitives and linear search for objects.

function uniq(a) {
    var prims = {"boolean":{}, "number":{}, "string":{}}, objs = [];

    return a.filter(function(item) {
        var type = typeof item;
        if(type in prims)
            return prims[type].hasOwnProperty(item) ? false : (prims[type][item] = true);
        else
            return objs.indexOf(item) >= 0 ? false : objs.push(item);
    });
}

sort | uniq

Another option is to sort the array first, and then remove each element equal to the preceding one:

function uniq(a) {
    return a.sort().filter(function(item, pos, ary) {
        return !pos || item != ary[pos - 1];
    });
}

Again, this doesn't work with objects (because all objects are equal for sort). Additionally, we silently change the original array as a side effect - not good! However, if your input is already sorted, this is the way to go (just remove sort from the above).

Unique by...

Sometimes it's desired to uniquify a list based on some criteria other than just equality, for example, to filter out objects that are different, but share some property. This can be done elegantly by passing a callback. This "key" callback is applied to each element, and elements with equal "keys" are removed. Since key is expected to return a primitive, hash table will work fine here:

function uniqBy(a, key) {
    var seen = {};
    return a.filter(function(item) {
        var k = key(item);
        return seen.hasOwnProperty(k) ? false : (seen[k] = true);
    })
}

A particularly useful key() is JSON.stringify which will remove objects that are physically different, but "look" the same:

a = [[1,2,3], [4,5,6], [1,2,3]]
b = uniqBy(a, JSON.stringify)
console.log(b) // [[1,2,3], [4,5,6]]

If the key is not primitive, you have to resort to the linear search:

function uniqBy(a, key) {
    var index = [];
    return a.filter(function (item) {
        var k = key(item);
        return index.indexOf(k) >= 0 ? false : index.push(k);
    });
}

In ES6 you can use a Set:

function uniqBy(a, key) {
    let seen = new Set();
    return a.filter(item => {
        let k = key(item);
        return seen.has(k) ? false : seen.add(k);
    });
}

or a Map:

function uniqBy(a, key) {
    return [
        ...new Map(
            a.map(x => [key(x), x])
        ).values()
    ]
}

which both also work with non-primitive keys.

First or last?

When removing objects by a key, you might to want to keep the first of "equal" objects or the last one.

Use the Set variant above to keep the first, and the Map to keep the last:

function uniqByKeepFirst(a, key) {_x000D_
    let seen = new Set();_x000D_
    return a.filter(item => {_x000D_
        let k = key(item);_x000D_
        return seen.has(k) ? false : seen.add(k);_x000D_
    });_x000D_
}_x000D_
_x000D_
_x000D_
function uniqByKeepLast(a, key) {_x000D_
    return [_x000D_
        ...new Map(_x000D_
            a.map(x => [key(x), x])_x000D_
        ).values()_x000D_
    ]_x000D_
}_x000D_
_x000D_
//_x000D_
_x000D_
data = [_x000D_
    {a:1, u:1},_x000D_
    {a:2, u:2},_x000D_
    {a:3, u:3},_x000D_
    {a:4, u:1},_x000D_
    {a:5, u:2},_x000D_
    {a:6, u:3},_x000D_
];_x000D_
_x000D_
console.log(uniqByKeepFirst(data, it => it.u))_x000D_
console.log(uniqByKeepLast(data, it => it.u))

Libraries

Both underscore and Lo-Dash provide uniq methods. Their algorithms are basically similar to the first snippet above and boil down to this:

var result = [];
a.forEach(function(item) {
     if(result.indexOf(item) < 0) {
         result.push(item);
     }
});

This is quadratic, but there are nice additional goodies, like wrapping native indexOf, ability to uniqify by a key (iteratee in their parlance), and optimizations for already sorted arrays.

If you're using jQuery and can't stand anything without a dollar before it, it goes like this:

  $.uniqArray = function(a) {
        return $.grep(a, function(item, pos) {
            return $.inArray(item, a) === pos;
        });
  }

which is, again, a variation of the first snippet.

Performance

Function calls are expensive in JavaScript, therefore the above solutions, as concise as they are, are not particularly efficient. For maximal performance, replace filter with a loop and get rid of other function calls:

function uniq_fast(a) {
    var seen = {};
    var out = [];
    var len = a.length;
    var j = 0;
    for(var i = 0; i < len; i++) {
         var item = a[i];
         if(seen[item] !== 1) {
               seen[item] = 1;
               out[j++] = item;
         }
    }
    return out;
}

This chunk of ugly code does the same as the snippet #3 above, but an order of magnitude faster (as of 2017 it's only twice as fast - JS core folks are doing a great job!)

function uniq(a) {_x000D_
    var seen = {};_x000D_
    return a.filter(function(item) {_x000D_
        return seen.hasOwnProperty(item) ? false : (seen[item] = true);_x000D_
    });_x000D_
}_x000D_
_x000D_
function uniq_fast(a) {_x000D_
    var seen = {};_x000D_
    var out = [];_x000D_
    var len = a.length;_x000D_
    var j = 0;_x000D_
    for(var i = 0; i < len; i++) {_x000D_
         var item = a[i];_x000D_
         if(seen[item] !== 1) {_x000D_
               seen[item] = 1;_x000D_
               out[j++] = item;_x000D_
         }_x000D_
    }_x000D_
    return out;_x000D_
}_x000D_
_x000D_
/////_x000D_
_x000D_
var r = [0,1,2,3,4,5,6,7,8,9],_x000D_
    a = [],_x000D_
    LEN = 1000,_x000D_
    LOOPS = 1000;_x000D_
_x000D_
while(LEN--)_x000D_
    a = a.concat(r);_x000D_
_x000D_
var d = new Date();_x000D_
for(var i = 0; i < LOOPS; i++)_x000D_
    uniq(a);_x000D_
document.write('<br>uniq, ms/loop: ' + (new Date() - d)/LOOPS)_x000D_
_x000D_
var d = new Date();_x000D_
for(var i = 0; i < LOOPS; i++)_x000D_
    uniq_fast(a);_x000D_
document.write('<br>uniq_fast, ms/loop: ' + (new Date() - d)/LOOPS)

ES6

ES6 provides the Set object, which makes things a whole lot easier:

function uniq(a) {
   return Array.from(new Set(a));
}

or

let uniq = a => [...new Set(a)];

Note that, unlike in python, ES6 sets are iterated in insertion order, so this code preserves the order of the original array.

However, if you need an array with unique elements, why not use sets right from the beginning?

Generators

A "lazy", generator-based version of uniq can be built on the same basis:

• take the next value from the argument
• if it's been seen already, skip it
• otherwise, yield it and add it to the set of already seen values

function* uniqIter(a) {_x000D_
    let seen = new Set();_x000D_
_x000D_
    for (let x of a) {_x000D_
        if (!seen.has(x)) {_x000D_
            seen.add(x);_x000D_
            yield x;_x000D_
        }_x000D_
    }_x000D_
}_x000D_
_x000D_
// example:_x000D_
_x000D_
function* randomsBelow(limit) {_x000D_
    while (1)_x000D_
        yield Math.floor(Math.random() * limit);_x000D_
}_x000D_
_x000D_
// note that randomsBelow is endless_x000D_
_x000D_
count = 20;_x000D_
limit = 30;_x000D_
_x000D_
for (let r of uniqIter(randomsBelow(limit))) {_x000D_
    console.log(r);_x000D_
    if (--count === 0)_x000D_
        break_x000D_
}_x000D_
_x000D_
// exercise for the reader: what happens if we set `limit` less than `count` and why