logo Netzschleuder network catalog, repository, and centrifuge

History of netflix_prize

The changes to the source code of this entry in the repository, newest first, each with the differences it made to the entry's files. They are also listed on GitLab.

[entries] New entry: netflix_prize by Tiago de Paula Peixoto 4371a41
diff --git a/entries/netflix_prize/__init__.py b/entries/netflix_prize/__init__.py
new file mode 100644
index 0000000..c9d799f
--- /dev/null
+++ b/entries/netflix_prize/__init__.py
@@ -0,0 +1,229 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2020-2026 Tiago de Paula Peixoto <tiago@skewed.de>
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+from .. import *
+
+import hashlib
+
+title = "Netflix Prize ratings (1998-2005)"
+description = """Bipartite networks of the customers and movies of the DVD rental and streaming service Netflix, from the data set that Netflix released in October 2006 for the Netflix Prize, an open competition for the best algorithm to predict user ratings of movies. The data set contains 100,480,507 ratings, from 1 to 5 stars (whole stars only), that 480,189 customers, chosen at random, gave to 17,770 movies between November 1999 and December 2005 (October 1998 to December 2005, according to the documentation of the data set), with the date of each rating. The customer ids were replaced by random ids. The movies are given with their titles and years of release (which may be those of the DVD release), and their ids are not those used by Netflix.
+
+- `training`: the training set of the competition, where each edge is the rating that a customer gave to a movie (a customer rates a movie at most once). The 1,408,395 ratings of the probe set, a subset of the training set that has the same properties as the qualifying set, are marked.
+- `qualifying`: the qualifying set of the competition, 2,817,131 pairs of customers and movies with the dates of the ratings (from January 2000 to December 2005), whose ratings were withheld (the ratings that the competitors had to predict). These ratings were never released.
+
+Both networks contain all the customers and movies of the data set, in the same order.
+
+Shortly after the release, Narayanan and Shmatikov [showed](https://doi.org/10.1109/SP.2008.33) that some of the customers in the data set could be re-identified by matching their ratings and dates with public ratings on the Internet Movie Database (IMDb). In December 2009, a class-action lawsuit was filed against Netflix over the release of the data, and in March 2010 Netflix settled it and canceled a planned second competition, which was to use data with demographic information; the release of the data to the UCI Machine Learning Repository after the competition, announced in its rules, did not take place, and Netflix no longer distributes the data set. The files of the upstream source used here are a copy of the original data set uploaded to the Internet Archive in 2016.
+
+The networks have the following properties:
+
+- Nodes: `is_user`, whether the node is a customer (otherwise a movie); `user_id`, the anonymized id of the customer (-1 for movies); `movie_id`, the id of the movie in the data set (-1 for customers); `title` and `year`, the title of the movie and its year of release (-1 if not given, and for customers).
+- Edges: `time`, the date of the rating, as Unix time (at midnight UTC).
+- `training`, edges: `rating`, the rating in stars; `probe`, whether the rating is in the probe set.
+
+The files of this entry are available only with an access token, since the license of the data set forbids redistributing it without permission from Netflix."""
+tags = ['Social', 'Online', 'Preferences', 'Weighted', 'Timestamps', 'Metadata']
+url = ['https://archive.org/details/nf_prize_dataset.tar']
+citation = [('J. Bennett, S. Lanning, "The Netflix Prize." Proceedings of KDD Cup and Workshop 2007, 3-6 (2007)', 'https://www.cs.uic.edu/~liub/KDD-cup-2007/proceedings/The-Netflix-Prize-Bennett.pdf')]
+icon_hash = None
+upstream_license = ("Netflix Prize data license (no redistribution)", "https://archive.org/details/nf_prize_dataset.tar")
+restricted = True
+# The data set, as distributed by Netflix for the Netflix Prize, is no longer
+# available from Netflix; this is a copy uploaded to the Internet Archive in
+# 2016, checked against the MD5 sum listed for the item
+upstream_prefix = 'https://archive.org/download/nf_prize_dataset.tar'
+fname = 'nf_prize_dataset.tar.gz'
+md5sum = 'a8f23d2d76461211c6b4c0ca6df2547d'
+files = [((f'{fname}:download/training_set.tar', f'{fname}:download/movie_titles.txt',
+           f'{fname}:download/probe.txt'), 'training', None),
+         ((f'{fname}:download/qualifying.txt', f'{fname}:download/movie_titles.txt'),
+          'qualifying', None)]
+
+def fetch_upstream(force=False):
+    name = __name__.split(".")[-1]
+    path = f"cache/upstream/{name}/{fname}"
+    if not force and os.path.exists(path):
+        return
+    os.makedirs(os.path.dirname(path), exist_ok=True)
+    def download():
+        h = hashlib.md5()
+        with open_url(f"{upstream_prefix}/{fname}") as f, \
+             open(path + ".part", "wb") as out:
+            while True:
+                buf = f.read(1 << 24)
+                if not buf:
+                    break
+                h.update(buf)
+                out.write(buf)
+        return h.hexdigest()
+    if retry_download(download, tries=3) != md5sum:
+        raise RuntimeError(f"unexpected content for {fname} (left in {path}.part)")
+    os.replace(path + ".part", path)
+
+def read_titles(name, fname):
+    "Movie ids, years of release (-1 if not given) and titles."
+    ids, years, titles = [], [], []
+    with open_upstream_file(name, fname, "r", encoding="latin1") as f:
+        for line in f:
+            line = line.rstrip("\r\n")
+            if line == "":
+                continue
+            mid, year, mtitle = line.split(",", 2)
+            ids.append(int(mid))
+            years.append(-1 if year == "NULL" else int(year))
+            titles.append(mtitle)
+    assert ids == list(range(1, len(ids) + 1))
+    return years, titles
+
+def read_movie_blocks(f):
+    """Pairs (movie id, lines) of a file of blocks "<movie id>:" followed by
+    lines (as in qualifying.txt, probe.txt and the training files)."""
+    mid = None
+    rows = []
+    for line in f:
+        line = line.strip()
+        if line == "":
+            continue
+        if line.endswith(":"):
+            if mid is not None:
+                yield mid, rows
+            mid = int(line[:-1])
+            rows = []
+        else:
+            rows.append(line)
+    if mid is not None:
+        yield mid, rows
+
+def to_unix_days(dates):
+    "Unix time (at midnight UTC) of dates in the format YYYY-MM-DD."
+    return (numpy.array(dates, dtype="datetime64[D]")
+            .astype("int64") * 86400)
+
+# For checks only: if set, only this many movies are read from the training set
+_check_movies = None
+
+def new_graph(user_ids, years, titles):
+    """Bipartite graph of the movies (vertices 0..M-1, by movie id) and
+    users (the remaining ones, by user id), without edges."""
+    M = len(titles)
+    g = Graph(directed=False)
+    g.add_vertex(M + len(user_ids))
+    g.vp.is_user = g.new_vp("bool")
+    g.vp.is_user.a[M:] = True
+    g.vp.user_id = g.new_vp("int32_t", val=-1)
+    g.vp.user_id.a[M:] = user_ids
+    g.vp.movie_id = g.new_vp("int32_t", val=-1)
+    g.vp.movie_id.a[:M] = numpy.arange(1, M + 1)
+    g.vp.year = g.new_vp("int16_t", val=-1)
+    g.vp.year.a[:M] = years
+    g.vp.title = g.new_vp("string")
+    for v, t in enumerate(titles):
+        g.vp.title[v] = t
+    return g
+
+@cache_network()
+@coerce_props()
+@annotate()
+def parse(alts=None):
+    global files
+    name = __name__.split(".")[-1]
+    years, titles = read_titles(name, files[0][0][1])
+    M = len(titles)
+
+    # Probe set: pairs of the training set, as movie id -> user ids
+    probe = {}
+    with open_upstream_file(name, files[0][0][2], "r") as f:
+        for mid, rows in read_movie_blocks(f):
+            probe[mid] = numpy.array(rows, dtype="int32")
+    n_probe = sum(len(x) for x in probe.values())
+
+    # Training set: one file per movie, "<movie id>:" followed by lines
+    # "<user id>,<rating>,<date>"
+    movies, users, ratings, times, in_probe = [], [], [], [], []
+    with open_upstream_file(name, files[0][0][0], "rb") as f:
+        with tarfile.open(fileobj=f, mode="r|") as tf:
+            for i, m in enumerate(tf):
+                if not m.isfile():
+                    continue
+                if _check_movies is not None and len(movies) >= _check_movies:
+                    break
+                # (read whole: the members of a streamed archive are not
+                # seekable, as io.TextIOWrapper expects)
+                with tf.extractfile(m) as mf:
+                    text = mf.read().decode("ascii")
+                blocks = list(read_movie_blocks(text.splitlines()))
+                assert len(blocks) == 1
+                mid, rows = blocks[0]
+                assert m.name.endswith(f"mv_{mid:07d}.txt"), (m.name, mid)
+                u, r, d = zip(*(row.split(",") for row in rows))
+                movies.append(numpy.full(len(u), mid - 1, dtype="int32"))
+                u = numpy.array(u, dtype="int32")
+                users.append(u)
+                in_probe.append(numpy.isin(u, probe.get(mid, [])))
+                ratings.append(numpy.array(r, dtype="int8"))
+                times.append(to_unix_days(d).astype("int32"))
+    movies = numpy.concatenate(movies)
+    users = numpy.concatenate(users)
+    ratings = numpy.concatenate(ratings)
+    times = numpy.concatenate(times)
+    in_probe = numpy.concatenate(in_probe)
+    del probe
+    user_ids = numpy.unique(users)
+    users = numpy.searchsorted(user_ids, users).astype("int32")
+
+    for fnames, alt, fmt in files:
+        if alts is not None and alt not in alts:
+            continue
+        g = new_graph(user_ids, years, titles)
+        if alt == "training":
+            g.ep.rating = g.new_ep("int16_t")
+            g.ep.time = g.new_ep("int32_t")
+            g.ep.probe = g.new_ep("bool")
+            # Edges added in chunks, to keep the memory use low
+            step = 10 ** 7
+            for i in range(0, len(movies), step):
+                sl = slice(i, i + step)
+                g.add_edge_list(numpy.column_stack((movies[sl], M + users[sl])))
+            g.ep.rating.a = ratings
+            g.ep.time.a = times
+            g.ep.probe.a = in_probe
+            if _check_movies is None:
+                assert in_probe.sum() == n_probe
+            # (not needed any more)
+            movies = users = ratings = times = in_probe = None
+        else:
+            # Qualifying set: "<user id>,<date>" lines, ratings withheld
+            qm, qu, qt = [], [], []
+            with open_upstream_file(name, fnames[0], "r") as f:
+                for mid, rows in read_movie_blocks(f):
+                    u, d = zip(*(row.split(",") for row in rows))
+                    qm.append(numpy.full(len(u), mid - 1, dtype="int64"))
+                    qu.append(numpy.array(u, dtype="int64"))
+                    qt.append(to_unix_days(d))
+            qm, qu, qt = (numpy.concatenate(x) for x in (qm, qu, qt))
+            uidx = numpy.searchsorted(user_ids, qu)
+            if _check_movies is None:
+                assert (user_ids[uidx] == qu).all()
+            else:
+                ok = (uidx < len(user_ids))
+                ok[ok] = user_ids[uidx[ok]] == qu[ok]
+                qm, uidx, qt = qm[ok], uidx[ok], qt[ok]
+            g.ep.time = g.new_ep("int32_t")
+            g.add_edge_list(numpy.column_stack((qm, M + uidx)))
+            g.ep.time.a = qt
+        yield alt, g