History of movies_tmdb
The changes to the source code of this entry in the repository, newest first, each with the differences it made to the entry's files. They are also listed on GitLab.
[entries] New entry: movies_tmdb 9ea4d1c
diff --git a/entries/movies_tmdb/__init__.py b/entries/movies_tmdb/__init__.py
new file mode 100644
index 0000000..3e7d5b9
--- /dev/null
+++ b/entries/movies_tmdb/__init__.py
@@ -0,0 +1,203 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2020-2026 Tiago de Paula Peixoto <tiago@skewed.de>
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+
+from .. import *
+import ast
+
+title = "The Movies Dataset: TMDB cast, crew and keywords (2017)"
+description = """Bipartite networks of movies and the people credited in them, and of movies and the keywords describing them, from "The Movies Dataset" assembled by R. Banik and published on Kaggle (version 7, November 2017). The dataset covers the about 45,000 movies listed in the full MovieLens dataset of GroupLens (movies released on or before July 2017); their metadata and credits were collected from the API of The Movie Database (TMDB), a community-edited online movie database. Persons are identified by their TMDB person ids, movies by their TMDB movie ids.
+
+- `cast`: a node is either a movie or an actor, and an edge connects an actor to a movie in whose cast the actor is credited.
+- `crew`: a node is either a movie or a crew member (director, writer, producer, composer, editor, etc.), and an edge connects a person to a movie in which the person has a crew credit.
+- `keywords`: a node is either a movie or a keyword (a short descriptive tag assigned to movies by TMDB contributors, e.g. "based on novel", "time travel", "duringcreditsstinger"), and an edge connects a keyword to a movie tagged with it.
+
+In `cast` and `crew`, each edge is one TMDB credit; a person credited more than once in the same movie (e.g. playing two characters, or as both director and writer) is connected to it by parallel edges. In `keywords`, each edge is one keyword assignment (there are no parallel edges). Only movies with at least one credit or keyword of the respective kind are included (2,415 of the 45,433 movies with credits have an empty cast list, and 772 an empty crew list; 14,340 of the 45,432 movies in the keyword file have no keywords). Movie metadata is given as in the source (the values of `budget`, `revenue` and `runtime` are 0 when unknown); the TMDB genres of a movie are given in `genres`.
+
+The networks have the following properties:
+
+- All nodes: `is_movie`, whether the node is a movie (otherwise it is a person, or a keyword in `keywords`); `tmdb_id`, the TMDB id of the movie, person or keyword; `name`, the title of the movie (in English, as in TMDB), the name of the person, or the keyword.
+- Person nodes (`cast` and `crew` only): `gender`, as coded by TMDB (0: not specified, 1: female, 2: male); -1 for movies.
+- Movie nodes: `original_title`; `imdb_id`, the IMDb identifier; `release_date` (YYYY-MM-DD) and `year`, the year of release (-1 if unknown, or for persons and keywords); `genres`, the list of TMDB genres; `original_language` (ISO 639-1 code); `runtime` in minutes; `budget` and `revenue` in US dollars; `vote_average` and `vote_count`, the average rating (0-10) and number of ratings by TMDB users; `popularity`, TMDB's popularity score at collection time; `adult`, whether the movie is flagged as adult content.
+- `cast` edges: `character`, the name of the role; `order`, the position of the credit in the movie's cast list (0 for the top-billed actor); `credit_id`, the TMDB credit identifier.
+- `crew` edges: `job` (e.g. Director, Screenplay, Original Music Composer); `department` (e.g. Directing, Writing, Sound); `credit_id`, the TMDB credit identifier.
+
+**Relation to the hypergraphx movie-cast datasets.** The `movies-cast-<genre>` datasets and the `movies-cast-genre-multiplex` dataset of the hypergraphx-data repository were built from the same files. To obtain the actors and casts of one genre, take the movie nodes of the `cast` network whose `genres` contain it and that have at least two distinct adjacent actors, together with these actors; hypergraphx additionally merges the movies of a genre with exactly the same set of actors into a single group weighted by their number (ignoring repeated credits of an actor in a movie). This reproduces exactly the node and group counts of all 20 hypergraphx genre datasets and of the multiplex dataset (the union of the genre layers). Minor differences: in the source file of movie metadata, 29 movies appear twice (identical rows, or differing only in the popularity score or vote count; the first row is used here), which hypergraphx counts twice in the weights, whereas here each movie appears once; and three rows of that file are broken in two by a formatting error, which hypergraphx reads as rows without title and release date, whereas here they are repaired.
+
+Note that TMDB is community-edited, so the credits, keywords and metadata are of varying completeness. This product uses the TMDB API but is not endorsed or certified by TMDB."""
+tags = ['Economic', 'Employment', 'Affiliation', 'Unweighted', 'Multigraph', 'Metadata']
+url = 'https://www.kaggle.com/datasets/rounakbanik/the-movies-dataset'
+citation = [('R. Banik, "The Movies Dataset." Kaggle, version 7 (2017)', 'https://www.kaggle.com/datasets/rounakbanik/the-movies-dataset')]
+icon_hash = None
+upstream_license = ('Creative Commons CC0 1.0 Universal', 'https://creativecommons.org/publicdomain/zero/1.0/')
+upstream_prefix = 'kaggle://rounakbanik/the-movies-dataset/7'
+files = [(('the-movies-dataset.zip:movies_metadata.csv',
+ 'the-movies-dataset.zip:credits.csv'), 'cast', None),
+ (('the-movies-dataset.zip:movies_metadata.csv',
+ 'the-movies-dataset.zip:credits.csv'), 'crew', None),
+ (('the-movies-dataset.zip:movies_metadata.csv',
+ 'the-movies-dataset.zip:keywords.csv'), 'keywords', None)]
+
+def fetch_upstream(force=False):
+ return fetch_upstream_files(__name__.split(".")[-1], upstream_prefix, files,
+ force)
+
+def read_movies(f):
+ """Movie metadata by TMDB id (first occurrence of duplicated ids), with
+ the rows broken in two by an embedded line break joined back."""
+ rows = []
+ reader = csv.reader(f)
+ header = next(reader)
+ iid = header.index("id")
+ ipop = header.index("popularity")
+ iov = header.index("overview")
+ for row in reader:
+ if not row[iid].isdigit():
+ # Continuation of the previous row: its first field continues
+ # the overview, and the rest are the fields after it
+ prev = rows[-1]
+ prev[iov] += "\n" + row[0]
+ prev[ipop:] = row[1:1 + len(header) - ipop]
+ continue
+ row += [""] * (len(header) - len(row))
+ rows.append(row)
+ movies = {}
+ for row in rows:
+ m = dict(zip(header, row))
+ movies.setdefault(int(m["id"]), m)
+ return movies
+
+def read_credits(f, kind):
+ """Credits of `kind` ("cast", "crew" or "keywords") by TMDB movie id
+ (first occurrence of duplicated ids)."""
+ credits = {}
+ for row in csv.DictReader(f):
+ mid = int(row["id"])
+ if mid in credits:
+ continue
+ credits[mid] = ast.literal_eval(row[kind])
+ return credits
+
+def to_float(x):
+ try:
+ return float(x)
+ except (TypeError, ValueError):
+ return numpy.nan
+
+def to_int(x, default=0):
+ try:
+ return int(float(x))
+ except (TypeError, ValueError):
+ return default
+
+@cache_network()
+@coerce_props()
+@annotate()
+def parse(alts=None):
+ global files
+ name = __name__.split(".")[-1]
+ for fnames, alt, fmt in files:
+ if alts is not None and alt not in alts:
+ continue
+ with open_upstream_file(name, fnames[0], "r") as f:
+ movies = read_movies(f)
+ with open_upstream_file(name, fnames[1], "r") as f:
+ credits = read_credits(f, alt)
+
+ g = Graph(directed=False)
+ vp = {}
+ vprops = [("is_movie", "bool"), ("tmdb_id", "int"),
+ ("name", "string")]
+ if alt != "keywords":
+ vprops.append(("gender", "int"))
+ vprops += [("original_title", "string"), ("imdb_id", "string"),
+ ("release_date", "string"), ("year", "int"),
+ ("genres", "vector<string>"),
+ ("original_language", "string"), ("runtime", "double"),
+ ("budget", "int64_t"), ("revenue", "int64_t"),
+ ("vote_average", "double"), ("vote_count", "int"),
+ ("popularity", "double"), ("adult", "bool")]
+ for k, t in vprops:
+ vp[k] = g.vp[k] = g.new_vp(t)
+ if alt == "cast":
+ ekeys = ["character", "order", "credit_id"]
+ g.ep.character = g.new_ep("string")
+ g.ep.order = g.new_ep("int")
+ elif alt == "crew":
+ ekeys = ["job", "department", "credit_id"]
+ g.ep.job = g.new_ep("string")
+ g.ep.department = g.new_ep("string")
+ else:
+ ekeys = []
+ if alt != "keywords":
+ g.ep.credit_id = g.new_ep("string")
+
+ person_v = {}
+ elist = []
+ for mid in sorted(credits):
+ clist = credits[mid]
+ if len(clist) == 0:
+ continue
+ m = movies[mid]
+ v = g.add_vertex()
+ vp["is_movie"][v] = True
+ vp["tmdb_id"][v] = mid
+ vp["name"][v] = m["title"]
+ if "gender" in vp:
+ vp["gender"][v] = -1
+ vp["original_title"][v] = m["original_title"]
+ vp["imdb_id"][v] = m["imdb_id"]
+ vp["release_date"][v] = m["release_date"]
+ vp["year"][v] = to_int(m["release_date"][:4], -1)
+ vp["genres"][v] = [x["name"] for x in ast.literal_eval(m["genres"])]
+ vp["original_language"][v] = m["original_language"]
+ vp["runtime"][v] = to_float(m["runtime"])
+ vp["budget"][v] = to_int(m["budget"])
+ vp["revenue"][v] = to_int(m["revenue"])
+ vp["vote_average"][v] = to_float(m["vote_average"])
+ vp["vote_count"][v] = to_int(m["vote_count"])
+ vp["popularity"][v] = to_float(m["popularity"])
+ vp["adult"][v] = m["adult"] == "True"
+ if alt == "keywords":
+ for c in clist:
+ kid = c["id"]
+ u = person_v.get(kid)
+ if u is None:
+ u = person_v[kid] = g.add_vertex()
+ vp["tmdb_id"][u] = kid
+ vp["name"][u] = c["name"]
+ vp["year"][u] = -1
+ elist.append((int(u), int(v)))
+ continue
+ seen = set()
+ for c in clist:
+ if c["credit_id"] in seen:
+ continue
+ seen.add(c["credit_id"])
+ pid = c["id"]
+ u = person_v.get(pid)
+ if u is None:
+ u = person_v[pid] = g.add_vertex()
+ vp["tmdb_id"][u] = pid
+ vp["name"][u] = c["name"]
+ vp["gender"][u] = c["gender"]
+ vp["year"][u] = -1
+ elif vp["gender"][u] == 0:
+ vp["gender"][u] = c["gender"]
+ elist.append((int(u), int(v)) + tuple(c[k] for k in ekeys))
+ g.add_edge_list(elist, eprops=[g.ep[k] for k in ekeys])
+ yield alt, g