History of yambda
The changes to the source code of this entry in the repository, newest first, each with the differences it made to the entry's files. They are also listed on GitLab.
[entries] Add the date when each entry was added, and remove icon_hash beff7c9
The dates are those of the commits that first added the entries' files (2020-08-11 for the entries of the initial commit); renamed entries keep the date of their first name. Changed: none
diff --git a/entries/yambda/__init__.py b/entries/yambda/__init__.py
index 79d0085..c5ccc53 100644
--- a/entries/yambda/__init__.py
+++ b/entries/yambda/__init__.py
@@ -44,7 +44,7 @@ The networks have the following properties:
tags = ['Social', 'Online', 'Preferences', 'Bipartite', 'Unweighted', 'Multigraph', 'Timestamps', 'Temporal', 'Metadata']
url = 'https://huggingface.co/datasets/yandex/yambda'
citation = [('A. Ploshkin, V. Tytskiy, A. Pismenny, V. Baikalov, E. Taychinov, A. Permiakov, D. Burlakov, E. Krofto, N. Savushkin, "Yambda-5B -- A Large-Scale Multi-modal Dataset for Ranking And Retrieval." arXiv:2505.22238 (2025)', 'https://arxiv.org/abs/2505.22238')]
-icon_hash = None
+date_added = '2026-10-28'
upstream_license = ('Apache 2.0', 'https://huggingface.co/datasets/yandex/yambda/blob/main/LICENSE')
# networks of more than about 300 million edges: only stored and served in the
# gt format
[entries] Prefix the properties in the descriptions with (Node), (Edge) or (Graph) bc3dbbd
The lists of properties at the end of the descriptions marked edge properties with "Edge" but left node properties unmarked, or grouped them in "Nodes: `a`, ...; `b`, ..." items, so the kind of a property was often unclear. Every item now starts with (Node), (Edge) or (Graph), followed by the property names, a colon and the explanation, one item per property or group of properties; qualifications (kinds of nodes, networks) follow the names in parentheses. Properties that were only named got a short explanation, and a few items that named the wrong kind of property were corrected.
diff --git a/entries/yambda/__init__.py b/entries/yambda/__init__.py
index 2224873..79d0085 100644
--- a/entries/yambda/__init__.py
+++ b/entries/yambda/__init__.py
@@ -32,15 +32,15 @@ The largest sample (`5b`, one million users and 4.79 billion interactions) is no
The networks have the following properties:
-- `id`: the number of the user or of the track in the data.
-- `is_user`: whether the node is a user (otherwise a track).
-- `track_length`: the length of the track, in seconds (-1 for users, and for tracks without listens).
-- `artists`: the numbers of the artists of the track (empty for users).
-- `albums`: the numbers of the albums containing the track (empty for users).
-- Edge `time`: the time of the interaction, in seconds since the beginning of the data (in steps of 5 seconds).
-- Edge `event`: the kind of interaction: 0 for a listen, 1 for a like, 2 for a dislike, 3 for an unlike and 4 for an undislike.
-- Edge `organic`: whether the interaction was initiated by the user (otherwise it followed a recommendation).
-- Edge `played_ratio`: for listens, the percentage of the track that was played (it can exceed 100); -1 for the other interactions."""
+- (Node) `id`: the number of the user or of the track in the data.
+- (Node) `is_user`: whether the node is a user (otherwise a track).
+- (Node) `track_length`: the length of the track, in seconds (-1 for users, and for tracks without listens).
+- (Node) `artists`: the numbers of the artists of the track (empty for users).
+- (Node) `albums`: the numbers of the albums containing the track (empty for users).
+- (Edge) `time`: the time of the interaction, in seconds since the beginning of the data (in steps of 5 seconds).
+- (Edge) `event`: the kind of interaction: 0 for a listen, 1 for a like, 2 for a dislike, 3 for an unlike and 4 for an undislike.
+- (Edge) `organic`: whether the interaction was initiated by the user (otherwise it followed a recommendation).
+- (Edge) `played_ratio`: for listens, the percentage of the track that was played (it can exceed 100); -1 for the other interactions."""
tags = ['Social', 'Online', 'Preferences', 'Bipartite', 'Unweighted', 'Multigraph', 'Timestamps', 'Temporal', 'Metadata']
url = 'https://huggingface.co/datasets/yandex/yambda'
citation = [('A. Ploshkin, V. Tytskiy, A. Pismenny, V. Baikalov, E. Taychinov, A. Permiakov, D. Burlakov, E. Krofto, N. Savushkin, "Yambda-5B -- A Large-Scale Multi-modal Dataset for Ranking And Retrieval." arXiv:2505.22238 (2025)', 'https://arxiv.org/abs/2505.22238')]
[entries] New entry: yambda 3f7e176
diff --git a/entries/yambda/__init__.py b/entries/yambda/__init__.py
new file mode 100644
index 0000000..2224873
--- /dev/null
+++ b/entries/yambda/__init__.py
@@ -0,0 +1,199 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2026 Tiago de Paula Peixoto <tiago@skewed.de>
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+
+import http.client
+import time
+
+from .. import *
+import process_entry
+
+title = "Yandex Music user-track interactions (Yambda)"
+description = r"""Bipartite networks of the users of the music streaming service Yandex Music and the tracks they listened to or rated, from the Yambda dataset released by Yandex. The data are the interactions of a sample of users with the tracks over about four years (the dates are not given): listens (with the fraction of the track played) and the explicit feedback of the users, i.e. likes and dislikes and their withdrawal (unlikes and undislikes). Each interaction is marked as organic (initiated by the user) or as the result of a recommendation of the service. Each node is a user or a track, and an edge connects a user to a track for each interaction (a multigraph). Yambda comes in three sizes, from nested samples of users; there is one network per size:
+
+- `50m`: 10,000 users, 47.8 million interactions.
+- `500m`: 100,000 users, 480.3 million interactions (only available in the gt format).
+
+The largest sample (`5b`, one million users and 4.79 billion interactions) is not included. The times of the interactions are given relative to the beginning of the data (not as dates), in bins of 5 seconds. The audio embeddings of the tracks are not included.
+
+The networks have the following properties:
+
+- `id`: the number of the user or of the track in the data.
+- `is_user`: whether the node is a user (otherwise a track).
+- `track_length`: the length of the track, in seconds (-1 for users, and for tracks without listens).
+- `artists`: the numbers of the artists of the track (empty for users).
+- `albums`: the numbers of the albums containing the track (empty for users).
+- Edge `time`: the time of the interaction, in seconds since the beginning of the data (in steps of 5 seconds).
+- Edge `event`: the kind of interaction: 0 for a listen, 1 for a like, 2 for a dislike, 3 for an unlike and 4 for an undislike.
+- Edge `organic`: whether the interaction was initiated by the user (otherwise it followed a recommendation).
+- Edge `played_ratio`: for listens, the percentage of the track that was played (it can exceed 100); -1 for the other interactions."""
+tags = ['Social', 'Online', 'Preferences', 'Bipartite', 'Unweighted', 'Multigraph', 'Timestamps', 'Temporal', 'Metadata']
+url = 'https://huggingface.co/datasets/yandex/yambda'
+citation = [('A. Ploshkin, V. Tytskiy, A. Pismenny, V. Baikalov, E. Taychinov, A. Permiakov, D. Burlakov, E. Krofto, N. Savushkin, "Yambda-5B -- A Large-Scale Multi-modal Dataset for Ranking And Retrieval." arXiv:2505.22238 (2025)', 'https://arxiv.org/abs/2505.22238')]
+icon_hash = None
+upstream_license = ('Apache 2.0', 'https://huggingface.co/datasets/yandex/yambda/blob/main/LICENSE')
+# networks of more than about 300 million edges: only stored and served in the
+# gt format
+skip_conversion = ['500m']
+
+revision = 'dd6f3a19eef5866e346c3270e098baa641a44948'
+upstream_prefix = f'https://huggingface.co/datasets/yandex/yambda/resolve/{revision}'
+_maps = ('artist_item_mapping.parquet', 'album_item_mapping.parquet')
+files = [(('flat/50m/multi_event.parquet',) + _maps, '50m', None),
+ (('flat/500m/multi_event.parquet',) + _maps, '500m', None)]
+
+def _upstream_dir():
+ return f"cache/upstream/{__name__.split('.')[-1]}"
+
+def _download(url, target, tries=20):
+ """Download `url` to `target`, resuming (with range requests) when the
+ connection is closed early, which can end a download without an error;
+ the size is checked against the one announced by the server."""
+ part = target + ".part"
+ size = None
+ for i in range(tries):
+ have = os.path.getsize(part) if os.path.exists(part) else 0
+ if size is not None and have == size:
+ break
+ headers = {"Range": f"bytes={have}-"} if have > 0 else {}
+ try:
+ with open_url(url, headers=headers) as r:
+ if have > 0 and r.status != 206:
+ have = 0 # the server ignored the range
+ total = r.headers.get("Content-Range")
+ size = (int(total.rsplit("/", 1)[1]) if total else
+ int(r.headers["Content-Length"]))
+ with open(part, "ab" if have > 0 else "wb") as out:
+ shutil.copyfileobj(r, out, 1 << 24)
+ except (OSError, http.client.HTTPException):
+ if i == tries - 1:
+ raise
+ time.sleep(10 * (i + 1))
+ if size is None or os.path.getsize(part) != size:
+ raise IOError(f"incomplete download of {url}")
+ os.replace(part, target)
+
+def fetch_upstream(force=False):
+ # The parquet files (already compressed) are stored as they are
+ fnames = list(dict.fromkeys(f for fs, alt, fmt in files for f in fs))
+ if process_entry._check_health:
+ for f in process_entry.health_sample(fnames):
+ with open_url(f"{upstream_prefix}/{f}") as r:
+ r.read(1)
+ return
+ for f in fnames:
+ target = f"{_upstream_dir()}/{f}"
+ if not force and os.path.exists(target):
+ continue
+ os.makedirs(os.path.dirname(target), exist_ok=True)
+ _download(f"{upstream_prefix}/{f}", target)
+
+# For checks only: if set, only (about) this many events are read
+_check_rows = None
+
+_EVENTS = {"listen": 0, "like": 1, "dislike": 2, "unlike": 3, "undislike": 4}
+
+def _batches(path, columns):
+ import pyarrow.parquet as pq
+ n = 0
+ for b in pq.ParquetFile(path).iter_batches(batch_size=1 << 22,
+ columns=columns):
+ yield b
+ n += b.num_rows
+ if _check_rows is not None and n >= _check_rows:
+ break
+
+@cache_network()
+@coerce_props()
+@annotate()
+def parse(alts=None):
+ import pyarrow
+ import pyarrow.compute as pc
+ import pyarrow.parquet as pq
+ pyarrow.set_cpu_count(4)
+ pyarrow.set_io_thread_count(4)
+ for fnames, alt, fmt in files:
+ if alts is not None and alt not in alts:
+ continue
+ path = f"{_upstream_dir()}/{fnames[0]}"
+ # the users and tracks (a first pass), in the order of their numbers
+ users = numpy.zeros(1, dtype="bool")
+ items = numpy.zeros(1, dtype="bool")
+ for b in _batches(path, ["uid", "item_id"]):
+ u = b.column("uid").to_numpy()
+ i = b.column("item_id").to_numpy()
+ if u.max() >= len(users):
+ users = numpy.r_[users, numpy.zeros(u.max() + 1 - len(users), dtype="bool")]
+ if i.max() >= len(items):
+ items = numpy.r_[items, numpy.zeros(i.max() + 1 - len(items), dtype="bool")]
+ users[u] = True
+ items[i] = True
+ uids = numpy.flatnonzero(users)
+ iids = numpy.flatnonzero(items)
+ nu = len(uids)
+ uidx = numpy.cumsum(users) - 1
+ iidx = numpy.cumsum(items) - 1 + nu
+ del users, items
+
+ g = Graph(directed=False)
+ g.add_vertex(nu + len(iids))
+ g.vp.id = g.new_vp("int64_t", vals=numpy.r_[uids, iids])
+ g.vp.is_user = g.new_vp("bool")
+ g.vp.is_user.a[:nu] = True
+ g.vp.track_length = g.new_vp("int32_t", val=-1)
+ # (the properties have the smallest types that hold the values, to
+ # save memory)
+ g.ep.time = g.new_ep("int32_t")
+ g.ep.event = g.new_ep("int16_t")
+ g.ep.organic = g.new_ep("bool")
+ g.ep.played_ratio = g.new_ep("int16_t")
+ eprops = [g.ep.time, g.ep.event, g.ep.organic, g.ep.played_ratio]
+ for b in _batches(path, ["uid", "item_id", "timestamp", "is_organic",
+ "event_type", "played_ratio_pct",
+ "track_length_seconds"]):
+ ev = b.column("event_type")
+ codes = numpy.array([_EVENTS[x] for x in ev.dictionary.to_pylist()],
+ dtype="int64")
+ ev = codes[ev.indices.to_numpy()]
+ s = uidx[b.column("uid").to_numpy()]
+ t = iidx[b.column("item_id").to_numpy()]
+ ratio = pc.fill_null(b.column("played_ratio_pct").cast("int64"), -1)
+ g.add_edge_list(numpy.column_stack(
+ [s, t, b.column("timestamp").to_numpy().astype("int64") * 5,
+ ev, b.column("is_organic").to_numpy(),
+ ratio.to_numpy()]), eprops=eprops)
+ tl = b.column("track_length_seconds")
+ sel = numpy.flatnonzero(pc.is_valid(tl).to_numpy(zero_copy_only=False))
+ g.vp.track_length.a[t[sel]] = tl.take(sel).to_numpy()
+ del b, s, t, ev, ratio
+
+ # the artists and albums of the tracks
+ for k, fname in (("artists", fnames[1]), ("albums", fnames[2])):
+ col = "artist_id" if k == "artists" else "album_id"
+ m = pq.read_table(f"{_upstream_dir()}/{fname}").sort_by([("item_id", "ascending"), (col, "ascending")])
+ it = m.column("item_id").to_numpy()
+ ids = m.column(col).to_numpy()
+ ok = it < len(iidx)
+ it, ids = it[ok], ids[ok]
+ pos = numpy.searchsorted(iids, it)
+ pos[pos == len(iids)] = 0
+ ok = iids[pos] == it
+ p = g.new_vp("vector<int32_t>")
+ for v, x in zip(pos[ok] + nu, ids[ok]):
+ p[v].append(x)
+ g.vp[k] = p
+ yield alt, g