History of kuairand
The changes to the source code of this entry in the repository, newest first, each with the differences it made to the entry's files. They are also listed on GitLab.
[entries] Add the date when each entry was added, and remove icon_hash beff7c9
The dates are those of the commits that first added the entries' files (2020-08-11 for the entries of the initial commit); renamed entries keep the date of their first name. Changed: none
diff --git a/entries/kuairand/__init__.py b/entries/kuairand/__init__.py
index 1a29b48..935a2c3 100644
--- a/entries/kuairand/__init__.py
+++ b/entries/kuairand/__init__.py
@@ -67,7 +67,7 @@ tags = ['Social', 'Online', 'Preferences', 'Bipartite', 'Multigraph', 'Timestamp
url = ['https://kuairand.com/', 'https://doi.org/10.5281/zenodo.10439422']
citation = [('C. Gao, S. Li, Y. Zhang, J. Chen, B. Li, W. Lei, P. Jiang, X. He, "KuaiRand: An Unbiased Sequential Recommendation Dataset with Randomly Exposed Videos." Proceedings of the 31st ACM International Conference on Information and Knowledge Management (CIKM), 3953-3957 (2022)',
'https://doi.org/10.1145/3511808.3557624')]
-icon_hash = None
+date_added = '2026-10-28'
upstream_license = ('CC BY-SA 4.0', 'https://github.com/chongminggao/KuaiRand/blob/f8dbf6678b3c9594050e3e813aeff0c942260ec4/LICENSE')
upstream_prefix = 'https://zenodo.org/records/10439422/files'
# networks of more than about 300 million edges: only stored and served in the
[entries] Prefix the properties in the descriptions with (Node), (Edge) or (Graph) bc3dbbd
The lists of properties at the end of the descriptions marked edge properties with "Edge" but left node properties unmarked, or grouped them in "Nodes: `a`, ...; `b`, ..." items, so the kind of a property was often unclear. Every item now starts with (Node), (Edge) or (Graph), followed by the property names, a colon and the explanation, one item per property or group of properties; qualifications (kinds of nodes, networks) follow the names in parentheses. Properties that were only named got a short explanation, and a few items that named the wrong kind of property were corrected.
diff --git a/entries/kuairand/__init__.py b/entries/kuairand/__init__.py
index 275fa76..1a29b48 100644
--- a/entries/kuairand/__init__.py
+++ b/entries/kuairand/__init__.py
@@ -37,32 +37,32 @@ The user and video identifiers were re-indexed by the source. The anonymized cat
The networks have the following properties:
-- `is_user`: whether the node is a user (otherwise it is a video).
-- `id`: the identifier of the user or video in the dataset.
-- `active_degree` (users): the activity level of the user, e.g. `full_active`, `high_active`, `middle_active`, `2_14_day_new`, `30day_retention`, `UNKNOWN`.
-- `is_lowactive_period`, `is_live_streamer`, `is_video_author` (users): flags of the user, as given by the source (with a few values other than 0 and 1, e.g. -124).
-- `follow_user_num`, `fans_user_num`, `friend_user_num` (users): the numbers of users that the user follows, of followers, and of friends.
-- `register_days` (users): the number of days since the user registered.
-- `onehot_features` (users): the 18 anonymized categorical features of the user (`onehot_feat0` to `onehot_feat17`; NaN if missing).
-- `author_id` (videos): the identifier of the author of the video (-1 if unknown).
-- `video_type` (videos): `NORMAL` or `AD`.
-- `upload_time` (videos): the upload date of the video, as Unix time (-1 if unknown).
-- `upload_type` (videos): the way the video was uploaded, e.g. `LongImport`, `ShortImport`, `Web`, `Kmovie`.
-- `visible_status` (videos): the visibility status of the video (NaN if unknown).
-- `duration` (videos): the duration of the video, in milliseconds (NaN if unknown).
-- `width`, `height` (videos): the size of the video, in pixels (NaN if unknown).
-- `music_id` (videos): the identifier of the background music of the video (-1 if unknown).
-- `music_type` (videos): the type of the background music (NaN if unknown).
-- `tags` (videos): the ids of the categories of the video, comma-separated, e.g. `20,43`.
-- Edge `time`: the time of the exposure, as Unix time (with milliseconds).
-- Edge `click`: whether the user clicked on the video (in the two-column interface), or played it validly (in the single-column interface: for at least its duration, if it lasts at most 7 seconds, or for more than 7 seconds otherwise).
-- Edge `long_view`: whether the user watched the video for at least its duration, if it lasts at most 18 seconds, or for at least 18 seconds otherwise.
-- Edge `like`, `follow`, `comment`, `forward`, `hate`: whether the user liked the video, followed its author, commented on it, forwarded it, or marked it as disliked.
-- Edge `profile_enter`: whether the user visited the profile of the author.
-- Edge `play_time`: how long the user watched the video, in milliseconds.
-- Edge `profile_stay_time`, `comment_stay_time`: how long the user stayed on the profile of the author and in the comment section of the video, in milliseconds.
-- Edge `random`: whether the video was shown by the random intervention.
-- Edge `tab`: the interface (tab of the app) where the video was shown, a number from 0 to 14."""
+- (Node) `is_user`: whether the node is a user (otherwise it is a video).
+- (Node) `id`: the identifier of the user or video in the dataset.
+- (Node) `active_degree` (users): the activity level of the user, e.g. `full_active`, `high_active`, `middle_active`, `2_14_day_new`, `30day_retention`, `UNKNOWN`.
+- (Node) `is_lowactive_period`, `is_live_streamer`, `is_video_author` (users): flags of the user, as given by the source (with a few values other than 0 and 1, e.g. -124).
+- (Node) `follow_user_num`, `fans_user_num`, `friend_user_num` (users): the numbers of users that the user follows, of followers, and of friends.
+- (Node) `register_days` (users): the number of days since the user registered.
+- (Node) `onehot_features` (users): the 18 anonymized categorical features of the user (`onehot_feat0` to `onehot_feat17`; NaN if missing).
+- (Node) `author_id` (videos): the identifier of the author of the video (-1 if unknown).
+- (Node) `video_type` (videos): `NORMAL` or `AD`.
+- (Node) `upload_time` (videos): the upload date of the video, as Unix time (-1 if unknown).
+- (Node) `upload_type` (videos): the way the video was uploaded, e.g. `LongImport`, `ShortImport`, `Web`, `Kmovie`.
+- (Node) `visible_status` (videos): the visibility status of the video (NaN if unknown).
+- (Node) `duration` (videos): the duration of the video, in milliseconds (NaN if unknown).
+- (Node) `width`, `height` (videos): the size of the video, in pixels (NaN if unknown).
+- (Node) `music_id` (videos): the identifier of the background music of the video (-1 if unknown).
+- (Node) `music_type` (videos): the type of the background music (NaN if unknown).
+- (Node) `tags` (videos): the ids of the categories of the video, comma-separated, e.g. `20,43`.
+- (Edge) `time`: the time of the exposure, as Unix time (with milliseconds).
+- (Edge) `click`: whether the user clicked on the video (in the two-column interface), or played it validly (in the single-column interface: for at least its duration, if it lasts at most 7 seconds, or for more than 7 seconds otherwise).
+- (Edge) `long_view`: whether the user watched the video for at least its duration, if it lasts at most 18 seconds, or for at least 18 seconds otherwise.
+- (Edge) `like`, `follow`, `comment`, `forward`, `hate`: whether the user liked the video, followed its author, commented on it, forwarded it, or marked it as disliked.
+- (Edge) `profile_enter`: whether the user visited the profile of the author.
+- (Edge) `play_time`: how long the user watched the video, in milliseconds.
+- (Edge) `profile_stay_time`, `comment_stay_time`: how long the user stayed on the profile of the author and in the comment section of the video, in milliseconds.
+- (Edge) `random`: whether the video was shown by the random intervention.
+- (Edge) `tab`: the interface (tab of the app) where the video was shown, a number from 0 to 14."""
tags = ['Social', 'Online', 'Preferences', 'Bipartite', 'Multigraph', 'Timestamps', 'Metadata']
url = ['https://kuairand.com/', 'https://doi.org/10.5281/zenodo.10439422']
citation = [('C. Gao, S. Li, Y. Zhang, J. Chen, B. Li, W. Lei, P. Jiang, X. He, "KuaiRand: An Unbiased Sequential Recommendation Dataset with Randomly Exposed Videos." Proceedings of the 31st ACM International Conference on Information and Knowledge Management (CIKM), 3953-3957 (2022)',
[entries] New entry: kuairand a74a7c4
diff --git a/entries/kuairand/__init__.py b/entries/kuairand/__init__.py
new file mode 100644
index 0000000..275fa76
--- /dev/null
+++ b/entries/kuairand/__init__.py
@@ -0,0 +1,272 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2026 Tiago de Paula Peixoto <tiago@skewed.de>
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+
+import pyarrow
+import pyarrow.csv
+import pyarrow.compute as pc
+
+from .. import *
+
+title = "Kuaishou short video recommendation logs with random exposure (KuaiRand, 2022)"
+description = r"""Networks of the users of the short video sharing app Kuaishou (快手) and the videos that its recommender system showed them, from the KuaiRand dataset, collected by C. Gao and colleagues (University of Science and Technology of China and Kuaishou) from the recommendation logs of the app between 8 April and 8 May 2022. In the last two and a half weeks of this period (22 April to 8 May), each video recommended to the users had a small probability of being replaced by a video drawn at random from a pool of 7,583 videos; about 0.37% of the exposures were replaced. The logs record, for each exposure, whether the user clicked (or, in the single-column interface, validly played) the video, liked it, followed its author, commented on it, forwarded it, or marked it as disliked, how long it was watched, and whether the user visited the author's profile.
+
+A directed edge from a user to a video is one exposure of the video to the user, with the user's feedback. The networks are bipartite and are multigraphs (a video may be shown to the same user several times). They combine the three log files of each version: the standard recommendations of 8-21 April and of 22 April - 8 May, and the random exposures of 22 April - 8 May (marked by the edge property `random`).
+
+There are three versions, released by the authors, each a network of this entry:
+
+- `27k`: the complete dataset, with all 27,285 users and the 32 million videos shown to them (322 million exposures).
+- `1k`: 1,000 users drawn at random from the complete dataset, with all their exposures.
+- `pure`: all users, but only the exposures of the 7,583 videos of the pool of random videos.
+
+The user and video identifiers were re-indexed by the source. The anonymized categorical user features of the source (`onehot_feat0` to `onehot_feat17`) are kept as a vector property; the binned versions of the counts of the users (e.g. `follow_user_num_range`), and the date and hour of the exposures (redundant with the timestamps) are left out. The daily statistics of the videos (counts of plays, likes, comments, shares, ... of each video during the period, in the source's `video_features_statistic` files, with 50 columns) are also left out. The per-exposure duration of the videos in the logs agrees with the duration of the video given in the video features, except for very few exposures (e.g. 350 of the 11.7 million of `1k`); it is used for the videos whose duration is missing there, and otherwise left out.
+
+The networks have the following properties:
+
+- `is_user`: whether the node is a user (otherwise it is a video).
+- `id`: the identifier of the user or video in the dataset.
+- `active_degree` (users): the activity level of the user, e.g. `full_active`, `high_active`, `middle_active`, `2_14_day_new`, `30day_retention`, `UNKNOWN`.
+- `is_lowactive_period`, `is_live_streamer`, `is_video_author` (users): flags of the user, as given by the source (with a few values other than 0 and 1, e.g. -124).
+- `follow_user_num`, `fans_user_num`, `friend_user_num` (users): the numbers of users that the user follows, of followers, and of friends.
+- `register_days` (users): the number of days since the user registered.
+- `onehot_features` (users): the 18 anonymized categorical features of the user (`onehot_feat0` to `onehot_feat17`; NaN if missing).
+- `author_id` (videos): the identifier of the author of the video (-1 if unknown).
+- `video_type` (videos): `NORMAL` or `AD`.
+- `upload_time` (videos): the upload date of the video, as Unix time (-1 if unknown).
+- `upload_type` (videos): the way the video was uploaded, e.g. `LongImport`, `ShortImport`, `Web`, `Kmovie`.
+- `visible_status` (videos): the visibility status of the video (NaN if unknown).
+- `duration` (videos): the duration of the video, in milliseconds (NaN if unknown).
+- `width`, `height` (videos): the size of the video, in pixels (NaN if unknown).
+- `music_id` (videos): the identifier of the background music of the video (-1 if unknown).
+- `music_type` (videos): the type of the background music (NaN if unknown).
+- `tags` (videos): the ids of the categories of the video, comma-separated, e.g. `20,43`.
+- Edge `time`: the time of the exposure, as Unix time (with milliseconds).
+- Edge `click`: whether the user clicked on the video (in the two-column interface), or played it validly (in the single-column interface: for at least its duration, if it lasts at most 7 seconds, or for more than 7 seconds otherwise).
+- Edge `long_view`: whether the user watched the video for at least its duration, if it lasts at most 18 seconds, or for at least 18 seconds otherwise.
+- Edge `like`, `follow`, `comment`, `forward`, `hate`: whether the user liked the video, followed its author, commented on it, forwarded it, or marked it as disliked.
+- Edge `profile_enter`: whether the user visited the profile of the author.
+- Edge `play_time`: how long the user watched the video, in milliseconds.
+- Edge `profile_stay_time`, `comment_stay_time`: how long the user stayed on the profile of the author and in the comment section of the video, in milliseconds.
+- Edge `random`: whether the video was shown by the random intervention.
+- Edge `tab`: the interface (tab of the app) where the video was shown, a number from 0 to 14."""
+tags = ['Social', 'Online', 'Preferences', 'Bipartite', 'Multigraph', 'Timestamps', 'Metadata']
+url = ['https://kuairand.com/', 'https://doi.org/10.5281/zenodo.10439422']
+citation = [('C. Gao, S. Li, Y. Zhang, J. Chen, B. Li, W. Lei, P. Jiang, X. He, "KuaiRand: An Unbiased Sequential Recommendation Dataset with Randomly Exposed Videos." Proceedings of the 31st ACM International Conference on Information and Knowledge Management (CIKM), 3953-3957 (2022)',
+ 'https://doi.org/10.1145/3511808.3557624')]
+icon_hash = None
+upstream_license = ('CC BY-SA 4.0', 'https://github.com/chongminggao/KuaiRand/blob/f8dbf6678b3c9594050e3e813aeff0c942260ec4/LICENSE')
+upstream_prefix = 'https://zenodo.org/records/10439422/files'
+# networks of more than about 300 million edges: only stored and served in the
+# gt format
+skip_conversion = ['27k']
+# Huge: the `27k` network has 322 million edges and 32 million videos with
+# properties (estimated peak memory of the parse: 50-70 GB, from the 3.9 GB of
+# `1k`, with 11.8 million edges and 4.4 million videos)
+huge = True
+
+# The versions: (name, archive, prefix of the members, suffix, log files)
+VERSIONS = [
+ ("27k", "KuaiRand-27K.tar.gz", "KuaiRand-27K/data", "27k",
+ ["log_standard_4_08_to_4_21_27k_part1.csv",
+ "log_standard_4_08_to_4_21_27k_part2.csv",
+ "log_standard_4_22_to_5_08_27k_part1.csv",
+ "log_standard_4_22_to_5_08_27k_part2.csv",
+ "log_random_4_22_to_5_08_27k.csv"]),
+ ("1k", "KuaiRand-1K.tar.gz", "KuaiRand-1K/data", "1k",
+ ["log_standard_4_08_to_4_21_1k.csv",
+ "log_standard_4_22_to_5_08_1k.csv",
+ "log_random_4_22_to_5_08_1k.csv"]),
+ ("pure", "KuaiRand-Pure.tar.gz", "KuaiRand-Pure/data", "pure",
+ ["log_standard_4_08_to_4_21_pure.csv",
+ "log_standard_4_22_to_5_08_pure.csv",
+ "log_random_4_22_to_5_08_pure.csv"])]
+
+files = [(tuple(f"{archive}:{prefix}/{f}" for f in
+ [f"user_features_{suf}.csv", f"video_features_basic_{suf}.csv"] + logs),
+ alt, None)
+ for alt, archive, prefix, suf, logs in VERSIONS]
+
+def fetch_upstream(force=False):
+ # check=False: only the existence of the archives is checked (opening a
+ # member of the large tar archives means decompressing them up to it)
+ return fetch_upstream_files(__name__.split(".")[-1], upstream_prefix, files,
+ force, check=False)
+
+# For checks only: if set, only (about) this many records of each log file and
+# of the video features are read
+_check_rows = None
+
+def read_csv(name, fname, types=None, cols=None, limit=True):
+ "Iterate over the record batches of the CSV member `fname`."
+ n = 0
+ with open_upstream_file(name, fname, "rb") as f:
+ reader = pyarrow.csv.open_csv(
+ f,
+ read_options=pyarrow.csv.ReadOptions(block_size=1 << 26),
+ convert_options=pyarrow.csv.ConvertOptions(
+ column_types=types or {}, include_columns=cols or []))
+ for batch in reader:
+ if limit and _check_rows is not None and n >= _check_rows:
+ break
+ n += batch.num_rows
+ yield batch
+
+def read_table(name, fname, types=None, limit=True):
+ return pyarrow.Table.from_batches(list(read_csv(name, fname, types,
+ limit=limit)))
+
+def fill(a, default):
+ "The array `a` as numpy, with nulls replaced by `default`."
+ return pc.fill_null(a, default).to_numpy(zero_copy_only=False)
+
+LOG_COLS = ["user_id", "video_id", "time_ms", "is_click", "is_like",
+ "is_follow", "is_comment", "is_forward", "is_hate", "long_view",
+ "play_time_ms", "duration_ms", "profile_stay_time",
+ "comment_stay_time", "is_profile_enter", "is_rand", "tab"]
+FLAGS = [("is_click", "click"), ("long_view", "long_view"),
+ ("is_like", "like"), ("is_follow", "follow"),
+ ("is_comment", "comment"), ("is_forward", "forward"),
+ ("is_hate", "hate"), ("is_profile_enter", "profile_enter"),
+ ("is_rand", "random")]
+TIMES = [("play_time_ms", "play_time"),
+ ("profile_stay_time", "profile_stay_time"),
+ ("comment_stay_time", "comment_stay_time")]
+
+@cache_network()
+@coerce_props()
+@annotate()
+def parse(alts=None):
+ name = __name__.split(".")[-1]
+ for fnames, alt, fmt in files:
+ if alts is not None and alt not in alts:
+ continue
+ users = read_table(name, fnames[0], limit=False)
+ videos = read_table(name, fnames[1],
+ {"upload_dt": pyarrow.string(),
+ "tag": pyarrow.string()})
+ uids = users["user_id"].to_numpy()
+ U = len(uids)
+ vids = videos["video_id"].to_numpy()
+
+ # The exposures, read in blocks into growing numpy arrays
+ cols = {c: [] for c in LOG_COLS}
+ for fname in fnames[2:]:
+ for batch in read_csv(name, fname, cols=LOG_COLS):
+ for c in LOG_COLS:
+ x = batch.column(c).to_numpy()
+ if c in ("user_id", "video_id", "time_ms"):
+ x = x.astype("int64")
+ elif c in dict(FLAGS) or c == "tab":
+ x = x.astype("int8")
+ else:
+ x = x.astype("int32")
+ cols[c].append(x)
+ cols = {c: numpy.concatenate(v) for c, v in cols.items()}
+
+ # Users first, then videos (also those only in the logs)
+ all_uids = numpy.union1d(uids, cols["user_id"])
+ all_vids = numpy.union1d(vids, cols["video_id"])
+ assert len(all_uids) == U, "users missing from the user features"
+ U = len(all_uids)
+ N = U + len(all_vids)
+
+ g = Graph(directed=True)
+ g.add_vertex(N)
+ src = numpy.searchsorted(all_uids, cols.pop("user_id"))
+ tgt = U + numpy.searchsorted(all_vids, cols.pop("video_id"))
+ g.add_edge_list(numpy.column_stack((src, tgt)))
+ del src
+
+ g.vp.is_user = g.new_vp("bool")
+ g.vp.is_user.a[:U] = 1
+ g.vp.id = g.new_vp("int64_t")
+ g.vp.id.a[:U] = all_uids
+ g.vp.id.a[U:] = all_vids
+
+ # User features
+ ui = numpy.searchsorted(all_uids, uids)
+ vals = [""] * N
+ for i, x in zip(ui, users["user_active_degree"].to_pylist()):
+ vals[i] = x or ""
+ g.vp.active_degree = g.new_vp("string", vals=vals)
+ for c, t in [("is_lowactive_period", "int"),
+ ("is_live_streamer", "int"), ("is_video_author", "int"),
+ ("follow_user_num", "int"), ("fans_user_num", "int"),
+ ("friend_user_num", "int"), ("register_days", "int")]:
+ p = g.new_vp(t)
+ p.a[ui] = fill(users[c], -1)
+ g.vp[c] = p
+ oh = numpy.column_stack([fill(users[f"onehot_feat{i}"].cast(pyarrow.float64()),
+ numpy.nan) for i in range(18)])
+ p = g.new_vp("vector<double>")
+ for i, row in zip(ui, oh):
+ p[i] = row
+ g.vp.onehot_features = p
+ del users, oh
+
+ # Video features
+ vi = U + numpy.searchsorted(all_vids, vids)
+ for c, pname, t, default in [("author_id", "author_id", "int64_t", -1),
+ ("visible_status", "visible_status", "double", numpy.nan),
+ ("video_duration", "duration", "double", numpy.nan),
+ ("server_width", "width", "double", numpy.nan),
+ ("server_height", "height", "double", numpy.nan),
+ ("music_id", "music_id", "int64_t", -1),
+ ("music_type", "music_type", "double", numpy.nan)]:
+ p = g.new_vp(t)
+ if t == "double":
+ p.a[:] = numpy.nan
+ p.a[vi] = fill(videos[c].cast(pyarrow.float64()), default)
+ else:
+ p.a[:] = default
+ p.a[vi] = fill(videos[c], default)
+ g.vp[pname] = p
+
+ # The duration of the videos, from the logs where missing
+ dur = cols.pop("duration_ms")
+ d = g.vp.duration.a
+ known = ~numpy.isnan(d[tgt])
+ mismatch = int((d[tgt][known] != dur[known]).sum())
+ if mismatch:
+ print(f"{alt}: {mismatch} exposures with a duration different from the video's")
+ miss = ~known
+ d[tgt[miss]] = dur[miss]
+ del dur, known, miss, tgt
+
+ up = pc.strptime(videos["upload_dt"], format="%Y-%m-%d", unit="s",
+ error_is_null=True)
+ up = fill(up.cast(pyarrow.int64()), -1)
+ g.vp.upload_time = g.new_vp("int64_t")
+ g.vp.upload_time.a[:] = -1
+ g.vp.upload_time.a[vi] = up
+ for c, pname in [("video_type", "video_type"),
+ ("upload_type", "upload_type"), ("tag", "tags")]:
+ vals = [""] * N
+ for i, x in zip(vi, videos[c].to_pylist()):
+ vals[i] = x or ""
+ g.vp[pname] = g.new_vp("string", vals=vals)
+ del vals
+ del videos
+
+ # Exposures
+ g.ep.time = g.new_ep("double", vals=cols.pop("time_ms") / 1000)
+ for c, pname in FLAGS:
+ g.ep[pname] = g.new_ep("bool", vals=cols.pop(c))
+ for c, pname in TIMES:
+ g.ep[pname] = g.new_ep("int", vals=cols.pop(c))
+ g.ep.tab = g.new_ep("int16_t", vals=cols.pop("tab"))
+ yield alt, g