History of twitch_viewers
The changes to the source code of this entry in the repository, newest first, each with the differences it made to the entry's files. They are also listed on GitLab.
[entries] New entry: twitch_viewers 930df96
diff --git a/entries/twitch_viewers/__init__.py b/entries/twitch_viewers/__init__.py
new file mode 100644
index 0000000..0201e1a
--- /dev/null
+++ b/entries/twitch_viewers/__init__.py
@@ -0,0 +1,156 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2026 Tiago de Paula Peixoto <tiago@skewed.de>
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+import pyarrow
+import pyarrow.compute as pc
+import pyarrow.csv
+
+from .. import *
+
+
+title = "Twitch stream viewers (2019)"
+description = """A bipartite network of the users and the streamers (channels) of the live streaming platform Twitch, collected by J. Rappaz, J. McAuley and K. Aberer in July and August 2019 to study recommendation on live-streaming platforms. Every 10 minutes during 43 days, the authors listed all the live streams with the public Twitch API, and the users who were logged in to the chat of each stream. The source data have one record per user and broadcast (a stream of a streamer), with the first and the last round (of 10 minutes) at which the user was seen in its chat; the user names were anonymized by the source.
+
+Here a user is connected to a streamer if the user was present in the chat of at least one broadcast of the streamer, and the edges are weighted by the number of these broadcasts; the individual records (about 475 million, according to the authors) are aggregated into these weighted edges. The subset of 100,000 users provided by the source for benchmarks (`100k_a.csv`) is not included.
+
+The network has the following properties:
+
+- `is_user`: whether the node is a user (otherwise a streamer).
+- `user_id`: the (anonymized) id of the user in the source data (-1 for streamers).
+- `name`: the name of the channel of the streamer (empty for users).
+- Edge `weight`: the number of broadcasts of the streamer in whose chat the user was seen.
+- Edge `rounds`: the total number of 10-minute rounds the user was seen in the chats of these broadcasts (the sum of the differences between the last and the first round of each).
+- Edges `first` and `last`: the first and the last round at which the user was seen in a chat of the streamer (rounds of 10 minutes, numbered from 0 to 6148)."""
+tags = ['Social', 'Online', 'Preferences', 'Weighted', 'Metadata']
+url = ['https://cseweb.ucsd.edu/~jmcauley/datasets.html#twitch',
+ 'https://github.com/JRappaz/liverec']
+citation = [('J. Rappaz, J. McAuley, K. Aberer, "Recommendation on Live-Streaming Platforms: Dynamic Availability and Repeat Consumption." Proceedings of the 15th ACM Conference on Recommender Systems (RecSys), 390-399 (2021)', 'https://doi.org/10.1145/3460231.3474267')]
+icon_hash = None
+upstream_license = None
+upstream_prefix = 'https://mcauleylab.ucsd.edu/public_datasets/gdrive/twitch'
+files = [('full_a.csv.gz', None, None)]
+
+def fetch_upstream(force=False):
+ return fetch_upstream_files(__name__.split(".")[-1], upstream_prefix, files,
+ force)
+
+# For checks only: if set, only (about) this many records are read
+_check_rows = None
+
+def read_records(name, fname):
+ """The records "<user>,<stream id>,<streamer>,<first round>,<last
+ round>", in blocks of pyarrow record batches."""
+ n = 0
+ with open_upstream_file(name, fname, "rb") as f:
+ reader = pyarrow.csv.open_csv(
+ f,
+ read_options=pyarrow.csv.ReadOptions(
+ column_names=["user", "stream", "streamer", "start", "stop"],
+ block_size=1 << 26),
+ convert_options=pyarrow.csv.ConvertOptions(
+ column_types={"user": pyarrow.int64(),
+ "stream": pyarrow.int64(),
+ "streamer": pyarrow.string(),
+ "start": pyarrow.int64(),
+ "stop": pyarrow.int64()}))
+ for batch in reader:
+ if _check_rows is not None and n >= _check_rows:
+ break
+ n += batch.num_rows
+ yield batch
+
+def aggregate(u, s, start, stop):
+ """Aggregate the records by (user, streamer): returns the users,
+ streamers, numbers of records, sums of rounds, first and last rounds."""
+ key = u * (1 << 24) + s
+ keys, inv = numpy.unique(key, return_inverse=True)
+ n = len(keys)
+ count = numpy.bincount(inv, minlength=n)
+ rounds = numpy.bincount(inv, weights=stop - start, minlength=n).astype("int64")
+ first = numpy.full(n, numpy.iinfo("int64").max)
+ numpy.minimum.at(first, inv, start)
+ last = numpy.full(n, -1)
+ numpy.maximum.at(last, inv, stop)
+ return (keys >> 24).astype("int32"), (keys & ((1 << 24) - 1)).astype("int32"), \
+ count.astype("int32"), rounds.astype("int32"), first.astype("int16"), \
+ last.astype("int16")
+
+@cache_network()
+@coerce_props()
+@annotate()
+def parse(alts=None):
+ global files
+ name = __name__.split(".")[-1]
+ for fname, alt, fmt in files:
+ if alts is not None and alt not in alts:
+ continue
+ streamers = {} # name -> index
+ blocks = [] # aggregated blocks
+ carry = None # the records of the last user of a block
+ prev = -1
+ def process(u, s, start, stop):
+ if len(u) > 0:
+ blocks.append(aggregate(u, s, start, stop))
+ for batch in read_records(name, fname):
+ u = batch.column(0).to_numpy()
+ # the records are grouped by user (in increasing order), so that
+ # all the records of a pair (user, streamer) are aggregated in
+ # the same block, except for the last user of a block, whose
+ # records are carried over to the next one
+ assert (numpy.diff(u) >= 0).all() and u[0] >= prev
+ prev = u[-1]
+ d = pc.dictionary_encode(batch.column(2))
+ sidx = numpy.array([streamers.setdefault(x, len(streamers))
+ for x in d.dictionary.to_pylist()], dtype="int64")
+ s = sidx[d.indices.to_numpy()]
+ start = batch.column(3).to_numpy()
+ stop = batch.column(4).to_numpy()
+ assert (stop >= start).all()
+ cols = [u, s, start, stop]
+ if carry is not None:
+ cols = [numpy.concatenate([c, x]) for c, x in zip(carry, cols)]
+ last = cols[0] == cols[0][-1]
+ carry = [c[last] for c in cols]
+ process(*[c[~last] for c in cols])
+ del batch, cols, u, s, start, stop, d
+ if carry is not None:
+ process(*carry)
+ assert len(streamers) < (1 << 24)
+ users = numpy.concatenate([b[0] for b in blocks])
+ # Users are numbered in the order of their ids, then the streamers
+ uids = numpy.unique(users)
+ U = len(uids)
+ g = Graph(directed=False)
+ g.add_vertex(U + len(streamers))
+ for p in ["weight", "rounds"]:
+ g.ep[p] = g.new_ep("int32_t")
+ for p in ["first", "last"]:
+ g.ep[p] = g.new_ep("int16_t")
+ for b in blocks:
+ e = numpy.column_stack([numpy.searchsorted(uids, b[0]), U + b[1]] +
+ list(b[2:]))
+ g.add_edge_list(e, eprops=[g.ep.weight, g.ep.rounds, g.ep.first,
+ g.ep.last])
+ del blocks, users, e
+ g.vp.is_user = g.new_vp("bool")
+ g.vp.is_user.a[:U] = True
+ g.vp.user_id = g.new_vp("int64_t", val=-1)
+ g.vp.user_id.a[:U] = uids
+ g.vp.name = g.new_vp("string")
+ for x, i in streamers.items():
+ g.vp.name[U + i] = x
+ yield alt, g