History of hacker_news
The changes to the source code of this entry in the repository, newest first, each with the differences it made to the entry's files. They are also listed on GitLab.
[entries] Add the date when each entry was added, and remove icon_hash beff7c9
The dates are those of the commits that first added the entries' files (2020-08-11 for the entries of the initial commit); renamed entries keep the date of their first name. Changed: none
diff --git a/entries/hacker_news/__init__.py b/entries/hacker_news/__init__.py
index 8997334..5a55b98 100644
--- a/entries/hacker_news/__init__.py
+++ b/entries/hacker_news/__init__.py
@@ -50,7 +50,7 @@ url = ['https://news.ycombinator.com/',
'https://huggingface.co/datasets/open-index/hacker-news']
citation = [('Y Combinator, "Hacker News API."', 'https://github.com/HackerNewsAPI/HN-API'),
('Open Index, "Hacker News - Complete Archive." Hugging Face dataset (2026)', 'https://huggingface.co/datasets/open-index/hacker-news')]
-icon_hash = None
+date_added = '2026-10-28'
upstream_license = ('ODC-BY', 'https://opendatacommons.org/licenses/by/1-0/')
# A pinned revision of the archive (the monthly files are rewritten while the
[entries] Prefix the properties in the descriptions with (Node), (Edge) or (Graph) bc3dbbd
The lists of properties at the end of the descriptions marked edge properties with "Edge" but left node properties unmarked, or grouped them in "Nodes: `a`, ...; `b`, ..." items, so the kind of a property was often unclear. Every item now starts with (Node), (Edge) or (Graph), followed by the property names, a colon and the explanation, one item per property or group of properties; qualifications (kinds of nodes, networks) follow the names in parentheses. Properties that were only named got a short explanation, and a few items that named the wrong kind of property were corrected.
diff --git a/entries/hacker_news/__init__.py b/entries/hacker_news/__init__.py
index 4c870b3..8997334 100644
--- a/entries/hacker_news/__init__.py
+++ b/entries/hacker_news/__init__.py
@@ -32,19 +32,19 @@ Items deleted by their authors or by the moderators have no author in the data:
The networks have the following properties:
-- `name`: the user name, or, in `discussions`, the title of the story (for stories).
-- `is_user` (`discussions` only): whether the node is a user (otherwise a story).
-- `item` (`discussions` only): the item number of the story (-1 for users), as in its address `https://news.ycombinator.com/item?id=<item>`.
-- `author` (`discussions` only): the user name of the author of the story (empty for users).
-- `kind` (`discussions` only): `story`, `poll` or `job` (empty for users).
-- `score` (`discussions` only): the score (points) of the story (0 for users).
-- `time` (`discussions` only): the submission time of the story, in Unix time (-1 for users).
-- Edge `time`: in `replies`, the time of the comment; in `discussions`, the time of the first comment of the user in the discussion; in Unix time.
-- Edge `comment` (`replies` only): the item number of the comment.
-- Edge `parent` (`replies` only): the item number of the item replied to.
-- Edge `story` (`replies` only): the item number of the story at the root of the discussion.
-- Edge `dead` (`replies` only): whether the comment is dead (killed by the moderators or flagged).
-- Edge `weight` (`discussions` only): the number of comments of the user in the discussion."""
+- (Node) `name`: the user name, or, in `discussions`, the title of the story (for stories).
+- (Node) `is_user` (`discussions` only): whether the node is a user (otherwise a story).
+- (Node) `item` (`discussions` only): the item number of the story (-1 for users), as in its address `https://news.ycombinator.com/item?id=<item>`.
+- (Node) `author` (`discussions` only): the user name of the author of the story (empty for users).
+- (Node) `kind` (`discussions` only): `story`, `poll` or `job` (empty for users).
+- (Node) `score` (`discussions` only): the score (points) of the story (0 for users).
+- (Node) `time` (`discussions` only): the submission time of the story, in Unix time (-1 for users).
+- (Edge) `time`: in `replies`, the time of the comment; in `discussions`, the time of the first comment of the user in the discussion; in Unix time.
+- (Edge) `comment` (`replies` only): the item number of the comment.
+- (Edge) `parent` (`replies` only): the item number of the item replied to.
+- (Edge) `story` (`replies` only): the item number of the story at the root of the discussion.
+- (Edge) `dead` (`replies` only): whether the comment is dead (killed by the moderators or flagged).
+- (Edge) `weight` (`discussions` only): the number of comments of the user in the discussion."""
tags = ['Social', 'Online', 'Communication', 'Multigraph', 'Timestamps', 'Temporal', 'Bipartite', 'Weighted', 'Metadata']
url = ['https://news.ycombinator.com/',
'https://huggingface.co/datasets/open-index/hacker-news']
[entries] New entry: hacker_news a473b38
diff --git a/entries/hacker_news/__init__.py b/entries/hacker_news/__init__.py
new file mode 100644
index 0000000..4c870b3
--- /dev/null
+++ b/entries/hacker_news/__init__.py
@@ -0,0 +1,271 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2026 Tiago de Paula Peixoto <tiago@skewed.de>
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+
+import http.client
+import time
+
+from .. import *
+import process_entry
+
+title = "Hacker News discussions (2006-2026)"
+description = r"""Networks of the discussions on Hacker News, the news aggregation and discussion website of the startup accelerator Y Combinator, from its first item in October 2006 to the end of July 2026, from the complete archive of the items of the site (stories, comments, polls and jobs) published on Hugging Face by Open Index, a mirror of the official Hacker News API (via the ClickHouse public dataset). There are two networks:
+
+- `replies`: each node is a user, and a directed edge from user $i$ to user $j$ is a comment of $i$ replying to an item (a story, a comment, a poll or a job posting) of $j$, with one edge per comment (a multigraph). Users replying to themselves give self-loops.
+- `discussions`: a bipartite network of users and stories (including polls and job postings), where an edge connects a user to a story if the user commented somewhere in the discussion of the story (the tree of the comments below it), weighted by the number of comments. Only the users and stories with such comments are included.
+
+Items deleted by their authors or by the moderators have no author in the data: the comments that are deleted, or that reply to a deleted item, are left out of `replies`, and the deleted comments are left out of `discussions` (the comments in a discussion below a deleted comment are kept). Comments killed by the moderators or flagged by users ("dead") are included and marked. Comments whose parent item is not in the archive are left out. The texts of the items, the URLs of the stories and the poll options are not included. The archive is updated continuously; the data are those of a pinned revision (of 23 August 2026), and the last, incomplete month (August 2026) is left out.
+
+The networks have the following properties:
+
+- `name`: the user name, or, in `discussions`, the title of the story (for stories).
+- `is_user` (`discussions` only): whether the node is a user (otherwise a story).
+- `item` (`discussions` only): the item number of the story (-1 for users), as in its address `https://news.ycombinator.com/item?id=<item>`.
+- `author` (`discussions` only): the user name of the author of the story (empty for users).
+- `kind` (`discussions` only): `story`, `poll` or `job` (empty for users).
+- `score` (`discussions` only): the score (points) of the story (0 for users).
+- `time` (`discussions` only): the submission time of the story, in Unix time (-1 for users).
+- Edge `time`: in `replies`, the time of the comment; in `discussions`, the time of the first comment of the user in the discussion; in Unix time.
+- Edge `comment` (`replies` only): the item number of the comment.
+- Edge `parent` (`replies` only): the item number of the item replied to.
+- Edge `story` (`replies` only): the item number of the story at the root of the discussion.
+- Edge `dead` (`replies` only): whether the comment is dead (killed by the moderators or flagged).
+- Edge `weight` (`discussions` only): the number of comments of the user in the discussion."""
+tags = ['Social', 'Online', 'Communication', 'Multigraph', 'Timestamps', 'Temporal', 'Bipartite', 'Weighted', 'Metadata']
+url = ['https://news.ycombinator.com/',
+ 'https://huggingface.co/datasets/open-index/hacker-news']
+citation = [('Y Combinator, "Hacker News API."', 'https://github.com/HackerNewsAPI/HN-API'),
+ ('Open Index, "Hacker News - Complete Archive." Hugging Face dataset (2026)', 'https://huggingface.co/datasets/open-index/hacker-news')]
+icon_hash = None
+upstream_license = ('ODC-BY', 'https://opendatacommons.org/licenses/by/1-0/')
+
+# A pinned revision of the archive (the monthly files are rewritten while the
+# month is ongoing)
+revision = 'c54ced5973b95d80cd934b8921f44ec8d87c3cd1'
+upstream_prefix = f'https://huggingface.co/datasets/open-index/hacker-news/resolve/{revision}'
+months = [f"data/{y}/{y}-{m:02d}.parquet"
+ for y in range(2006, 2027) for m in range(1, 13)
+ if (2006, 10) <= (y, m) <= (2026, 7)]
+files = [(tuple(months), "replies", None),
+ (tuple(months), "discussions", None)]
+
+def _upstream_dir():
+ return f"cache/upstream/{__name__.split('.')[-1]}"
+
+def _download(url, target, tries=20):
+ """Download `url` to `target`, resuming (with range requests) when the
+ connection is closed early, which can end a download without an error;
+ the size is checked against the one announced by the server."""
+ part = target + ".part"
+ size = None
+ for i in range(tries):
+ have = os.path.getsize(part) if os.path.exists(part) else 0
+ if size is not None and have == size:
+ break
+ headers = {"Range": f"bytes={have}-"} if have > 0 else {}
+ try:
+ with open_url(url, headers=headers) as r:
+ if have > 0 and r.status != 206:
+ have = 0 # the server ignored the range
+ total = r.headers.get("Content-Range")
+ size = (int(total.rsplit("/", 1)[1]) if total else
+ int(r.headers["Content-Length"]))
+ with open(part, "ab" if have > 0 else "wb") as out:
+ shutil.copyfileobj(r, out, 1 << 24)
+ except (OSError, http.client.HTTPException):
+ if i == tries - 1:
+ raise
+ time.sleep(10 * (i + 1))
+ if size is None or os.path.getsize(part) != size:
+ raise IOError(f"incomplete download of {url}")
+ os.replace(part, target)
+
+def fetch_upstream(force=False):
+ # The parquet files (already compressed) are stored as they are
+ if process_entry._check_health:
+ for f in process_entry.health_sample(months):
+ with open_url(f"{upstream_prefix}/{f}") as r:
+ r.read(1)
+ return
+ for f in months:
+ target = f"{_upstream_dir()}/{f}"
+ if not force and os.path.exists(target):
+ continue
+ os.makedirs(os.path.dirname(target), exist_ok=True)
+ _download(f"{upstream_prefix}/{f}", target)
+
+# Item types
+_STORY, _COMMENT, _POLL, _POLLOPT, _JOB = 1, 2, 3, 4, 5
+
+def _read_items():
+ """The items of all the months, as arrays (sorted by id), and the user
+ names (the authors are indices into them, -1 if there is none)."""
+ import pyarrow
+ import pyarrow.parquet as pq
+ import pyarrow.compute as pc
+ pyarrow.set_cpu_count(4)
+ pyarrow.set_io_thread_count(4)
+ users = {}
+ cols = {k: [] for k in ["id", "type", "by", "time", "parent", "dead",
+ "score"]}
+ tids, titles = [], [] # the titles of the stories (polls, jobs)
+ for f in months:
+ t = pq.read_table(f"{_upstream_dir()}/{f}",
+ columns=["id", "type", "by", "time", "parent",
+ "dead", "score", "title"])
+ by = t.column("by").fill_null("")
+ enc = pc.dictionary_encode(by).combine_chunks()
+ uidx = numpy.array([users.setdefault(u, len(users)) if u else -1
+ for u in enc.dictionary.to_pylist()],
+ dtype="int32")
+ cols["by"].append(uidx[enc.indices.to_numpy()] if len(uidx) else
+ numpy.zeros(0, dtype="int32"))
+ ids = t.column("id").to_numpy().astype("int32")
+ typ = t.column("type").to_numpy().astype("int8")
+ cols["id"].append(ids)
+ cols["type"].append(typ)
+ cols["time"].append(t.column("time").cast("timestamp[s]").cast("int64")
+ .to_numpy(zero_copy_only=False))
+ cols["parent"].append(t.column("parent").to_numpy().astype("int32"))
+ cols["dead"].append(t.column("dead").to_numpy().astype("bool"))
+ cols["score"].append(t.column("score").fill_null(0).to_numpy().astype("int32"))
+ sel = numpy.flatnonzero((typ == _STORY) | (typ == _POLL) | (typ == _JOB))
+ tids.append(ids[sel])
+ titles.append(t.column("title").take(sel).fill_null("").combine_chunks())
+ del t
+ cols = {k: numpy.concatenate(v) for k, v in cols.items()}
+ if (numpy.diff(cols["id"]) < 0).any(): # (the files are sorted by id)
+ order = numpy.argsort(cols["id"], kind="stable")
+ cols = {k: v[order] for k, v in cols.items()}
+ del order
+ if (numpy.diff(cols["id"]) == 0).any():
+ raise ValueError("repeated items")
+ stitles = (numpy.concatenate(tids), pyarrow.concat_arrays(titles))
+ return cols, list(users.keys()), stitles
+
+@cache_network()
+@coerce_props()
+@annotate()
+def parse(alts=None):
+ items = None
+ for fnames, alt, fmt in files:
+ if alts is not None and alt not in alts:
+ continue
+ if items is None:
+ items, unames, stitles = _read_items()
+ ids, typ, by = items["id"], items["type"], items["by"]
+ n = len(ids)
+ # the position of the parent of each item (-1 if not in the archive)
+ pos = numpy.searchsorted(ids, items["parent"]).astype("int32")
+ pos[pos == n] = 0
+ ppos = numpy.where((ids[pos] == items["parent"]) &
+ (items["parent"] > 0), pos, -1).astype("int32")
+ del pos
+ com = numpy.flatnonzero((typ == _COMMENT) & (ppos >= 0))
+ # the root of each comment's discussion, by pointer jumping
+ root = numpy.arange(n, dtype="int32")
+ isc = typ == _COMMENT
+ root[isc] = numpy.where(ppos[isc] >= 0, ppos[isc], numpy.flatnonzero(isc))
+ del isc
+ while True:
+ nroot = root[root]
+ if (nroot == root).all():
+ break
+ root = nroot
+ isstory = (typ == _STORY) | (typ == _POLL) | (typ == _JOB)
+
+ if alt == "replies":
+ sel = com[(by[com] >= 0) & (by[ppos[com]] >= 0)]
+ g = Graph(directed=True)
+ g.add_vertex(len(unames))
+ g.vp.name = g.new_vp("string", vals=unames)
+ eprops = []
+ for k in ["time", "comment", "parent", "story"]:
+ g.ep[k] = g.new_ep("int64_t")
+ eprops.append(g.ep[k])
+ g.ep.dead = g.new_ep("bool")
+ eprops.append(g.ep.dead)
+ # (added in pieces, to save memory)
+ step = 1 << 23
+ for i in range(0, len(sel), step):
+ c = sel[i:i + step]
+ r = root[c]
+ story = numpy.where(isstory[r], ids[r], -1)
+ g.add_edge_list(numpy.column_stack([by[c], by[ppos[c]],
+ items["time"][c], ids[c],
+ ids[ppos[c]], story,
+ items["dead"][c]]),
+ eprops=eprops)
+ del sel
+ else:
+ sel = com[(by[com] >= 0) & isstory[root[com]]]
+ # only the users and stories with comments
+ cusers = numpy.unique(by[sel])
+ nu = len(cusers)
+ uidx = numpy.full(len(unames), -1, dtype="int64")
+ uidx[cusers] = numpy.arange(nu)
+ stories = numpy.unique(root[sel])
+ sidx = numpy.full(n, -1, dtype="int64")
+ sidx[stories] = numpy.arange(len(stories)) + nu
+ u = uidx[by[sel]]
+ s = sidx[root[sel]]
+ t = items["time"][sel]
+ # one edge per user and story, with the number of comments and the
+ # first time
+ key = u * (len(stories) + nu) + s
+ order = numpy.lexsort((t, key))
+ key, t = key[order], t[order]
+ first = numpy.flatnonzero(numpy.r_[True, key[1:] != key[:-1]])
+ weight = numpy.diff(numpy.r_[first, len(key)])
+ key, t = key[first], t[first]
+ g = Graph(directed=False)
+ g.add_vertex(nu + len(stories))
+ # the titles of the stories
+ sids, stitle = stitles
+ o = numpy.argsort(sids, kind="stable")
+ p = numpy.searchsorted(sids, ids[stories], sorter=o)
+ p[p == len(sids)] = 0
+ p = o[p]
+ found = sids[p] == ids[stories]
+ tl = stitle.take(p).to_pylist()
+ g.vp.name = g.new_vp("string", vals=[unames[i] for i in cusers] +
+ [x if ok else "" for x, ok in zip(tl, found)])
+ del tl
+ g.vp.is_user = g.new_vp("bool")
+ g.vp.is_user.a[:nu] = True
+ g.vp.item = g.new_vp("int64_t", val=-1)
+ g.vp.item.a[nu:] = ids[stories]
+ g.vp.author = g.new_vp("string")
+ g.vp.kind = g.new_vp("string")
+ g.vp.score = g.new_vp("int32_t")
+ g.vp.score.a[nu:] = items["score"][stories]
+ g.vp.time = g.new_vp("int64_t", val=-1)
+ g.vp.time.a[nu:] = items["time"][stories]
+ kinds = {_STORY: "story", _POLL: "poll", _JOB: "job"}
+ for v, a, k in zip(range(nu, nu + len(stories)), by[stories],
+ typ[stories]):
+ if a >= 0:
+ g.vp.author[v] = unames[a]
+ g.vp.kind[v] = kinds[k]
+ g.ep.weight = g.new_ep("int32_t")
+ g.ep.time = g.new_ep("int64_t")
+ g.add_edge_list(numpy.column_stack([key // (len(stories) + nu),
+ key % (len(stories) + nu),
+ weight, t]),
+ eprops=[g.ep.weight, g.ep.time])
+ yield alt, g
+ del g