History of conceptnet
The changes to the source code of this entry in the repository, newest first, each with the differences it made to the entry's files. They are also listed on GitLab.
[entries] Add the date when each entry was added, and remove icon_hash beff7c9
The dates are those of the commits that first added the entries' files (2020-08-11 for the entries of the initial commit); renamed entries keep the date of their first name. Changed: none
diff --git a/entries/conceptnet/__init__.py b/entries/conceptnet/__init__.py
index 484a396..2bb5429 100644
--- a/entries/conceptnet/__init__.py
+++ b/entries/conceptnet/__init__.py
@@ -46,7 +46,7 @@ url = ['https://conceptnet.io/',
'https://github.com/commonsense/conceptnet5/wiki/Downloads']
citation = [('R. Speer, J. Chin, C. Havasi, "ConceptNet 5.5: An Open Multilingual Graph of General Knowledge." Proceedings of the AAAI Conference on Artificial Intelligence 31(1), 4444-4451 (2017)',
'https://doi.org/10.1609/aaai.v31i1.11164')]
-icon_hash = None
+date_added = '2026-10-28'
upstream_license = ("CC BY-SA 4.0 (some assertions CC BY 4.0)",
"https://github.com/commonsense/conceptnet5/wiki/Copying-and-sharing-ConceptNet")
upstream_prefix = 'https://s3.amazonaws.com/conceptnet/downloads'
[entries] Prefix the properties in the descriptions with (Node), (Edge) or (Graph) bc3dbbd
The lists of properties at the end of the descriptions marked edge properties with "Edge" but left node properties unmarked, or grouped them in "Nodes: `a`, ...; `b`, ..." items, so the kind of a property was often unclear. Every item now starts with (Node), (Edge) or (Graph), followed by the property names, a colon and the explanation, one item per property or group of properties; qualifications (kinds of nodes, networks) follow the names in parentheses. Properties that were only named got a short explanation, and a few items that named the wrong kind of property were corrected.
diff --git a/entries/conceptnet/__init__.py b/entries/conceptnet/__init__.py
index 372ee92..484a396 100644
--- a/entries/conceptnet/__init__.py
+++ b/entries/conceptnet/__init__.py
@@ -31,16 +31,16 @@ The assertions of the relation `/r/ExternalURL`, which link terms to the URLs of
The networks have the following properties:
-- `uri`: the ConceptNet URI of the term, e.g. `/c/en/ice_cream`.
-- `name`: the text of the term, with underscores instead of spaces, e.g. `ice_cream`.
-- `language`: the language code of the term, e.g. `en`, `fr`, `zh`.
-- Edge `relation`: the relation of the assertion, e.g. `/r/IsA`.
-- Edge `weight`: the weight of the assertion (1 for most; higher for assertions with more supporting evidence, and lower for less reliable ones).
-- Edge `dataset`: the dataset the assertion comes from, e.g. `/d/wiktionary/en`, `/d/conceptnet/4/en`, `/d/wordnet/3.1`.
-- Edge `license`: the license of the assertion, `cc:by/4.0` or `cc:by-sa/4.0`.
-- Edge `start_sense`, `end_sense`: the part of the URI of the start (end) node of the assertion that refines the term, e.g. `n` (a noun) or `n/wn/animal` (a WordNet sense), or empty if the assertion refers to the term as a whole.
-- `release` (graph): the release of ConceptNet.
-- `external_url_assertions` (graph): the number of `/r/ExternalURL` assertions, which are left out."""
+- (Node) `uri`: the ConceptNet URI of the term, e.g. `/c/en/ice_cream`.
+- (Node) `name`: the text of the term, with underscores instead of spaces, e.g. `ice_cream`.
+- (Node) `language`: the language code of the term, e.g. `en`, `fr`, `zh`.
+- (Edge) `relation`: the relation of the assertion, e.g. `/r/IsA`.
+- (Edge) `weight`: the weight of the assertion (1 for most; higher for assertions with more supporting evidence, and lower for less reliable ones).
+- (Edge) `dataset`: the dataset the assertion comes from, e.g. `/d/wiktionary/en`, `/d/conceptnet/4/en`, `/d/wordnet/3.1`.
+- (Edge) `license`: the license of the assertion, `cc:by/4.0` or `cc:by-sa/4.0`.
+- (Edge) `start_sense`, `end_sense`: the part of the URI of the start (end) node of the assertion that refines the term, e.g. `n` (a noun) or `n/wn/animal` (a WordNet sense), or empty if the assertion refers to the term as a whole.
+- (Node) `release` (graph): the release of ConceptNet.
+- (Node) `external_url_assertions` (graph): the number of `/r/ExternalURL` assertions, which are left out."""
tags = ['Informational', 'Language', 'Weighted', 'Multigraph', 'Metadata']
url = ['https://conceptnet.io/',
'https://github.com/commonsense/conceptnet5/wiki/Downloads']
[entries] New entry: conceptnet 2a2ac30
diff --git a/entries/conceptnet/__init__.py b/entries/conceptnet/__init__.py
new file mode 100644
index 0000000..372ee92
--- /dev/null
+++ b/entries/conceptnet/__init__.py
@@ -0,0 +1,181 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2026 Tiago de Paula Peixoto <tiago@skewed.de>
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+
+import pyarrow
+import pyarrow.csv
+import pyarrow.compute as pc
+
+from .. import *
+
+title = "ConceptNet commonsense knowledge graph (2017-2019)"
+description = r"""ConceptNet is a multilingual knowledge graph of general (commonsense) knowledge, which connects words and phrases of natural language with labeled and weighted relations. It combines knowledge collected by crowdsourcing (the Open Mind Common Sense project and its sister projects in other languages), games with a purpose (Verbosity, nadya.jp), expert-created resources (WordNet, Open Multilingual WordNet, JMDict, CC-CEDICT, OpenCyc, Unicode CLDR), Wikipedia infoboxes (via DBpedia) and the Wiktionaries of several languages. Each assertion is a directed edge from a term to another, with one of 34 relations, e.g. `/r/IsA` ("a dog is an animal"), `/r/PartOf`, `/r/UsedFor`, `/r/CapableOf`, `/r/Synonym`, `/r/Antonym`, `/r/RelatedTo`, `/r/FormOf`, `/r/DerivedFrom`, `/r/EtymologicallyRelatedTo` or `/r/Translation`-like relations between languages (`/r/Synonym`), and a weight that expresses the strength of the evidence (e.g. how many sources assert it).
+
+Nodes are terms, identified by their ConceptNet URI `/c/<language>/<text>` (e.g. `/c/en/ice_cream`); assertions that refer to a particular sense of a term (e.g. `/c/en/dog/n` for the noun, or `/c/en/dog/n/wn/animal` for a WordNet sense) are attached to the term, and the sense is kept as an edge property. This follows ConceptNet's own convention, in which terms are the nodes of the graph and the senses are refinements of them. Each network corresponds to a release of ConceptNet: `5.7.0` (2019), `5.6.0` (2018) and `5.5.5` (2017).
+
+The assertions of the relation `/r/ExternalURL`, which link terms to the URLs of the same concept in other linked-data resources (e.g. DBpedia, Wikidata, WordNet RDF, OpenCyc) rather than to other terms, are left out; their number is given by the graph property `external_url_assertions`. The provenance of each assertion beyond its dataset (the contributors and processes in its `sources`) and its surface text (the natural-language sentence it was collected from, if any) are also left out.
+
+The networks have the following properties:
+
+- `uri`: the ConceptNet URI of the term, e.g. `/c/en/ice_cream`.
+- `name`: the text of the term, with underscores instead of spaces, e.g. `ice_cream`.
+- `language`: the language code of the term, e.g. `en`, `fr`, `zh`.
+- Edge `relation`: the relation of the assertion, e.g. `/r/IsA`.
+- Edge `weight`: the weight of the assertion (1 for most; higher for assertions with more supporting evidence, and lower for less reliable ones).
+- Edge `dataset`: the dataset the assertion comes from, e.g. `/d/wiktionary/en`, `/d/conceptnet/4/en`, `/d/wordnet/3.1`.
+- Edge `license`: the license of the assertion, `cc:by/4.0` or `cc:by-sa/4.0`.
+- Edge `start_sense`, `end_sense`: the part of the URI of the start (end) node of the assertion that refines the term, e.g. `n` (a noun) or `n/wn/animal` (a WordNet sense), or empty if the assertion refers to the term as a whole.
+- `release` (graph): the release of ConceptNet.
+- `external_url_assertions` (graph): the number of `/r/ExternalURL` assertions, which are left out."""
+tags = ['Informational', 'Language', 'Weighted', 'Multigraph', 'Metadata']
+url = ['https://conceptnet.io/',
+ 'https://github.com/commonsense/conceptnet5/wiki/Downloads']
+citation = [('R. Speer, J. Chin, C. Havasi, "ConceptNet 5.5: An Open Multilingual Graph of General Knowledge." Proceedings of the AAAI Conference on Artificial Intelligence 31(1), 4444-4451 (2017)',
+ 'https://doi.org/10.1609/aaai.v31i1.11164')]
+icon_hash = None
+upstream_license = ("CC BY-SA 4.0 (some assertions CC BY 4.0)",
+ "https://github.com/commonsense/conceptnet5/wiki/Copying-and-sharing-ConceptNet")
+upstream_prefix = 'https://s3.amazonaws.com/conceptnet/downloads'
+# The assertion files of the three releases (418-498 MB each), published once
+files = [("2019/edges/conceptnet-assertions-5.7.0.csv.gz", "5.7.0", None),
+ ("2018/edges/conceptnet-assertions-5.6.0.csv.gz", "5.6.0", None),
+ ("2017/edges/conceptnet-assertions-5.5.5.csv.gz", "5.5.5", None)]
+
+def fetch_upstream(force=False):
+ return fetch_upstream_files(__name__.split(".")[-1], upstream_prefix, files,
+ force)
+
+TERM = r"^/c/[^/]+/[^/]+"
+
+def split_uri(uris):
+ "The terms (/c/<lang>/<text>) of the URIs `uris`, and their senses."
+ terms = pc.extract_regex(uris, r"^(?P<term>/c/[^/]+/[^/]+)").field(0)
+ senses = pc.replace_substring_regex(uris, TERM + r"/?", "")
+ return terms, senses
+
+def json_field(meta, pattern):
+ "The field matching the regular expression `pattern` in the JSON column."
+ r = pc.extract_regex(meta, pattern)
+ return r.field(0), pc.is_valid(r)
+
+# For checks only: if set, only (about) this many assertions are read
+_check_rows = None
+
+class Codes(dict):
+ "Integer codes of strings."
+ def __call__(self, s):
+ c = self.get(s)
+ if c is None:
+ c = self[s] = len(self)
+ return c
+ def labels(self):
+ return [s for s, c in sorted(self.items(), key=lambda x: x[1])]
+ def encode(self, values):
+ "The codes of the string array `values` (nulls as empty strings)."
+ d = values.fill_null("").dictionary_encode()
+ m = numpy.array([self(x) for x in d.dictionary.to_pylist()],
+ dtype="int32")
+ return m[d.indices.to_numpy()]
+
+def categorical_ep(g, codes, labels, name):
+ "Store the integer codes `codes` of `labels` as the string property `name`."
+ c = g.new_ep("int", vals=codes)
+ p = g.new_ep("string")
+ map_property_values(c, p, lambda i: labels[i])
+ g.ep[name] = p
+
+@cache_network()
+@coerce_props()
+@annotate()
+def parse(alts=None):
+ name = __name__.split(".")[-1]
+ for fname, alt, fmt in files:
+ if alts is not None and alt not in alts:
+ continue
+ codes = {k: Codes() for k in ("sense", "rel", "ds", "lic")}
+ cols = {k: [] for k in ("s", "t", "ss", "ts", "rel", "w", "ds", "lic")}
+ n_ext = n_read = 0
+ with open_upstream_file(name, fname, "rb") as f:
+ # <assertion uri> <relation> <start> <end> <JSON metadata>
+ reader = pyarrow.csv.open_csv(
+ f,
+ read_options=pyarrow.csv.ReadOptions(
+ column_names=["uri", "rel", "start", "end", "meta"],
+ block_size=1 << 26),
+ parse_options=pyarrow.csv.ParseOptions(delimiter="\t",
+ quote_char=False,
+ double_quote=False,
+ escape_char=False),
+ convert_options=pyarrow.csv.ConvertOptions(
+ include_columns=["rel", "start", "end", "meta"],
+ column_types={k: pyarrow.string() for k in
+ ("uri", "rel", "start", "end", "meta")}))
+ for batch in reader:
+ ext = pc.equal(batch.column("rel"), "/r/ExternalURL")
+ n_ext += pc.sum(ext).as_py() or 0
+ batch = batch.filter(pc.invert(ext))
+ s, ss = split_uri(batch.column("start"))
+ t, ts = split_uri(batch.column("end"))
+ assert s.null_count == 0 and t.null_count == 0
+ meta = batch.column("meta")
+ w, ok = json_field(meta, r'"weight": (?P<v>[-0-9.eE]+)')
+ assert pc.all(ok).as_py() is not False
+ ds, _ = json_field(meta, r'"dataset": "(?P<v>[^"]*)"')
+ lic, _ = json_field(meta, r'"license": "(?P<v>[^"]*)"')
+ cols["s"].append(s)
+ cols["t"].append(t)
+ cols["ss"].append(codes["sense"].encode(ss))
+ cols["ts"].append(codes["sense"].encode(ts))
+ cols["rel"].append(codes["rel"].encode(batch.column("rel")))
+ cols["ds"].append(codes["ds"].encode(ds))
+ cols["lic"].append(codes["lic"].encode(lic))
+ cols["w"].append(w.cast("double").to_numpy())
+ del batch, meta, s, t, ss, ts, w, ds, lic
+ # free the memory of the processed block (otherwise kept by
+ # the allocator of pyarrow)
+ pyarrow.default_memory_pool().release_unused()
+ n_read += len(cols["w"][-1])
+ if _check_rows is not None and n_read >= _check_rows:
+ break
+ s = pyarrow.chunked_array(cols.pop("s"))
+ t = pyarrow.chunked_array(cols.pop("t"))
+ cols = {k: numpy.concatenate(v) for k, v in cols.items()}
+ terms = pc.unique(pyarrow.chunked_array(s.chunks + t.chunks))
+ terms = pc.take(terms, pc.sort_indices(terms))
+ s = pc.index_in(s, value_set=terms).to_numpy()
+ t = pc.index_in(t, value_set=terms).to_numpy()
+
+ g = Graph(directed=True)
+ g.add_vertex(len(terms))
+ g.add_edge_list(numpy.column_stack([s, t]))
+ del s, t
+ g.vp.uri = g.new_vp("string", vals=terms.to_pylist())
+ parts = pc.split_pattern(terms, "/")
+ g.vp.name = g.new_vp("string", vals=pc.list_element(parts, 3).to_pylist())
+ g.vp.language = g.new_vp("string", vals=pc.list_element(parts, 2).to_pylist())
+ del parts, terms
+ categorical_ep(g, cols.pop("rel"), codes["rel"].labels(), "relation")
+ g.ep.weight = g.new_ep("double", vals=cols.pop("w"))
+ categorical_ep(g, cols.pop("ds"), codes["ds"].labels(), "dataset")
+ categorical_ep(g, cols.pop("lic"), codes["lic"].labels(), "license")
+ senses = codes["sense"].labels()
+ categorical_ep(g, cols.pop("ss"), senses, "start_sense")
+ categorical_ep(g, cols.pop("ts"), senses, "end_sense")
+ del senses
+ g.gp.release = g.new_gp("string", val=alt)
+ g.gp.external_url_assertions = g.new_gp("int64_t", val=n_ext)
+ yield alt, g