History of littlesis
The changes to the source code of this entry in the repository, newest first, each with the differences it made to the entry's files. They are also listed on GitLab.
[entries] Add the date when each entry was added, and remove icon_hash beff7c9
The dates are those of the commits that first added the entries' files (2020-08-11 for the entries of the initial commit); renamed entries keep the date of their first name. Changed: none
diff --git a/entries/littlesis/__init__.py b/entries/littlesis/__init__.py
index 008f65e..d881e51 100644
--- a/entries/littlesis/__init__.py
+++ b/entries/littlesis/__init__.py
@@ -58,7 +58,7 @@ url = ['https://littlesis.org/database/bulk_data', 'https://web.archive.org/web/
citation = [('Public Accountability Initiative, "LittleSis: a free database of who-knows-who at the heights of business and government." Bulk data (2022, 2024, 2026)', 'https://littlesis.org/database/bulk_data')]
upstream_license = ('Creative Commons Attribution-ShareAlike 4.0 International (CC BY-SA 4.0)',
'https://creativecommons.org/licenses/by-sa/4.0/')
-icon_hash = None
+date_added = '2026-10-31'
# The bulk files of LittleSis are refused to scripts by a bot check of the
# site, and are replaced in place; they are downloaded from the snapshots of
# the Internet Archive: those of 2022-08-09 and 2024-01-23 (all those of the
[entries] Prefix the properties in the descriptions with (Node), (Edge) or (Graph) bc3dbbd
The lists of properties at the end of the descriptions marked edge properties with "Edge" but left node properties unmarked, or grouped them in "Nodes: `a`, ...; `b`, ..." items, so the kind of a property was often unclear. Every item now starts with (Node), (Edge) or (Graph), followed by the property names, a colon and the explanation, one item per property or group of properties; qualifications (kinds of nodes, networks) follow the names in parentheses. Properties that were only named got a short explanation, and a few items that named the wrong kind of property were corrected.
diff --git a/entries/littlesis/__init__.py b/entries/littlesis/__init__.py
index e2f50a8..008f65e 100644
--- a/entries/littlesis/__init__.py
+++ b/entries/littlesis/__init__.py
@@ -28,8 +28,31 @@ The bulk files of LittleSis (`entities.json.gz` and `relationships.json.gz`) are
The networks have the following properties:
-- Nodes: `entity_id`, the id of the entity in LittleSis (its page is `https://littlesis.org/entities/<entity_id>`); `has_record`, whether the entity is in the entities file; `name`; `kind`, `Person` or `Org`; `types`, its types (e.g. `Business Person`, `Lobbyist`, `Public Company`, `Political Fundraising Committee`); `blurb`, a short description; `summary`, a longer one; `aliases`, its other names; `website`; `start_date` and `end_date`, e.g. the dates of birth and death or of foundation and dissolution (as given, `YYYY-MM-DD` with `00` for unknown months and days); `parent_id`, the id of the parent organization (-1 if none); `updated_at`, the time of the last update of the record (Unix time); `extensions`, the attributes specific to its types (e.g. the parts of the names, gender and party ids, net worth, revenue, stock ticker, SEC CIK), as a JSON object; in `2024` and `2026` also `tags`, the tags of the entity (thematic research projects, e.g. `fracking`).
-- Edges: `relationship_id`, the id of the relationship in LittleSis; `category` and `category_id`, its category (see above, ids 1 to 12 in that order); `description`, a sentence describing it (e.g. "Linda S Wolf has a position (Director) at Walmart Inc."); `description1` and `description2`, the roles of the two entities (e.g. `Director`, `Contractor`/`Client`); `amount` and `currency`, the amount of money involved (NaN if not given); `goods`, the goods involved; `filings`, the number of filings on which a donation or transaction is based (-1 if not given); `start_date` and `end_date`, as for the nodes; `is_current`, whether the relationship is current (1, 0, or -1 if unknown); `updated_at`, the time of the last update (Unix time); `category_attributes`, the attributes specific to the category (e.g. `is_board` and `compensation` of positions, `degree_id` of education, `percent_stake` of ownership), as a JSON object."""
+- (Node) `entity_id`: the id of the entity in LittleSis (its page is `https://littlesis.org/entities/<entity_id>`).
+- (Node) `has_record`: whether the entity is in the entities file.
+- (Node) `name`: the name of the entity.
+- (Node) `kind`: `Person` or `Org`.
+- (Node) `types`: its types (e.g. `Business Person`, `Lobbyist`, `Public Company`, `Political Fundraising Committee`).
+- (Node) `blurb`: a short description.
+- (Node) `summary`: a longer one.
+- (Node) `aliases`: its other names.
+- (Node) `website`: its website.
+- (Node) `start_date`, `end_date`: e.g. the dates of birth and death or of foundation and dissolution (as given, `YYYY-MM-DD` with `00` for unknown months and days).
+- (Node) `parent_id`: the id of the parent organization (-1 if none).
+- (Node) `updated_at`: the time of the last update of the record (Unix time).
+- (Node) `extensions`: the attributes specific to its types (e.g. the parts of the names, gender and party ids, net worth, revenue, stock ticker, SEC CIK), as a JSON object.
+- (Node) `tags` (`2024` and `2026`): the tags of the entity (thematic research projects, e.g. `fracking`).
+- (Edge) `relationship_id`: the id of the relationship in LittleSis.
+- (Edge) `category` and `category_id`: its category (see above, ids 1 to 12 in that order).
+- (Edge) `description`: a sentence describing it (e.g. "Linda S Wolf has a position (Director) at Walmart Inc.").
+- (Edge) `description1` and `description2`: the roles of the two entities (e.g. `Director`, `Contractor`/`Client`).
+- (Edge) `amount` and `currency`: the amount of money involved (NaN if not given).
+- (Edge) `goods`: the goods involved.
+- (Edge) `filings`: the number of filings on which a donation or transaction is based (-1 if not given).
+- (Edge) `start_date` and `end_date`: as for the nodes.
+- (Edge) `is_current`: whether the relationship is current (1, 0, or -1 if unknown).
+- (Edge) `updated_at`: the time of the last update (Unix time).
+- (Edge) `category_attributes`: the attributes specific to the category (e.g. `is_board` and `compensation` of positions, `degree_id` of education, `percent_stake` of ownership), as a JSON object."""
tags = ['Social', 'Economic', 'Political', 'Affiliation', 'Multigraph', 'Weighted', 'Metadata']
url = ['https://littlesis.org/database/bulk_data', 'https://web.archive.org/web/20261019163047/https://littlesis.org/database/public_data/entities.json.gz']
citation = [('Public Accountability Initiative, "LittleSis: a free database of who-knows-who at the heights of business and government." Bulk data (2022, 2024, 2026)', 'https://littlesis.org/database/bulk_data')]
[entries] New entry: littlesis 9825ff6
diff --git a/entries/littlesis/__init__.py b/entries/littlesis/__init__.py
new file mode 100644
index 0000000..e2f50a8
--- /dev/null
+++ b/entries/littlesis/__init__.py
@@ -0,0 +1,200 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2026 Tiago de Paula Peixoto <tiago@skewed.de>
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+
+from .. import *
+import json
+
+title = "LittleSis network of power (2022, 2024, 2026)"
+description = """The LittleSis database ("the opposite of Big Brother"), a collaborative database of the relationships between powerful people and organizations in the United States (and beyond), maintained by the Public Accountability Initiative. Its entries are edited by researchers, journalists and volunteers from public sources (news, government filings, corporate reports, campaign finance and lobbying disclosures, which are also imported in bulk), and record "who knows who" among business executives, politicians, lobbyists, financiers and their affiliated institutions.
+
+Each node is an entity of the database, a person or an organization, and each directed edge is a relationship between two entities, from `entity1` to `entity2` of the record, of one of twelve categories: `position` (a person holds a position in an organization, e.g. as director), `education` (a person studied at a school), `membership` (an entity is a member of an organization), `family`, `donation` (the first entity gave money or goods to the second, e.g. a campaign contribution), `transaction` (the two entities did business, e.g. as contractor and client), `lobbying` (the first entity lobbied the second), `social`, `professional`, `ownership` (the first entity owns a share of the second), `hierarchy` (between organizations, e.g. a subsidiary) and `generic`. The relationships are directed as recorded, even those of symmetric categories (e.g. family). The network is a multigraph: two entities can have several relationships, e.g. donations in several election cycles (some entities are also related to themselves).
+
+The bulk files of LittleSis (`entities.json.gz` and `relationships.json.gz`) are replaced in place by new versions, and the site refuses them to scripts. The networks are built from copies archived by the Internet Archive: of August 9, 2022 (`2022`), January 23, 2024 (`2024`) and October 19, 2026 (`2026`, the latest; two earlier copies of 2026, of April and May, are left out). A few relationships (11 in `2022`, 8 in `2024`, 3 in `2026`) involve entities that are missing from the entities file (presumably deleted); these are included as nodes without attributes (`has_record` false).
+
+The networks have the following properties:
+
+- Nodes: `entity_id`, the id of the entity in LittleSis (its page is `https://littlesis.org/entities/<entity_id>`); `has_record`, whether the entity is in the entities file; `name`; `kind`, `Person` or `Org`; `types`, its types (e.g. `Business Person`, `Lobbyist`, `Public Company`, `Political Fundraising Committee`); `blurb`, a short description; `summary`, a longer one; `aliases`, its other names; `website`; `start_date` and `end_date`, e.g. the dates of birth and death or of foundation and dissolution (as given, `YYYY-MM-DD` with `00` for unknown months and days); `parent_id`, the id of the parent organization (-1 if none); `updated_at`, the time of the last update of the record (Unix time); `extensions`, the attributes specific to its types (e.g. the parts of the names, gender and party ids, net worth, revenue, stock ticker, SEC CIK), as a JSON object; in `2024` and `2026` also `tags`, the tags of the entity (thematic research projects, e.g. `fracking`).
+- Edges: `relationship_id`, the id of the relationship in LittleSis; `category` and `category_id`, its category (see above, ids 1 to 12 in that order); `description`, a sentence describing it (e.g. "Linda S Wolf has a position (Director) at Walmart Inc."); `description1` and `description2`, the roles of the two entities (e.g. `Director`, `Contractor`/`Client`); `amount` and `currency`, the amount of money involved (NaN if not given); `goods`, the goods involved; `filings`, the number of filings on which a donation or transaction is based (-1 if not given); `start_date` and `end_date`, as for the nodes; `is_current`, whether the relationship is current (1, 0, or -1 if unknown); `updated_at`, the time of the last update (Unix time); `category_attributes`, the attributes specific to the category (e.g. `is_board` and `compensation` of positions, `degree_id` of education, `percent_stake` of ownership), as a JSON object."""
+tags = ['Social', 'Economic', 'Political', 'Affiliation', 'Multigraph', 'Weighted', 'Metadata']
+url = ['https://littlesis.org/database/bulk_data', 'https://web.archive.org/web/20261019163047/https://littlesis.org/database/public_data/entities.json.gz']
+citation = [('Public Accountability Initiative, "LittleSis: a free database of who-knows-who at the heights of business and government." Bulk data (2022, 2024, 2026)', 'https://littlesis.org/database/bulk_data')]
+upstream_license = ('Creative Commons Attribution-ShareAlike 4.0 International (CC BY-SA 4.0)',
+ 'https://creativecommons.org/licenses/by-sa/4.0/')
+icon_hash = None
+# The bulk files of LittleSis are refused to scripts by a bot check of the
+# site, and are replaced in place; they are downloaded from the snapshots of
+# the Internet Archive: those of 2022-08-09 and 2024-01-23 (all those of the
+# old path, public_data/), and the latest one of the new path
+# (database/public_data/), of 2026-10-19 (the earlier ones of 2026, of April
+# and May, are left out)
+snapshots = {"2022": ("20220809013919", "20220809013909", "public_data"),
+ "2024": ("20240123041748", "20240123041818", "public_data"),
+ "2026": ("20261019163047", "20261019163124",
+ "database/public_data")}
+archived_urls = {}
+files = []
+for alt, (ts_ent, ts_rel, path) in snapshots.items():
+ for kind, ts in [("entities", ts_ent), ("relationships", ts_rel)]:
+ archived_urls[f"{kind}_{alt}.json.gz"] = \
+ f"https://web.archive.org/web/{ts}id_/https://littlesis.org/{path}/{kind}.json.gz"
+ files.append(((f"entities_{alt}.json.gz", f"relationships_{alt}.json.gz"),
+ alt, None))
+
+def fetch_upstream(force=False):
+ return fetch_archived_files(__name__.split(".")[-1], archived_urls, force,
+ tries=3)
+
+def iter_json_array(f, chunk=1 << 22):
+ "Iterate over the elements of a JSON array in the text file `f`, streamed."
+ dec = json.JSONDecoder()
+ buf = f.read(chunk).lstrip()
+ if not buf.startswith("["):
+ raise ValueError("not a JSON array")
+ pos = 1
+ eof = False
+ ws = " \t\r\n,"
+ while True:
+ while pos < len(buf) and buf[pos] in ws:
+ pos += 1
+ if pos < len(buf) and buf[pos] == "]":
+ return
+ try:
+ if pos >= len(buf):
+ raise json.JSONDecodeError("end of buffer", buf, pos)
+ obj, end = dec.raw_decode(buf, pos)
+ except json.JSONDecodeError:
+ if eof:
+ raise
+ # read more (keeping only the unparsed rest of the buffer)
+ more = f.read(chunk)
+ eof = not more
+ buf = buf[pos:] + more
+ pos = 0
+ continue
+ yield obj
+ pos = end
+
+CATEGORIES = {1: "position", 2: "education", 3: "membership", 4: "family",
+ 5: "donation", 6: "transaction", 7: "lobbying", 8: "social",
+ 9: "professional", 10: "ownership", 11: "hierarchy",
+ 12: "generic"}
+
+def _time(x):
+ "ISO time stamps as Unix time (-1 if missing)."
+ if not x:
+ return -1
+ return int(datetime.fromisoformat(x.replace("Z", "+00:00")).timestamp())
+
+def _tri(x):
+ return -1 if x is None else int(bool(x))
+
+def _num(x):
+ return numpy.nan if x is None else float(x)
+
+@cache_network()
+@coerce_props()
+@annotate()
+def parse(alts=None):
+
+ name = __name__.split(".")[-1]
+ for fnames, alt, fmt in files:
+ if alts is not None and alt not in alts:
+ continue
+ vp = defaultdict(list)
+ idx = {}
+ with open_upstream_file(name, fnames[0], "rb") as f, \
+ io.TextIOWrapper(f, "utf8") as f:
+ for o in iter_json_array(f):
+ a = o["attributes"]
+ idx[a["id"]] = len(idx)
+ vp["entity_id"].append(a["id"])
+ for k in ["name", "blurb", "summary", "website", "start_date",
+ "end_date"]:
+ vp[k].append(a[k] or "")
+ vp["kind"].append(a["primary_ext"] or "")
+ vp["parent_id"].append(a["parent_id"] or -1)
+ vp["updated_at"].append(_time(a["updated_at"]))
+ vp["aliases"].append(a["aliases"] or [])
+ vp["types"].append(a["types"] or [])
+ if "tags" in a:
+ vp["tags"].append(a["tags"] or [])
+ vp["extensions"].append(json.dumps(a["extensions"] or {},
+ separators=(",", ":"),
+ ensure_ascii=False))
+ missing = 0
+ edges = []
+ ep = defaultdict(list)
+ with open_upstream_file(name, fnames[1], "rb") as f, \
+ io.TextIOWrapper(f, "utf8") as f:
+ for o in iter_json_array(f):
+ a = o["attributes"]
+ e = []
+ for k in ["entity1_id", "entity2_id"]:
+ if a[k] not in idx:
+ # entities missing from the file (presumably
+ # deleted): kept as nodes without attributes
+ idx[a[k]] = len(idx)
+ missing += 1
+ e.append(idx[a[k]])
+ edges.append(e)
+ ep["relationship_id"].append(a["id"])
+ ep["category_id"].append(a["category_id"])
+ ep["category"].append(CATEGORIES.get(a["category_id"], ""))
+ for k in ["description", "description1", "description2",
+ "currency", "goods", "start_date", "end_date"]:
+ ep[k].append(" ".join((a[k] or "").split()))
+ ep["amount"].append(_num(a["amount"]))
+ ep["filings"].append(-1 if a["filings"] is None else a["filings"])
+ ep["is_current"].append(_tri(a["is_current"]))
+ ep["updated_at"].append(_time(a["updated_at"]))
+ ep["category_attributes"].append(
+ json.dumps(a.get("category_attributes") or {},
+ separators=(",", ":"), ensure_ascii=False))
+ g = Graph(directed=True)
+ g.add_vertex(len(idx))
+ g.add_edge_list(edges)
+ del edges
+ n = len(vp["entity_id"])
+ g.vp.entity_id = g.new_vp("int64_t", vals=list(idx))
+ g.vp.has_record = g.new_vp("bool")
+ g.vp.has_record.a[:n] = True
+ for k, vals in vp.items():
+ if k == "entity_id":
+ continue
+ if k in ("parent_id", "updated_at"):
+ vals = vals + [-1] * missing
+ t = "int64_t"
+ elif k in ("aliases", "types", "tags"):
+ vals = vals + [[]] * missing
+ t = "vector<string>"
+ else:
+ vals = vals + [""] * missing
+ t = "string"
+ g.vp[k] = g.new_vp(t, vals=vals)
+ del vp
+ for k, vals in ep.items():
+ if k in ("relationship_id", "category_id", "filings", "is_current",
+ "updated_at"):
+ t = "int64_t"
+ elif k == "amount":
+ t = "double"
+ else:
+ t = "string"
+ g.ep[k] = g.new_ep(t, vals=vals)
+ del ep
+ yield alt, g