History of offshore_leaks
The changes to the source code of this entry in the repository, newest first, each with the differences it made to the entry's files. They are also listed on GitLab.
[entries] New entry: offshore_leaks bb34256
diff --git a/entries/offshore_leaks/__init__.py b/entries/offshore_leaks/__init__.py
new file mode 100644
index 0000000..acc31b1
--- /dev/null
+++ b/entries/offshore_leaks/__init__.py
@@ -0,0 +1,120 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2026 Tiago de Paula Peixoto <tiago@skewed.de>
+#
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+from .. import *
+from datetime import timezone
+
+title = "ICIJ Offshore Leaks database"
+description = """The Offshore Leaks database of the International Consortium of Investigative Journalists (ICIJ): the offshore companies, foundations and trusts, the people and companies behind them, their intermediaries and their addresses, as revealed by the leaked documents of five investigations: the Offshore Leaks (2013), the Panama Papers (2016, from the law firm Mossack Fonseca), the Bahamas Leaks (2016), the Paradise Papers (2017, from the law firm Appleby and several corporate registries) and the Pandora Papers (2021, from several offshore service providers). The data cover offshore entities in over 200 countries and territories, over more than 80 years up to 2020. They are not complete records of the service providers, and the ICIJ notes that there are legitimate uses of offshore entities, and that the presence of a person or company in the database does not imply wrongdoing.
+
+Each node is an entity (814,344: offshore companies, trusts and foundations), an officer (771,315: people or companies with a role in an entity, e.g. directors, shareholders and beneficiaries), an intermediary (26,768: e.g. lawyers, banks and agents that set up entities), an address (402,246), or another kind of node (2,989, e.g. companies from corporate registries). The 1,139 nodes that are listed both as officers and as intermediaries are single nodes. Each directed edge is a relationship as given in the database, e.g. from an officer to an entity of which it is a director (`officer_of`), from an intermediary to an entity (`intermediary_of`), from a node to its address (`registered_address`), or between nodes that the ICIJ found to be probably the same (e.g. `same_name_as`, `similar`, `same_as`); several relationships between the same nodes appear as multiple edges. The network contains all the 2,016,523 nodes and 3,339,267 relationships of the bulk download of the database generated on 9 September 2026.
+
+The network has the following properties:
+
+- Nodes: `node_id`, the id of the node in the database; `node_type`, its kind (`entity`, `officer`, `intermediary`, `address` or `other`; `officer; intermediary` for the nodes listed as both); `name`; and, as given for the nodes (and empty otherwise): `original_name`, `former_name`, `address`, `jurisdiction` and `jurisdiction_description` (where the entity was incorporated), `company_type`, `type`, `status`, `service_provider`, `internal_id`, `ibcRUC` (a registration number), `incorporation_date`, `inactivation_date`, `struck_off_date`, `dorm_date`, `closed_date` (dates as given, mostly as DD-MON-YYYY), `countries` and `country_codes` (the countries the node is linked to, separated by semicolons), `source` (the leak and, for some, the service provider or registry), `valid_until` (how recent the data of the source are) and `note`.
+- Edges: `rel_type`, the type of relationship (e.g. `officer_of`); `link`, its description (e.g. "shareholder of", "director of"); `status` (e.g. "Resigned"); `start_date` and `end_date`, as given (in several formats), and `start_time` and `end_time`, the same as Unix time for the dates given as DD-MON-YYYY or YYYY-MM-DD (NaN otherwise); `source`, the leak (empty for 518,982 relationships)."""
+tags = ['Economic', 'Social', 'Offline', 'Unweighted', 'Multigraph', 'Metadata']
+url = ['https://offshoreleaks.icij.org/pages/database',
+ 'https://web.archive.org/web/20260924142228/https://offshoreleaks-data.icij.org/offshoreleaks/csv/full-oldb.LATEST.zip']
+citation = [('International Consortium of Investigative Journalists (ICIJ), "Offshore Leaks Database." (2026)', 'https://offshoreleaks.icij.org')]
+icon_hash = None
+upstream_license = ("ODbL (database), CC BY-SA (contents)", "https://offshoreleaks.icij.org/pages/database")
+
+# The bulk CSV export is republished in place as "full-oldb.LATEST.zip";
+# this is the export generated on 9 September 2026, from a pinned Internet
+# Archive snapshot
+archive = 'full-oldb.20260909.zip'
+archived_urls = {archive: 'https://web.archive.org/web/20260924142228id_/https://offshoreleaks-data.icij.org/offshoreleaks/csv/full-oldb.LATEST.zip'}
+node_files = [('entity', 'nodes-entities.csv'), ('officer', 'nodes-officers.csv'),
+ ('intermediary', 'nodes-intermediaries.csv'),
+ ('address', 'nodes-addresses.csv'), ('other', 'nodes-others.csv')]
+files = [(tuple(f'{archive}:{f}' for t, f in node_files) +
+ (f'{archive}:relationships.csv',), None, None)]
+
+# CSV columns stored as vertex properties (all as strings), under the same
+# names, except sourceID (stored as `source`)
+vcols = ['name', 'original_name', 'former_name', 'address', 'jurisdiction',
+ 'jurisdiction_description', 'company_type', 'type', 'status',
+ 'service_provider', 'internal_id', 'ibcRUC', 'incorporation_date',
+ 'inactivation_date', 'struck_off_date', 'dorm_date', 'closed_date',
+ 'countries', 'country_codes', 'sourceID', 'valid_until', 'note']
+
+def fetch_upstream(force=False):
+ return fetch_archived_files(__name__.split(".")[-1], archived_urls, force,
+ tries=3)
+
+def parse_date(s):
+ "Unix time of a date given as DD-MON-YYYY or YYYY-MM-DD, NaN otherwise."
+ for fmt in ("%d-%b-%Y", "%Y-%m-%d"):
+ try:
+ d = datetime.strptime(s, fmt).replace(tzinfo=timezone.utc)
+ return d.timestamp()
+ except ValueError:
+ pass
+ return numpy.nan
+
+@cache_network()
+@coerce_props()
+@annotate()
+def parse(alts=None):
+ global files
+ name = __name__.split(".")[-1]
+ for fnames, alt, fmt in files:
+ if alts is not None and alt not in alts:
+ continue
+ g = Graph(directed=True)
+ g.vp.node_id = g.new_vp("int64_t")
+ g.vp.node_type = g.new_vp("string")
+ for c in vcols:
+ g.vp["source" if c == "sourceID" else c] = g.new_vp("string")
+ vs = {}
+ for (ntype, f), fname in zip(node_files, fnames):
+ with open_upstream_file(name, fname, "r") as fs:
+ for r in csv.DictReader(fs):
+ nid = int(r["node_id"])
+ v = vs.get(nid)
+ if v is None:
+ v = vs[nid] = g.add_vertex()
+ g.vp.node_id[v] = nid
+ g.vp.node_type[v] = ntype
+ else:
+ # a node listed in two files (e.g. both an officer
+ # and an intermediary)
+ g.vp.node_type[v] += "; " + ntype
+ for c in vcols:
+ x = r.get(c)
+ if x:
+ g.vp["source" if c == "sourceID" else c][v] = x
+ eprops = [("rel_type", "string"), ("link", "string"),
+ ("status", "string"), ("start_date", "string"),
+ ("end_date", "string"), ("start_time", "double"),
+ ("end_time", "double"), ("source", "string")]
+ for p, t in eprops:
+ g.ep[p] = g.new_ep(t)
+ edges = []
+ with open_upstream_file(name, fnames[-1], "r") as fs:
+ for r in csv.DictReader(fs):
+ edges.append((int(vs[int(r["node_id_start"])]),
+ int(vs[int(r["node_id_end"])]),
+ r["rel_type"], r["link"], r["status"],
+ r["start_date"], r["end_date"],
+ parse_date(r["start_date"]),
+ parse_date(r["end_date"]), r["sourceID"]))
+ g.add_edge_list(edges, eprops=[g.ep[p] for p, t in eprops])
+ yield alt, g