History of eu_procurements
The changes to the source code of this entry in the repository, newest first, each with the differences it made to the entry's files. They are also listed on GitLab.
[entries] Add the date when each entry was added, and remove icon_hash beff7c9
The dates are those of the commits that first added the entries' files (2020-08-11 for the entries of the initial commit); renamed entries keep the date of their first name. Changed: none
diff --git a/entries/eu_procurements/__init__.py b/entries/eu_procurements/__init__.py
index 65f2e50..d39ff58 100644
--- a/entries/eu_procurements/__init__.py
+++ b/entries/eu_procurements/__init__.py
@@ -77,7 +77,7 @@ url = ['https://zenodo.org/record/3537986#.Xis4mC2ZNGV',
'https://data.open-contracting.org/en/search/']
citation = [('J. Wachs, M. Fazekas, and J. Kertész, "Corruption Risk in Contracting Markets: A Network Science Perspective." International Journal of Data Science and Analytics, pp 1–16 (2020)', 'https://doi.org/10.1007/s41060-019-00204-1'),
('Government Transparency Institute, "OpenTender." OCDS publications of the OCP Data Registry (2026)', 'https://opentender.eu/')]
-icon_hash = ['5e2b3bc478691ca4060bdee7', '5e2bd0a978691ca4060bdee9']
+date_added = '2020-08-11'
# (no license given for the data of Wachs et al.)
upstream_license = ("CC BY-NC-SA 4.0 (OpenTender networks)", "https://creativecommons.org/licenses/by-nc-sa/4.0/")
upstream_prefix = 'https://zenodo.org/record/3537986/files'
[entries] Prefix the properties in the descriptions with (Node), (Edge) or (Graph) bc3dbbd
The lists of properties at the end of the descriptions marked edge properties with "Edge" but left node properties unmarked, or grouped them in "Nodes: `a`, ...; `b`, ..." items, so the kind of a property was often unclear. Every item now starts with (Node), (Edge) or (Graph), followed by the property names, a colon and the explanation, one item per property or group of properties; qualifications (kinds of nodes, networks) follow the names in parentheses. Properties that were only named got a short explanation, and a few items that named the wrong kind of property were corrected.
diff --git a/entries/eu_procurements/__init__.py b/entries/eu_procurements/__init__.py
index 7ee40ef..65f2e50 100644
--- a/entries/eu_procurements/__init__.py
+++ b/entries/eu_procurements/__init__.py
@@ -29,45 +29,47 @@ description = r"""Networks of public procurement contracts in 26 European countr
The `contracts` network has the following properties, taken from the upstream data dictionary:
-- `id_final`: deduplicated identifier of the issuer or winner, ending in `_i` for issuers and `_w` for winners.
-- `is_issuer`: whether the node is an issuer (True) or a winner (False).
-- `name`: raw name; `canonical_name`: most common name after deduplication.
-- `country`, `address`, `postcode`, `town`: location of the issuer or winner (raw text).
-- `type` and `sector`: type (e.g. "3. Regional or local authorities") and sector (e.g. "Health") of the issuer.
-- `tender_id` (edges): contract identifier.
-- `year` and `date_dispatched` (edges): year and date (YYYY-MM-DD) of the contract.
-- `est_value` and `contract_value` (edges): raw estimated and final value of the contract (currency and VAT treatment unclear).
-- `contract_nuts` and `processed_nuts` (edges): raw and (imperfectly) cleaned NUTS geographic code of where the contract is performed.
-- `contract_cpv` and `cpv2` (edges): Common Procurement Vocabulary (CPV) code of the purchased goods or services, and its 2-digit prefix.
-- `contract_eu_funded` (edges): 1 if the contract was supported by EU funding.
-- `contract_type` (edges): works, services or supplies.
-- `procedure` (edges): the award procedure (e.g. open, restricted).
-- `n_bids` (edges): number of bids received.
-- `nocft` (edges): 1 if no call for tenders was found.
-- `single_bidder` (edges): 1 if there was no competition for the contract (single bid), an indicator of corruption risk; missing values were imputed with the country mean.
+- (Node) `id_final`: deduplicated identifier of the issuer or winner, ending in `_i` for issuers and `_w` for winners.
+- (Node) `is_issuer`: whether the node is an issuer (True) or a winner (False).
+- (Node) `name`: raw name.
+- (Node) `canonical_name`: most common name after deduplication.
+- (Node) `country`, `address`, `postcode`, `town`: location of the issuer or winner (raw text).
+- (Node) `type` and `sector`: type (e.g. "3. Regional or local authorities") and sector (e.g. "Health") of the issuer.
+- (Edge) `tender_id`: contract identifier.
+- (Edge) `year` and `date_dispatched`: year and date (YYYY-MM-DD) of the contract.
+- (Edge) `est_value` and `contract_value`: raw estimated and final value of the contract (currency and VAT treatment unclear).
+- (Edge) `contract_nuts` and `processed_nuts`: raw and (imperfectly) cleaned NUTS geographic code of where the contract is performed.
+- (Edge) `contract_cpv` and `cpv2`: Common Procurement Vocabulary (CPV) code of the purchased goods or services, and its 2-digit prefix.
+- (Edge) `contract_eu_funded`: 1 if the contract was supported by EU funding.
+- (Edge) `contract_type`: works, services or supplies.
+- (Edge) `procedure`: the award procedure (e.g. open, restricted).
+- (Edge) `n_bids`: number of bids received.
+- (Edge) `nocft`: 1 if no call for tenders was found.
+- (Edge) `single_bidder`: 1 if there was no competition for the contract (single bid), an indicator of corruption risk; missing values were imputed with the country mean.
The country-year networks have the following properties:
-- `label`: node identifier, ending in `_i` for issuers and `_w` for winners.
-- `is_issuer`: whether the node is an issuer (True) or a winner (False).
-- `count` (edges): the number of contracts between the issuer and the winner in the given year.
-- `pctSingleBid` (edges): the fraction of these contracts that were awarded without competition, i.e. with a single bidder or to a sole supplier, an elementary indicator of corruption risk. Missing single-bidding information was imputed, as described in the paper.
+- (Node) `label`: node identifier, ending in `_i` for issuers and `_w` for winners.
+- (Node) `is_issuer`: whether the node is an issuer (True) or a winner (False).
+- (Edge) `count`: the number of contracts between the issuer and the winner in the given year.
+- (Edge) `pctSingleBid`: the fraction of these contracts that were awarded without competition, i.e. with a single bidder or to a sole supplier, an elementary indicator of corruption risk. Missing single-bidding information was imputed, as described in the paper.
The OpenTender networks have the following properties:
-- `body_id`: the body id of OpenTender.
-- `name`: the name of the buyer or supplier (the first one found).
-- `organization_ids`: its other identifiers, as given (e.g. national registration numbers).
-- `country`: the country of its address (two-letter code, as given; empty if none).
-- `is_buyer`, `is_supplier`: whether it appears as a buyer or as a supplier.
-- Edge `ocid`: the open contracting id of the contracting process.
-- Edge `award_id`: the id of the award within the process.
-- Edge `amount`, `currency`: the value of the award (NaN if not given).
-- Edge `date`: the date of the award (or of the release, if not given), ISO 8601.
-- Edge `method`: the procurement method (`open`, `selective`, `limited`, `direct`; empty if not given); `method_details`: the procedure in the terms of OpenTender.
-- Edge `category`: the main procurement category (`goods`, `works`, `services`; empty if not given).
-- Edge `cpv`: the CPV codes of the items of the award (or of the tender, if the award has none).
-- Edge `title`: the title of the tender."""
+- (Node) `body_id`: the body id of OpenTender.
+- (Node) `name`: the name of the buyer or supplier (the first one found).
+- (Node) `organization_ids`: its other identifiers, as given (e.g. national registration numbers).
+- (Node) `country`: the country of its address (two-letter code, as given; empty if none).
+- (Node) `is_buyer`, `is_supplier`: whether it appears as a buyer or as a supplier.
+- (Edge) `ocid`: the open contracting id of the contracting process.
+- (Edge) `award_id`: the id of the award within the process.
+- (Edge) `amount`, `currency`: the value of the award (NaN if not given).
+- (Edge) `date`: the date of the award (or of the release, if not given), ISO 8601.
+- (Edge) `method`: the procurement method (`open`, `selective`, `limited`, `direct`; empty if not given).
+- (Edge) `method_details`: the procedure in the terms of OpenTender.
+- (Edge) `category`: the main procurement category (`goods`, `works`, `services`; empty if not given).
+- (Edge) `cpv`: the CPV codes of the items of the award (or of the tender, if the award has none).
+- (Edge) `title`: the title of the tender."""
tags = ['Economic', 'Commerce', 'Weighted', 'Temporal', 'Metadata']
url = ['https://zenodo.org/record/3537986#.Xis4mC2ZNGV',
'https://zenodo.org/record/3627216#.XivHNy2ZNGV',
[entries] Remove unneeded code from the entries 2c69f19
Unneeded "global files" statements, unused imports and variables, f-strings without placeholders, duplicate keys in foodweb_little_rock (keeping the order and the values used), and the second definition of get_lines() in twitter_higgs. Nothing changes in the networks or descriptions. Changed: none
diff --git a/entries/eu_procurements/__init__.py b/entries/eu_procurements/__init__.py
index 5a838cf..7ee40ef 100644
--- a/entries/eu_procurements/__init__.py
+++ b/entries/eu_procurements/__init__.py
@@ -317,7 +317,6 @@ def parse_contracts(name, fnames):
@coerce_props()
@annotate()
def parse(alts=None):
- global files
name = __name__.split(".")[-1]
for fnames, alt, fmt in files:
if alts is not None and alt not in alts:
[entries] eu_procurements: add the OpenTender networks of 34 countries, the EU and TED a95cacc
The entry had the procurement networks of the TED and national data sets. It now also has the buyer-supplier networks of the OpenTender (DIGIWHIST) data of 34 countries, of the EU institutions and of TED, as networks opentender_<code>. The existing networks are unchanged. Changed: upstream eu_procurements
diff --git a/entries/eu_procurements/__init__.py b/entries/eu_procurements/__init__.py
index 3fe6d0e..5a838cf 100644
--- a/entries/eu_procurements/__init__.py
+++ b/entries/eu_procurements/__init__.py
@@ -17,12 +17,15 @@
# with this program. If not, see <http://www.gnu.org/licenses/>.
from .. import *
+import json
+import process_entry
-title = "EU procurement contract networks (2008-2016)"
-description = r"""Networks of public procurement contracts in 26 European countries (25 EU member states and Norway) from 2008 to 2016. The data were collected by J. Wachs, M. Fazekas and J. Kertész from Tenders Electronic Daily (TED), the official EU portal where all contracts above an estimated value of 5.2 million Euro (works) or 135 thousand Euro (services and supplies) must be published. Contracts from Luxembourg, Croatia and EU institutions were excluded. The identities of issuers and winners were deduplicated algorithmically per country, using string similarity and clustering with the Dedupe software. The entry contains two kinds of networks built from these data:
+title = "European procurement contract networks (2004-2025)"
+description = r"""Networks of public procurement contracts in 26 European countries (25 EU member states and Norway) from 2008 to 2016. The data were collected by J. Wachs, M. Fazekas and J. Kertész from Tenders Electronic Daily (TED), the official EU portal where all contracts above an estimated value of 5.2 million Euro (works) or 135 thousand Euro (services and supplies) must be published. Contracts from Luxembourg, Croatia and EU institutions were excluded. The identities of issuers and winners were deduplicated algorithmically per country, using string similarity and clustering with the Dedupe software. The entry contains two kinds of networks built from these data, and a third kind from the later data of OpenTender:
- `contracts`: a bipartite multigraph of all contracts, in which one set of nodes are issuers (buyers: public institutions such as ministries, hospitals or city halls), the other are winners (suppliers, generally private firms), and each edge is a single awarded contract, so that two nodes may be connected by many edges.
- 234 networks of the annual national procurement markets of each country and year, named by the two-letter country code and the year (e.g. `HU_2012`), published separately by the authors. Each is a bipartite graph between issuers and winners, in which an edge connects an issuer and a winner if at least one contract was awarded between them in that year. Their node identifiers are consistent across years within a country, but have been randomly generated and do not correspond to any official identifiers, nor to those of the `contracts` network.
+- 36 networks of the procurement data of OpenTender (opentender.eu), the platform of the Government Transparency Institute (successor of the DIGIWHIST project, in which the data above were collected), which collects and cleans the contracting data of TED and of the national procurement portals (thus also contracts below the EU thresholds) of 35 jurisdictions, from 2004 to 2025 (most from 2006 to 2024), as republished in the Open Contracting Data Standard (OCDS) by the OCP Data Registry of the Open Contracting Partnership (retrieved by the registry in June 2026, January 2026 for Poland and April 2026 for the EU institutions). They are named `opentender_<code>`, by the two-letter code of the country as above (`opentender_AT`, ..., `opentender_UK`, and `opentender_CH`, `opentender_GE`, `opentender_IS`, `opentender_MK`, `opentender_RS` for Switzerland, Georgia, Iceland, North Macedonia and Serbia), `opentender_EU` for the procurement of the EU institutions, and `opentender_TED` for all the notices of TED. Each node is a buyer or a supplier, identified by the de-duplicated body id of OpenTender (the same in all processes of a network), and each directed edge is the award of a contract by a buyer to a supplier, one edge for each supplier of each award (a multigraph). The tenders without awards, the bids, lots, items and the indicators of OpenTender are not included.
The `contracts` network has the following properties, taken from the upstream data dictionary:
@@ -48,13 +51,33 @@ The country-year networks have the following properties:
- `label`: node identifier, ending in `_i` for issuers and `_w` for winners.
- `is_issuer`: whether the node is an issuer (True) or a winner (False).
- `count` (edges): the number of contracts between the issuer and the winner in the given year.
-- `pctSingleBid` (edges): the fraction of these contracts that were awarded without competition, i.e. with a single bidder or to a sole supplier, an elementary indicator of corruption risk. Missing single-bidding information was imputed, as described in the paper."""
+- `pctSingleBid` (edges): the fraction of these contracts that were awarded without competition, i.e. with a single bidder or to a sole supplier, an elementary indicator of corruption risk. Missing single-bidding information was imputed, as described in the paper.
+
+The OpenTender networks have the following properties:
+
+- `body_id`: the body id of OpenTender.
+- `name`: the name of the buyer or supplier (the first one found).
+- `organization_ids`: its other identifiers, as given (e.g. national registration numbers).
+- `country`: the country of its address (two-letter code, as given; empty if none).
+- `is_buyer`, `is_supplier`: whether it appears as a buyer or as a supplier.
+- Edge `ocid`: the open contracting id of the contracting process.
+- Edge `award_id`: the id of the award within the process.
+- Edge `amount`, `currency`: the value of the award (NaN if not given).
+- Edge `date`: the date of the award (or of the release, if not given), ISO 8601.
+- Edge `method`: the procurement method (`open`, `selective`, `limited`, `direct`; empty if not given); `method_details`: the procedure in the terms of OpenTender.
+- Edge `category`: the main procurement category (`goods`, `works`, `services`; empty if not given).
+- Edge `cpv`: the CPV codes of the items of the award (or of the tender, if the award has none).
+- Edge `title`: the title of the tender."""
tags = ['Economic', 'Commerce', 'Weighted', 'Temporal', 'Metadata']
url = ['https://zenodo.org/record/3537986#.Xis4mC2ZNGV',
- 'https://zenodo.org/record/3627216#.XivHNy2ZNGV']
-citation = [('J. Wachs, M. Fazekas, and J. Kertész, "Corruption Risk in Contracting Markets: A Network Science Perspective." International Journal of Data Science and Analytics, pp 1–16 (2020)', 'https://doi.org/10.1007/s41060-019-00204-1')]
+ 'https://zenodo.org/record/3627216#.XivHNy2ZNGV',
+ 'https://opentender.eu/',
+ 'https://data.open-contracting.org/en/search/']
+citation = [('J. Wachs, M. Fazekas, and J. Kertész, "Corruption Risk in Contracting Markets: A Network Science Perspective." International Journal of Data Science and Analytics, pp 1–16 (2020)', 'https://doi.org/10.1007/s41060-019-00204-1'),
+ ('Government Transparency Institute, "OpenTender." OCDS publications of the OCP Data Registry (2026)', 'https://opentender.eu/')]
icon_hash = ['5e2b3bc478691ca4060bdee7', '5e2bd0a978691ca4060bdee9']
-upstream_license = None
+# (no license given for the data of Wachs et al.)
+upstream_license = ("CC BY-NC-SA 4.0 (OpenTender networks)", "https://creativecommons.org/licenses/by-nc-sa/4.0/")
upstream_prefix = 'https://zenodo.org/record/3537986/files'
files = [('data.zip:data/contracts.csv', 'contracts', None)]
@@ -111,10 +134,131 @@ country_year_files = \
"UK_2013", "UK_2014", "UK_2015", "UK_2016"]]
files += country_year_files
+# The OpenTender publications of the OCP Data Registry (publication id):
+# their files are replaced in place when the registry retrieves the data
+# again (every six months)
+opentender_prefix = 'https://data.open-contracting.org/en/publication'
+_opentender = {"AT": 46, "BE": 47, "BG": 44, "HR": 45, "CY": 48, "CZ": 12,
+ "DK": 49, "EE": 50, "EU": 94, "TED": 150, "FI": 51, "FR": 54,
+ "GE": 52, "DE": 53, "GR": 55, "HU": 56, "IS": 57, "IE": 58,
+ "IT": 87, "LV": 60, "LT": 68, "LU": 69, "MT": 70, "NL": 71,
+ "MK": 138, "NO": 72, "PL": 73, "PT": 74, "RO": 75, "RS": 139,
+ "SK": 88, "SI": 93, "ES": 89, "SE": 90, "CH": 91, "UK": 92}
+opentender_files = [(f"opentender/{i}_full.jsonl.gz", f"opentender_{c}", None)
+ for c, i in sorted(_opentender.items())]
+files += opentender_files
+
def fetch_upstream(force=False):
name = __name__.split(".")[-1]
fetch_upstream_files(name, upstream_prefix, files[:1], force)
fetch_upstream_files(name, country_year_prefix, country_year_files, force)
+ urls = {fn: f"{opentender_prefix}/{os.path.basename(fn).split('_')[0]}/download?name=full.jsonl.gz"
+ for fn, alt, fmt in opentender_files}
+ if process_entry._check_health:
+ for fn in health_sample(list(urls)):
+ with open_url(urls[fn]) as f:
+ check_not_html(fn, f.read(1 << 10))
+ return
+ for fn, url in urls.items():
+ if not force and os.path.exists(f"cache/upstream/{name}/{fn}"):
+ continue
+ with open_url(url) as f:
+ head = f.read(1 << 10)
+ check_not_html(fn, head)
+ with open_upstream_file(name, fn, "wb") as output:
+ output.write(head)
+ copy_download(f, output, len(head), label=fn)
+
+# For checks only: if set, only this many processes are read per OpenTender
+# network
+_check_rows = None
+
+def _norm(x):
+ return " ".join(str(x or "").split())
+
+def parse_opentender(name, fname):
+ """The network of the awards of an OpenTender publication (OCDS compiled
+ releases, one per line)."""
+ idx = {}
+ attrs = []
+
+ def node(party, parties):
+ if not isinstance(party, dict) or not party.get("id"):
+ return None
+ v = idx.get(party["id"])
+ if v is None:
+ v = idx[party["id"]] = len(attrs)
+ full = parties.get(party["id"], {})
+ addr = full.get("address") or party.get("address") or {}
+ ids = [f"{_norm(i.get('scheme'))}:{_norm(i.get('id'))}"
+ for i in (full.get("additionalIdentifiers") or [])
+ if isinstance(i, dict) and i.get("id")]
+ attrs.append(dict(body_id=party["id"],
+ name=_norm(party.get("name") or full.get("name")),
+ organization_ids=ids,
+ country=_norm(addr.get("countryName")),
+ is_buyer=False, is_supplier=False))
+ return v
+
+ edges = []
+ ep = defaultdict(list)
+ with open_upstream_file(name, fname, "r") as f:
+ for n, line in enumerate(f):
+ if _check_rows is not None and n >= _check_rows:
+ break
+ o = json.loads(line)
+ parties = {p.get("id"): p for p in o.get("parties") or []
+ if isinstance(p, dict)}
+ tender = o.get("tender") or {}
+ buyer = o.get("buyer") or tender.get("procuringEntity")
+ if not any(a.get("suppliers") for a in o.get("awards") or []):
+ continue
+ b = node(buyer, parties)
+ if b is None:
+ continue
+ tcpv = [i["classification"]["id"] for i in tender.get("items") or []
+ if isinstance(i.get("classification"), dict)]
+ for a in o.get("awards") or []:
+ cpv = [i["classification"]["id"] for i in a.get("items") or []
+ if isinstance(i.get("classification"), dict)] or tcpv
+ for s in a.get("suppliers") or []:
+ t = node(s, parties)
+ if t is None:
+ continue
+ attrs[b]["is_buyer"] = True
+ attrs[t]["is_supplier"] = True
+ edges.append((b, t))
+ val = a.get("value") or {}
+ try:
+ amt = float(val.get("amount"))
+ except (TypeError, ValueError):
+ amt = numpy.nan
+ ep["ocid"].append(_norm(o.get("ocid")))
+ ep["award_id"].append(_norm(a.get("id")))
+ ep["amount"].append(amt)
+ ep["currency"].append(_norm(val.get("currency")))
+ ep["date"].append(_norm(a.get("date") or o.get("date")))
+ ep["method"].append(_norm(tender.get("procurementMethod")))
+ ep["method_details"].append(_norm(tender.get("procurementMethodDetails")))
+ ep["category"].append(_norm(tender.get("mainProcurementCategory")))
+ ep["cpv"].append(cpv)
+ ep["title"].append(_norm(tender.get("title")))
+ del idx
+ g = Graph(directed=True)
+ g.add_vertex(len(attrs))
+ g.add_edge_list(edges)
+ del edges
+ for k in ["body_id", "name", "country"]:
+ g.vp[k] = g.new_vp("string", vals=[a[k] for a in attrs])
+ g.vp.organization_ids = g.new_vp("vector<string>", vals=[a["organization_ids"] for a in attrs])
+ for k in ["is_buyer", "is_supplier"]:
+ g.vp[k] = g.new_vp("bool", vals=[a[k] for a in attrs])
+ del attrs
+ for k in ["ocid", "award_id", "amount", "currency", "date", "method",
+ "method_details", "category", "cpv", "title"]:
+ t = {"amount": "double", "cpv": "vector<string>"}.get(k, "string")
+ g.ep[k] = g.new_ep(t, vals=ep[k])
+ return g
def parse_contracts(name, fnames):
with ExitStack() as stack:
@@ -180,7 +324,9 @@ def parse(alts=None):
continue
if isinstance(fnames, str):
fnames = [fnames]
- if alt == "contracts":
+ if alt.startswith("opentender_"):
+ g = parse_opentender(name, fnames[0])
+ elif alt == "contracts":
g = parse_contracts(name, fnames)
else:
with ExitStack() as stack:
[entries] Merge duplicate entries and add cross-references between related ones 9bc6033
- Removed duplicates, merged into the entries holding the same data: dblp_coauthor_snap (soc_net_comms), cattle and bison (dom; bison's breeding data move there, as do the wolves' age and sex), football_tsevans (football, with both conference assignments), eu_procurements_alt (eu_procurements), foursquare_friendships (foursquare_global). - software_dependencies: colt and scolt are one network; ambassador: the duplicate 2001 network is dropped and the three same-edge networks combined; dom: the dataset count corrected (436). - macaque_neural: Young's 71-area network added (young71; the 47-area one is now cocomac47), both from archived copies. - collins_yeast, celegans_interactomes: from archived copies of the dead CCSB site (byte-identical files). - Cross-references between entries holding versions of the same data (food webs, Wikipedia, Slashdot, physicians, SNAP/Mislove, CCSB). - ICON ids of duplicate records added; macaque_neural's corrected. - prosper, dbpedia_all: string properties given in edge-index order (needs graph-tool with the fixed vals ordering).
diff --git a/entries/eu_procurements/__init__.py b/entries/eu_procurements/__init__.py
index 908f573..3fe6d0e 100644
--- a/entries/eu_procurements/__init__.py
+++ b/entries/eu_procurements/__init__.py
@@ -19,9 +19,12 @@
from .. import *
title = "EU procurement contract networks (2008-2016)"
-description = r"""A bipartite multigraph of public procurement contracts in 26 European countries (25 EU member states and Norway) from 2008 to 2016. One set of nodes are issuers (buyers: public institutions such as ministries, hospitals or city halls), the other are winners (suppliers, generally private firms), and each edge is a single awarded contract, so that two nodes may be connected by many edges. The data were collected by J. Wachs, M. Fazekas and J. Kertész from Tenders Electronic Daily (TED), the official EU portal where all contracts above an estimated value of 5.2 million Euro (works) or 135 thousand Euro (services and supplies) must be published. Contracts from Luxembourg, Croatia and EU institutions were excluded. The identities of issuers and winners were deduplicated algorithmically per country, using string similarity and clustering with the Dedupe software. See `eu_procurements_alt` for the aggregated networks of each country and year.
+description = r"""Networks of public procurement contracts in 26 European countries (25 EU member states and Norway) from 2008 to 2016. The data were collected by J. Wachs, M. Fazekas and J. Kertész from Tenders Electronic Daily (TED), the official EU portal where all contracts above an estimated value of 5.2 million Euro (works) or 135 thousand Euro (services and supplies) must be published. Contracts from Luxembourg, Croatia and EU institutions were excluded. The identities of issuers and winners were deduplicated algorithmically per country, using string similarity and clustering with the Dedupe software. The entry contains two kinds of networks built from these data:
-The network has the following properties, taken from the upstream data dictionary:
+- `contracts`: a bipartite multigraph of all contracts, in which one set of nodes are issuers (buyers: public institutions such as ministries, hospitals or city halls), the other are winners (suppliers, generally private firms), and each edge is a single awarded contract, so that two nodes may be connected by many edges.
+- 234 networks of the annual national procurement markets of each country and year, named by the two-letter country code and the year (e.g. `HU_2012`), published separately by the authors. Each is a bipartite graph between issuers and winners, in which an edge connects an issuer and a winner if at least one contract was awarded between them in that year. Their node identifiers are consistent across years within a country, but have been randomly generated and do not correspond to any official identifiers, nor to those of the `contracts` network.
+
+The `contracts` network has the following properties, taken from the upstream data dictionary:
- `id_final`: deduplicated identifier of the issuer or winner, ending in `_i` for issuers and `_w` for winners.
- `is_issuer`: whether the node is an issuer (True) or a winner (False).
@@ -38,18 +41,133 @@ The network has the following properties, taken from the upstream data dictionar
- `procedure` (edges): the award procedure (e.g. open, restricted).
- `n_bids` (edges): number of bids received.
- `nocft` (edges): 1 if no call for tenders was found.
-- `single_bidder` (edges): 1 if there was no competition for the contract (single bid), an indicator of corruption risk; missing values were imputed with the country mean."""
+- `single_bidder` (edges): 1 if there was no competition for the contract (single bid), an indicator of corruption risk; missing values were imputed with the country mean.
+
+The country-year networks have the following properties:
+
+- `label`: node identifier, ending in `_i` for issuers and `_w` for winners.
+- `is_issuer`: whether the node is an issuer (True) or a winner (False).
+- `count` (edges): the number of contracts between the issuer and the winner in the given year.
+- `pctSingleBid` (edges): the fraction of these contracts that were awarded without competition, i.e. with a single bidder or to a sole supplier, an elementary indicator of corruption risk. Missing single-bidding information was imputed, as described in the paper."""
tags = ['Economic', 'Commerce', 'Weighted', 'Temporal', 'Metadata']
-url = 'https://zenodo.org/record/3537986#.Xis4mC2ZNGV'
+url = ['https://zenodo.org/record/3537986#.Xis4mC2ZNGV',
+ 'https://zenodo.org/record/3627216#.XivHNy2ZNGV']
citation = [('J. Wachs, M. Fazekas, and J. Kertész, "Corruption Risk in Contracting Markets: A Network Science Perspective." International Journal of Data Science and Analytics, pp 1–16 (2020)', 'https://doi.org/10.1007/s41060-019-00204-1')]
-icon_hash = '5e2b3bc478691ca4060bdee7'
+icon_hash = ['5e2b3bc478691ca4060bdee7', '5e2bd0a978691ca4060bdee9']
upstream_license = None
upstream_prefix = 'https://zenodo.org/record/3537986/files'
-files = [('data.zip:data/contracts.csv', None, None)]
+files = [('data.zip:data/contracts.csv', 'contracts', None)]
+
+# per-country, per-year networks (a separate Zenodo record)
+country_year_prefix = 'https://zenodo.org/record/3627216/files'
+country_year_files = \
+ [(f'country_year_networks.zip:country_year_networks/{net}.gml', net, "gml")
+ for net in ["AT_2008", "AT_2009", "AT_2010", "AT_2011", "AT_2012",
+ "AT_2013", "AT_2014", "AT_2015", "AT_2016", "BE_2008",
+ "BE_2009", "BE_2010", "BE_2011", "BE_2012", "BE_2013",
+ "BE_2014", "BE_2015", "BE_2016", "BG_2008", "BG_2009",
+ "BG_2010", "BG_2011", "BG_2012", "BG_2013", "BG_2014",
+ "BG_2015", "BG_2016", "CY_2008", "CY_2009", "CY_2010",
+ "CY_2011", "CY_2012", "CY_2013", "CY_2014", "CY_2015",
+ "CY_2016", "CZ_2008", "CZ_2009", "CZ_2010", "CZ_2011",
+ "CZ_2012", "CZ_2013", "CZ_2014", "CZ_2015", "CZ_2016",
+ "DE_2008", "DE_2009", "DE_2010", "DE_2011", "DE_2012",
+ "DE_2013", "DE_2014", "DE_2015", "DE_2016", "DK_2008",
+ "DK_2009", "DK_2010", "DK_2011", "DK_2012", "DK_2013",
+ "DK_2014", "DK_2015", "DK_2016", "EE_2008", "EE_2009",
+ "EE_2010", "EE_2011", "EE_2012", "EE_2013", "EE_2014",
+ "EE_2015", "EE_2016", "ES_2008", "ES_2009", "ES_2010",
+ "ES_2011", "ES_2012", "ES_2013", "ES_2014", "ES_2015",
+ "ES_2016", "FI_2008", "FI_2009", "FI_2010", "FI_2011",
+ "FI_2012", "FI_2013", "FI_2014", "FI_2015", "FI_2016",
+ "FR_2008", "FR_2009", "FR_2010", "FR_2011", "FR_2012",
+ "FR_2013", "FR_2014", "FR_2015", "FR_2016", "GR_2008",
+ "GR_2009", "GR_2010", "GR_2011", "GR_2012", "GR_2013",
+ "GR_2014", "GR_2015", "GR_2016", "HU_2008", "HU_2009",
+ "HU_2010", "HU_2011", "HU_2012", "HU_2013", "HU_2014",
+ "HU_2015", "HU_2016", "IE_2008", "IE_2009", "IE_2010",
+ "IE_2011", "IE_2012", "IE_2013", "IE_2014", "IE_2015",
+ "IE_2016", "IT_2008", "IT_2009", "IT_2010", "IT_2011",
+ "IT_2012", "IT_2013", "IT_2014", "IT_2015", "IT_2016",
+ "LT_2008", "LT_2009", "LT_2010", "LT_2011", "LT_2012",
+ "LT_2013", "LT_2014", "LT_2015", "LT_2016", "LV_2008",
+ "LV_2009", "LV_2010", "LV_2011", "LV_2012", "LV_2013",
+ "LV_2014", "LV_2015", "LV_2016", "NL_2008", "NL_2009",
+ "NL_2010", "NL_2011", "NL_2012", "NL_2013", "NL_2014",
+ "NL_2015", "NL_2016", "NO_2008", "NO_2009", "NO_2010",
+ "NO_2011", "NO_2012", "NO_2013", "NO_2014", "NO_2015",
+ "NO_2016", "PL_2008", "PL_2009", "PL_2010", "PL_2011",
+ "PL_2012", "PL_2013", "PL_2014", "PL_2015", "PL_2016",
+ "PT_2008", "PT_2009", "PT_2010", "PT_2011", "PT_2012",
+ "PT_2013", "PT_2014", "PT_2015", "PT_2016", "RO_2008",
+ "RO_2009", "RO_2010", "RO_2011", "RO_2012", "RO_2013",
+ "RO_2014", "RO_2015", "RO_2016", "SE_2008", "SE_2009",
+ "SE_2010", "SE_2011", "SE_2012", "SE_2013", "SE_2014",
+ "SE_2015", "SE_2016", "SI_2008", "SI_2009", "SI_2010",
+ "SI_2011", "SI_2012", "SI_2013", "SI_2014", "SI_2015",
+ "SI_2016", "SK_2008", "SK_2009", "SK_2010", "SK_2011",
+ "SK_2012", "SK_2013", "SK_2014", "SK_2015", "SK_2016",
+ "UK_2008", "UK_2009", "UK_2010", "UK_2011", "UK_2012",
+ "UK_2013", "UK_2014", "UK_2015", "UK_2016"]]
+files += country_year_files
def fetch_upstream(force=False):
- return fetch_upstream_files(__name__.split(".")[-1], upstream_prefix, files,
- force)
+ name = __name__.split(".")[-1]
+ fetch_upstream_files(name, upstream_prefix, files[:1], force)
+ fetch_upstream_files(name, country_year_prefix, country_year_files, force)
+
+def parse_contracts(name, fnames):
+ with ExitStack() as stack:
+ fs = [stack.enter_context(open_upstream_file(name, fn, "r")) for fn in fnames]
+ g = Graph(directed=False)
+
+ reader = csv.reader(fs[0], delimiter=",", quotechar='"')
+ cols = next(reader)[1:] # first column is a row index
+ idx = {c: i for i, c in enumerate(cols)}
+ iid = idx["issuer_id_final"]
+ wid = idx["winner_id_final"]
+ vcols = ["name", "canonical_name", "country", "address",
+ "postcode", "town", "type", "sector"]
+ iidx = [idx.get("issuer_" + c) for c in vcols]
+ widx = [idx.get("winner_" + c) for c in vcols]
+ ecols = [c for c in cols if not c.startswith("issuer_") and
+ not c.startswith("winner_")]
+ eidx = [idx[c] for c in ecols]
+ # numeric columns with missing values
+ numcols = {idx[c] for c in ["est_value", "contract_value",
+ "contract_eu_funded", "n_bids"]}
+ eprops = [g.new_ep("string") for c in ecols]
+ for c, p in zip(ecols, eprops):
+ g.ep[c] = p
+
+ def stripzero(x):
+ if x.endswith(".0"):
+ return x[:-2]
+ return x
+
+ vinfo = {}
+ def get_lines():
+ for row in reader:
+ row = row[1:]
+ s = row[iid]
+ t = row[wid]
+ vinfo[s] = [stripzero(row[i]) if i is not None else "" for i in iidx]
+ vinfo[t] = [stripzero(row[i]) if i is not None else "" for i in widx]
+ yield [s, t] + [stripzero(row[i]) if (row[i] != "" or
+ i not in numcols)
+ else "nan" for i in eidx]
+ g.vp.id_final = g.add_edge_list(get_lines(), hashed=True,
+ hash_type="string", eprops=eprops)
+ for c in vcols:
+ g.vp[c] = g.new_vp("string")
+ vprops = [g.vp[c] for c in vcols]
+ for v in g.vertices():
+ for p, x in zip(vprops, vinfo[g.vp.id_final[v]]):
+ p[v] = x
+
+ # issuer ids end in "_i", winner ids in "_w"
+ g.vp.is_issuer = g.new_vp("bool", vals=[x.endswith("_i") for x in g.vp.id_final])
+ return g
@cache_network()
@coerce_props()
@@ -62,55 +180,11 @@ def parse(alts=None):
continue
if isinstance(fnames, str):
fnames = [fnames]
- with ExitStack() as stack:
- fs = [stack.enter_context(open_upstream_file(name, fn, "r")) for fn in fnames]
- g = Graph(directed=False)
-
- reader = csv.reader(fs[0], delimiter=",", quotechar='"')
- cols = next(reader)[1:] # first column is a row index
- idx = {c: i for i, c in enumerate(cols)}
- iid = idx["issuer_id_final"]
- wid = idx["winner_id_final"]
- vcols = ["name", "canonical_name", "country", "address",
- "postcode", "town", "type", "sector"]
- iidx = [idx.get("issuer_" + c) for c in vcols]
- widx = [idx.get("winner_" + c) for c in vcols]
- ecols = [c for c in cols if not c.startswith("issuer_") and
- not c.startswith("winner_")]
- eidx = [idx[c] for c in ecols]
- # numeric columns with missing values
- numcols = {idx[c] for c in ["est_value", "contract_value",
- "contract_eu_funded", "n_bids"]}
- eprops = [g.new_ep("string") for c in ecols]
- for c, p in zip(ecols, eprops):
- g.ep[c] = p
-
- def stripzero(x):
- if x.endswith(".0"):
- return x[:-2]
- return x
-
- vinfo = {}
- def get_lines():
- for row in reader:
- row = row[1:]
- s = row[iid]
- t = row[wid]
- vinfo[s] = [stripzero(row[i]) if i is not None else "" for i in iidx]
- vinfo[t] = [stripzero(row[i]) if i is not None else "" for i in widx]
- yield [s, t] + [stripzero(row[i]) if (row[i] != "" or
- i not in numcols)
- else "nan" for i in eidx]
- g.vp.id_final = g.add_edge_list(get_lines(), hashed=True,
- hash_type="string", eprops=eprops)
- for c in vcols:
- g.vp[c] = g.new_vp("string")
- vprops = [g.vp[c] for c in vcols]
- for v in g.vertices():
- for p, x in zip(vprops, vinfo[g.vp.id_final[v]]):
- p[v] = x
-
- # issuer ids end in "_i", winner ids in "_w"
- g.vp.is_issuer = g.new_vp("bool", vals=[x.endswith("_i") for x in g.vp.id_final])
-
+ if alt == "contracts":
+ g = parse_contracts(name, fnames)
+ else:
+ with ExitStack() as stack:
+ fs = [stack.enter_context(open_upstream_file(name, fn, "rb")) for fn in fnames]
+ g = parse_graph(fs, fmt, directed=True)
+ g.vp.is_issuer = g.new_vp("bool", vals=[l.endswith("_i") for l in g.vp.label])
yield alt, g
[entries] Fix parse bugs and improve the metadata of existing entries c996a0e
From a review of the dataset descriptions against the upstream sources and papers, and from user reports: - parse fixes (wrong edge direction or directedness, shifted or misnamed properties, merged node identities, duplicated or missing edges and nodes, wrong types), e.g. arxiv_citation (the HepTh dates were a copy of the HepPh ones; now from the KDD Cup 2003 data), eu_procurements, foursquare_global, hiv_transmission, genetic_multiplex, gnutella (adds the 05 snapshot), roadnet, macaque_neural, python_dependency - descriptions rewritten from the upstream sources, documenting every property - corrected titles, citations and DOIs, licences (ustream_license typo, the data's own terms), tags and ICON ids - url lists for entries with several sources or dataset pages - SocioPatterns entries moved to the site's new data location, with the daily networks of sp_infectious and sp_primary_school - the Michael Fire entries marked as orphans (served from the mirror) - redundant type forcings removed after the graph-tool coercion fixes
diff --git a/entries/eu_procurements/__init__.py b/entries/eu_procurements/__init__.py
index d06fc13..908f573 100644
--- a/entries/eu_procurements/__init__.py
+++ b/entries/eu_procurements/__init__.py
@@ -19,13 +19,31 @@
from .. import *
title = "EU procurement contract networks (2008-2016)"
-description = """A bipartite network of public EU procurement contracts, from 2008 to 2016, between issuing buyers (public institutions such as a ministry or city hall) and supplying winners (a private firm). Contracts are aggregated into annual snapshots, edges are annotated with contract value information. Nodes are annotated with location information, including country of origin.[^icon]
-[^icon]: Description obtained from the [ICON](https://icon.colorado.edu) project."""
+description = r"""A bipartite multigraph of public procurement contracts in 26 European countries (25 EU member states and Norway) from 2008 to 2016. One set of nodes are issuers (buyers: public institutions such as ministries, hospitals or city halls), the other are winners (suppliers, generally private firms), and each edge is a single awarded contract, so that two nodes may be connected by many edges. The data were collected by J. Wachs, M. Fazekas and J. Kertész from Tenders Electronic Daily (TED), the official EU portal where all contracts above an estimated value of 5.2 million Euro (works) or 135 thousand Euro (services and supplies) must be published. Contracts from Luxembourg, Croatia and EU institutions were excluded. The identities of issuers and winners were deduplicated algorithmically per country, using string similarity and clustering with the Dedupe software. See `eu_procurements_alt` for the aggregated networks of each country and year.
+
+The network has the following properties, taken from the upstream data dictionary:
+
+- `id_final`: deduplicated identifier of the issuer or winner, ending in `_i` for issuers and `_w` for winners.
+- `is_issuer`: whether the node is an issuer (True) or a winner (False).
+- `name`: raw name; `canonical_name`: most common name after deduplication.
+- `country`, `address`, `postcode`, `town`: location of the issuer or winner (raw text).
+- `type` and `sector`: type (e.g. "3. Regional or local authorities") and sector (e.g. "Health") of the issuer.
+- `tender_id` (edges): contract identifier.
+- `year` and `date_dispatched` (edges): year and date (YYYY-MM-DD) of the contract.
+- `est_value` and `contract_value` (edges): raw estimated and final value of the contract (currency and VAT treatment unclear).
+- `contract_nuts` and `processed_nuts` (edges): raw and (imperfectly) cleaned NUTS geographic code of where the contract is performed.
+- `contract_cpv` and `cpv2` (edges): Common Procurement Vocabulary (CPV) code of the purchased goods or services, and its 2-digit prefix.
+- `contract_eu_funded` (edges): 1 if the contract was supported by EU funding.
+- `contract_type` (edges): works, services or supplies.
+- `procedure` (edges): the award procedure (e.g. open, restricted).
+- `n_bids` (edges): number of bids received.
+- `nocft` (edges): 1 if no call for tenders was found.
+- `single_bidder` (edges): 1 if there was no competition for the contract (single bid), an indicator of corruption risk; missing values were imputed with the country mean."""
tags = ['Economic', 'Commerce', 'Weighted', 'Temporal', 'Metadata']
url = 'https://zenodo.org/record/3537986#.Xis4mC2ZNGV'
citation = [('J. Wachs, M. Fazekas, and J. Kertész, "Corruption Risk in Contracting Markets: A Network Science Perspective." International Journal of Data Science and Analytics, pp 1–16 (2020)', 'https://doi.org/10.1007/s41060-019-00204-1')]
icon_hash = '5e2b3bc478691ca4060bdee7'
-ustream_license = None
+upstream_license = None
upstream_prefix = 'https://zenodo.org/record/3537986/files'
files = [('data.zip:data/contracts.csv', None, None)]
@@ -48,48 +66,51 @@ def parse(alts=None):
fs = [stack.enter_context(open_upstream_file(name, fn, "r")) for fn in fnames]
g = Graph(directed=False)
- header = next(fs[0])
- props = header.split(",")[1:]
-
- vprops = []
- for i in range(4,4+8):
- g.vp[props[i].replace("issuer_", "")] = p = g.new_vp("string")
- vprops.append(p)
-
- eprops = []
- for p in props:
- if "issuer_" in p or "winner_" in p:
- continue
- g.ep[p] = g.new_ep("string")
- eprops.append(g.ep[p])
-
reader = csv.reader(fs[0], delimiter=",", quotechar='"')
+ cols = next(reader)[1:] # first column is a row index
+ idx = {c: i for i, c in enumerate(cols)}
+ iid = idx["issuer_id_final"]
+ wid = idx["winner_id_final"]
+ vcols = ["name", "canonical_name", "country", "address",
+ "postcode", "town", "type", "sector"]
+ iidx = [idx.get("issuer_" + c) for c in vcols]
+ widx = [idx.get("winner_" + c) for c in vcols]
+ ecols = [c for c in cols if not c.startswith("issuer_") and
+ not c.startswith("winner_")]
+ eidx = [idx[c] for c in ecols]
+ # numeric columns with missing values
+ numcols = {idx[c] for c in ["est_value", "contract_value",
+ "contract_eu_funded", "n_bids"]}
+ eprops = [g.new_ep("string") for c in ecols]
+ for c, p in zip(ecols, eprops):
+ g.ep[c] = p
+
def stripzero(x):
if x.endswith(".0"):
return x[:-2]
return x
+
+ vinfo = {}
def get_lines():
for row in reader:
- del row[0]
- s = row[3]
- t = row[12]
- del row[12]
- del row[3]
- yield [s,t] + [stripzero(x) for x in row]
- g.vp.id_final = g.add_edge_list(get_lines(), hashed=True, hash_type="string",
- eprops=eprops)
- vs = {g.vp.id_final[v] : v for v in g.vertices()}
+ row = row[1:]
+ s = row[iid]
+ t = row[wid]
+ vinfo[s] = [stripzero(row[i]) if i is not None else "" for i in iidx]
+ vinfo[t] = [stripzero(row[i]) if i is not None else "" for i in widx]
+ yield [s, t] + [stripzero(row[i]) if (row[i] != "" or
+ i not in numcols)
+ else "nan" for i in eidx]
+ g.vp.id_final = g.add_edge_list(get_lines(), hashed=True,
+ hash_type="string", eprops=eprops)
+ for c in vcols:
+ g.vp[c] = g.new_vp("string")
+ vprops = [g.vp[c] for c in vcols]
+ for v in g.vertices():
+ for p, x in zip(vprops, vinfo[g.vp.id_final[v]]):
+ p[v] = x
- fs[0].seek(0)
- next(fs[0])
- reader = csv.reader(fs[0], delimiter=",", quotechar='"')
- for row in reader:
- del row[0]
- row = row[3:3+16+2]
- s = vs[row[0]]
- t = vs[row[9]]
- for i, p in enumerate(vprops):
- p[s] = stripzero(row[1+i])
- p[t] = stripzero(row[10+i])
+ # issuer ids end in "_i", winner ids in "_w"
+ g.vp.is_issuer = g.new_vp("bool", vals=[x.endswith("_i") for x in g.vp.id_final])
yield alt, g
Update copyright information 73a4f0d
diff --git a/entries/eu_procurements/__init__.py b/entries/eu_procurements/__init__.py
index 771d895..d06fc13 100644
--- a/entries/eu_procurements/__init__.py
+++ b/entries/eu_procurements/__init__.py
@@ -1,7 +1,7 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
#
-# Copyright (C) 2020-2024 Tiago de Paula Peixoto <tiago@skewed.de>
+# Copyright (C) 2020-2026 Tiago de Paula Peixoto <tiago@skewed.de>
#
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
Update copyright information 3d288ea
diff --git a/entries/eu_procurements/__init__.py b/entries/eu_procurements/__init__.py
index 3fbd456..771d895 100644
--- a/entries/eu_procurements/__init__.py
+++ b/entries/eu_procurements/__init__.py
@@ -1,7 +1,7 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
#
-# Copyright (C) 2020-2023 Tiago de Paula Peixoto <tiago@skewed.de>
+# Copyright (C) 2020-2024 Tiago de Paula Peixoto <tiago@skewed.de>
#
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
Update copyright information 8d005ae
diff --git a/entries/eu_procurements/__init__.py b/entries/eu_procurements/__init__.py
index 0d4c656..3fbd456 100644
--- a/entries/eu_procurements/__init__.py
+++ b/entries/eu_procurements/__init__.py
@@ -1,7 +1,7 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
#
-# Copyright (C) 2021 Tiago de Paula Peixoto <tiago@skewed.de>
+# Copyright (C) 2020-2023 Tiago de Paula Peixoto <tiago@skewed.de>
#
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
Update copyright info db2efcc
diff --git a/entries/eu_procurements/__init__.py b/entries/eu_procurements/__init__.py
index 3db1512..0d4c656 100644
--- a/entries/eu_procurements/__init__.py
+++ b/entries/eu_procurements/__init__.py
@@ -1,7 +1,7 @@
#! /usr/bin/env python
# -*- coding: utf-8 -*-
#
-# Copyright (C) 2020 Tiago de Paula Peixoto <tiago@skewed.de>
+# Copyright (C) 2021 Tiago de Paula Peixoto <tiago@skewed.de>
#
# This program is free software: you can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the Free
Update citations to use DOIs e878874
diff --git a/entries/eu_procurements/__init__.py b/entries/eu_procurements/__init__.py
index c0889c4..3db1512 100644
--- a/entries/eu_procurements/__init__.py
+++ b/entries/eu_procurements/__init__.py
@@ -23,7 +23,7 @@ description = """A bipartite network of public EU procurement contracts, from 20
[^icon]: Description obtained from the [ICON](https://icon.colorado.edu) project."""
tags = ['Economic', 'Commerce', 'Weighted', 'Temporal', 'Metadata']
url = 'https://zenodo.org/record/3537986#.Xis4mC2ZNGV'
-citation = [('J. Wachs, M. Fazekas, and J. Kertész, "Corruption Risk in Contracting Markets: A Network Science Perspective." International Journal of Data Science and Analytics, pp 1–16 (2020)', 'https://link.springer.com/article/10.1007%2Fs41060-019-00204-1')]
+citation = [('J. Wachs, M. Fazekas, and J. Kertész, "Corruption Risk in Contracting Markets: A Network Science Perspective." International Journal of Data Science and Analytics, pp 1–16 (2020)', 'https://doi.org/10.1007/s41060-019-00204-1')]
icon_hash = '5e2b3bc478691ca4060bdee7'
ustream_license = None
upstream_prefix = 'https://zenodo.org/record/3537986/files'
Change license from GPLv3 to AGPLv3 d754be5
diff --git a/entries/eu_procurements/__init__.py b/entries/eu_procurements/__init__.py
index 01f958e..c0889c4 100644
--- a/entries/eu_procurements/__init__.py
+++ b/entries/eu_procurements/__init__.py
@@ -3,18 +3,18 @@
#
# Copyright (C) 2020 Tiago de Paula Peixoto <tiago@skewed.de>
#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see <http://www.gnu.org/licenses/>.
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
from .. import *
Add explicit attribution to descriptions obtained from ICON 1645314
diff --git a/entries/eu_procurements/__init__.py b/entries/eu_procurements/__init__.py
index 302456a..01f958e 100644
--- a/entries/eu_procurements/__init__.py
+++ b/entries/eu_procurements/__init__.py
@@ -19,7 +19,8 @@
from .. import *
title = "EU procurement contract networks (2008-2016)"
-description = "A bipartite network of public EU procurement contracts, from 2008 to 2016, between issuing buyers (public institutions such as a ministry or city hall) and supplying winners (a private firm). Contracts are aggregated into annual snapshots, edges are annotated with contract value information. Nodes are annotated with location information, including country of origin."
+description = """A bipartite network of public EU procurement contracts, from 2008 to 2016, between issuing buyers (public institutions such as a ministry or city hall) and supplying winners (a private firm). Contracts are aggregated into annual snapshots, edges are annotated with contract value information. Nodes are annotated with location information, including country of origin.[^icon]
+[^icon]: Description obtained from the [ICON](https://icon.colorado.edu) project."""
tags = ['Economic', 'Commerce', 'Weighted', 'Temporal', 'Metadata']
url = 'https://zenodo.org/record/3537986#.Xis4mC2ZNGV'
citation = [('J. Wachs, M. Fazekas, and J. Kertész, "Corruption Risk in Contracting Markets: A Network Science Perspective." International Journal of Data Science and Analytics, pp 1–16 (2020)', 'https://link.springer.com/article/10.1007%2Fs41060-019-00204-1')]
Initial commit cc85872
diff --git a/entries/eu_procurements/__init__.py b/entries/eu_procurements/__init__.py
new file mode 100644
index 0000000..302456a
--- /dev/null
+++ b/entries/eu_procurements/__init__.py
@@ -0,0 +1,94 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2020 Tiago de Paula Peixoto <tiago@skewed.de>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+from .. import *
+
+title = "EU procurement contract networks (2008-2016)"
+description = "A bipartite network of public EU procurement contracts, from 2008 to 2016, between issuing buyers (public institutions such as a ministry or city hall) and supplying winners (a private firm). Contracts are aggregated into annual snapshots, edges are annotated with contract value information. Nodes are annotated with location information, including country of origin."
+tags = ['Economic', 'Commerce', 'Weighted', 'Temporal', 'Metadata']
+url = 'https://zenodo.org/record/3537986#.Xis4mC2ZNGV'
+citation = [('J. Wachs, M. Fazekas, and J. Kertész, "Corruption Risk in Contracting Markets: A Network Science Perspective." International Journal of Data Science and Analytics, pp 1–16 (2020)', 'https://link.springer.com/article/10.1007%2Fs41060-019-00204-1')]
+icon_hash = '5e2b3bc478691ca4060bdee7'
+ustream_license = None
+upstream_prefix = 'https://zenodo.org/record/3537986/files'
+files = [('data.zip:data/contracts.csv', None, None)]
+
+def fetch_upstream(force=False):
+ return fetch_upstream_files(__name__.split(".")[-1], upstream_prefix, files,
+ force)
+
+@cache_network()
+@coerce_props()
+@annotate()
+def parse(alts=None):
+ global files
+ name = __name__.split(".")[-1]
+ for fnames, alt, fmt in files:
+ if alts is not None and alt not in alts:
+ continue
+ if isinstance(fnames, str):
+ fnames = [fnames]
+ with ExitStack() as stack:
+ fs = [stack.enter_context(open_upstream_file(name, fn, "r")) for fn in fnames]
+ g = Graph(directed=False)
+
+ header = next(fs[0])
+ props = header.split(",")[1:]
+
+ vprops = []
+ for i in range(4,4+8):
+ g.vp[props[i].replace("issuer_", "")] = p = g.new_vp("string")
+ vprops.append(p)
+
+ eprops = []
+ for p in props:
+ if "issuer_" in p or "winner_" in p:
+ continue
+ g.ep[p] = g.new_ep("string")
+ eprops.append(g.ep[p])
+
+ reader = csv.reader(fs[0], delimiter=",", quotechar='"')
+ def stripzero(x):
+ if x.endswith(".0"):
+ return x[:-2]
+ return x
+ def get_lines():
+ for row in reader:
+ del row[0]
+ s = row[3]
+ t = row[12]
+ del row[12]
+ del row[3]
+ yield [s,t] + [stripzero(x) for x in row]
+ g.vp.id_final = g.add_edge_list(get_lines(), hashed=True, hash_type="string",
+ eprops=eprops)
+ vs = {g.vp.id_final[v] : v for v in g.vertices()}
+
+ fs[0].seek(0)
+ next(fs[0])
+ reader = csv.reader(fs[0], delimiter=",", quotechar='"')
+ for row in reader:
+ del row[0]
+ row = row[3:3+16+2]
+ s = vs[row[0]]
+ t = vs[row[9]]
+ for i, p in enumerate(vprops):
+ p[s] = stripzero(row[1+i])
+ p[t] = stripzero(row[10+i])
+
+ yield alt, g