API Documentation
This part of the project documentation focuses on
an information-oriented approach. Use it as a
reference for the technical implementation of the
pyutils project code.
histogis.histogis
main module of this package
HistoGis
Main class to interact with HistoGIS TempSpatial Objects
Source code in histogis/histogis.py
class HistoGis:
"""Main class to interact with HistoGIS TempSpatial Objects"""
def test_connection(self):
"""Checks if a GET request to histogis_url returns status code 200"""
try:
r = requests.get(self.histogis_url)
except Exception as e:
print(e)
return False
if r.status_code == 200:
return True
else:
print(r.status_code)
return False
def count(self):
"""returns the number of available TempSpatial objects"""
r = requests.get(self.list_endpoint)
result = r.json()["count"]
return result
def query(self, lat=48.2894, lng=14.304, when="1860-12-12", polygon=False):
"""Makes a spatial lookup and returns a list of matching objects
:param lat: Latitude of the place to query for
:param lng: Langitude of the place to query for
:param when: An iso-date string
:param polygon: If true, the whole geojson is returned, only the properties
:return: A list of matching objects
"""
if when is not None:
# ToDo check if when casts to iso date
url = "{}?lat={}&lng={}&when={}".format(self.query_endpoint, lat, lng, when)
else:
url = "{}?lat={}&lng={}&format".format(self.query_endpoint, lat, lng)
url = "{}&format=json".format(url)
r = requests.get(url)
if polygon:
return r.json()
else:
try:
return r.json()["features"][0]["properties"]
except IndexError:
return self.empty_result
def fetch_geonames_data(self, geonames="http://www.geonames.org/2772400/linz.html"):
"""returns name and coordinates for the passed in geonames identifier
:param geonames: Any kind of geonames id, can be just the geonames id (as string)\
or any geonames URI like http://www.geonames.org/2772400/linz.html
:return: a dict with keys 'name', 'lat' and lng.
"""
geonames_id = re.search(r"\d+", geonames).group()
url = "http://sws.geonames.org/{}/about.rdf".format(geonames_id)
rdf = ET.parse(url)
lat = rdf.xpath(".//wgs84_pos:lat/text()", namespaces=self.nsmap)[0]
lng = rdf.xpath(".//wgs84_pos:long/text()", namespaces=self.nsmap)[0]
name = rdf.xpath(".//gn:name/text()", namespaces=self.nsmap)[0]
return {"name": name, "lat": lat, "lng": lng}
def fetch_gnd_data(self, gnd="http://d-nb.info/gnd/4066009-6/about/lds.rdf"):
"""returns name and coordinates for the passed in gnd identifier
:param gnd: Any kind of gnd id, can be just the gnd id (as string)\
or any geonames URI like http://d-nb.info/gnd/4066009-6/about/lds.rdf
:return: a dict with keys 'name', 'lat' and lng.
"""
gnd_id = re.search("(\d+[A-Z0-9\-]+)", gnd).group()
url = "http://d-nb.info/gnd/{}/about/lds.rdf".format(gnd_id)
res = requests.get(url)
rdf = ET.fromstring(res.content)
lat_long = rdf.xpath(".//geo:asWKT/text()", namespaces=self.nsmap)[0]
lat_long_match = re.search("\+0?([0-9\.]+)\s+\+0?([0-9\.]+)", lat_long)
if not lat_long_match:
raise ValueError("GND RDF did not contain coordinates.")
lng = lat_long_match.group(1)
lat = lat_long_match.group(2)
name = rdf.xpath(
".//gndo:preferredNameForThePlaceOrGeographicName/text()",
namespaces=self.nsmap,
)[0]
return {"name": name, "lat": lat, "lng": lng}
def fetch_wikidata_data(self, wikidata="https://www.wikidata.org/entity/Q41329"):
"""returns name and coordinates for the passed in wikidata identifier
:param wikidata: Any kind of wikidata id, can be just the wikidata id (as string)\
or any wikidata URI like https://www.wikidata.org/entity/Q41329
:return: a dict with keys 'name', 'lat' and lng.
"""
wikidata_id = re.search("(Q\d+)", wikidata).group()
url = "https://wikidata.org/entity/{}.rdf".format(wikidata_id)
res = requests.get(url)
rdf = ET.fromstring(res.content)
lat_long = rdf.xpath(".//wdt:P625/text()", namespaces=self.nsmap)[0]
lat_long_match = re.search("0?([0-9\.]+)\s+0?([0-9\.]+)", lat_long)
if not lat_long_match:
raise ValueError("Wikidata RDF did not contain coordinates.")
lng = lat_long_match.group(1)
lat = lat_long_match.group(2)
name = rdf.xpath(
f".//rdf:Description[@rdf:about='http://www.wikidata.org/entity/{wikidata_id}']/rdfs:label[@xml:lang='de']/text()", # noqa:E501
namespaces=self.nsmap,
)[0]
return {"name": name, "lat": lat, "lng": lng}
def query_by_service_id(
self,
service=None,
id="https://www.geonames.org/2772400/",
when="1860-12-12",
polygon=False,
):
"""Makes a spatial lookup by geonames id a list of matching objects
:param gnd: Any kind of geonames id, can be just the geonames id (as string)\
or any geonames URI like http://www.geonames.org/2772400/linz.html
:param when: An iso-date string
:param polygon: If true, the whole geojson is returned, only the properties
:return: A list of matching objects
"""
if service is None:
for s in self.map_url_service.keys():
if s in id:
service = self.map_url_service[s]
coords = getattr(self, "fetch_{}_data".format(service))(id)
return self.query(
lat=coords["lat"], lng=coords["lng"], when=when, polygon=polygon
)
def dump_all(self, verbose=True):
"""Dumps HistoGIS data to GeoJSONL; a GeoJSON per line. Can take quite a while. So only/
use if really necessary. Alternatively go to ??? to download the latest data dump
:return: A text file.
"""
file = "histogis_dump__{date:%Y-%m-%d__%H-%M-%S}.txt".format(
date=datetime.datetime.now()
)
next_ft = self.list_endpoint
with open(file, "w", encoding="utf-8") as f:
while next_ft:
r = requests.get(next_ft)
ft = r.json()["features"]
f.write("{}\n".format(json.dumps(ft)))
if verbose:
print("writing to file: {}".format(next_ft))
try:
next_ft = r.json()["next"]
except JSONDecodeError:
next_ft = None
return file
def dump_all_file_per_feature(self, verbose=True, path="."):
"""Dumps HistoGIS data to GeoJSON; one file per object. Can take quite a while. So only/
use if really necessary. Alternatively go to https://doi.org/10.5281/zenodo.2615387\
to download the latest data dump
:return: A text file.
"""
next_ft = self.list_endpoint
counter = 0
while next_ft:
r = requests.get(next_ft)
ft = r.json()["features"][0]
ft_slugged = ft["properties"]["slugged_name"]
file = "{}.geojson".format(ft_slugged)
file = os.path.join(path, file)
counter += 1
if verbose:
print("{}".format(file))
with open(file, "w", encoding="utf-8") as f:
f.write("{}".format(json.dumps(ft)))
try:
next_ft = r.json()["next"]
except JSONDecodeError:
print("error")
next_ft = None
return "done: downloaded {} items".format(counter)
def __init__(self, histogis_url=HISTOGIS_URL):
"""__init__
:param histogis_url: Url pointing to a HistoGIS-TempSpatial endpoint
"""
if histogis_url.endswith("/"):
self.histogis_url = histogis_url
else:
self.histogis_url = "{}/".format(histogis_url)
self.status = self.test_connection()
self.list_endpoint = "{}tempspatial/?format=json".format(self.histogis_url)
self.query_endpoint = "{}where-was/".format(self.histogis_url)
self.empty_result = {
"count": 0,
"features": [],
"next": None,
"previous": None,
"type": "FeatureCollection",
}
self.map_url_service = {
"geonames": "geonames",
"d-nb": "gnd",
"wikidata": "wikidata",
}
self.nsmap = {
"wgs84_pos": "http://www.w3.org/2003/01/geo/wgs84_pos#",
"gn": "http://www.geonames.org/ontology#",
"rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
"rdfs": "http://www.w3.org/2000/01/rdf-schema#",
"tei": "http://www.tei-c.org/ns/1.0",
"xml": "http://www.w3.org/XML/1998/namespace",
"geo": "http://www.opengis.net/ont/geosparql#",
"gndo": "http://d-nb.info/standards/elementset/gnd#",
"wdt": "http://www.wikidata.org/prop/direct/",
}
__init__(histogis_url=HISTOGIS_URL)
init
:param histogis_url: Url pointing to a HistoGIS-TempSpatial endpoint
Source code in histogis/histogis.py
def __init__(self, histogis_url=HISTOGIS_URL):
"""__init__
:param histogis_url: Url pointing to a HistoGIS-TempSpatial endpoint
"""
if histogis_url.endswith("/"):
self.histogis_url = histogis_url
else:
self.histogis_url = "{}/".format(histogis_url)
self.status = self.test_connection()
self.list_endpoint = "{}tempspatial/?format=json".format(self.histogis_url)
self.query_endpoint = "{}where-was/".format(self.histogis_url)
self.empty_result = {
"count": 0,
"features": [],
"next": None,
"previous": None,
"type": "FeatureCollection",
}
self.map_url_service = {
"geonames": "geonames",
"d-nb": "gnd",
"wikidata": "wikidata",
}
self.nsmap = {
"wgs84_pos": "http://www.w3.org/2003/01/geo/wgs84_pos#",
"gn": "http://www.geonames.org/ontology#",
"rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
"rdfs": "http://www.w3.org/2000/01/rdf-schema#",
"tei": "http://www.tei-c.org/ns/1.0",
"xml": "http://www.w3.org/XML/1998/namespace",
"geo": "http://www.opengis.net/ont/geosparql#",
"gndo": "http://d-nb.info/standards/elementset/gnd#",
"wdt": "http://www.wikidata.org/prop/direct/",
}
count()
returns the number of available TempSpatial objects
Source code in histogis/histogis.py
def count(self):
"""returns the number of available TempSpatial objects"""
r = requests.get(self.list_endpoint)
result = r.json()["count"]
return result
dump_all(verbose=True)
Dumps HistoGIS data to GeoJSONL; a GeoJSON per line. Can take quite a while. So only/ use if really necessary. Alternatively go to ??? to download the latest data dump :return: A text file.
Source code in histogis/histogis.py
def dump_all(self, verbose=True):
"""Dumps HistoGIS data to GeoJSONL; a GeoJSON per line. Can take quite a while. So only/
use if really necessary. Alternatively go to ??? to download the latest data dump
:return: A text file.
"""
file = "histogis_dump__{date:%Y-%m-%d__%H-%M-%S}.txt".format(
date=datetime.datetime.now()
)
next_ft = self.list_endpoint
with open(file, "w", encoding="utf-8") as f:
while next_ft:
r = requests.get(next_ft)
ft = r.json()["features"]
f.write("{}\n".format(json.dumps(ft)))
if verbose:
print("writing to file: {}".format(next_ft))
try:
next_ft = r.json()["next"]
except JSONDecodeError:
next_ft = None
return file
dump_all_file_per_feature(verbose=True, path='.')
Dumps HistoGIS data to GeoJSON; one file per object. Can take quite a while. So only/ use if really necessary. Alternatively go to https://doi.org/10.5281/zenodo.2615387 to download the latest data dump :return: A text file.
Source code in histogis/histogis.py
def dump_all_file_per_feature(self, verbose=True, path="."):
"""Dumps HistoGIS data to GeoJSON; one file per object. Can take quite a while. So only/
use if really necessary. Alternatively go to https://doi.org/10.5281/zenodo.2615387\
to download the latest data dump
:return: A text file.
"""
next_ft = self.list_endpoint
counter = 0
while next_ft:
r = requests.get(next_ft)
ft = r.json()["features"][0]
ft_slugged = ft["properties"]["slugged_name"]
file = "{}.geojson".format(ft_slugged)
file = os.path.join(path, file)
counter += 1
if verbose:
print("{}".format(file))
with open(file, "w", encoding="utf-8") as f:
f.write("{}".format(json.dumps(ft)))
try:
next_ft = r.json()["next"]
except JSONDecodeError:
print("error")
next_ft = None
return "done: downloaded {} items".format(counter)
fetch_geonames_data(geonames='http://www.geonames.org/2772400/linz.html')
returns name and coordinates for the passed in geonames identifier :param geonames: Any kind of geonames id, can be just the geonames id (as string) or any geonames URI like http://www.geonames.org/2772400/linz.html :return: a dict with keys 'name', 'lat' and lng.
Source code in histogis/histogis.py
def fetch_geonames_data(self, geonames="http://www.geonames.org/2772400/linz.html"):
"""returns name and coordinates for the passed in geonames identifier
:param geonames: Any kind of geonames id, can be just the geonames id (as string)\
or any geonames URI like http://www.geonames.org/2772400/linz.html
:return: a dict with keys 'name', 'lat' and lng.
"""
geonames_id = re.search(r"\d+", geonames).group()
url = "http://sws.geonames.org/{}/about.rdf".format(geonames_id)
rdf = ET.parse(url)
lat = rdf.xpath(".//wgs84_pos:lat/text()", namespaces=self.nsmap)[0]
lng = rdf.xpath(".//wgs84_pos:long/text()", namespaces=self.nsmap)[0]
name = rdf.xpath(".//gn:name/text()", namespaces=self.nsmap)[0]
return {"name": name, "lat": lat, "lng": lng}
fetch_gnd_data(gnd='http://d-nb.info/gnd/4066009-6/about/lds.rdf')
returns name and coordinates for the passed in gnd identifier :param gnd: Any kind of gnd id, can be just the gnd id (as string) or any geonames URI like http://d-nb.info/gnd/4066009-6/about/lds.rdf :return: a dict with keys 'name', 'lat' and lng.
Source code in histogis/histogis.py
def fetch_gnd_data(self, gnd="http://d-nb.info/gnd/4066009-6/about/lds.rdf"):
"""returns name and coordinates for the passed in gnd identifier
:param gnd: Any kind of gnd id, can be just the gnd id (as string)\
or any geonames URI like http://d-nb.info/gnd/4066009-6/about/lds.rdf
:return: a dict with keys 'name', 'lat' and lng.
"""
gnd_id = re.search("(\d+[A-Z0-9\-]+)", gnd).group()
url = "http://d-nb.info/gnd/{}/about/lds.rdf".format(gnd_id)
res = requests.get(url)
rdf = ET.fromstring(res.content)
lat_long = rdf.xpath(".//geo:asWKT/text()", namespaces=self.nsmap)[0]
lat_long_match = re.search("\+0?([0-9\.]+)\s+\+0?([0-9\.]+)", lat_long)
if not lat_long_match:
raise ValueError("GND RDF did not contain coordinates.")
lng = lat_long_match.group(1)
lat = lat_long_match.group(2)
name = rdf.xpath(
".//gndo:preferredNameForThePlaceOrGeographicName/text()",
namespaces=self.nsmap,
)[0]
return {"name": name, "lat": lat, "lng": lng}
fetch_wikidata_data(wikidata='https://www.wikidata.org/entity/Q41329')
returns name and coordinates for the passed in wikidata identifier :param wikidata: Any kind of wikidata id, can be just the wikidata id (as string) or any wikidata URI like https://www.wikidata.org/entity/Q41329 :return: a dict with keys 'name', 'lat' and lng.
Source code in histogis/histogis.py
def fetch_wikidata_data(self, wikidata="https://www.wikidata.org/entity/Q41329"):
"""returns name and coordinates for the passed in wikidata identifier
:param wikidata: Any kind of wikidata id, can be just the wikidata id (as string)\
or any wikidata URI like https://www.wikidata.org/entity/Q41329
:return: a dict with keys 'name', 'lat' and lng.
"""
wikidata_id = re.search("(Q\d+)", wikidata).group()
url = "https://wikidata.org/entity/{}.rdf".format(wikidata_id)
res = requests.get(url)
rdf = ET.fromstring(res.content)
lat_long = rdf.xpath(".//wdt:P625/text()", namespaces=self.nsmap)[0]
lat_long_match = re.search("0?([0-9\.]+)\s+0?([0-9\.]+)", lat_long)
if not lat_long_match:
raise ValueError("Wikidata RDF did not contain coordinates.")
lng = lat_long_match.group(1)
lat = lat_long_match.group(2)
name = rdf.xpath(
f".//rdf:Description[@rdf:about='http://www.wikidata.org/entity/{wikidata_id}']/rdfs:label[@xml:lang='de']/text()", # noqa:E501
namespaces=self.nsmap,
)[0]
return {"name": name, "lat": lat, "lng": lng}
query(lat=48.2894, lng=14.304, when='1860-12-12', polygon=False)
Makes a spatial lookup and returns a list of matching objects :param lat: Latitude of the place to query for :param lng: Langitude of the place to query for :param when: An iso-date string :param polygon: If true, the whole geojson is returned, only the properties :return: A list of matching objects
Source code in histogis/histogis.py
def query(self, lat=48.2894, lng=14.304, when="1860-12-12", polygon=False):
"""Makes a spatial lookup and returns a list of matching objects
:param lat: Latitude of the place to query for
:param lng: Langitude of the place to query for
:param when: An iso-date string
:param polygon: If true, the whole geojson is returned, only the properties
:return: A list of matching objects
"""
if when is not None:
# ToDo check if when casts to iso date
url = "{}?lat={}&lng={}&when={}".format(self.query_endpoint, lat, lng, when)
else:
url = "{}?lat={}&lng={}&format".format(self.query_endpoint, lat, lng)
url = "{}&format=json".format(url)
r = requests.get(url)
if polygon:
return r.json()
else:
try:
return r.json()["features"][0]["properties"]
except IndexError:
return self.empty_result
query_by_service_id(service=None, id='https://www.geonames.org/2772400/', when='1860-12-12', polygon=False)
Makes a spatial lookup by geonames id a list of matching objects :param gnd: Any kind of geonames id, can be just the geonames id (as string) or any geonames URI like http://www.geonames.org/2772400/linz.html :param when: An iso-date string :param polygon: If true, the whole geojson is returned, only the properties :return: A list of matching objects
Source code in histogis/histogis.py
def query_by_service_id(
self,
service=None,
id="https://www.geonames.org/2772400/",
when="1860-12-12",
polygon=False,
):
"""Makes a spatial lookup by geonames id a list of matching objects
:param gnd: Any kind of geonames id, can be just the geonames id (as string)\
or any geonames URI like http://www.geonames.org/2772400/linz.html
:param when: An iso-date string
:param polygon: If true, the whole geojson is returned, only the properties
:return: A list of matching objects
"""
if service is None:
for s in self.map_url_service.keys():
if s in id:
service = self.map_url_service[s]
coords = getattr(self, "fetch_{}_data".format(service))(id)
return self.query(
lat=coords["lat"], lng=coords["lng"], when=when, polygon=polygon
)
test_connection()
Checks if a GET request to histogis_url returns status code 200
Source code in histogis/histogis.py
def test_connection(self):
"""Checks if a GET request to histogis_url returns status code 200"""
try:
r = requests.get(self.histogis_url)
except Exception as e:
print(e)
return False
if r.status_code == 200:
return True
else:
print(r.status_code)
return False
merge_single_files(file='histogis-dump.jsonl', source_path='single_files', verbose=True)
writes all geojson from the given folder into a JSONL file :path: Path to folder containing the .geojson files :return: A JSONL file.
Source code in histogis/histogis.py
def merge_single_files(
file="histogis-dump.jsonl", source_path="single_files", verbose=True
):
"""
writes all geojson from the given folder into a JSONL file
:path: Path to folder containing the .geojson files
:return: A JSONL file.
"""
files = sorted(glob.glob(f"./{source_path}/*.geojson"))
with open(file, "w", encoding="utf-8") as f:
for x in files:
with open(x) as json_file:
data = json.load(json_file)
f.write(f"{json.dumps(data)}\n")
return file