readme
This commit is contained in:
randogoth
parent
fc91d3c884
commit
fc00e77b33
3 changed files with 136 additions and 94 deletions
94
import.py
94
import.py
|
|
@ -1,94 +0,0 @@
|
|||
import duckdb
|
||||
from pathlib import Path
|
||||
import geohash
|
||||
import osmium
|
||||
import subprocess
|
||||
from pprint import pprint
|
||||
from typing import List, Tuple, Dict
|
||||
|
||||
def remove_points(input_file: str, output_file: str):
|
||||
try:
|
||||
input_path, output_path = Path(input_file), Path(output_file)
|
||||
if not input_path.exists():
|
||||
raise FileNotFoundError(f"File '{input_file}' not found.")
|
||||
|
||||
con = duckdb.connect()
|
||||
con.execute("INSTALL spatial; LOAD spatial;")
|
||||
|
||||
con.execute(f"""
|
||||
CREATE VIEW geoparquet_filtered AS
|
||||
SELECT * FROM read_parquet('{input_path}')
|
||||
WHERE ST_GeometryType(geometry) IS NOT NULL
|
||||
AND ST_GeometryType(geometry) != 'POINT'
|
||||
""")
|
||||
|
||||
if con.execute("SELECT COUNT(*) FROM geoparquet_filtered").fetchone()[0] == 0:
|
||||
raise ValueError("No non-POINT geometries found.")
|
||||
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
con.execute(f"COPY geoparquet_filtered TO '{output_path}' (FORMAT PARQUET)")
|
||||
print(f"✅ Saved to {output_path}")
|
||||
|
||||
except Exception as e:
|
||||
print(f"❌ Error: {e}")
|
||||
|
||||
finally:
|
||||
if 'con' in locals():
|
||||
con.close()
|
||||
|
||||
def bbox(file_path: str) -> Dict[str, float]:
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["osmium", "fileinfo", "-e", file_path],
|
||||
capture_output=True, text=True, check=True
|
||||
)
|
||||
for line in result.stdout.split("\n"):
|
||||
if "Bounding box:" in line:
|
||||
bbox_str = line.strip().replace("Bounding box:", "").strip()
|
||||
# Convert to a tuple of floats
|
||||
bbox_tuple = tuple(map(float, bbox_str.strip("()").split(",")))
|
||||
return bbox_tuple
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(f"❌ Error running osmium: {e}")
|
||||
|
||||
def check_data(file: str):
|
||||
con = duckdb.connect()
|
||||
con.execute("INSTALL spatial; LOAD spatial;")
|
||||
con.execute(f"CREATE VIEW geoparquet AS SELECT * FROM read_parquet('{file}')")
|
||||
columns = con.execute("DESCRIBE geoparquet").fetchdf()
|
||||
print("Columns in the file:\n", columns)
|
||||
dc = con.execute("SELECT count(*) FROM geoparquet LIMIT 10").fetchdf()
|
||||
print("\nTotal rows:", dc)
|
||||
df = con.execute("SELECT * FROM geoparquet LIMIT 10").fetchdf()
|
||||
print("\nFirst 10 rows:\n", df)
|
||||
try:
|
||||
geo_types = con.execute("SELECT DISTINCT ST_GeometryType(geometry) AS geom_type FROM geoparquet").fetchdf()
|
||||
print("\nDetected Geometry Types:\n", geo_types)
|
||||
except Exception as e:
|
||||
print("\nCould not determine geometry type. Possible reason: No valid geometry column found.\n", e)
|
||||
con.close()
|
||||
|
||||
def bbox_geohashes(bbox: Tuple[float, float, float, float]) -> List[str]:
|
||||
min_x, min_y, max_x, max_y = bbox
|
||||
geohashes = set()
|
||||
|
||||
# Define step size based on 4-character Geohash precision (~39km × 19.5km)
|
||||
step_size = 0.25 # Approximate step size in degrees (varies slightly by latitude)
|
||||
|
||||
lat = min_y
|
||||
while lat <= max_y:
|
||||
lon = min_x
|
||||
while lon <= max_x:
|
||||
gh = geohash.encode(lat, lon, precision=4)
|
||||
geohashes.add(gh)
|
||||
lon += step_size # Move eastward
|
||||
lat += step_size # Move northward
|
||||
|
||||
return sorted(geohashes)
|
||||
|
||||
# remove_points('compact.parquet', 'processed.parquet')
|
||||
# bbox = bbox('israel.pbf')
|
||||
# hashes = bbox_geohashes(bbox)
|
||||
# print(hashes)
|
||||
|
||||
check_data("geohash/sv2c.parquet")
|
||||
136
readme.md
Normal file
136
readme.md
Normal file
|
|
@ -0,0 +1,136 @@
|
|||
<image src="https://blog.therugseller.co.uk/wp-content/uploads/2017/10/2-10.png" width="400" />
|
||||
<p> <p>
|
||||
|
||||
# dopecarpet
|
||||
|
||||
Generate GeoHash parquet files from OpenStreetMap PBF files with custom tag filter to look up metadata for coordinates.
|
||||
|
||||
## Dependencies
|
||||
|
||||
Installation (Fedora)
|
||||
|
||||
```bash
|
||||
sudo dnf install osmium-tool jq python3-devel pip
|
||||
pip install quackosm[cli] psycopg2-binary pyosmium geoalchemy2 geojson shapely pygeohash pyarrow pandas click
|
||||
```
|
||||
|
||||
## Basic Use
|
||||
|
||||
### 1. Customize Filter
|
||||
|
||||
Edit the `tags.json` to specify which OSM tags you want to include in the metadata. This affects what geometries are extracted from the OSM data and has a direct influence on the file sizes.
|
||||
|
||||
```json
|
||||
{
|
||||
"name:en": true,
|
||||
"name": true,
|
||||
"aeroway": true,
|
||||
"boundary": ["aboriginal_lands", "border_zone", "forest", "hazard", "national_park", "protected_area", "disputed"],
|
||||
"geological": true,
|
||||
"highway": ["motorway", "trunk", "primary", "raceway"],
|
||||
"historic": true,
|
||||
"landuse": true,
|
||||
"leisure": ["dog_park", "garden", "miniature_golf", "nature_reserve", "park", "playground", "stadium", "swimming_pool", "water_park"],
|
||||
"military": ["airfield", "base", "bunker", "barracks", "danger_area", "nuclear_explosion_site", "range", "training_area"],
|
||||
"natural": true,
|
||||
"place": ["plot", "farm", "island", "islet", "sea", "ocean"],
|
||||
"power": ["plant"],
|
||||
"railway": ["abandoned", "construction", "disused", "rail"],
|
||||
"tourism": ["attraction", "yes"],
|
||||
"water": true,
|
||||
"waterway": true
|
||||
}
|
||||
```
|
||||
|
||||
### 2. Generate GeoHash Files
|
||||
|
||||
This extracts the desired data from the PBF file and generates four digit geohash parquet files in the `geohash/` directory.
|
||||
|
||||
```bash
|
||||
just process <osm_data.pbf>
|
||||
```
|
||||
|
||||
### 3. Lookup Coordinate Metadata
|
||||
|
||||
Prints metadata found for a coordinate as JSON object list.
|
||||
|
||||
```bash
|
||||
just lookup <latitude>, <longitude>
|
||||
```
|
||||
|
||||
Sample Output: `just lookup 52.496846793890256 13.435128880554208`
|
||||
|
||||
```json
|
||||
[
|
||||
{
|
||||
"type": "polygon",
|
||||
"leisure": "park",
|
||||
"name": "Görlitzer Park"
|
||||
},
|
||||
{
|
||||
"type": "polygon",
|
||||
"landuse": "forest"
|
||||
},
|
||||
{
|
||||
"type": "polygon",
|
||||
"name": "Luisenstadt"
|
||||
},
|
||||
{
|
||||
"type": "polygon",
|
||||
"name": "Jugendverkehrsschule Wiener Straße"
|
||||
},
|
||||
{
|
||||
"type": "polygon",
|
||||
"name": "Friedrichshain-Kreuzberg"
|
||||
},
|
||||
{
|
||||
"type": "polygon",
|
||||
"name": "Kreuzberg"
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
## Advanced Use
|
||||
|
||||
### Extract and Filter Data
|
||||
|
||||
To just extract the filtered data from an OSM PBF file and save it as parquet file
|
||||
|
||||
```bash
|
||||
just extract <osm_data.pbf> <extracted_data.parquet>
|
||||
```
|
||||
|
||||
### Download OSM Data File for Region
|
||||
|
||||
Automatically download a file from Geofabrik based on the name of a region, filter it, and save it as parquet file.
|
||||
|
||||
```bash
|
||||
just download '<name of the region>' <extracted_data.parquet>
|
||||
```
|
||||
|
||||
The original PBF file remains saved in the `files/` folder.
|
||||
|
||||
### Split Parquet File to GeoHash Parquet Files
|
||||
|
||||
```bash
|
||||
just convert <extracted_data.parquet> <geohash_parquet_folder>
|
||||
```
|
||||
|
||||
### Plot Parquet File as PNG
|
||||
|
||||
Plot the geometry of a parquet file as a PNG with the same filename.
|
||||
|
||||
```bash
|
||||
just plot <file.parquet>
|
||||
```
|
||||
|
||||
Sample Output:
|
||||
|
||||
|
||||
## TO DO
|
||||
|
||||
- [X] GeoHash parquet file generation
|
||||
- [X] Append data to existing parquet files
|
||||
- [ ] Create web API server
|
||||
- [ ] Dockerize it
|
||||
- [ ] Scheduled update routine
|
||||
BIN
u33d.png
Normal file
BIN
u33d.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 3.9 MiB |
Loading…
Add table
Add a link
Reference in a new issue