From fa7e5b1c629bceddb603c178c516c924fbcc9905 Mon Sep 17 00:00:00 2001 From: randogoth Date: Tue, 4 Mar 2025 20:50:16 +0200 Subject: [PATCH] cache management --- README.md | 106 +++++++++++++++--- lib/rugreefer.dart | 270 +++++++++++++++++++++++++++++---------------- 2 files changed, 268 insertions(+), 108 deletions(-) diff --git a/README.md b/README.md index e0280f3..ea07004 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ # Rugreefer -**Rugreefer** is a Dart library for retrieving metadata about geographic coordinates from a [dopecarpet](http://github.com/TheRandonauts/dopecarpet) server. It converts coordinates into a geohash, fetches a gzipped GeoJSON file from the API, and extracts relevant spatial data such as polygons and lines. The library also supports caching with optional expiration to improve performance. +Rugreefer is a Dart library for retrieving metadata about geographic coordinates from a [dopecarpet](http://github.com/TheRandonauts/dopecarpet) server. It converts coordinates into a geohash, fetches a gzipped GeoJSON file from the API, and extracts relevant spatial data such as polygons and lines. The library also supports caching with optional expiration to improve performance. ## Features - Converts latitude/longitude to geohash (precision: 4 characters) @@ -10,10 +10,10 @@ - Extracts relevant metadata from polygons and lines - Supports caching for performance optimization - Optional cache expiration (default: 14 days) -- Works in **Flutter** (mobile, desktop, web) +- Works in Flutter (mobile, desktop, web) ## Installation -Add **rugreefer** to your `pubspec.yaml`: +Add rugreefer to your `pubspec.yaml`: ```yaml dependencies: @@ -29,17 +29,23 @@ flutter pub get ## Usage -### **Basic Example** +### Basic Example + +To create an instance of `RugReefer`, provide the required parameters: + ```dart import 'package:rugreefer/rugreefer.dart'; void main() async { - final lookup = RugReefer( - apiUrl: 'http://localhost:8000/get_geodata/?format=geojson&geohash=', - enableCacheExpiration: true, - ); + final rugReefer = RugReefer( + apiUrl: 'https://example.com/api', // API endpoint for geospatial data + categoryMap: 'assets/categories.json', // Path to the JSON asset for category mapping + enableCacheExpiration: true, // Optional: Enable cache expiration (default: false) + cacheExpirationDuration: Duration(days: 14), // Optional: Cache expiration duration (default: 14 days) + cacheSizeThreshold: 20 * 1024 * 1024, // Optional: Cache size threshold in bytes (default: 20MB) +); - final results = await lookup.lookup(30.5735040, 34.453125); + final results = await rugReefer.lookup(30.5735040, 34.453125); print(results); } ``` @@ -62,13 +68,83 @@ Example Output: ] ``` -### **Custom Cache Settings** +## Available Methods + +### 1. Lookup Metadata +Perform a geospatial metadata lookup for a given latitude and longitude. + ```dart -final lookup = RugReefer( - apiUrl: 'http://your-api.com/get_geodata/?format=geojson&geohash=', - enableCacheExpiration: true, - cacheExpirationDuration: Duration(days: 7), -); +Future>>> lookup( + double latitude, + double longitude, { + double distanceThreshold = 10.0, // Optional: Distance threshold in meters (default: 10.0) +}); +``` + +Example: +```dart +final metadata = await rugReefer.lookup(37.7749, -122.4194); +print(metadata); +``` + +--- + +### 2. Get Current Cache Size +Get the total size of the cache in bytes. + +```dart +Future getCurrentCacheSize(); +``` + +Example: +```dart +final cacheSize = await rugReefer.getCurrentCacheSize(); +print("Cache size: ${cacheSize / 1024 / 1024} MB"); +``` + +--- + +### 3. List Cached Files +List all currently cached files with their sizes and last access times. + +```dart +Future>> listCachedFiles(); +``` + +Example: +```dart +final cachedFiles = await rugReefer.listCachedFiles(); +for (var file in cachedFiles) { + print("File: ${file['file']}, Size: ${file['size']} bytes, Last Access: ${file['lastAccess']}"); +} +``` + +--- + +### 4. Delete a Specific Cached File +Delete a specific cached file by its geohash. + +```dart +Future deleteCachedFile(String geohash); +``` + +Example: +```dart +await rugReefer.deleteCachedFile('u4pru'); +``` + +--- + +### 5. Clear the Entire Cache +Delete all cached files and remove their last access times. + +```dart +Future clearCache(); +``` + +Example: +```dart +await rugReefer.clearCache(); ``` ## Author diff --git a/lib/rugreefer.dart b/lib/rugreefer.dart index 27e3681..3e893a3 100644 --- a/lib/rugreefer.dart +++ b/lib/rugreefer.dart @@ -14,6 +14,7 @@ class RugReefer { final Duration cacheExpirationDuration; final http.Client httpClient; final String categoryMap; + final int cacheSizeThreshold; // Cache size threshold in bytes (default: 20MB) Directory? cacheDir; RugReefer({ @@ -21,6 +22,7 @@ class RugReefer { required this.categoryMap, this.enableCacheExpiration = false, this.cacheExpirationDuration = const Duration(days: 14), + this.cacheSizeThreshold = 20 * 1024 * 1024, // 20MB in bytes http.Client? client, }) : httpClient = client ?? http.Client() { _initCacheDir(); @@ -29,87 +31,107 @@ class RugReefer { Future _initCacheDir() async { if (!kIsWeb) { cacheDir = await getApplicationDocumentsDirectory(); - if (enableCacheExpiration) { - await _cleanExpiredCache(); - } + if (enableCacheExpiration) await _cleanExpiredCache(); } } Future>>> lookup(double latitude, double longitude, {double distanceThreshold = 10.0}) async { - final String geohash = GeoHasher().encode(longitude, latitude, precision: 4); - print("Generated geohash: $geohash"); - + final geohash = GeoHasher().encode(longitude, latitude, precision: 4); final jsonMap = await loadJsonFromAssets(categoryMap); - print("Loaded JSON Map from Assets"); try { - List> rawData = kIsWeb - ? await _lookupWeb(geohash, latitude, longitude, distanceThreshold) - : await _lookupFile(geohash, latitude, longitude, distanceThreshold); - - print("Raw Data from lookup: ${jsonEncode(rawData)}"); - - final processedData = _processMetadata(rawData, jsonMap, latitude, longitude, distanceThreshold); - print("Processed Data: ${jsonEncode(processedData)}"); - - return processedData; + final rawData = kIsWeb + ? await _lookupWeb(geohash) + : await _lookupFile(geohash); + return _processMetadata(rawData, jsonMap, Position(longitude, latitude), distanceThreshold); } catch (e, stackTrace) { - print("Error in lookup: $e"); - print("StackTrace: $stackTrace"); + print("Error in lookup: $e\nStackTrace: $stackTrace"); return {}; } } - - Future>> _lookupFile(String geohash, double latitude, double longitude, double distanceThreshold) async { + Future>> _lookupFile(String geohash) async { await _initCacheDir(); - final File cacheFile = File('${cacheDir?.path}/$geohash.geojson.gz'); - - print("Checking cache file: ${cacheFile.path}"); + final cacheFile = File('${cacheDir?.path}/$geohash.geojson.gz'); if (!cacheFile.existsSync() || await _isCacheExpired(geohash)) { - print("Cache file missing or expired, downloading..."); await _downloadGeoJson(geohash, cacheFile); } else { - print("Cache file found, updating last access time"); await _updateLastAccessTime(geohash); } - final String geoJsonStr = utf8.decode(GZipCodec().decode(cacheFile.readAsBytesSync())); - print("Read from cache: ${geoJsonStr.length} characters"); - - // ✅ Explicitly cast to List> - final List decodedJson = jsonDecode(geoJsonStr)['features']; - return decodedJson.cast>(); + final geoJsonStr = utf8.decode(GZipCodec().decode(cacheFile.readAsBytesSync())); + return (jsonDecode(geoJsonStr)['features'] as List).cast>(); } - Future>> _lookupWeb(String geohash, double latitude, double longitude, double distanceThreshold) async { + Future>> _lookupWeb(String geohash) async { final prefs = await SharedPreferences.getInstance(); - final String? cachedGeoJson = prefs.getString('geojson_$geohash'); + final cachedGeoJson = prefs.getString('geojson_$geohash'); + if (cachedGeoJson == null || await _isCacheExpired(geohash)) { final response = await httpClient.get(Uri.parse('$apiUrl?geohash=$geohash&format=geojson')); if (response.statusCode == 200) { final geoJsonStr = utf8.decode(response.bodyBytes); await prefs.setString('geojson_$geohash', geoJsonStr); await _updateLastAccessTime(geohash); - return jsonDecode(geoJsonStr)['features']; + return (jsonDecode(geoJsonStr)['features'] as List).cast>(); } throw Exception('Failed to download GeoJSON for geohash: $geohash'); } + await _updateLastAccessTime(geohash); - return jsonDecode(cachedGeoJson)['features']; + return (jsonDecode(cachedGeoJson)['features'] as List).cast>(); } Future _downloadGeoJson(String geohash, File cacheFile) async { final response = await httpClient.get(Uri.parse('$apiUrl?geohash=$geohash&format=geojson')); if (response.statusCode == 200) { - cacheFile.writeAsBytesSync(GZipCodec().encode(response.bodyBytes)); + final bytes = response.bodyBytes; + cacheFile.writeAsBytesSync(GZipCodec().encode(bytes)); await _updateLastAccessTime(geohash); + await _ensureCacheSizeWithinThreshold(bytes.length); // Check and manage cache size } else { throw Exception('Failed to download GeoJSON for geohash: $geohash'); } } + Future _ensureCacheSizeWithinThreshold(int newFileSize) async { + if (kIsWeb || cacheDir == null) return; // Skip for web or if cacheDir is not initialized + + final files = await cacheDir!.list().where((file) => file.path.endsWith('.geojson.gz')).toList(); + final fileSizes = await Future.wait(files.map((file) => file.stat().then((stat) => stat.size))); + + int totalSize = fileSizes.fold(0, (sum, size) => sum + size); + print("Current cache size: ${totalSize / 1024 / 1024} MB"); + + if (totalSize + newFileSize <= cacheSizeThreshold) return; // Within threshold + + // Sort files by last access time (oldest first) + final prefs = await SharedPreferences.getInstance(); + final sortedFiles = await Future.wait(files.map((file) async { + final geohash = file.path.split('/').last.replaceAll('.geojson.gz', ''); + final lastAccess = prefs.getInt('cache_last_access_$geohash') ?? 0; + return {'file': file, 'lastAccess': lastAccess}; + })); + + sortedFiles.sort((a, b) => a['lastAccess'].compareTo(b['lastAccess'])); + + // Delete oldest files until we're within the threshold + for (var entry in sortedFiles) { + final file = entry['file'] as File; + final size = await file.length(); + + file.deleteSync(); + totalSize -= size; + + print("Deleted file: ${file.path} (${size / 1024 / 1024} MB)"); + + if (totalSize + newFileSize <= cacheSizeThreshold) break; + } + + print("New cache size: ${totalSize / 1024 / 1024} MB"); + } + Future _updateLastAccessTime(String geohash) async { final prefs = await SharedPreferences.getInstance(); await prefs.setInt('cache_last_access_$geohash', DateTime.now().millisecondsSinceEpoch); @@ -118,7 +140,7 @@ class RugReefer { Future _isCacheExpired(String geohash) async { if (!enableCacheExpiration) return false; final prefs = await SharedPreferences.getInstance(); - final int? lastAccess = prefs.getInt('cache_last_access_$geohash'); + final lastAccess = prefs.getInt('cache_last_access_$geohash'); return lastAccess == null || DateTime.now().difference(DateTime.fromMillisecondsSinceEpoch(lastAccess)) > cacheExpirationDuration; } @@ -128,76 +150,110 @@ class RugReefer { for (var key in keys) { final geohash = key.replaceFirst('cache_last_access_', ''); if (await _isCacheExpired(geohash)) { - final File cacheFile = File('${cacheDir?.path}/$geohash.geojson.gz'); + final cacheFile = File('${cacheDir?.path}/$geohash.geojson.gz'); if (cacheFile.existsSync()) cacheFile.deleteSync(); await prefs.remove(key); } } } + Future clearCache() async { + if (kIsWeb || cacheDir == null) return; // Skip for web or if cacheDir is not initialized + + final prefs = await SharedPreferences.getInstance(); + + // Delete all cached files + final files = await cacheDir!.list().where((file) => file.path.endsWith('.geojson.gz')).toList(); + for (var file in files) { + if (file is File) { + file.deleteSync(); + print("Deleted file: ${file.path}"); + } + } + + // Remove all last access times from SharedPreferences + final keys = prefs.getKeys().where((key) => key.startsWith('cache_last_access_')).toList(); + for (var key in keys) { + await prefs.remove(key); + print("Removed cache access time for key: $key"); + } + + print("Cache cleared successfully."); + } + + /// Returns the total size of the cache in bytes. + Future getCurrentCacheSize() async { + if (kIsWeb || cacheDir == null) return 0; // Skip for web or if cacheDir is not initialized + + final files = await cacheDir!.list().where((file) => file.path.endsWith('.geojson.gz')).toList(); + final fileSizes = await Future.wait(files.map((file) => file.stat().then((stat) => stat.size))); + return fileSizes.fold(0, (sum, size) => sum + size); + } + + /// Returns a list of currently cached files with their sizes and last access times. + Future>> listCachedFiles() async { + if (kIsWeb || cacheDir == null) return []; // Skip for web or if cacheDir is not initialized + + final prefs = await SharedPreferences.getInstance(); + final files = await cacheDir!.list().where((file) => file.path.endsWith('.geojson.gz')).toList(); + + final fileDetails = await Future.wait(files.map((file) async { + final geohash = file.path.split('/').last.replaceAll('.geojson.gz', ''); + final lastAccess = prefs.getInt('cache_last_access_$geohash') ?? 0; + final size = await file.stat().then((stat) => stat.size); + + return { + 'file': file.path, + 'size': size, + 'lastAccess': DateTime.fromMillisecondsSinceEpoch(lastAccess), + }; + })); + + return fileDetails; + } + + /// Deletes a specific cached file by its geohash. + Future deleteCachedFile(String geohash) async { + if (kIsWeb || cacheDir == null) return; // Skip for web or if cacheDir is not initialized + + final prefs = await SharedPreferences.getInstance(); + final cacheFile = File('${cacheDir?.path}/$geohash.geojson.gz'); + + if (cacheFile.existsSync()) { + cacheFile.deleteSync(); + print("Deleted file: ${cacheFile.path}"); + } + + // Remove the last access time for this geohash + await prefs.remove('cache_last_access_$geohash'); + print("Removed cache access time for geohash: $geohash"); + } + Future> loadJsonFromAssets(String assetPath) async { - final jsonString = await rootBundle.loadString(assetPath); - return jsonDecode(jsonString); + return jsonDecode(await rootBundle.loadString(assetPath)); } Map>> _processMetadata( List> features, Map jsonMap, - double latitude, - double longitude, + Position point, double distanceThreshold) { - final Position point = Position(longitude, latitude); - Map>> groupedObjects = {"in": {}, "by": {}}; + final groupedObjects = {"in": >{}, "by": >{}}; for (var feature in features) { final geometry = feature['geometry']; final metadata = Map.from(feature['properties']['tags'] ?? {})..removeWhere((_, v) => v == null); - String? mainKey; - - // Determine spatial category: "in" (Polygon) or "by" (LineString) - switch (geometry['type']) { - case 'Polygon': - final polygon = Polygon( - coordinates: (geometry['coordinates'] as List) - .map((ring) => (ring as List) - .map((p) => Position(p[0].toDouble(), p[1].toDouble())) - .toList()) - .toList(), - ); - if (booleanPointInPolygon(point, polygon)) mainKey = 'in'; - break; - case 'LineString': - final line = LineString( - coordinates: (geometry['coordinates'] as List) - .map((p) => Position(p[0].toDouble(), p[1].toDouble())) - .toList(), - ); - if (_isPointNearLine(Point(coordinates: point), line, distanceThreshold)) mainKey = 'by'; - break; - } + final mainKey = _determineSpatialCategory(geometry, point, distanceThreshold); if (mainKey == null) continue; for (var key in metadata.keys) { - var value = metadata[key]; - + final value = metadata[key]; for (var category in jsonMap.keys) { - var categoryMap = jsonMap[category]; - - if (categoryMap is Map && categoryMap.containsKey(key)) { - var rule = categoryMap[key]; - - if (rule is String && rule == value) { - groupedObjects[mainKey]!.putIfAbsent(category, () => []); - groupedObjects[mainKey]![category]!.add(value); - } else if (rule is List && rule.contains(value)) { - groupedObjects[mainKey]!.putIfAbsent(category, () => []); - groupedObjects[mainKey]![category]!.add(value); - } else if (rule == true) { - groupedObjects[mainKey]!.putIfAbsent(category, () => []); - groupedObjects[mainKey]![category]!.add(value); - } + final rule = jsonMap[category] is Map ? jsonMap[category][key] : null; + if (_matchesRule(rule, value)) { + groupedObjects[mainKey]!.putIfAbsent(category, () => []).add(value); } } } @@ -206,15 +262,43 @@ class RugReefer { return groupedObjects; } - - - bool _isPointNearLine(Point point, LineString line, double distanceThreshold) { - for (final Position p in line.coordinates) { - if (distance(point, Point(coordinates: p), Unit.meters) <= distanceThreshold) { - return true; - } + String? _determineSpatialCategory(Map geometry, Position point, double distanceThreshold) { + switch (geometry['type']) { + case 'Polygon': + final polygon = Polygon( + coordinates: (geometry['coordinates'] as List) + .map((ring) => (ring as List) + .map((p) => Position(p[0].toDouble(), p[1].toDouble())) + .toList()) + .toList(), + ); + return booleanPointInPolygon(point, polygon) ? 'in' : null; + case 'LineString': + final line = LineString( + coordinates: (geometry['coordinates'] as List) + .map((p) => Position(p[0].toDouble(), p[1].toDouble())) + .toList(), + ); + return _isPointNearLine(Point(coordinates: point), line, distanceThreshold) ? 'by' : null; + default: + return null; } + } + + bool _matchesRule(dynamic rule, dynamic value) { + if (rule == true) return true; + if (rule is String && rule == value) return true; + if (rule is List && rule.contains(value)) return true; return false; } -} + bool _isPointNearLine(Point point, LineString line, double distanceThreshold) { + for (int i = 0; i < line.coordinates.length - 1; i++) { + final p1 = line.coordinates[i]; + final p2 = line.coordinates[i + 1]; + final dist = distance(p1, point, Unit.meter); + if (dist <= distanceThreshold) return true; + } + return false; + } +} \ No newline at end of file