unified Z-Score

src/lib.rs

@@ -2,7 +2,7 @@ use pyo3::prelude::*;
 use pyo3::types::PyList;
 use pyo3::wrap_pyfunction;
 use serde::Serialize;
-use serde_json;
+use serde_json::to_string_pretty;
 
 #[derive(Clone, Debug, Serialize)]
 struct Point {
@@ -106,66 +106,77 @@ fn calculate_densities_and_gaps(dataset: &[Point], factor: f32, min_cluster_size
     results
 }
 
-/// A Python-exposed function that analyzes a list of numerical values to identify clusters and significant gaps,
+/// Analyzes a dataset of integers to identify clusters and gaps, then calculates Z-scores 
-/// calculates z-scores for each identified cluster/gap, and returns the analysis results as a JSON string.
+/// for each based on their deviation from mean metrics. The analysis aims to highlight 
-///
+/// significant clusters of closely grouped points and notable gaps between them, providing 
-/// This function takes a list of integers (representing a dataset), a factor to adjust clustering and gap detection thresholds,
+/// a statistical measure of their significance through Z-scores. The results, including 
-/// and a minimum cluster size. It calculates the mean distance and standard deviation across the dataset,
+/// clusters, gaps, and their Z-scores, are serialized into a JSON string.
-/// identifies clusters and significant gaps based on these metrics, calculates z-scores for each cluster/gap,
-/// and returns a JSON string representing the analysis results.
 ///
 /// # Arguments
-/// * `_py`: The Python interpreter, used for Python-Rust interactions. Not directly used in the function body.
+/// * `_py` - The Python interpreter instance, used for Python-Rust interoperability. 
-/// * `int_list`: A Python list of integers representing the dataset to be analyzed.
+///     This argument is necessary for functions exposed to Python via PyO3 but is not 
-/// * `factor`: A floating-point value used to adjust the sensitivity of cluster and gap detection.
+///     directly used within the function.
-///   Lower values result in tighter clustering and wider gaps, while higher values do the opposite.
+/// * `int_list` - A Python list of integers representing the dataset to be analyzed. 
-/// * `min_cluster_size`: The minimum number of contiguous points required to be considered a cluster.
+///     This list is converted into a Vec<Point> for internal processing.
+/// * `factor` - A floating-point value used as a threshold factor to adjust the sensitivity 
+///     of cluster and gap detection. This factor influences the identification of clusters 
+///     by defining the minimum density or separation required.
+/// * `min_cluster_size` - An integer specifying the minimum number of contiguous points 
+///     required for a group of points to be considered a cluster. This parameter helps 
+///     filter out noise by defining a threshold for the minimum cluster size.
 ///
 /// # Returns
-/// A `PyResult<String>` which is either:
+/// Returns a `PyResult<String>` containing a JSON-formatted string of the analysis results. 
-/// * Ok containing a JSON-formatted string of the analysis results, including clusters and gaps with their z-scores.
+/// The JSON string includes detailed information about each identified cluster and gap, 
-/// * Err containing a Python exception if an error occurs during processing or JSON serialization.
+/// such as their span length, number of elements (if applicable), centroid, and calculated 
+/// Z-score. In case of an error during processing or serialization, a Python exception is 
+/// returned.
 ///
 #[pyfunction]
 fn lyagushka(_py: Python, int_list: &PyList, factor: f32, min_cluster_size: usize) -> PyResult<String> {
+    // Extract integers from a Python list and create a vector of Point structs.
+    let dataset: Vec<Point> = int_list.extract::<Vec<u32>>()?
+                                      .into_iter()
+                                      .map(Point::new)
+                                      .collect();
 
-    // Convert the Python list of integers into a Rust Vec of Point structs.
+    // Calculate clusters and gaps from the dataset using predefined criteria.
-    let dataset: Vec<Point> = int_list.into_iter()
-                                      .map(|py_any| py_any.extract::<u32>().map(Point::new))
-                                      .collect::<PyResult<Vec<Point>>>()?;
-
-    // Analyze the dataset to identify clusters and significant gaps.
     let mut cluster_gap_infos = calculate_densities_and_gaps(&dataset, factor, min_cluster_size);
 
-    // Calculate the mean distance between consecutive points in the dataset.
+    // Calculate the mean density of clusters in the dataset for comparison.
-    let mean_distance: f32 = dataset.windows(2)
+    let mean_density: f32 = cluster_gap_infos.iter()
-                                    .map(|w| w[1].value as f32 - w[0].value as f32)
+                                             .filter(|info| info.num_elements > 0)
-                                    .sum::<f32>() / (dataset.len() - 1) as f32;
+                                             .map(|info| info.num_elements as f32 / info.span_length)
+                                             .sum::<f32>() / cluster_gap_infos.iter().filter(|info| info.num_elements > 0).count() as f32;
 
-    // Calculate the standard deviation of distances between consecutive points.
+    // Calculate the standard deviation of cluster densities to evaluate variation.
-    let std_deviation: f32 = (dataset.windows(2)
+    let variance_density: f32 = cluster_gap_infos.iter()
-                                      .map(|w| w[1].value as f32 - w[0].value as f32 - mean_distance)
+                                                 .filter(|info| info.num_elements > 0)
-                                      .map(|d| d * d)
+                                                 .map(|info| info.num_elements as f32 / info.span_length)
-                                      .sum::<f32>() / (dataset.len() - 1) as f32)
+                                                 .map(|density| (density - mean_density).powi(2))
-                                      .sqrt();
+                                                 .sum::<f32>() / cluster_gap_infos.iter().filter(|info| info.num_elements > 0).count() as f32;
+    let std_dev_density = variance_density.sqrt();
 
-    // Calculate and assign z-scores for each cluster/gap based on their centroid or span length.
+    // Calculate the average span of all clusters and gaps to assess gap significance.
-    for info in cluster_gap_infos.iter_mut() {
+    let average_span: f32 = cluster_gap_infos.iter().map(|info| info.span_length).sum::<f32>() / cluster_gap_infos.len() as f32;
-        info.z_score = Some(if info.num_elements > 0 {
+
-            // For clusters, use the centroid for z-score calculation.
+    // Update Z-scores for both clusters and gaps based on their deviation from mean metrics.
-            (info.centroid - mean_distance) / std_deviation
+    for info in &mut cluster_gap_infos {
+        if info.num_elements > 0 {
+            // Calculate and update Z-score for clusters based on density deviation.
+            let cluster_density = info.num_elements as f32 / info.span_length;
+            info.z_score = Some((cluster_density - mean_density) / std_dev_density);
         } else {
-            // For gaps, use the span length for z-score calculation.
+            // Calculate and update Z-score for gaps based on span length deviation.
-            (info.span_length - mean_distance) / std_deviation
+            info.z_score = Some((info.span_length - average_span) / std_dev_density);
-        });
+        }
     }
 
-    // Serialize the analysis results into a JSON string and return it.
+    // Serialize the updated cluster and gap information, including Z-scores, to a JSON string.
-    serde_json::to_string_pretty(&cluster_gap_infos)
+    to_string_pretty(&cluster_gap_infos)
         .map_err(|e| PyErr::new::<pyo3::exceptions::PyException, _>(format!("JSON Serialization Error: {}", e)))
 }
 
-
 #[pymodule]
 fn pyagushka(_py: Python, m: &PyModule) -> PyResult<()> {
     m.add_function(wrap_pyfunction!(lyagushka, m)?)?;

test.py

@@ -1,13 +1,74 @@
 from pyagushka import lyagushka
 from randonautentropy import rndo
 import json
+import matplotlib.pyplot as plt
+import numpy as np
+from scipy.interpolate import interp1d
 
-random_data = []
+def generate_random_data(size=1024, max_value=100):
 
-with open('random_values.txt', 'r') as file:
+    random_data = []
-    for line in file:
+    max_value_bytes = (max_value.bit_length() + 7) // 8
-        random_data.append(int(line.strip()))
+    max_int_for_bytes = 2**(max_value_bytes * 8) - 1
+    min_bytes_needed =  max_value_bytes * size
+    mod_cutoff = max_int_for_bytes - (max_int_for_bytes % max_value) - 1
 
-anomalies = json.loads(lyagushka(random_data, 1.0, 5))
+    # Populate the 'random_data' array
+    while len(random_data) < size:
+        hex_data = rndo.get(length=min_bytes_needed)
+        hex_chunks = list((hex_data[0+i:2 * max_value_bytes+i] for i in range(0, len(hex_data), 2 * max_value_bytes)))
+        for i in hex_chunks:
+            num = int(i, 16)
+            if num <= mod_cutoff and len(random_data) < size:
+                random_data.append( num % (max_value + 1) )
 
-print(anomalies)
+    return random_data
+
+# load the random test data
+dataset = []
+# with open('random_values.txt', 'r') as file:
+#     for line in file:
+#         random_data.append(int(line.strip()))
+
+dataset = generate_random_data(1024, 1024)
+dataset.sort()
+
+# calculate the anomalies in the data
+analysis_results = json.loads(lyagushka(dataset, 3.0, 7))
+print(analysis_results)
+print(len(analysis_results))
+
+# Initialize plot
+plt.figure(figsize=(10, 6))
+
+# Plot dataset points
+for point in dataset:
+    plt.plot(point, 0, 'ko')  # Plot dataset as black dots at y=0
+
+# Process each cluster/gap for plotting
+for result in analysis_results:
+    if result['num_elements'] > 0:  # It's a cluster
+        color = 'blue'  # Color for clusters
+    else:  # It's a gap
+        color = 'green'  # Color for gaps
+
+    # Generate start and end points for the segment
+    start = result['centroid'] - result['span_length'] / 2
+    end = result['centroid'] + result['span_length'] / 2
+    z_score = result['z_score'] if result['z_score'] is not None else 0
+
+    # Plot a line segment for the cluster/gap
+    plt.plot([start, end], [z_score, z_score], color=color, linewidth=2)
+
+# Enhancements for visualization
+plt.xlabel('Integer Value')
+plt.ylabel('Z-Score')
+plt.title('Cluster and Gap Analysis with Distinct Z-Score Curves')
+plt.grid(True)
+
+# Custom legend
+plt.plot([], [], color='blue', label='Clusters')
+plt.plot([], [], color='green', label='Gaps')
+plt.legend()
+
+plt.show()