unified Z-Score
This commit is contained in:
randogoth
parent
517b7d55f1
commit
1189d70db1
2 changed files with 121 additions and 49 deletions
97
src/lib.rs
97
src/lib.rs
|
|
@ -2,7 +2,7 @@ use pyo3::prelude::*;
|
|||
use pyo3::types::PyList;
|
||||
use pyo3::wrap_pyfunction;
|
||||
use serde::Serialize;
|
||||
use serde_json;
|
||||
use serde_json::to_string_pretty;
|
||||
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
struct Point {
|
||||
|
|
@ -106,66 +106,77 @@ fn calculate_densities_and_gaps(dataset: &[Point], factor: f32, min_cluster_size
|
|||
results
|
||||
}
|
||||
|
||||
/// A Python-exposed function that analyzes a list of numerical values to identify clusters and significant gaps,
|
||||
/// calculates z-scores for each identified cluster/gap, and returns the analysis results as a JSON string.
|
||||
///
|
||||
/// This function takes a list of integers (representing a dataset), a factor to adjust clustering and gap detection thresholds,
|
||||
/// and a minimum cluster size. It calculates the mean distance and standard deviation across the dataset,
|
||||
/// identifies clusters and significant gaps based on these metrics, calculates z-scores for each cluster/gap,
|
||||
/// and returns a JSON string representing the analysis results.
|
||||
/// Analyzes a dataset of integers to identify clusters and gaps, then calculates Z-scores
|
||||
/// for each based on their deviation from mean metrics. The analysis aims to highlight
|
||||
/// significant clusters of closely grouped points and notable gaps between them, providing
|
||||
/// a statistical measure of their significance through Z-scores. The results, including
|
||||
/// clusters, gaps, and their Z-scores, are serialized into a JSON string.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `_py`: The Python interpreter, used for Python-Rust interactions. Not directly used in the function body.
|
||||
/// * `int_list`: A Python list of integers representing the dataset to be analyzed.
|
||||
/// * `factor`: A floating-point value used to adjust the sensitivity of cluster and gap detection.
|
||||
/// Lower values result in tighter clustering and wider gaps, while higher values do the opposite.
|
||||
/// * `min_cluster_size`: The minimum number of contiguous points required to be considered a cluster.
|
||||
/// * `_py` - The Python interpreter instance, used for Python-Rust interoperability.
|
||||
/// This argument is necessary for functions exposed to Python via PyO3 but is not
|
||||
/// directly used within the function.
|
||||
/// * `int_list` - A Python list of integers representing the dataset to be analyzed.
|
||||
/// This list is converted into a Vec<Point> for internal processing.
|
||||
/// * `factor` - A floating-point value used as a threshold factor to adjust the sensitivity
|
||||
/// of cluster and gap detection. This factor influences the identification of clusters
|
||||
/// by defining the minimum density or separation required.
|
||||
/// * `min_cluster_size` - An integer specifying the minimum number of contiguous points
|
||||
/// required for a group of points to be considered a cluster. This parameter helps
|
||||
/// filter out noise by defining a threshold for the minimum cluster size.
|
||||
///
|
||||
/// # Returns
|
||||
/// A `PyResult<String>` which is either:
|
||||
/// * Ok containing a JSON-formatted string of the analysis results, including clusters and gaps with their z-scores.
|
||||
/// * Err containing a Python exception if an error occurs during processing or JSON serialization.
|
||||
/// Returns a `PyResult<String>` containing a JSON-formatted string of the analysis results.
|
||||
/// The JSON string includes detailed information about each identified cluster and gap,
|
||||
/// such as their span length, number of elements (if applicable), centroid, and calculated
|
||||
/// Z-score. In case of an error during processing or serialization, a Python exception is
|
||||
/// returned.
|
||||
///
|
||||
#[pyfunction]
|
||||
fn lyagushka(_py: Python, int_list: &PyList, factor: f32, min_cluster_size: usize) -> PyResult<String> {
|
||||
|
||||
// Convert the Python list of integers into a Rust Vec of Point structs.
|
||||
let dataset: Vec<Point> = int_list.into_iter()
|
||||
.map(|py_any| py_any.extract::<u32>().map(Point::new))
|
||||
.collect::<PyResult<Vec<Point>>>()?;
|
||||
// Extract integers from a Python list and create a vector of Point structs.
|
||||
let dataset: Vec<Point> = int_list.extract::<Vec<u32>>()?
|
||||
.into_iter()
|
||||
.map(Point::new)
|
||||
.collect();
|
||||
|
||||
// Analyze the dataset to identify clusters and significant gaps.
|
||||
// Calculate clusters and gaps from the dataset using predefined criteria.
|
||||
let mut cluster_gap_infos = calculate_densities_and_gaps(&dataset, factor, min_cluster_size);
|
||||
|
||||
// Calculate the mean distance between consecutive points in the dataset.
|
||||
let mean_distance: f32 = dataset.windows(2)
|
||||
.map(|w| w[1].value as f32 - w[0].value as f32)
|
||||
.sum::<f32>() / (dataset.len() - 1) as f32;
|
||||
// Calculate the mean density of clusters in the dataset for comparison.
|
||||
let mean_density: f32 = cluster_gap_infos.iter()
|
||||
.filter(|info| info.num_elements > 0)
|
||||
.map(|info| info.num_elements as f32 / info.span_length)
|
||||
.sum::<f32>() / cluster_gap_infos.iter().filter(|info| info.num_elements > 0).count() as f32;
|
||||
|
||||
// Calculate the standard deviation of distances between consecutive points.
|
||||
let std_deviation: f32 = (dataset.windows(2)
|
||||
.map(|w| w[1].value as f32 - w[0].value as f32 - mean_distance)
|
||||
.map(|d| d * d)
|
||||
.sum::<f32>() / (dataset.len() - 1) as f32)
|
||||
.sqrt();
|
||||
// Calculate the standard deviation of cluster densities to evaluate variation.
|
||||
let variance_density: f32 = cluster_gap_infos.iter()
|
||||
.filter(|info| info.num_elements > 0)
|
||||
.map(|info| info.num_elements as f32 / info.span_length)
|
||||
.map(|density| (density - mean_density).powi(2))
|
||||
.sum::<f32>() / cluster_gap_infos.iter().filter(|info| info.num_elements > 0).count() as f32;
|
||||
let std_dev_density = variance_density.sqrt();
|
||||
|
||||
// Calculate and assign z-scores for each cluster/gap based on their centroid or span length.
|
||||
for info in cluster_gap_infos.iter_mut() {
|
||||
info.z_score = Some(if info.num_elements > 0 {
|
||||
// For clusters, use the centroid for z-score calculation.
|
||||
(info.centroid - mean_distance) / std_deviation
|
||||
// Calculate the average span of all clusters and gaps to assess gap significance.
|
||||
let average_span: f32 = cluster_gap_infos.iter().map(|info| info.span_length).sum::<f32>() / cluster_gap_infos.len() as f32;
|
||||
|
||||
// Update Z-scores for both clusters and gaps based on their deviation from mean metrics.
|
||||
for info in &mut cluster_gap_infos {
|
||||
if info.num_elements > 0 {
|
||||
// Calculate and update Z-score for clusters based on density deviation.
|
||||
let cluster_density = info.num_elements as f32 / info.span_length;
|
||||
info.z_score = Some((cluster_density - mean_density) / std_dev_density);
|
||||
} else {
|
||||
// For gaps, use the span length for z-score calculation.
|
||||
(info.span_length - mean_distance) / std_deviation
|
||||
});
|
||||
// Calculate and update Z-score for gaps based on span length deviation.
|
||||
info.z_score = Some((info.span_length - average_span) / std_dev_density);
|
||||
}
|
||||
}
|
||||
|
||||
// Serialize the analysis results into a JSON string and return it.
|
||||
serde_json::to_string_pretty(&cluster_gap_infos)
|
||||
// Serialize the updated cluster and gap information, including Z-scores, to a JSON string.
|
||||
to_string_pretty(&cluster_gap_infos)
|
||||
.map_err(|e| PyErr::new::<pyo3::exceptions::PyException, _>(format!("JSON Serialization Error: {}", e)))
|
||||
}
|
||||
|
||||
|
||||
#[pymodule]
|
||||
fn pyagushka(_py: Python, m: &PyModule) -> PyResult<()> {
|
||||
m.add_function(wrap_pyfunction!(lyagushka, m)?)?;
|
||||
|
|
|
|||
73
test.py
73
test.py
|
|
@ -1,13 +1,74 @@
|
|||
from pyagushka import lyagushka
|
||||
from randonautentropy import rndo
|
||||
import json
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
from scipy.interpolate import interp1d
|
||||
|
||||
random_data = []
|
||||
def generate_random_data(size=1024, max_value=100):
|
||||
|
||||
with open('random_values.txt', 'r') as file:
|
||||
for line in file:
|
||||
random_data.append(int(line.strip()))
|
||||
random_data = []
|
||||
max_value_bytes = (max_value.bit_length() + 7) // 8
|
||||
max_int_for_bytes = 2**(max_value_bytes * 8) - 1
|
||||
min_bytes_needed = max_value_bytes * size
|
||||
mod_cutoff = max_int_for_bytes - (max_int_for_bytes % max_value) - 1
|
||||
|
||||
anomalies = json.loads(lyagushka(random_data, 1.0, 5))
|
||||
# Populate the 'random_data' array
|
||||
while len(random_data) < size:
|
||||
hex_data = rndo.get(length=min_bytes_needed)
|
||||
hex_chunks = list((hex_data[0+i:2 * max_value_bytes+i] for i in range(0, len(hex_data), 2 * max_value_bytes)))
|
||||
for i in hex_chunks:
|
||||
num = int(i, 16)
|
||||
if num <= mod_cutoff and len(random_data) < size:
|
||||
random_data.append( num % (max_value + 1) )
|
||||
|
||||
print(anomalies)
|
||||
return random_data
|
||||
|
||||
# load the random test data
|
||||
dataset = []
|
||||
# with open('random_values.txt', 'r') as file:
|
||||
# for line in file:
|
||||
# random_data.append(int(line.strip()))
|
||||
|
||||
dataset = generate_random_data(1024, 1024)
|
||||
dataset.sort()
|
||||
|
||||
# calculate the anomalies in the data
|
||||
analysis_results = json.loads(lyagushka(dataset, 3.0, 7))
|
||||
print(analysis_results)
|
||||
print(len(analysis_results))
|
||||
|
||||
# Initialize plot
|
||||
plt.figure(figsize=(10, 6))
|
||||
|
||||
# Plot dataset points
|
||||
for point in dataset:
|
||||
plt.plot(point, 0, 'ko') # Plot dataset as black dots at y=0
|
||||
|
||||
# Process each cluster/gap for plotting
|
||||
for result in analysis_results:
|
||||
if result['num_elements'] > 0: # It's a cluster
|
||||
color = 'blue' # Color for clusters
|
||||
else: # It's a gap
|
||||
color = 'green' # Color for gaps
|
||||
|
||||
# Generate start and end points for the segment
|
||||
start = result['centroid'] - result['span_length'] / 2
|
||||
end = result['centroid'] + result['span_length'] / 2
|
||||
z_score = result['z_score'] if result['z_score'] is not None else 0
|
||||
|
||||
# Plot a line segment for the cluster/gap
|
||||
plt.plot([start, end], [z_score, z_score], color=color, linewidth=2)
|
||||
|
||||
# Enhancements for visualization
|
||||
plt.xlabel('Integer Value')
|
||||
plt.ylabel('Z-Score')
|
||||
plt.title('Cluster and Gap Analysis with Distinct Z-Score Curves')
|
||||
plt.grid(True)
|
||||
|
||||
# Custom legend
|
||||
plt.plot([], [], color='blue', label='Clusters')
|
||||
plt.plot([], [], color='green', label='Gaps')
|
||||
plt.legend()
|
||||
|
||||
plt.show()
|
||||
Loading…
Add table
Add a link
Reference in a new issue