randogoth/lyagushka
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fixed Z-Scores

Browse source
This commit is contained in:
randogoth 2024-03-01 18:35:31 +02:00
parent 58eb8a2f40
commit a04dfc45ca
3 changed files with 93 additions and 78 deletions
Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -1 +1,2 @@
/target /target
*.json

146
src/lib.rs
View file

@ -4,44 +4,41 @@ use pyo3::wrap_pyfunction;
use serde::Serialize; use serde::Serialize;
use serde_json::to_string_pretty; use serde_json::to_string_pretty;
#[derive(Clone, Debug, Serialize)]
struct Point {
value: u32,
}
impl Point {
fn new(value: u32) -> Self {
Point { value }
}
}
#[derive(Debug, Clone, Serialize)] #[derive(Debug, Clone, Serialize)]
struct ClusterGapInfo { struct Anomaly {
span_length: f32, elements: Vec<i32>,
start: i32,
end: i32,
span_length: i32,
num_elements: usize, num_elements: usize,
centroid: f32, centroid: f32,
z_score: Option<f32>, z_score: Option<f32>,
} }
fn create_cluster_info(cluster: &[Point]) -> ClusterGapInfo { fn anomaly_info(cluster: &[i32]) -> Anomaly {
let num_elements: usize = cluster.len();
let num_elements = cluster.len(); let start: i32 = *cluster.first().expect("Cluster has no start");
let span_length = (cluster.last().unwrap().value as f32) - (cluster.first().unwrap().value as f32); let end: i32 = *cluster.last().expect("Cluster has no end");
let centroid = cluster.iter().map(|p| p.value as f32).sum::<f32>() / num_elements as f32; let span_length: i32 = end - start;
let centroid: f32 = start as f32 + span_length as f32 / 2.0;
ClusterGapInfo { Anomaly {
elements: cluster.to_vec(),
start,
end,
span_length, span_length,
num_elements, num_elements,
centroid, centroid,
z_score: None, z_score: None, // Placeholder for actual Z-score calculation
} }
} }
/// Calculates the densities (clusters) and significant gaps between points in a dataset. /// Calculates the densities (clusters) and significant gaps between points in a dataset.
/// ///
/// This function iterates over a dataset of points, identifying clusters based on a distance threshold /// This function iterates over a dataset of points, identifying clusters based on a distance threshold
/// (calculated from the mean distance between points and adjusted by a given factor) and identifying significant gaps /// (calculated from the mean distance between points and adjusted by a given factor) and identifying significant gaps
/// that exceed a certain threshold. Each cluster or significant gap identified is summarized in a `ClusterGapInfo` object. /// that exceed a certain threshold. Each cluster or significant gap identified is summarized in a `Anomaly` object.
/// ///
/// # Arguments /// # Arguments
/// * `dataset`: A slice of `Point` objects representing the dataset to be analyzed. /// * `dataset`: A slice of `Point` objects representing the dataset to be analyzed.
@ -50,57 +47,60 @@ fn create_cluster_info(cluster: &[Point]) -> ClusterGapInfo {
/// * `min_cluster_size`: The minimum number of points required for a group of points to be considered a cluster. /// * `min_cluster_size`: The minimum number of points required for a group of points to be considered a cluster.
/// ///
/// # Returns /// # Returns
/// A vector of `ClusterGapInfo` objects, each representing either a cluster of points or a significant gap between points. /// A vector of `Anomaly` objects, each representing either a cluster of points or a significant gap between points.
/// ///
fn calculate_densities_and_gaps(dataset: &[Point], factor: f32, min_cluster_size: usize) -> Vec<ClusterGapInfo> { fn scan_anomalies(dataset: &[i32], factor: f32, min_cluster_size: usize) -> Vec<Anomaly> {
// Return early if the dataset is too small to form any clusters or gaps. // Return early if the dataset is too small to form any clusters or gaps.
if dataset.len() < 2 { return Vec::new(); } if dataset.len() < 2 { return Vec::new(); }
// Calculate the mean distance between consecutive points in the dataset. // Calculate the mean distance between consecutive points in the dataset.
let mean_distance = dataset.windows(2) let mean_distance: f32 = dataset.windows(2)
.map(|w| w[1].value as f32 - w[0].value as f32) .map(|w| (w[1] - w[0]) as f32)
.sum::<f32>() / (dataset.len() - 1) as f32; .sum::<f32>() / (dataset.len() - 1) as f32;
// Define thresholds for clustering and gap identification based on the mean distance and factor. // Define thresholds for clustering and gap identification based on the mean distance and factor.
let cluster_threshold = mean_distance / factor; let cluster_threshold: f32 = mean_distance / factor;
let gap_threshold = factor * mean_distance; let gap_threshold: f32 = factor * mean_distance;
let mut results: Vec<ClusterGapInfo> = Vec::new(); // Stores the resulting clusters and gaps. let mut results: Vec<Anomaly> = Vec::new(); // Stores the resulting clusters and gaps.
let mut current_cluster: Vec<Point> = Vec::new(); // Temporary storage for points in the current cluster. let mut current_cluster: Vec<i32> = Vec::new(); // Temporary storage for points in the current cluster.
// Iterate through pairs of consecutive points to find clusters and significant gaps. // Iterate through pairs of consecutive points to find clusters and significant gaps.
for window in dataset.windows(2) { for window in dataset.windows(2) {
let gap_distance = window[1].value as f32 - window[0].value as f32; let gap_size: f32 = (window[1] - window[0]) as f32;
// If the distance between points is within the cluster threshold, add to current cluster. if gap_size <= cluster_threshold {
if gap_distance <= cluster_threshold { // Add points to the current cluster
if current_cluster.is_empty() { if current_cluster.is_empty() {
current_cluster.push(window[0].clone()); // Start a new cluster with the first point. current_cluster.push(window[0]); // Start a new cluster with the first point
} }
current_cluster.push(window[1].clone()); // Add the second point to the cluster. current_cluster.push(window[1]); // Add the second point to the cluster
} else { } else {
// If the current cluster is large enough, finalize it and prepare for a new cluster. // End the current cluster and start a new gap
if !current_cluster.is_empty() && current_cluster.len() >= min_cluster_size { if !current_cluster.is_empty() && current_cluster.len() >= min_cluster_size {
results.push(create_cluster_info(&current_cluster)); results.push(anomaly_info(&current_cluster));
current_cluster.clear(); current_cluster.clear();
} }
// If the gap between points is significant, record it as a gap. // Record the gap
if gap_distance > gap_threshold { if gap_size > gap_threshold {
results.push(ClusterGapInfo { results.push(Anomaly {
span_length: gap_distance, elements: Vec::new(), // No elements in a gap
num_elements: 0, // Indicating this is a gap, not a cluster. start: window[0],
centroid: (window[0].value as f32 + window[1].value as f32) / 2.0, end: window[1],
z_score: None, // Z-score will be calculated later if necessary. span_length: gap_size as i32,
num_elements: 0,
centroid: (window[0] as f32 + window[1] as f32) / 2.0,
z_score: None,
}); });
} }
} }
} }
// Finalize the last cluster if it meets the size requirement. // Finalize the last cluster if applicable
if !current_cluster.is_empty() && current_cluster.len() >= min_cluster_size { if !current_cluster.is_empty() && current_cluster.len() >= min_cluster_size {
results.push(create_cluster_info(&current_cluster)); results.push(anomaly_info(&current_cluster));
} }
results results
@ -134,54 +134,60 @@ fn calculate_densities_and_gaps(dataset: &[Point], factor: f32, min_cluster_size
/// ///
#[pyfunction] #[pyfunction]
fn lyagushka(_py: Python, int_list: &PyList, factor: f32, min_cluster_size: usize) -> PyResult<String> { fn lyagushka(_py: Python, int_list: &PyList, factor: f32, min_cluster_size: usize) -> PyResult<String> {
// Extract integers from a Python list and create a vector.
// Extract integers from a Python list and create a vector of Point structs. let mut dataset: Vec<i32> = int_list.extract::<Vec<i32>>()?;
let mut dataset: Vec<Point> = int_list.extract::<Vec<u32>>()?
.into_iter()
.map(Point::new)
.collect();
// Sort the vector // Sort the vector
dataset.sort_by_key(|p| p.value); dataset.sort_unstable();
// Calculate clusters and gaps from the dataset using predefined criteria. // Calculate clusters and gaps from the dataset using predefined criteria.
let mut cluster_gap_infos = calculate_densities_and_gaps(&dataset, factor, min_cluster_size); let mut anomalies: Vec<Anomaly> = scan_anomalies(&dataset, factor, min_cluster_size);
// Calculate the mean density of clusters in the dataset for comparison. // Calculate the mean density of clusters in the dataset for comparison.
let mean_density: f32 = cluster_gap_infos.iter() let mean_density: f32 = anomalies.iter()
.filter(|info| info.num_elements > 0) .filter(|info: &&Anomaly| info.num_elements > 0)
.map(|info| info.num_elements as f32 / info.span_length) .map(|info: &Anomaly| info.num_elements as f32 / info.span_length as f32)
.sum::<f32>() / cluster_gap_infos.iter().filter(|info| info.num_elements > 0).count() as f32; .sum::<f32>() / anomalies.iter().filter(|info: &&Anomaly| info.num_elements > 0).count() as f32;
// Calculate the standard deviation of cluster densities to evaluate variation. // Calculate the standard deviation of cluster densities to evaluate variation.
let variance_density: f32 = cluster_gap_infos.iter() let variance_density: f32 = anomalies.iter()
.filter(|info| info.num_elements > 0) .filter(|info: &&Anomaly| info.num_elements > 0)
.map(|info| info.num_elements as f32 / info.span_length) .map(|info: &Anomaly| info.num_elements as f32 / info.span_length as f32)
.map(|density| (density - mean_density).powi(2)) .map(|density| (density - mean_density).powi(2))
.sum::<f32>() / cluster_gap_infos.iter().filter(|info| info.num_elements > 0).count() as f32; .sum::<f32>() / anomalies.iter().filter(|info: &&Anomaly| info.num_elements > 0).count() as f32;
let std_dev_density = variance_density.sqrt(); let std_dev_density = variance_density.sqrt();
// Calculate the average span of all clusters and gaps to assess gap significance. // Calculate mean span length
let average_span: f32 = cluster_gap_infos.iter().map(|info| info.span_length).sum::<f32>() / cluster_gap_infos.len() as f32; let mean_span_length: f32 = anomalies.iter()
.map(|info: &Anomaly| info.span_length as f32)
.sum::<f32>() / anomalies.len() as f32;
// Calculate variance
let variance: f32 = anomalies.iter()
.map(|info: &Anomaly| (info.span_length as f32 - mean_span_length).powi(2))
.sum::<f32>() / anomalies.len() as f32;
// Standard deviation is the square root of variance
let std_dev_span_length: f32 = variance.sqrt();
// Update Z-scores for both clusters and gaps based on their deviation from mean metrics. // Update Z-scores for both clusters and gaps based on their deviation from mean metrics.
for info in &mut cluster_gap_infos { for info in anomalies.iter_mut() {
if info.num_elements > 0 { if info.num_elements > 0 {
// Calculate and update Z-score for clusters based on density deviation. // Calculate and update Z-score for clusters based on density deviation.
let cluster_density = info.num_elements as f32 / info.span_length; let cluster_density: f32 = info.num_elements as f32 / info.span_length as f32;
info.z_score = Some((cluster_density - mean_density) / std_dev_density); info.z_score = Some((cluster_density - mean_density) / std_dev_density);
} else { } else {
// Calculate and update Z-score for gaps based on span length deviation. // Calculate and update Z-score for gaps based on span length deviation.
info.z_score = Some((info.span_length - average_span) / std_dev_density); info.z_score = Some((info.span_length as f32 / std_dev_span_length) * -1.0);
} }
} }
// Serialize the updated cluster and gap information, including Z-scores, to a JSON string. // Serialize the updated cluster and gap information, including Z-scores, to a JSON string.
to_string_pretty(&cluster_gap_infos) to_string_pretty(&anomalies)
.map_err(|e| PyErr::new::<pyo3::exceptions::PyException, _>(format!("JSON Serialization Error: {}", e))) .map_err(|e| PyErr::new::<pyo3::exceptions::PyException, _>(format!("JSON Serialization Error: {}", e)))
} }
#[pymodule] #[pymodule]
fn pyagushka(_py: Python, m: &PyModule) -> PyResult<()> { fn pyagushka(_py: Python, m: &PyModule) -> PyResult<()> {
m.add_function(wrap_pyfunction!(lyagushka, m)?)?; m.add_function(wrap_pyfunction!(lyagushka, m)?)?;

24
test.py
View file

@ -24,8 +24,12 @@ def generate_random_data(size=1024, max_value=100):
return random_data return random_data
def filter_by_z_score(data, z_score_threshold):
filtered_data = [item for item in data if item['z_score'] is not None and abs(item['z_score']) >= z_score_threshold]
return filtered_data
# load the random test data # load the random test data
dataset = [] # dataset = []
# with open('random_values.txt', 'r') as file: # with open('random_values.txt', 'r') as file:
# for line in file: # for line in file:
# random_data.append(int(line.strip())) # random_data.append(int(line.strip()))
@ -33,8 +37,15 @@ dataset = []
dataset = generate_random_data(1024, 1024) dataset = generate_random_data(1024, 1024)
dataset.sort() dataset.sort()
with open('dataset.json', 'w') as r:
r.write(json.dumps(dataset, indent=4))
# calculate the anomalies in the data # calculate the anomalies in the data
analysis_results = json.loads(lyagushka(dataset, 3.0, 7)) analysis_results = json.loads(lyagushka(dataset, 4.0, 7))
analysis_results = filter_by_z_score(analysis_results, 1.0)
with open('result.json', 'w') as r:
r.write(json.dumps(analysis_results, indent=4))
# Initialize plot # Initialize plot
plt.figure(figsize=(10, 6)) plt.figure(figsize=(10, 6))
@ -45,15 +56,12 @@ colors = plt.cm.jet(np.linspace(0, 1, len(analysis_results)))
# Plot dataset points and assign colors based on cluster membership # Plot dataset points and assign colors based on cluster membership
for i, result in enumerate(analysis_results): for i, result in enumerate(analysis_results):
if result['num_elements'] > 0: # It's a cluster if result['num_elements'] > 0: # It's a cluster
points_in_cluster = [point for point in dataset if for point in dataset:
result['centroid'] - result['span_length'] / 2 <= point <=
result['centroid'] + result['span_length'] / 2]
for point in points_in_cluster:
plt.plot(point, 0, 'o', color=colors[i]) # Plot points in cluster with the same color plt.plot(point, 0, 'o', color=colors[i]) # Plot points in cluster with the same color
# Plot a line segment for the cluster/gap Z-score in the same color # Plot a line segment for the cluster/gap Z-score in the same color
start = result['centroid'] - result['span_length'] / 2 start = result['start']
end = result['centroid'] + result['span_length'] / 2 end = result['end']
z_score = result['z_score'] if result['z_score'] is not None else 0 z_score = result['z_score'] if result['z_score'] is not None else 0
plt.plot([start, end], [z_score, z_score], color=colors[i], linewidth=2) plt.plot([start, end], [z_score, z_score], color=colors[i], linewidth=2)