diff --git a/.gitignore b/.gitignore index ea8c4bf..cb7f85d 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ /target +*.json \ No newline at end of file diff --git a/src/lib.rs b/src/lib.rs index 367344d..23e086d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,44 +4,41 @@ use pyo3::wrap_pyfunction; use serde::Serialize; use serde_json::to_string_pretty; -#[derive(Clone, Debug, Serialize)] -struct Point { - value: u32, -} - -impl Point { - fn new(value: u32) -> Self { - Point { value } - } -} - #[derive(Debug, Clone, Serialize)] -struct ClusterGapInfo { - span_length: f32, +struct Anomaly { + elements: Vec, + start: i32, + end: i32, + span_length: i32, num_elements: usize, centroid: f32, z_score: Option, } -fn create_cluster_info(cluster: &[Point]) -> ClusterGapInfo { - - let num_elements = cluster.len(); - let span_length = (cluster.last().unwrap().value as f32) - (cluster.first().unwrap().value as f32); - let centroid = cluster.iter().map(|p| p.value as f32).sum::() / num_elements as f32; +fn anomaly_info(cluster: &[i32]) -> Anomaly { + let num_elements: usize = cluster.len(); + let start: i32 = *cluster.first().expect("Cluster has no start"); + let end: i32 = *cluster.last().expect("Cluster has no end"); + let span_length: i32 = end - start; + let centroid: f32 = start as f32 + span_length as f32 / 2.0; - ClusterGapInfo { + Anomaly { + elements: cluster.to_vec(), + start, + end, span_length, num_elements, centroid, - z_score: None, + z_score: None, // Placeholder for actual Z-score calculation } } + /// Calculates the densities (clusters) and significant gaps between points in a dataset. /// /// This function iterates over a dataset of points, identifying clusters based on a distance threshold /// (calculated from the mean distance between points and adjusted by a given factor) and identifying significant gaps -/// that exceed a certain threshold. Each cluster or significant gap identified is summarized in a `ClusterGapInfo` object. +/// that exceed a certain threshold. Each cluster or significant gap identified is summarized in a `Anomaly` object. /// /// # Arguments /// * `dataset`: A slice of `Point` objects representing the dataset to be analyzed. @@ -50,57 +47,60 @@ fn create_cluster_info(cluster: &[Point]) -> ClusterGapInfo { /// * `min_cluster_size`: The minimum number of points required for a group of points to be considered a cluster. /// /// # Returns -/// A vector of `ClusterGapInfo` objects, each representing either a cluster of points or a significant gap between points. +/// A vector of `Anomaly` objects, each representing either a cluster of points or a significant gap between points. /// -fn calculate_densities_and_gaps(dataset: &[Point], factor: f32, min_cluster_size: usize) -> Vec { +fn scan_anomalies(dataset: &[i32], factor: f32, min_cluster_size: usize) -> Vec { // Return early if the dataset is too small to form any clusters or gaps. if dataset.len() < 2 { return Vec::new(); } // Calculate the mean distance between consecutive points in the dataset. - let mean_distance = dataset.windows(2) - .map(|w| w[1].value as f32 - w[0].value as f32) - .sum::() / (dataset.len() - 1) as f32; + let mean_distance: f32 = dataset.windows(2) + .map(|w| (w[1] - w[0]) as f32) + .sum::() / (dataset.len() - 1) as f32; // Define thresholds for clustering and gap identification based on the mean distance and factor. - let cluster_threshold = mean_distance / factor; - let gap_threshold = factor * mean_distance; + let cluster_threshold: f32 = mean_distance / factor; + let gap_threshold: f32 = factor * mean_distance; - let mut results: Vec = Vec::new(); // Stores the resulting clusters and gaps. - let mut current_cluster: Vec = Vec::new(); // Temporary storage for points in the current cluster. + let mut results: Vec = Vec::new(); // Stores the resulting clusters and gaps. + let mut current_cluster: Vec = Vec::new(); // Temporary storage for points in the current cluster. // Iterate through pairs of consecutive points to find clusters and significant gaps. for window in dataset.windows(2) { - let gap_distance = window[1].value as f32 - window[0].value as f32; + let gap_size: f32 = (window[1] - window[0]) as f32; - // If the distance between points is within the cluster threshold, add to current cluster. - if gap_distance <= cluster_threshold { + if gap_size <= cluster_threshold { + // Add points to the current cluster if current_cluster.is_empty() { - current_cluster.push(window[0].clone()); // Start a new cluster with the first point. + current_cluster.push(window[0]); // Start a new cluster with the first point } - current_cluster.push(window[1].clone()); // Add the second point to the cluster. + current_cluster.push(window[1]); // Add the second point to the cluster } else { - // If the current cluster is large enough, finalize it and prepare for a new cluster. + // End the current cluster and start a new gap if !current_cluster.is_empty() && current_cluster.len() >= min_cluster_size { - results.push(create_cluster_info(¤t_cluster)); + results.push(anomaly_info(¤t_cluster)); current_cluster.clear(); } - // If the gap between points is significant, record it as a gap. - if gap_distance > gap_threshold { - results.push(ClusterGapInfo { - span_length: gap_distance, - num_elements: 0, // Indicating this is a gap, not a cluster. - centroid: (window[0].value as f32 + window[1].value as f32) / 2.0, - z_score: None, // Z-score will be calculated later if necessary. + // Record the gap + if gap_size > gap_threshold { + results.push(Anomaly { + elements: Vec::new(), // No elements in a gap + start: window[0], + end: window[1], + span_length: gap_size as i32, + num_elements: 0, + centroid: (window[0] as f32 + window[1] as f32) / 2.0, + z_score: None, }); } } } - // Finalize the last cluster if it meets the size requirement. + // Finalize the last cluster if applicable if !current_cluster.is_empty() && current_cluster.len() >= min_cluster_size { - results.push(create_cluster_info(¤t_cluster)); + results.push(anomaly_info(¤t_cluster)); } results @@ -134,54 +134,60 @@ fn calculate_densities_and_gaps(dataset: &[Point], factor: f32, min_cluster_size /// #[pyfunction] fn lyagushka(_py: Python, int_list: &PyList, factor: f32, min_cluster_size: usize) -> PyResult { - - // Extract integers from a Python list and create a vector of Point structs. - let mut dataset: Vec = int_list.extract::>()? - .into_iter() - .map(Point::new) - .collect(); - + // Extract integers from a Python list and create a vector. + let mut dataset: Vec = int_list.extract::>()?; // Sort the vector - dataset.sort_by_key(|p| p.value); + dataset.sort_unstable(); // Calculate clusters and gaps from the dataset using predefined criteria. - let mut cluster_gap_infos = calculate_densities_and_gaps(&dataset, factor, min_cluster_size); + let mut anomalies: Vec = scan_anomalies(&dataset, factor, min_cluster_size); // Calculate the mean density of clusters in the dataset for comparison. - let mean_density: f32 = cluster_gap_infos.iter() - .filter(|info| info.num_elements > 0) - .map(|info| info.num_elements as f32 / info.span_length) - .sum::() / cluster_gap_infos.iter().filter(|info| info.num_elements > 0).count() as f32; + let mean_density: f32 = anomalies.iter() + .filter(|info: &&Anomaly| info.num_elements > 0) + .map(|info: &Anomaly| info.num_elements as f32 / info.span_length as f32) + .sum::() / anomalies.iter().filter(|info: &&Anomaly| info.num_elements > 0).count() as f32; // Calculate the standard deviation of cluster densities to evaluate variation. - let variance_density: f32 = cluster_gap_infos.iter() - .filter(|info| info.num_elements > 0) - .map(|info| info.num_elements as f32 / info.span_length) - .map(|density| (density - mean_density).powi(2)) - .sum::() / cluster_gap_infos.iter().filter(|info| info.num_elements > 0).count() as f32; + let variance_density: f32 = anomalies.iter() + .filter(|info: &&Anomaly| info.num_elements > 0) + .map(|info: &Anomaly| info.num_elements as f32 / info.span_length as f32) + .map(|density| (density - mean_density).powi(2)) + .sum::() / anomalies.iter().filter(|info: &&Anomaly| info.num_elements > 0).count() as f32; let std_dev_density = variance_density.sqrt(); - // Calculate the average span of all clusters and gaps to assess gap significance. - let average_span: f32 = cluster_gap_infos.iter().map(|info| info.span_length).sum::() / cluster_gap_infos.len() as f32; + // Calculate mean span length + let mean_span_length: f32 = anomalies.iter() + .map(|info: &Anomaly| info.span_length as f32) + .sum::() / anomalies.len() as f32; + + // Calculate variance + let variance: f32 = anomalies.iter() + .map(|info: &Anomaly| (info.span_length as f32 - mean_span_length).powi(2)) + .sum::() / anomalies.len() as f32; + + // Standard deviation is the square root of variance + let std_dev_span_length: f32 = variance.sqrt(); // Update Z-scores for both clusters and gaps based on their deviation from mean metrics. - for info in &mut cluster_gap_infos { + for info in anomalies.iter_mut() { if info.num_elements > 0 { // Calculate and update Z-score for clusters based on density deviation. - let cluster_density = info.num_elements as f32 / info.span_length; + let cluster_density: f32 = info.num_elements as f32 / info.span_length as f32; info.z_score = Some((cluster_density - mean_density) / std_dev_density); } else { // Calculate and update Z-score for gaps based on span length deviation. - info.z_score = Some((info.span_length - average_span) / std_dev_density); + info.z_score = Some((info.span_length as f32 / std_dev_span_length) * -1.0); } } // Serialize the updated cluster and gap information, including Z-scores, to a JSON string. - to_string_pretty(&cluster_gap_infos) + to_string_pretty(&anomalies) .map_err(|e| PyErr::new::(format!("JSON Serialization Error: {}", e))) } + #[pymodule] fn pyagushka(_py: Python, m: &PyModule) -> PyResult<()> { m.add_function(wrap_pyfunction!(lyagushka, m)?)?; diff --git a/test.py b/test.py index 0a84dc7..129515a 100644 --- a/test.py +++ b/test.py @@ -24,8 +24,12 @@ def generate_random_data(size=1024, max_value=100): return random_data +def filter_by_z_score(data, z_score_threshold): + filtered_data = [item for item in data if item['z_score'] is not None and abs(item['z_score']) >= z_score_threshold] + return filtered_data + # load the random test data -dataset = [] +# dataset = [] # with open('random_values.txt', 'r') as file: # for line in file: # random_data.append(int(line.strip())) @@ -33,8 +37,15 @@ dataset = [] dataset = generate_random_data(1024, 1024) dataset.sort() +with open('dataset.json', 'w') as r: + r.write(json.dumps(dataset, indent=4)) + # calculate the anomalies in the data -analysis_results = json.loads(lyagushka(dataset, 3.0, 7)) +analysis_results = json.loads(lyagushka(dataset, 4.0, 7)) +analysis_results = filter_by_z_score(analysis_results, 1.0) + +with open('result.json', 'w') as r: + r.write(json.dumps(analysis_results, indent=4)) # Initialize plot plt.figure(figsize=(10, 6)) @@ -45,15 +56,12 @@ colors = plt.cm.jet(np.linspace(0, 1, len(analysis_results))) # Plot dataset points and assign colors based on cluster membership for i, result in enumerate(analysis_results): if result['num_elements'] > 0: # It's a cluster - points_in_cluster = [point for point in dataset if - result['centroid'] - result['span_length'] / 2 <= point <= - result['centroid'] + result['span_length'] / 2] - for point in points_in_cluster: + for point in dataset: plt.plot(point, 0, 'o', color=colors[i]) # Plot points in cluster with the same color # Plot a line segment for the cluster/gap Z-score in the same color - start = result['centroid'] - result['span_length'] / 2 - end = result['centroid'] + result['span_length'] / 2 + start = result['start'] + end = result['end'] z_score = result['z_score'] if result['z_score'] is not None else 0 plt.plot([start, end], [z_score, z_score], color=colors[i], linewidth=2)