diff --git a/Cargo.lock b/Cargo.lock index c5dc19e..8d8efb9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13,6 +13,30 @@ dependencies = [ "winapi", ] +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + [[package]] name = "hermit-abi" version = "0.1.19" @@ -34,15 +58,62 @@ version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" +[[package]] +name = "lyagushka" +version = "1.1.0" +dependencies = [ + "autocfg", + "scopeguard", +] + [[package]] name = "lyagushka" version = "1.1.0" dependencies = [ "atty", + "pyo3", "serde", "serde_json", ] +[[package]] +name = "memoffset" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "parking_lot" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets", +] + [[package]] name = "proc-macro2" version = "1.0.78" @@ -61,6 +132,15 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "redox_syscall" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +dependencies = [ + "bitflags", +] + [[package]] name = "ryu" version = "1.0.16" @@ -109,12 +189,24 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "target-lexicon" +version = "0.12.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69758bda2e78f098e4ccb393021a0963bb3442eac05f135c30f61b7370bbafae" + [[package]] name = "unicode-ident" version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" +[[package]] +name = "unindent" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" + [[package]] name = "winapi" version = "0.3.9" diff --git a/Cargo.toml b/Cargo.toml index e19a6c3..d6aedab 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,8 +2,10 @@ name = "lyagushka" version = "1.1.0" edition = "2021" +crate-type = ["cdylib"] [dependencies] atty = "0.2.14" +pyo3 = { version = "0.20.2", features = ["extension-module"] } serde = { version = "1.0.196", features = ["derive"] } -serde_json = "1.0.113" +serde_json = "1.0.113" \ No newline at end of file diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..dd09518 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,160 @@ +use pyo3::prelude::*; +use serde::Serialize; + +#[derive(Debug, Clone, Serialize)] +struct Anomaly { + elements: Vec, + start: i32, + end: i32, + span_length: i32, + num_elements: usize, + centroid: f32, + z_score: Option, +} + +impl Anomaly { + + pub fn new(cluster: &[i32]) -> Self { + let num_elements: usize = cluster.len(); + let start: i32 = *cluster.first().expect("Cluster has no start"); + let end: i32 = *cluster.last().expect("Cluster has no end"); + let span_length: i32 = end - start; + let centroid: f32 = start as f32 + span_length as f32 / 2.0; + + Anomaly { + elements: cluster.to_vec(), + start, + end, + span_length, + num_elements, + centroid, + z_score: None, + } + } +} + +#[pyclass] +pub struct Lyagushka { + dataset: Vec, + anomalies: Vec, +} + +#[pymethods] +impl Lyagushka { + + #[new] + pub fn new(dataset: Vec) -> Self { + Lyagushka { + dataset, + anomalies: vec![] + } + } + + fn scan_anomalies(&mut self, factor: f32, min_cluster_size: usize) { + + // Calculate the mean distance between consecutive points in the dataset. + let mean_distance: f32 = self.dataset.windows(2) + .map(|w| (w[1] - w[0]) as f32) + .sum::() / (self.dataset.len() - 1) as f32; + + // Define thresholds for clustering and gap identification based on the mean distance and factor. + let cluster_threshold: f32 = mean_distance / factor; + let gap_threshold: f32 = factor * mean_distance; + + let mut current_cluster: Vec = Vec::new(); // Temporary storage for points in the current cluster. + + // Iterate through pairs of consecutive points to find clusters and significant gaps. + for window in self.dataset.windows(2) { + let gap_size: f32 = (window[1] - window[0]) as f32; + + if gap_size <= cluster_threshold { + // Add points to the current cluster + if current_cluster.is_empty() { + current_cluster.push(window[0]); // Start a new cluster with the first point + } + current_cluster.push(window[1]); // Add the second point to the cluster + } else { + // End the current cluster and start a new gap + if !current_cluster.is_empty() && current_cluster.len() >= min_cluster_size { + self.anomalies.push(Anomaly::new(¤t_cluster)); + current_cluster.clear(); + } + + // Record the gap + if gap_size > gap_threshold { + self.anomalies.push(Anomaly { + elements: Vec::new(), // No elements in a gap + start: window[0], + end: window[1], + span_length: gap_size as i32, + num_elements: 0, + centroid: (window[0] as f32 + window[1] as f32) / 2.0, + z_score: None, + }); + } + } + } + + // Finalize the last cluster if applicable + if !current_cluster.is_empty() && current_cluster.len() >= min_cluster_size { + self.anomalies.push(Anomaly::new(¤t_cluster)); + } + + } + + pub fn search(&mut self, factor: f32, min_cluster_size: usize) -> String { + + // Sort the vector + self.dataset.sort_unstable(); + + // Calculate clusters and gaps from the dataset using predefined criteria. + self.scan_anomalies(factor, min_cluster_size); + + // Calculate the mean density of clusters in the dataset for comparison. + let mean_density: f32 = self.anomalies.iter() + .filter(|info: &&Anomaly| info.num_elements > 0) + .map(|info: &Anomaly| info.num_elements as f32 / info.span_length as f32) + .sum::() / self.anomalies.iter().filter(|info: &&Anomaly| info.num_elements > 0).count() as f32; + + // Calculate the standard deviation of cluster densities to evaluate variation. + let variance_density: f32 = self.anomalies.iter() + .filter(|info: &&Anomaly| info.num_elements > 0) + .map(|info: &Anomaly| info.num_elements as f32 / info.span_length as f32) + .map(|density: f32| (density - mean_density).powi(2)) + .sum::() / self.anomalies.iter().filter(|info: &&Anomaly| info.num_elements > 0).count() as f32; + let std_dev_density: f32 = variance_density.sqrt(); + + // Calculate mean span length + let mean_span_length: f32 = self.anomalies.iter() + .map(|info: &Anomaly| info.span_length as f32) + .sum::() / self.anomalies.len() as f32; + + // Calculate variance + let variance: f32 = self.anomalies.iter() + .map(|info: &Anomaly| (info.span_length as f32 - mean_span_length).powi(2)) + .sum::() / self.anomalies.len() as f32; + + // Standard deviation is the square root of variance + let std_dev_span_length: f32 = variance.sqrt(); + + // Update Z-scores for both clusters and gaps based on their deviation from mean metrics. + for info in self.anomalies.iter_mut() { + if info.num_elements > 0 { + // Calculate and update Z-score for clusters based on density deviation. + let cluster_density: f32 = info.num_elements as f32 / info.span_length as f32; + info.z_score = Some((cluster_density - mean_density) / std_dev_density); + } else { + // Calculate and update Z-score for gaps based on span length deviation. + info.z_score = Some((info.span_length as f32 / std_dev_span_length) * -1.0); + } + } + + serde_json::to_string_pretty(&self.anomalies).unwrap_or_else(|_| "Failed to serialize data".to_string()) + } +} + +#[pymodule] +fn lyagushka(_py: Python, m: &PyModule) -> PyResult<()> { + m.add_class::()?; + Ok(()) +} diff --git a/src/main.rs b/src/main.rs index d0179b1..0dfe6d2 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,157 +2,7 @@ use std::fs::File; use std::io::{self, BufRead, BufReader, stdin}; use std::env; use std::process; -use serde::Serialize; -use serde_json; - -#[derive(Debug, Clone, Serialize)] -struct Anomaly { - elements: Vec, - start: i32, - end: i32, - span_length: i32, - num_elements: usize, - centroid: f32, - z_score: Option, -} - -impl Anomaly { - - pub fn new(cluster: &[i32]) -> Self { - let num_elements: usize = cluster.len(); - let start: i32 = *cluster.first().expect("Cluster has no start"); - let end: i32 = *cluster.last().expect("Cluster has no end"); - let span_length: i32 = end - start; - let centroid: f32 = start as f32 + span_length as f32 / 2.0; - - Anomaly { - elements: cluster.to_vec(), - start, - end, - span_length, - num_elements, - centroid, - z_score: None, - } - } -} - -struct Lyagushka { - dataset: Vec, - anomalies: Vec, -} - -impl Lyagushka { - - pub fn new(dataset: Vec) -> Self { - Lyagushka { - dataset, - anomalies: vec![] - } - } - - fn scan_anomalies(&mut self, factor: f32, min_cluster_size: usize) { - - // Calculate the mean distance between consecutive points in the dataset. - let mean_distance: f32 = self.dataset.windows(2) - .map(|w| (w[1] - w[0]) as f32) - .sum::() / (self.dataset.len() - 1) as f32; - - // Define thresholds for clustering and gap identification based on the mean distance and factor. - let cluster_threshold: f32 = mean_distance / factor; - let gap_threshold: f32 = factor * mean_distance; - - let mut current_cluster: Vec = Vec::new(); // Temporary storage for points in the current cluster. - - // Iterate through pairs of consecutive points to find clusters and significant gaps. - for window in self.dataset.windows(2) { - let gap_size: f32 = (window[1] - window[0]) as f32; - - if gap_size <= cluster_threshold { - // Add points to the current cluster - if current_cluster.is_empty() { - current_cluster.push(window[0]); // Start a new cluster with the first point - } - current_cluster.push(window[1]); // Add the second point to the cluster - } else { - // End the current cluster and start a new gap - if !current_cluster.is_empty() && current_cluster.len() >= min_cluster_size { - self.anomalies.push(Anomaly::new(¤t_cluster)); - current_cluster.clear(); - } - - // Record the gap - if gap_size > gap_threshold { - self.anomalies.push(Anomaly { - elements: Vec::new(), // No elements in a gap - start: window[0], - end: window[1], - span_length: gap_size as i32, - num_elements: 0, - centroid: (window[0] as f32 + window[1] as f32) / 2.0, - z_score: None, - }); - } - } - } - - // Finalize the last cluster if applicable - if !current_cluster.is_empty() && current_cluster.len() >= min_cluster_size { - self.anomalies.push(Anomaly::new(¤t_cluster)); - } - - } - - pub fn search(&mut self, factor: f32, min_cluster_size: usize) -> String { - - // Sort the vector - self.dataset.sort_unstable(); - - // Calculate clusters and gaps from the dataset using predefined criteria. - self.scan_anomalies(factor, min_cluster_size); - - // Calculate the mean density of clusters in the dataset for comparison. - let mean_density: f32 = self.anomalies.iter() - .filter(|info: &&Anomaly| info.num_elements > 0) - .map(|info: &Anomaly| info.num_elements as f32 / info.span_length as f32) - .sum::() / self.anomalies.iter().filter(|info: &&Anomaly| info.num_elements > 0).count() as f32; - - // Calculate the standard deviation of cluster densities to evaluate variation. - let variance_density: f32 = self.anomalies.iter() - .filter(|info: &&Anomaly| info.num_elements > 0) - .map(|info: &Anomaly| info.num_elements as f32 / info.span_length as f32) - .map(|density: f32| (density - mean_density).powi(2)) - .sum::() / self.anomalies.iter().filter(|info: &&Anomaly| info.num_elements > 0).count() as f32; - let std_dev_density: f32 = variance_density.sqrt(); - - // Calculate mean span length - let mean_span_length: f32 = self.anomalies.iter() - .map(|info: &Anomaly| info.span_length as f32) - .sum::() / self.anomalies.len() as f32; - - // Calculate variance - let variance: f32 = self.anomalies.iter() - .map(|info: &Anomaly| (info.span_length as f32 - mean_span_length).powi(2)) - .sum::() / self.anomalies.len() as f32; - - // Standard deviation is the square root of variance - let std_dev_span_length: f32 = variance.sqrt(); - - // Update Z-scores for both clusters and gaps based on their deviation from mean metrics. - for info in self.anomalies.iter_mut() { - if info.num_elements > 0 { - // Calculate and update Z-score for clusters based on density deviation. - let cluster_density: f32 = info.num_elements as f32 / info.span_length as f32; - info.z_score = Some((cluster_density - mean_density) / std_dev_density); - } else { - // Calculate and update Z-score for gaps based on span length deviation. - info.z_score = Some((info.span_length as f32 / std_dev_span_length) * -1.0); - } - } - - serde_json::to_string_pretty(&self.anomalies).unwrap_or_else(|_| "Failed to serialize data".to_string()) - } -} +use lyagushka::Lyagushka; /// The entry point for the command-line tool that reads a dataset of integers from either a file or stdin, /// performs cluster and gap analysis using specified parameters, and prints the results as a JSON string. diff --git a/test.py b/test.py new file mode 100644 index 0000000..111ded7 --- /dev/null +++ b/test.py @@ -0,0 +1,75 @@ +from pyagushka import Lyagushka +from randonautentropy import rndo +import json +import matplotlib.pyplot as plt +import numpy as np +from scipy.interpolate import interp1d + +def generate_random_data(size=1024, max_value=100): + + random_data = [] + max_value_bytes = (max_value.bit_length() + 7) // 8 + max_int_for_bytes = 2**(max_value_bytes * 8) - 1 + min_bytes_needed = max_value_bytes * size + mod_cutoff = max_int_for_bytes - (max_int_for_bytes % max_value) - 1 + + # Populate the 'random_data' array + while len(random_data) < size: + hex_data = rndo.get(length=min_bytes_needed) + hex_chunks = list((hex_data[0+i:2 * max_value_bytes+i] for i in range(0, len(hex_data), 2 * max_value_bytes))) + for i in hex_chunks: + num = int(i, 16) + if num <= mod_cutoff and len(random_data) < size: + random_data.append( num % (max_value + 1) ) + + return random_data + +def filter_by_z_score(data, z_score_threshold): + filtered_data = [item for item in data if item['z_score'] is not None and abs(item['z_score']) >= z_score_threshold] + return filtered_data + +# load the random test data +# dataset = [] +# with open('random_values.txt', 'r') as file: +# for line in file: +# random_data.append(int(line.strip())) + +dataset = generate_random_data(1024, 1024) +dataset.sort() + +with open('dataset.json', 'w') as r: + r.write(json.dumps(dataset, indent=4)) + +# calculate the anomalies in the data +zhaba = Lyagushka(dataset) +analysis_results = json.loads(zhaba.search(4.0, 7)) +analysis_results = filter_by_z_score(analysis_results, 1.0) + +with open('result.json', 'w') as r: + r.write(json.dumps(analysis_results, indent=4)) + +# Initialize plot +plt.figure(figsize=(10, 6)) + +# Color palette for clusters and gaps +colors = plt.cm.jet(np.linspace(0, 1, len(analysis_results))) + +# Plot dataset points and assign colors based on cluster membership +for i, result in enumerate(analysis_results): + if result['num_elements'] > 0: # It's a cluster + for point in dataset: + plt.plot(point, 0, 'o', color=colors[i]) # Plot points in cluster with the same color + + # Plot a line segment for the cluster/gap Z-score in the same color + start = result['start'] + end = result['end'] + z_score = result['z_score'] if result['z_score'] is not None else 0 + plt.plot([start, end], [z_score, z_score], color=colors[i], linewidth=2) + +# Enhancements for visualization +plt.xlabel('Integer Value') +plt.ylabel('Z-Score') +plt.title('Cluster and Gap Analysis') +plt.grid(True) + +plt.show() \ No newline at end of file