randogoth/lyagushka
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

class

Browse source
This commit is contained in:
randogoth 2024-04-09 07:47:42 +03:00
parent 23a2da6233
commit 92c862bd4c
5 changed files with 136 additions and 213 deletions
Download patch file Download diff file Expand all files Collapse all files

45
Cargo.lock generated
View file

@ -2,17 +2,6 @@
# It is not intended for manual editing. # It is not intended for manual editing.
version = 3 version = 3
[[package]]
name = "atty"
version = "0.2.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
dependencies = [
"hermit-abi",
"libc",
"winapi",
]
[[package]] [[package]]
name = "autocfg" name = "autocfg"
version = "1.1.0" version = "1.1.0"
@ -37,15 +26,6 @@ version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
[[package]]
name = "hermit-abi"
version = "0.1.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
dependencies = [
"libc",
]
[[package]] [[package]]
name = "indoc" name = "indoc"
version = "2.0.4" version = "2.0.4"
@ -123,9 +103,8 @@ dependencies = [
[[package]] [[package]]
name = "pyagushka" name = "pyagushka"
version = "1.0.0" version = "1.1.0"
dependencies = [ dependencies = [
"atty",
"pyo3", "pyo3",
"serde", "serde",
"serde_json", "serde_json",
@ -288,28 +267,6 @@ version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce"
[[package]]
name = "winapi"
version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
dependencies = [
"winapi-i686-pc-windows-gnu",
"winapi-x86_64-pc-windows-gnu",
]
[[package]]
name = "winapi-i686-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
[[package]]
name = "winapi-x86_64-pc-windows-gnu"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]] [[package]]
name = "windows-targets" name = "windows-targets"
version = "0.48.5" version = "0.48.5"

3
Cargo.toml
View file

@ -1,6 +1,6 @@
[package] [package]
name = "pyagushka" name = "pyagushka"
version = "1.0.0" version = "1.1.0"
edition = "2021" edition = "2021"
[lib] [lib]
@ -8,7 +8,6 @@ name = "pyagushka"
crate-type = ["cdylib"] crate-type = ["cdylib"]
[dependencies] [dependencies]
atty = "0.2.14"
pyo3 = "0.20.2" pyo3 = "0.20.2"
serde = { version = "1.0.196", features = ["derive"] } serde = { version = "1.0.196", features = ["derive"] }
serde_json = "1.0.113" serde_json = "1.0.113"

7
readme.md
View file

@ -13,7 +13,7 @@ With a Rust/Cargo and Python3/Pip environment set up, run:
```sh ```sh
$ pip install maturin $ pip install maturin
$ maturin build --release $ maturin build --release
$ pip install target/wheels/pyagushka-1.0.0-*.whl $ pip install target/wheels/pyagushka-1.1.0-*.whl
``` ```
## Usage ## Usage
@ -58,14 +58,15 @@ The tool outputs a JSON string that includes details about the identified attrac
To analyze a dataset from a file, provide the filename as an argument, followed by the factor and minimum cluster size parameters To analyze a dataset from a file, provide the filename as an argument, followed by the factor and minimum cluster size parameters
```Python ```Python
from pyagushka import lyagushka from pyagushka import Lyagushka
dataset = [] dataset = []
with open('random_values.txt', 'r') as file: with open('random_values.txt', 'r') as file:
for line in file: for line in file:
random_data.append(int(line.strip())) random_data.append(int(line.strip()))
analysis_results = json.loads(lyagushka(dataset, 4.0, 7)) zhaba = Lyagushka(dataset)
analysis_results = json.loads(lyagushka.search(4.0, 7))
print(analysis_result) print(analysis_result)
``` ```

289
src/lib.rs
View file

@ -1,8 +1,5 @@
use pyo3::prelude::*; use pyo3::prelude::*;
use pyo3::types::PyList;
use pyo3::wrap_pyfunction;
use serde::Serialize; use serde::Serialize;
use serde_json::to_string_pretty;
#[derive(Debug, Clone, Serialize)] #[derive(Debug, Clone, Serialize)]
struct Anomaly { struct Anomaly {
@ -15,181 +12,149 @@ struct Anomaly {
z_score: Option<f32>, z_score: Option<f32>,
} }
fn anomaly_info(cluster: &[i32]) -> Anomaly { impl Anomaly {
let num_elements: usize = cluster.len();
let start: i32 = *cluster.first().expect("Cluster has no start");
let end: i32 = *cluster.last().expect("Cluster has no end");
let span_length: i32 = end - start;
let centroid: f32 = start as f32 + span_length as f32 / 2.0;
Anomaly { pub fn new(cluster: &[i32]) -> Self {
elements: cluster.to_vec(), let num_elements: usize = cluster.len();
start, let start: i32 = *cluster.first().expect("Cluster has no start");
end, let end: i32 = *cluster.last().expect("Cluster has no end");
span_length, let span_length: i32 = end - start;
num_elements, let centroid: f32 = start as f32 + span_length as f32 / 2.0;
centroid,
z_score: None, // Placeholder for actual Z-score calculation Anomaly {
elements: cluster.to_vec(),
start,
end,
span_length,
num_elements,
centroid,
z_score: None,
}
} }
} }
#[pyclass]
struct Lyagushka {
dataset: Vec<i32>,
anomalies: Vec<Anomaly>,
}
/// Calculates the densities (clusters) and significant gaps between points in a dataset. #[pymethods]
/// impl Lyagushka {
/// This function iterates over a dataset of points, identifying clusters based on a distance threshold
/// (calculated from the mean distance between points and adjusted by a given factor) and identifying significant gaps
/// that exceed a certain threshold. Each cluster or significant gap identified is summarized in a `Anomaly` object.
///
/// # Arguments
/// * `dataset`: A slice of `i32` objects representing the dataset to be analyzed.
/// * `factor`: A multiplier used to define the thresholds for clustering and gap identification.
/// A lower factor tightens the cluster threshold and widens the gap threshold, and vice versa.
/// * `min_cluster_size`: The minimum number of points required for a group of points to be considered a cluster.
///
/// # Returns
/// A vector of `Anomaly` objects, each representing either a cluster of points or a significant gap between points.
///
fn scan_anomalies(dataset: &[i32], factor: f32, min_cluster_size: usize) -> Vec<Anomaly> {
// Return early if the dataset is too small to form any clusters or gaps. #[new]
if dataset.len() < 2 { return Vec::new(); } pub fn new(dataset: Vec<i32>) -> Self {
Lyagushka {
// Calculate the mean distance between consecutive points in the dataset. dataset,
let mean_distance: f32 = dataset.windows(2) anomalies: vec![]
.map(|w| (w[1] - w[0]) as f32)
.sum::<f32>() / (dataset.len() - 1) as f32;
// Define thresholds for clustering and gap identification based on the mean distance and factor.
let cluster_threshold: f32 = mean_distance / factor;
let gap_threshold: f32 = factor * mean_distance;
let mut results: Vec<Anomaly> = Vec::new(); // Stores the resulting clusters and gaps.
let mut current_cluster: Vec<i32> = Vec::new(); // Temporary storage for points in the current cluster.
// Iterate through pairs of consecutive points to find clusters and significant gaps.
for window in dataset.windows(2) {
let gap_size: f32 = (window[1] - window[0]) as f32;
if gap_size <= cluster_threshold {
// Add points to the current cluster
if current_cluster.is_empty() {
current_cluster.push(window[0]); // Start a new cluster with the first point
}
current_cluster.push(window[1]); // Add the second point to the cluster
} else {
// End the current cluster and start a new gap
if !current_cluster.is_empty() && current_cluster.len() >= min_cluster_size {
results.push(anomaly_info(&current_cluster));
current_cluster.clear();
}
// Record the gap
if gap_size > gap_threshold {
results.push(Anomaly {
elements: Vec::new(), // No elements in a gap
start: window[0],
end: window[1],
span_length: gap_size as i32,
num_elements: 0,
centroid: (window[0] as f32 + window[1] as f32) / 2.0,
z_score: None,
});
}
} }
} }
// Finalize the last cluster if applicable fn scan_anomalies(&mut self, factor: f32, min_cluster_size: usize) {
if !current_cluster.is_empty() && current_cluster.len() >= min_cluster_size {
results.push(anomaly_info(&current_cluster));
}
results
}
/// Analyzes a dataset of integers to identify clusters and gaps, then calculates Z-scores
/// for each based on their deviation from mean metrics. The analysis aims to highlight
/// significant clusters of closely grouped points and notable gaps between them, providing
/// a statistical measure of their significance through Z-scores. The results, including
/// clusters, gaps, and their Z-scores, are serialized into a JSON string.
///
/// # Arguments
/// * `_py` - The Python interpreter instance, used for Python-Rust interoperability.
/// This argument is necessary for functions exposed to Python via PyO3 but is not
/// directly used within the function.
/// * `int_list` - A Python list of integers representing the dataset to be analyzed.
/// This list is converted into a Vec<Point> for internal processing.
/// * `factor` - A floating-point value used as a threshold factor to adjust the sensitivity
/// of cluster and gap detection. This factor influences the identification of clusters
/// by defining the minimum density or separation required.
/// * `min_cluster_size` - An integer specifying the minimum number of contiguous points
/// required for a group of points to be considered a cluster. This parameter helps
/// filter out noise by defining a threshold for the minimum cluster size.
///
/// # Returns
/// Returns a `PyResult<String>` containing a JSON-formatted string of the analysis results.
/// The JSON string includes detailed information about each identified cluster and gap,
/// such as their span length, number of elements (if applicable), centroid, and calculated
/// Z-score. In case of an error during processing or serialization, a Python exception is
/// returned.
///
#[pyfunction]
fn lyagushka(_py: Python, int_list: &PyList, factor: f32, min_cluster_size: usize) -> PyResult<String> {
// Extract integers from a Python list and create a vector.
let mut dataset: Vec<i32> = int_list.extract::<Vec<i32>>()?;
// Sort the vector // Calculate the mean distance between consecutive points in the dataset.
dataset.sort_unstable(); let mean_distance: f32 = self.dataset.windows(2)
.map(|w| (w[1] - w[0]) as f32)
// Calculate clusters and gaps from the dataset using predefined criteria. .sum::<f32>() / (self.dataset.len() - 1) as f32;
let mut anomalies: Vec<Anomaly> = scan_anomalies(&dataset, factor, min_cluster_size);
// Define thresholds for clustering and gap identification based on the mean distance and factor.
// Calculate the mean density of clusters in the dataset for comparison. let cluster_threshold: f32 = mean_distance / factor;
let mean_density: f32 = anomalies.iter() let gap_threshold: f32 = factor * mean_distance;
.filter(|info: &&Anomaly| info.num_elements > 0)
.map(|info: &Anomaly| info.num_elements as f32 / info.span_length as f32) let mut current_cluster: Vec<i32> = Vec::new(); // Temporary storage for points in the current cluster.
.sum::<f32>() / anomalies.iter().filter(|info: &&Anomaly| info.num_elements > 0).count() as f32;
// Iterate through pairs of consecutive points to find clusters and significant gaps.
// Calculate the standard deviation of cluster densities to evaluate variation. for window in self.dataset.windows(2) {
let variance_density: f32 = anomalies.iter() let gap_size: f32 = (window[1] - window[0]) as f32;
.filter(|info: &&Anomaly| info.num_elements > 0)
.map(|info: &Anomaly| info.num_elements as f32 / info.span_length as f32) if gap_size <= cluster_threshold {
.map(|density| (density - mean_density).powi(2)) // Add points to the current cluster
.sum::<f32>() / anomalies.iter().filter(|info: &&Anomaly| info.num_elements > 0).count() as f32; if current_cluster.is_empty() {
let std_dev_density = variance_density.sqrt(); current_cluster.push(window[0]); // Start a new cluster with the first point
}
// Calculate mean span length current_cluster.push(window[1]); // Add the second point to the cluster
let mean_span_length: f32 = anomalies.iter() } else {
.map(|info: &Anomaly| info.span_length as f32) // End the current cluster and start a new gap
.sum::<f32>() / anomalies.len() as f32; if !current_cluster.is_empty() && current_cluster.len() >= min_cluster_size {
self.anomalies.push(Anomaly::new(&current_cluster));
// Calculate variance current_cluster.clear();
let variance: f32 = anomalies.iter() }
.map(|info: &Anomaly| (info.span_length as f32 - mean_span_length).powi(2))
.sum::<f32>() / anomalies.len() as f32; // Record the gap
if gap_size > gap_threshold {
// Standard deviation is the square root of variance self.anomalies.push(Anomaly {
let std_dev_span_length: f32 = variance.sqrt(); elements: Vec::new(), // No elements in a gap
start: window[0],
// Update Z-scores for both clusters and gaps based on their deviation from mean metrics. end: window[1],
for info in anomalies.iter_mut() { span_length: gap_size as i32,
if info.num_elements > 0 { num_elements: 0,
// Calculate and update Z-score for clusters based on density deviation. centroid: (window[0] as f32 + window[1] as f32) / 2.0,
let cluster_density: f32 = info.num_elements as f32 / info.span_length as f32; z_score: None,
info.z_score = Some((cluster_density - mean_density) / std_dev_density); });
} else { }
// Calculate and update Z-score for gaps based on span length deviation. }
info.z_score = Some((info.span_length as f32 / std_dev_span_length) * -1.0);
} }
// Finalize the last cluster if applicable
if !current_cluster.is_empty() && current_cluster.len() >= min_cluster_size {
self.anomalies.push(Anomaly::new(&current_cluster));
}
} }
// Serialize the updated cluster and gap information, including Z-scores, to a JSON string. pub fn search(&mut self, factor: f32, min_cluster_size: usize) -> String {
to_string_pretty(&anomalies)
.map_err(|e| PyErr::new::<pyo3::exceptions::PyException, _>(format!("JSON Serialization Error: {}", e)))
}
// Sort the vector
self.dataset.sort_unstable();
// Calculate clusters and gaps from the dataset using predefined criteria.
self.scan_anomalies(factor, min_cluster_size);
// Calculate the mean density of clusters in the dataset for comparison.
let mean_density: f32 = self.anomalies.iter()
.filter(|info: &&Anomaly| info.num_elements > 0)
.map(|info: &Anomaly| info.num_elements as f32 / info.span_length as f32)
.sum::<f32>() / self.anomalies.iter().filter(|info: &&Anomaly| info.num_elements > 0).count() as f32;
// Calculate the standard deviation of cluster densities to evaluate variation.
let variance_density: f32 = self.anomalies.iter()
.filter(|info: &&Anomaly| info.num_elements > 0)
.map(|info: &Anomaly| info.num_elements as f32 / info.span_length as f32)
.map(|density: f32| (density - mean_density).powi(2))
.sum::<f32>() / self.anomalies.iter().filter(|info: &&Anomaly| info.num_elements > 0).count() as f32;
let std_dev_density: f32 = variance_density.sqrt();
// Calculate mean span length
let mean_span_length: f32 = self.anomalies.iter()
.map(|info: &Anomaly| info.span_length as f32)
.sum::<f32>() / self.anomalies.len() as f32;
// Calculate variance
let variance: f32 = self.anomalies.iter()
.map(|info: &Anomaly| (info.span_length as f32 - mean_span_length).powi(2))
.sum::<f32>() / self.anomalies.len() as f32;
// Standard deviation is the square root of variance
let std_dev_span_length: f32 = variance.sqrt();
// Update Z-scores for both clusters and gaps based on their deviation from mean metrics.
for info in self.anomalies.iter_mut() {
if info.num_elements > 0 {
// Calculate and update Z-score for clusters based on density deviation.
let cluster_density: f32 = info.num_elements as f32 / info.span_length as f32;
info.z_score = Some((cluster_density - mean_density) / std_dev_density);
} else {
// Calculate and update Z-score for gaps based on span length deviation.
info.z_score = Some((info.span_length as f32 / std_dev_span_length) * -1.0);
}
}
serde_json::to_string_pretty(&self.anomalies).unwrap_or_else(|_| "Failed to serialize data".to_string())
}
}
#[pymodule] #[pymodule]
fn pyagushka(_py: Python, m: &PyModule) -> PyResult<()> { fn pyagushka(_py: Python, m: &PyModule) -> PyResult<()> {
m.add_function(wrap_pyfunction!(lyagushka, m)?)?; m.add_class::<Lyagushka>()?;
Ok(()) Ok(())
} }

5
test.py
View file

@ -1,4 +1,4 @@
from pyagushka import lyagushka from pyagushka import Lyagushka
from randonautentropy import rndo from randonautentropy import rndo
import json import json
import matplotlib.pyplot as plt import matplotlib.pyplot as plt
@ -41,7 +41,8 @@ with open('dataset.json', 'w') as r:
r.write(json.dumps(dataset, indent=4)) r.write(json.dumps(dataset, indent=4))
# calculate the anomalies in the data # calculate the anomalies in the data
analysis_results = json.loads(lyagushka(dataset, 4.0, 7)) zhaba = Lyagushka(dataset)
analysis_results = json.loads(zhaba.search(4.0, 7))
analysis_results = filter_by_z_score(analysis_results, 1.0) analysis_results = filter_by_z_score(analysis_results, 1.0)
with open('result.json', 'w') as r: with open('result.json', 'w') as r: