randogoth/lyagushka
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'python'

Browse source
This commit is contained in:
randogoth 2024-04-09 08:39:31 +03:00
commit 276dd11a01
5 changed files with 331 additions and 152 deletions
Download patch file Download diff file Expand all files Collapse all files

92
Cargo.lock generated
View file

@ -13,6 +13,30 @@ dependencies = [
"winapi",
]
[[package]]
name = "autocfg"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
[[package]]
name = "bitflags"
version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "cfg-if"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
[[package]]
name = "heck"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
[[package]]
name = "hermit-abi"
version = "0.1.19"
@ -34,15 +58,62 @@ version = "0.2.153"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd"
[[package]]
name = "lyagushka"
version = "1.1.0"
dependencies = [
"autocfg",
"scopeguard",
]
[[package]]
name = "lyagushka"
version = "1.1.0"
dependencies = [
"atty",
"pyo3",
"serde",
"serde_json",
]
[[package]]
name = "memoffset"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c"
dependencies = [
"autocfg",
]
[[package]]
name = "once_cell"
version = "1.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
[[package]]
name = "parking_lot"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f"
dependencies = [
"lock_api",
"parking_lot_core",
]
[[package]]
name = "parking_lot_core"
version = "0.9.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e"
dependencies = [
"cfg-if",
"libc",
"redox_syscall",
"smallvec",
"windows-targets",
]
[[package]]
name = "proc-macro2"
version = "1.0.78"
@ -61,6 +132,15 @@ dependencies = [
"proc-macro2",
]
[[package]]
name = "redox_syscall"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa"
dependencies = [
"bitflags",
]
[[package]]
name = "ryu"
version = "1.0.16"
@ -109,12 +189,24 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "target-lexicon"
version = "0.12.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "69758bda2e78f098e4ccb393021a0963bb3442eac05f135c30f61b7370bbafae"
[[package]]
name = "unicode-ident"
version = "1.0.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
[[package]]
name = "unindent"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce"
[[package]]
name = "winapi"
version = "0.3.9"

4
Cargo.toml
View file

@ -2,8 +2,10 @@
name = "lyagushka"
version = "1.1.0"
edition = "2021"
crate-type = ["cdylib"]
[dependencies]
atty = "0.2.14"
pyo3 = { version = "0.20.2", features = ["extension-module"] }
serde = { version = "1.0.196", features = ["derive"] }
serde_json = "1.0.113"
serde_json = "1.0.113"

160
src/lib.rs Normal file
View file

@ -0,0 +1,160 @@
use pyo3::prelude::*;
use serde::Serialize;
#[derive(Debug, Clone, Serialize)]
struct Anomaly {
elements: Vec<i32>,
start: i32,
end: i32,
span_length: i32,
num_elements: usize,
centroid: f32,
z_score: Option<f32>,
}
impl Anomaly {
pub fn new(cluster: &[i32]) -> Self {
let num_elements: usize = cluster.len();
let start: i32 = *cluster.first().expect("Cluster has no start");
let end: i32 = *cluster.last().expect("Cluster has no end");
let span_length: i32 = end - start;
let centroid: f32 = start as f32 + span_length as f32 / 2.0;
Anomaly {
elements: cluster.to_vec(),
start,
end,
span_length,
num_elements,
centroid,
z_score: None,
}
}
}
#[pyclass]
pub struct Lyagushka {
dataset: Vec<i32>,
anomalies: Vec<Anomaly>,
}
#[pymethods]
impl Lyagushka {
#[new]
pub fn new(dataset: Vec<i32>) -> Self {
Lyagushka {
dataset,
anomalies: vec![]
}
}
fn scan_anomalies(&mut self, factor: f32, min_cluster_size: usize) {
// Calculate the mean distance between consecutive points in the dataset.
let mean_distance: f32 = self.dataset.windows(2)
.map(|w| (w[1] - w[0]) as f32)
.sum::<f32>() / (self.dataset.len() - 1) as f32;
// Define thresholds for clustering and gap identification based on the mean distance and factor.
let cluster_threshold: f32 = mean_distance / factor;
let gap_threshold: f32 = factor * mean_distance;
let mut current_cluster: Vec<i32> = Vec::new(); // Temporary storage for points in the current cluster.
// Iterate through pairs of consecutive points to find clusters and significant gaps.
for window in self.dataset.windows(2) {
let gap_size: f32 = (window[1] - window[0]) as f32;
if gap_size <= cluster_threshold {
// Add points to the current cluster
if current_cluster.is_empty() {
current_cluster.push(window[0]); // Start a new cluster with the first point
}
current_cluster.push(window[1]); // Add the second point to the cluster
} else {
// End the current cluster and start a new gap
if !current_cluster.is_empty() && current_cluster.len() >= min_cluster_size {
self.anomalies.push(Anomaly::new(&current_cluster));
current_cluster.clear();
}
// Record the gap
if gap_size > gap_threshold {
self.anomalies.push(Anomaly {
elements: Vec::new(), // No elements in a gap
start: window[0],
end: window[1],
span_length: gap_size as i32,
num_elements: 0,
centroid: (window[0] as f32 + window[1] as f32) / 2.0,
z_score: None,
});
}
}
}
// Finalize the last cluster if applicable
if !current_cluster.is_empty() && current_cluster.len() >= min_cluster_size {
self.anomalies.push(Anomaly::new(&current_cluster));
}
}
pub fn search(&mut self, factor: f32, min_cluster_size: usize) -> String {
// Sort the vector
self.dataset.sort_unstable();
// Calculate clusters and gaps from the dataset using predefined criteria.
self.scan_anomalies(factor, min_cluster_size);
// Calculate the mean density of clusters in the dataset for comparison.
let mean_density: f32 = self.anomalies.iter()
.filter(|info: &&Anomaly| info.num_elements > 0)
.map(|info: &Anomaly| info.num_elements as f32 / info.span_length as f32)
.sum::<f32>() / self.anomalies.iter().filter(|info: &&Anomaly| info.num_elements > 0).count() as f32;
// Calculate the standard deviation of cluster densities to evaluate variation.
let variance_density: f32 = self.anomalies.iter()
.filter(|info: &&Anomaly| info.num_elements > 0)
.map(|info: &Anomaly| info.num_elements as f32 / info.span_length as f32)
.map(|density: f32| (density - mean_density).powi(2))
.sum::<f32>() / self.anomalies.iter().filter(|info: &&Anomaly| info.num_elements > 0).count() as f32;
let std_dev_density: f32 = variance_density.sqrt();
// Calculate mean span length
let mean_span_length: f32 = self.anomalies.iter()
.map(|info: &Anomaly| info.span_length as f32)
.sum::<f32>() / self.anomalies.len() as f32;
// Calculate variance
let variance: f32 = self.anomalies.iter()
.map(|info: &Anomaly| (info.span_length as f32 - mean_span_length).powi(2))
.sum::<f32>() / self.anomalies.len() as f32;
// Standard deviation is the square root of variance
let std_dev_span_length: f32 = variance.sqrt();
// Update Z-scores for both clusters and gaps based on their deviation from mean metrics.
for info in self.anomalies.iter_mut() {
if info.num_elements > 0 {
// Calculate and update Z-score for clusters based on density deviation.
let cluster_density: f32 = info.num_elements as f32 / info.span_length as f32;
info.z_score = Some((cluster_density - mean_density) / std_dev_density);
} else {
// Calculate and update Z-score for gaps based on span length deviation.
info.z_score = Some((info.span_length as f32 / std_dev_span_length) * -1.0);
}
}
serde_json::to_string_pretty(&self.anomalies).unwrap_or_else(|_| "Failed to serialize data".to_string())
}
}
#[pymodule]
fn lyagushka(_py: Python, m: &PyModule) -> PyResult<()> {
m.add_class::<Lyagushka>()?;
Ok(())
}

152
src/main.rs
View file

@ -2,157 +2,7 @@ use std::fs::File;
use std::io::{self, BufRead, BufReader, stdin};
use std::env;
use std::process;
use serde::Serialize;
use serde_json;
#[derive(Debug, Clone, Serialize)]
struct Anomaly {
elements: Vec<i32>,
start: i32,
end: i32,
span_length: i32,
num_elements: usize,
centroid: f32,
z_score: Option<f32>,
}
impl Anomaly {
pub fn new(cluster: &[i32]) -> Self {
let num_elements: usize = cluster.len();
let start: i32 = *cluster.first().expect("Cluster has no start");
let end: i32 = *cluster.last().expect("Cluster has no end");
let span_length: i32 = end - start;
let centroid: f32 = start as f32 + span_length as f32 / 2.0;
Anomaly {
elements: cluster.to_vec(),
start,
end,
span_length,
num_elements,
centroid,
z_score: None,
}
}
}
struct Lyagushka {
dataset: Vec<i32>,
anomalies: Vec<Anomaly>,
}
impl Lyagushka {
pub fn new(dataset: Vec<i32>) -> Self {
Lyagushka {
dataset,
anomalies: vec![]
}
}
fn scan_anomalies(&mut self, factor: f32, min_cluster_size: usize) {
// Calculate the mean distance between consecutive points in the dataset.
let mean_distance: f32 = self.dataset.windows(2)
.map(|w| (w[1] - w[0]) as f32)
.sum::<f32>() / (self.dataset.len() - 1) as f32;
// Define thresholds for clustering and gap identification based on the mean distance and factor.
let cluster_threshold: f32 = mean_distance / factor;
let gap_threshold: f32 = factor * mean_distance;
let mut current_cluster: Vec<i32> = Vec::new(); // Temporary storage for points in the current cluster.
// Iterate through pairs of consecutive points to find clusters and significant gaps.
for window in self.dataset.windows(2) {
let gap_size: f32 = (window[1] - window[0]) as f32;
if gap_size <= cluster_threshold {
// Add points to the current cluster
if current_cluster.is_empty() {
current_cluster.push(window[0]); // Start a new cluster with the first point
}
current_cluster.push(window[1]); // Add the second point to the cluster
} else {
// End the current cluster and start a new gap
if !current_cluster.is_empty() && current_cluster.len() >= min_cluster_size {
self.anomalies.push(Anomaly::new(&current_cluster));
current_cluster.clear();
}
// Record the gap
if gap_size > gap_threshold {
self.anomalies.push(Anomaly {
elements: Vec::new(), // No elements in a gap
start: window[0],
end: window[1],
span_length: gap_size as i32,
num_elements: 0,
centroid: (window[0] as f32 + window[1] as f32) / 2.0,
z_score: None,
});
}
}
}
// Finalize the last cluster if applicable
if !current_cluster.is_empty() && current_cluster.len() >= min_cluster_size {
self.anomalies.push(Anomaly::new(&current_cluster));
}
}
pub fn search(&mut self, factor: f32, min_cluster_size: usize) -> String {
// Sort the vector
self.dataset.sort_unstable();
// Calculate clusters and gaps from the dataset using predefined criteria.
self.scan_anomalies(factor, min_cluster_size);
// Calculate the mean density of clusters in the dataset for comparison.
let mean_density: f32 = self.anomalies.iter()
.filter(|info: &&Anomaly| info.num_elements > 0)
.map(|info: &Anomaly| info.num_elements as f32 / info.span_length as f32)
.sum::<f32>() / self.anomalies.iter().filter(|info: &&Anomaly| info.num_elements > 0).count() as f32;
// Calculate the standard deviation of cluster densities to evaluate variation.
let variance_density: f32 = self.anomalies.iter()
.filter(|info: &&Anomaly| info.num_elements > 0)
.map(|info: &Anomaly| info.num_elements as f32 / info.span_length as f32)
.map(|density: f32| (density - mean_density).powi(2))
.sum::<f32>() / self.anomalies.iter().filter(|info: &&Anomaly| info.num_elements > 0).count() as f32;
let std_dev_density: f32 = variance_density.sqrt();
// Calculate mean span length
let mean_span_length: f32 = self.anomalies.iter()
.map(|info: &Anomaly| info.span_length as f32)
.sum::<f32>() / self.anomalies.len() as f32;
// Calculate variance
let variance: f32 = self.anomalies.iter()
.map(|info: &Anomaly| (info.span_length as f32 - mean_span_length).powi(2))
.sum::<f32>() / self.anomalies.len() as f32;
// Standard deviation is the square root of variance
let std_dev_span_length: f32 = variance.sqrt();
// Update Z-scores for both clusters and gaps based on their deviation from mean metrics.
for info in self.anomalies.iter_mut() {
if info.num_elements > 0 {
// Calculate and update Z-score for clusters based on density deviation.
let cluster_density: f32 = info.num_elements as f32 / info.span_length as f32;
info.z_score = Some((cluster_density - mean_density) / std_dev_density);
} else {
// Calculate and update Z-score for gaps based on span length deviation.
info.z_score = Some((info.span_length as f32 / std_dev_span_length) * -1.0);
}
}
serde_json::to_string_pretty(&self.anomalies).unwrap_or_else(|_| "Failed to serialize data".to_string())
}
}
use lyagushka::Lyagushka;
/// The entry point for the command-line tool that reads a dataset of integers from either a file or stdin,
/// performs cluster and gap analysis using specified parameters, and prints the results as a JSON string.

75
test.py Normal file
View file

@ -0,0 +1,75 @@
from pyagushka import Lyagushka
from randonautentropy import rndo
import json
import matplotlib.pyplot as plt
import numpy as np
from scipy.interpolate import interp1d
def generate_random_data(size=1024, max_value=100):
random_data = []
max_value_bytes = (max_value.bit_length() + 7) // 8
max_int_for_bytes = 2**(max_value_bytes * 8) - 1
min_bytes_needed = max_value_bytes * size
mod_cutoff = max_int_for_bytes - (max_int_for_bytes % max_value) - 1
# Populate the 'random_data' array
while len(random_data) < size:
hex_data = rndo.get(length=min_bytes_needed)
hex_chunks = list((hex_data[0+i:2 * max_value_bytes+i] for i in range(0, len(hex_data), 2 * max_value_bytes)))
for i in hex_chunks:
num = int(i, 16)
if num <= mod_cutoff and len(random_data) < size:
random_data.append( num % (max_value + 1) )
return random_data
def filter_by_z_score(data, z_score_threshold):
filtered_data = [item for item in data if item['z_score'] is not None and abs(item['z_score']) >= z_score_threshold]
return filtered_data
# load the random test data
# dataset = []
# with open('random_values.txt', 'r') as file:
# for line in file:
# random_data.append(int(line.strip()))
dataset = generate_random_data(1024, 1024)
dataset.sort()
with open('dataset.json', 'w') as r:
r.write(json.dumps(dataset, indent=4))
# calculate the anomalies in the data
zhaba = Lyagushka(dataset)
analysis_results = json.loads(zhaba.search(4.0, 7))
analysis_results = filter_by_z_score(analysis_results, 1.0)
with open('result.json', 'w') as r:
r.write(json.dumps(analysis_results, indent=4))
# Initialize plot
plt.figure(figsize=(10, 6))
# Color palette for clusters and gaps
colors = plt.cm.jet(np.linspace(0, 1, len(analysis_results)))
# Plot dataset points and assign colors based on cluster membership
for i, result in enumerate(analysis_results):
if result['num_elements'] > 0: # It's a cluster
for point in dataset:
plt.plot(point, 0, 'o', color=colors[i]) # Plot points in cluster with the same color
# Plot a line segment for the cluster/gap Z-score in the same color
start = result['start']
end = result['end']
z_score = result['z_score'] if result['z_score'] is not None else 0
plt.plot([start, end], [z_score, z_score], color=colors[i], linewidth=2)
# Enhancements for visualization
plt.xlabel('Integer Value')
plt.ylabel('Z-Score')
plt.title('Cluster and Gap Analysis')
plt.grid(True)
plt.show()