optimizations
This commit is contained in:
randogoth
parent
e4ea8d6ffc
commit
b307383f52
9 changed files with 33 additions and 10315 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -1,2 +1,3 @@
|
|||
/target
|
||||
*.json
|
||||
*.lock
|
||||
230
Cargo.lock
generated
230
Cargo.lock
generated
|
|
@ -1,230 +0,0 @@
|
|||
# This file is automatically @generated by Cargo.
|
||||
# It is not intended for manual editing.
|
||||
version = 3
|
||||
|
||||
[[package]]
|
||||
name = "atty"
|
||||
version = "0.2.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
|
||||
dependencies = [
|
||||
"hermit-abi",
|
||||
"libc",
|
||||
"winapi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "autocfg"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "1.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
|
||||
|
||||
[[package]]
|
||||
name = "cfg-if"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
|
||||
|
||||
[[package]]
|
||||
name = "heck"
|
||||
version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
|
||||
|
||||
[[package]]
|
||||
name = "hermit-abi"
|
||||
version = "0.1.19"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "itoa"
|
||||
version = "1.0.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c"
|
||||
|
||||
[[package]]
|
||||
name = "libc"
|
||||
version = "0.2.153"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd"
|
||||
|
||||
[[package]]
|
||||
name = "lyagushka"
|
||||
version = "1.1.0"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
"scopeguard",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lyagushka"
|
||||
version = "1.1.0"
|
||||
dependencies = [
|
||||
"atty",
|
||||
"pyo3",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "memoffset"
|
||||
version = "0.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c"
|
||||
dependencies = [
|
||||
"autocfg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "once_cell"
|
||||
version = "1.19.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot"
|
||||
version = "0.12.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f"
|
||||
dependencies = [
|
||||
"lock_api",
|
||||
"parking_lot_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "parking_lot_core"
|
||||
version = "0.9.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"redox_syscall",
|
||||
"smallvec",
|
||||
"windows-targets",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "proc-macro2"
|
||||
version = "1.0.78"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae"
|
||||
dependencies = [
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.35"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "redox_syscall"
|
||||
version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ryu"
|
||||
version = "1.0.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f98d2aa92eebf49b69786be48e4477826b256916e84a57ff2a4f21923b48eb4c"
|
||||
|
||||
[[package]]
|
||||
name = "serde"
|
||||
version = "1.0.196"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "870026e60fa08c69f064aa766c10f10b1d62db9ccd4d0abb206472bee0ce3b32"
|
||||
dependencies = [
|
||||
"serde_derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_derive"
|
||||
version = "1.0.196"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "33c85360c95e7d137454dc81d9a4ed2b8efd8fbe19cee57357b32b9771fccb67"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "serde_json"
|
||||
version = "1.0.113"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "69801b70b1c3dac963ecb03a364ba0ceda9cf60c71cfe475e99864759c8b8a79"
|
||||
dependencies = [
|
||||
"itoa",
|
||||
"ryu",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "syn"
|
||||
version = "2.0.48"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"unicode-ident",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "target-lexicon"
|
||||
version = "0.12.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "69758bda2e78f098e4ccb393021a0963bb3442eac05f135c30f61b7370bbafae"
|
||||
|
||||
[[package]]
|
||||
name = "unicode-ident"
|
||||
version = "1.0.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
|
||||
|
||||
[[package]]
|
||||
name = "unindent"
|
||||
version = "0.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce"
|
||||
|
||||
[[package]]
|
||||
name = "winapi"
|
||||
version = "0.3.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
|
||||
dependencies = [
|
||||
"winapi-i686-pc-windows-gnu",
|
||||
"winapi-x86_64-pc-windows-gnu",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi-i686-pc-windows-gnu"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
|
||||
|
||||
[[package]]
|
||||
name = "winapi-x86_64-pc-windows-gnu"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
|
||||
12
Cargo.toml
12
Cargo.toml
|
|
@ -2,10 +2,18 @@
|
|||
name = "lyagushka"
|
||||
version = "1.1.0"
|
||||
edition = "2021"
|
||||
crate-type = ["cdylib"]
|
||||
|
||||
[dependencies]
|
||||
atty = "0.2.14"
|
||||
pyo3 = { version = "0.20.2", features = ["extension-module"] }
|
||||
pyo3 = { version = "0.23.3", features = ["extension-module"] }
|
||||
serde = { version = "1.0.196", features = ["derive"] }
|
||||
serde_json = "1.0.113"
|
||||
|
||||
[lib]
|
||||
name = "lyagushka"
|
||||
path = "src/lib.rs"
|
||||
crate-type = ["cdylib", "lib"]
|
||||
|
||||
[[bin]]
|
||||
name = "lyagushka"
|
||||
path = "src/bin/bin.rs"
|
||||
12
Pipfile
Normal file
12
Pipfile
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
[[source]]
|
||||
url = "https://pypi.org/simple"
|
||||
verify_ssl = true
|
||||
name = "pypi"
|
||||
|
||||
[packages]
|
||||
maturin = "*"
|
||||
|
||||
[dev-packages]
|
||||
|
||||
[requires]
|
||||
python_version = "3.12"
|
||||
10000
random_values.txt
10000
random_values.txt
File diff suppressed because it is too large
Load diff
10
readme.md
10
readme.md
|
|
@ -11,14 +11,16 @@ It is an algorithm that analyzes a one-dimensional dataset of integers to identi
|
|||
With a Rust and Cargo environment set up, simply run:
|
||||
|
||||
```sh
|
||||
$ cargo build --release
|
||||
cargo build --release
|
||||
```
|
||||
|
||||
To also compile a Python wheel, you need Maturin set up. SImply run:
|
||||
To also compile a Python wheel, you need Maturin set up:
|
||||
|
||||
```sh
|
||||
$ maturin build --release
|
||||
$ pip install target/wheels/lyagushka-1.1.0*.whl
|
||||
pipenv install
|
||||
pipenv shell
|
||||
maturin build --release
|
||||
pip install target/wheels/lyagushka-1.1.0*.whl
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
|
|
|||
|
|
@ -154,7 +154,7 @@ impl Lyagushka {
|
|||
}
|
||||
|
||||
#[pymodule]
|
||||
fn lyagushka(_py: Python, m: &PyModule) -> PyResult<()> {
|
||||
fn lyagushka(m: &Bound<'_, PyModule>) -> PyResult<()> {
|
||||
m.add_class::<Lyagushka>()?;
|
||||
Ok(())
|
||||
}
|
||||
75
test.py
75
test.py
|
|
@ -1,75 +0,0 @@
|
|||
from pyagushka import Lyagushka
|
||||
from randonautentropy import rndo
|
||||
import json
|
||||
import matplotlib.pyplot as plt
|
||||
import numpy as np
|
||||
from scipy.interpolate import interp1d
|
||||
|
||||
def generate_random_data(size=1024, max_value=100):
|
||||
|
||||
random_data = []
|
||||
max_value_bytes = (max_value.bit_length() + 7) // 8
|
||||
max_int_for_bytes = 2**(max_value_bytes * 8) - 1
|
||||
min_bytes_needed = max_value_bytes * size
|
||||
mod_cutoff = max_int_for_bytes - (max_int_for_bytes % max_value) - 1
|
||||
|
||||
# Populate the 'random_data' array
|
||||
while len(random_data) < size:
|
||||
hex_data = rndo.get(length=min_bytes_needed)
|
||||
hex_chunks = list((hex_data[0+i:2 * max_value_bytes+i] for i in range(0, len(hex_data), 2 * max_value_bytes)))
|
||||
for i in hex_chunks:
|
||||
num = int(i, 16)
|
||||
if num <= mod_cutoff and len(random_data) < size:
|
||||
random_data.append( num % (max_value + 1) )
|
||||
|
||||
return random_data
|
||||
|
||||
def filter_by_z_score(data, z_score_threshold):
|
||||
filtered_data = [item for item in data if item['z_score'] is not None and abs(item['z_score']) >= z_score_threshold]
|
||||
return filtered_data
|
||||
|
||||
# load the random test data
|
||||
# dataset = []
|
||||
# with open('random_values.txt', 'r') as file:
|
||||
# for line in file:
|
||||
# random_data.append(int(line.strip()))
|
||||
|
||||
dataset = generate_random_data(1024, 1024)
|
||||
dataset.sort()
|
||||
|
||||
with open('dataset.json', 'w') as r:
|
||||
r.write(json.dumps(dataset, indent=4))
|
||||
|
||||
# calculate the anomalies in the data
|
||||
zhaba = Lyagushka(dataset)
|
||||
analysis_results = json.loads(zhaba.search(4.0, 7))
|
||||
analysis_results = filter_by_z_score(analysis_results, 1.0)
|
||||
|
||||
with open('result.json', 'w') as r:
|
||||
r.write(json.dumps(analysis_results, indent=4))
|
||||
|
||||
# Initialize plot
|
||||
plt.figure(figsize=(10, 6))
|
||||
|
||||
# Color palette for clusters and gaps
|
||||
colors = plt.cm.jet(np.linspace(0, 1, len(analysis_results)))
|
||||
|
||||
# Plot dataset points and assign colors based on cluster membership
|
||||
for i, result in enumerate(analysis_results):
|
||||
if result['num_elements'] > 0: # It's a cluster
|
||||
for point in dataset:
|
||||
plt.plot(point, 0, 'o', color=colors[i]) # Plot points in cluster with the same color
|
||||
|
||||
# Plot a line segment for the cluster/gap Z-score in the same color
|
||||
start = result['start']
|
||||
end = result['end']
|
||||
z_score = result['z_score'] if result['z_score'] is not None else 0
|
||||
plt.plot([start, end], [z_score, z_score], color=colors[i], linewidth=2)
|
||||
|
||||
# Enhancements for visualization
|
||||
plt.xlabel('Integer Value')
|
||||
plt.ylabel('Z-Score')
|
||||
plt.title('Cluster and Gap Analysis')
|
||||
plt.grid(True)
|
||||
|
||||
plt.show()
|
||||
Loading…
Add table
Add a link
Reference in a new issue