init
This commit is contained in:
commit
10ff2123ac
22 changed files with 1430 additions and 0 deletions
109
src/bin/main.rs
Normal file
109
src/bin/main.rs
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
use std::fs::File;
|
||||
use std::io::{self, Read};
|
||||
use std::path::Path;
|
||||
use statrs::distribution::{Normal, ContinuousCDF};
|
||||
|
||||
use onod3000::Onod;
|
||||
|
||||
fn main() -> io::Result<()> {
|
||||
let mut args = std::env::args();
|
||||
args.next(); // Skip program name
|
||||
|
||||
let mut input_data = Vec::new();
|
||||
let mut source = String::from("stdin");
|
||||
|
||||
if let Some(arg) = args.next() {
|
||||
if arg == "-f" {
|
||||
if let Some(file_path) = args.next() {
|
||||
source = file_path.clone();
|
||||
let path = Path::new(&file_path);
|
||||
let mut file = File::open(path)?;
|
||||
file.read_to_end(&mut input_data)?;
|
||||
} else {
|
||||
eprintln!("Error: No file path provided after -f.");
|
||||
std::process::exit(1);
|
||||
}
|
||||
} else {
|
||||
eprintln!("Usage: {} [-f <file_path>]", std::env::args().next().unwrap());
|
||||
std::process::exit(1);
|
||||
}
|
||||
} else {
|
||||
io::stdin().read_to_end(&mut input_data)?;
|
||||
}
|
||||
|
||||
if input_data.is_empty() {
|
||||
eprintln!("Error: No input data provided.");
|
||||
std::process::exit(1);
|
||||
}
|
||||
|
||||
println!("\nTesting {}", source);
|
||||
println!("Testing {} bytes.", input_data.len());
|
||||
println!("--------------------------------------");
|
||||
|
||||
let alpha = 0.01;
|
||||
let mut passed_tests = 0;
|
||||
|
||||
// Run each test with the appropriate handling
|
||||
let tests = [
|
||||
("Shannon", Onod::shannon(&input_data)),
|
||||
("Sanity", Onod::sanity(&input_data)),
|
||||
("Monobit", Onod::monobit(&input_data)),
|
||||
("ChiBit", Onod::chi_bit(&input_data)),
|
||||
("ChiByte", Onod::chi_byte(&input_data)),
|
||||
("MeanByte", Onod::mean_byte(&input_data)),
|
||||
("Compression", Onod::compression(&input_data)),
|
||||
("KS", Onod::ks(&input_data)),
|
||||
("Pi", Onod::pi(&input_data)),
|
||||
("Shells", Onod::shells(&input_data)),
|
||||
("Gaps", Onod::gaps(&input_data)),
|
||||
("Avalanche", Onod::avalanche(&input_data)),
|
||||
("Runs", Onod::runs(&input_data)),
|
||||
("RunUps", Onod::run_ups(&input_data)),
|
||||
("Prediction", Onod::prediction(&input_data)),
|
||||
("UnCorrelation", Onod::un_correlation(&input_data)),
|
||||
];
|
||||
|
||||
let mut p_values = Vec::new();
|
||||
|
||||
for (test_name, p_value) in tests.iter() {
|
||||
let result = if *p_value >= alpha {
|
||||
passed_tests += 1;
|
||||
"PASS"
|
||||
} else {
|
||||
"FAIL"
|
||||
};
|
||||
|
||||
println!("{:<20} p = {:.4}, {}", test_name, p_value, result);
|
||||
|
||||
p_values.push(*p_value); // Dereference the value and push it
|
||||
}
|
||||
|
||||
// Calculate combined p-value using Fisher's method
|
||||
let (combined_z_score, combined_p_value) = combined_score_stouffer(&p_values);
|
||||
let overall_result = if combined_p_value >= alpha { "PASS" } else { "FAIL" };
|
||||
|
||||
println!("--------------------------------------");
|
||||
println!("{}/{} tests passed.", passed_tests, tests.len());
|
||||
println!("--------------------------------------");
|
||||
println!("Combined Z-Score = {:.6}\nCombined P-Value = {:.6}\nOverall Result: {}", combined_z_score.abs(), combined_p_value, overall_result);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn combined_score_stouffer(p_values: &[f64]) -> (f64, f64) {
|
||||
let normal_dist = Normal::new(0.0, 1.0).expect("Failed to create Normal distribution");
|
||||
|
||||
// Calculate Z-scores for each p-value
|
||||
let z_scores: Vec<f64> = p_values
|
||||
.iter()
|
||||
.map(|&p| normal_dist.inverse_cdf(1.0 - p))
|
||||
.collect();
|
||||
|
||||
// Combine Z-scores
|
||||
let combined_z = z_scores.iter().sum::<f64>() / (p_values.len() as f64).sqrt();
|
||||
|
||||
// Convert combined Z-score back to a p-value
|
||||
let p_value = 2.0 * (1.0 - normal_dist.cdf(combined_z.abs())); // Two-tailed
|
||||
|
||||
(combined_z, p_value)
|
||||
}
|
||||
3
src/lib.rs
Normal file
3
src/lib.rs
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
pub struct Onod;
|
||||
|
||||
mod uniformity;
|
||||
52
src/uniformity/avalanche.rs
Normal file
52
src/uniformity/avalanche.rs
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
use statrs::distribution::{Normal, ContinuousCDF};
|
||||
|
||||
use crate::Onod;
|
||||
|
||||
impl Onod {
|
||||
|
||||
/// Avalanche randomness test
|
||||
/// Compares the bit-level differences between consecutive chunks of data and returns a p-value.
|
||||
pub fn avalanche(samples: &[u8]) -> f64 {
|
||||
|
||||
const XOR_WINDOW_SIZE: usize = 20; // Bytes. Equivalent to SHA-1 (160 bits).
|
||||
|
||||
if samples.len() < 2 * XOR_WINDOW_SIZE {
|
||||
return 0.0; // Not enough data for meaningful calculation
|
||||
}
|
||||
|
||||
let mut means = Vec::new();
|
||||
|
||||
for i in (0..samples.len() - (2 * XOR_WINDOW_SIZE)).step_by(2 * XOR_WINDOW_SIZE) {
|
||||
let a_start = i;
|
||||
let a_end = i + XOR_WINDOW_SIZE;
|
||||
let b_start = a_end;
|
||||
let b_end = b_start + XOR_WINDOW_SIZE;
|
||||
|
||||
let a_bytes = &samples[a_start..a_end];
|
||||
let b_bytes = &samples[b_start..b_end];
|
||||
|
||||
// XOR the two chunks and count differing bits
|
||||
let mut changed_bits = 0;
|
||||
for (a, b) in a_bytes.iter().zip(b_bytes.iter()) {
|
||||
changed_bits += (a ^ b).count_ones();
|
||||
}
|
||||
|
||||
means.push(changed_bits as f64);
|
||||
}
|
||||
|
||||
// Calculate the mean and standard deviation of bit differences
|
||||
let mean_observed = means.iter().sum::<f64>() / means.len() as f64;
|
||||
let mean_ref = (XOR_WINDOW_SIZE * 8) as f64 / 2.0; // Expected mean bits
|
||||
let std_dev_ref = 0.5 * ((XOR_WINDOW_SIZE * 8) as f64).sqrt(); // Expected standard deviation
|
||||
|
||||
// Calculate Z score
|
||||
let z_score = (mean_observed - mean_ref) / std_dev_ref;
|
||||
|
||||
// Convert Z score to p-value
|
||||
let normal_dist = Normal::new(0.0, 1.0).expect("Failed to create Normal distribution");
|
||||
let p_value = 2.0 * (1.0 - normal_dist.cdf(z_score.abs()));
|
||||
|
||||
p_value
|
||||
}
|
||||
|
||||
}
|
||||
62
src/uniformity/chi_bit.rs
Normal file
62
src/uniformity/chi_bit.rs
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
use statrs::distribution::{ChiSquared, ContinuousCDF};
|
||||
|
||||
use crate::Onod;
|
||||
|
||||
impl Onod {
|
||||
|
||||
pub fn chi_bit(samples: &[u8]) -> f64 {
|
||||
|
||||
if samples.is_empty() {
|
||||
return 0.0; // empty data
|
||||
}
|
||||
|
||||
// Lookup table for the number of set bits in each byte (Hamming weight)
|
||||
const SET_BITS_PER_BYTE: [usize; 256] = [
|
||||
0, 1, 1, 2, 1, 2, 2, 3, 1, 2, 2, 3, 2, 3, 3, 4, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 1, 2, 2, 3,
|
||||
2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 1, 2, 2, 3, 2, 3, 3, 4,
|
||||
2, 3, 3, 4, 3, 4, 4, 5, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5,
|
||||
4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 1, 2, 2, 3, 2, 3, 3, 4, 2, 3, 3, 4, 3, 4, 4, 5,
|
||||
2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5,
|
||||
4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 2, 3, 3, 4, 3, 4, 4, 5, 3, 4, 4, 5, 4, 5, 5, 6, 3, 4, 4, 5, 4, 5, 5, 6,
|
||||
4, 5, 5, 6, 5, 6, 6, 7, 3, 4, 4, 5, 4, 5, 5, 6, 4, 5, 5, 6, 5, 6, 6, 7, 4, 5, 5, 6, 5, 6, 6, 7, 5, 6, 6, 7,
|
||||
6, 7, 7, 8,
|
||||
];
|
||||
|
||||
// Expected number of occurrences for each Hamming weight
|
||||
const EXPECTED_NUMBER: [f64; 9] = [1.0, 8.0, 28.0, 56.0, 70.0, 56.0, 28.0, 8.0, 1.0];
|
||||
|
||||
// Frequency count of Hamming weights
|
||||
let mut frequency = vec![0; EXPECTED_NUMBER.len()];
|
||||
for &byte in samples {
|
||||
let hamming_weight = SET_BITS_PER_BYTE[byte as usize];
|
||||
frequency[hamming_weight] += 1;
|
||||
}
|
||||
|
||||
// Calculate observed and expected counts
|
||||
let total_samples = samples.len() as f64;
|
||||
let expected: Vec<f64> = EXPECTED_NUMBER
|
||||
.iter()
|
||||
.map(|&e| e / 256.0 * total_samples)
|
||||
.collect();
|
||||
|
||||
// Chi-squared statistic calculation
|
||||
let chi_squared_stat = frequency
|
||||
.iter()
|
||||
.zip(expected.iter())
|
||||
.map(|(&observed, &expected)| {
|
||||
if expected > 0.0 {
|
||||
(observed as f64 - expected).powi(2) / expected
|
||||
} else {
|
||||
0.0
|
||||
}
|
||||
})
|
||||
.sum();
|
||||
|
||||
// Degrees of freedom: 9 categories - 1
|
||||
let degrees_of_freedom = (EXPECTED_NUMBER.len() - 1) as f64;
|
||||
let chi_squared_dist = ChiSquared::new(degrees_of_freedom).expect("Failed to create ChiSquared distribution");
|
||||
let p_value = 1.0 - chi_squared_dist.cdf(chi_squared_stat);
|
||||
|
||||
p_value
|
||||
}
|
||||
}
|
||||
41
src/uniformity/chi_byte.rs
Normal file
41
src/uniformity/chi_byte.rs
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
use std::collections::HashMap;
|
||||
use statrs::distribution::{ChiSquared, ContinuousCDF};
|
||||
|
||||
use crate::Onod;
|
||||
|
||||
impl Onod {
|
||||
|
||||
/// ChiByte randomness test
|
||||
/// Evaluates the uniformity of byte values across the data and returns a p-value.
|
||||
pub fn chi_byte(samples: &[u8]) -> f64 {
|
||||
|
||||
if samples.is_empty() {
|
||||
return 0.0; // Default to perfect randomness for empty data
|
||||
}
|
||||
|
||||
// Count occurrences of each byte value (0-255)
|
||||
let mut counts = HashMap::new();
|
||||
for &byte in samples {
|
||||
*counts.entry(byte).or_insert(0) += 1;
|
||||
}
|
||||
|
||||
// Calculate expected count assuming uniform distribution
|
||||
let expected_count = samples.len() as f64 / 256.0;
|
||||
|
||||
// Calculate chi-squared statistic
|
||||
let mut chi_squared_stat = 0.0;
|
||||
for i in 0..256 {
|
||||
let observed = *counts.get(&(i as u8)).unwrap_or(&0) as f64;
|
||||
let diff = observed - expected_count;
|
||||
chi_squared_stat += (diff * diff) / expected_count;
|
||||
}
|
||||
|
||||
// Use chi-squared distribution to calculate p-value
|
||||
let degrees_of_freedom = 256.0 - 1.0; // 256 possible byte values - 1
|
||||
let chi_squared_dist = ChiSquared::new(degrees_of_freedom).expect("Failed to create ChiSquared distribution");
|
||||
let p_value = 1.0 - chi_squared_dist.cdf(chi_squared_stat);
|
||||
|
||||
p_value
|
||||
}
|
||||
|
||||
}
|
||||
42
src/uniformity/compression.rs
Normal file
42
src/uniformity/compression.rs
Normal file
|
|
@ -0,0 +1,42 @@
|
|||
use flate2::{write::DeflateEncoder, Compression};
|
||||
use std::io::Write;
|
||||
use statrs::distribution::{Normal, ContinuousCDF};
|
||||
|
||||
use crate::Onod;
|
||||
|
||||
impl Onod {
|
||||
|
||||
/// Compression randomness test
|
||||
/// Estimates randomness by the compressibility of the data and returns a p-value.
|
||||
pub fn compression(samples: &[u8]) -> f64 {
|
||||
if samples.is_empty() {
|
||||
return 0.0; // Perfect randomness for empty data
|
||||
}
|
||||
|
||||
// Compress the data using deflate
|
||||
let mut encoder = DeflateEncoder::new(Vec::new(), Compression::default());
|
||||
if encoder.write_all(samples).is_err() {
|
||||
return 0.0; // Compression failed
|
||||
}
|
||||
let compressed_data = match encoder.finish() {
|
||||
Ok(data) => data,
|
||||
Err(_) => return 0.0, // Compression failed
|
||||
};
|
||||
|
||||
// Calculate compression ratio
|
||||
let original_size = samples.len() as f64;
|
||||
let compressed_size = compressed_data.len() as f64;
|
||||
let compression_ratio = compressed_size / original_size;
|
||||
|
||||
// Z-score calculation
|
||||
let expected_mean = 1.0;
|
||||
let std_dev = 0.002;
|
||||
let z_score = (compression_ratio - expected_mean) / std_dev;
|
||||
|
||||
// Calculate p-value using the normal distribution
|
||||
let normal_dist = Normal::new(0.0, 1.0).expect("Failed to create Normal distribution");
|
||||
let p_value = 2.0 * (1.0 - normal_dist.cdf(z_score.abs())); // Two-tailed test
|
||||
|
||||
p_value
|
||||
}
|
||||
}
|
||||
79
src/uniformity/gaps.rs
Normal file
79
src/uniformity/gaps.rs
Normal file
|
|
@ -0,0 +1,79 @@
|
|||
use statrs::distribution::{ChiSquared, ContinuousCDF};
|
||||
|
||||
use crate::Onod;
|
||||
|
||||
impl Onod {
|
||||
|
||||
/// Gaps randomness test
|
||||
/// Analyzes the gaps between occurrences of a specific byte value and returns a p-value.
|
||||
pub fn gaps(samples: &[u8]) -> f64 {
|
||||
|
||||
if samples.is_empty() {
|
||||
return 0.0; // empty data
|
||||
}
|
||||
|
||||
// Bin edges and expected frequencies from the Java implementation
|
||||
let bin_edges = [0, 26, 56, 90, 130, 176, 233, 307, 410, 587, 1_000_000_000];
|
||||
let expected_frequencies = [
|
||||
0.10028324483130746,
|
||||
0.09967572210885733,
|
||||
0.09968386797087492,
|
||||
0.10149320063058609,
|
||||
0.09867042665174708,
|
||||
0.10001818308995936,
|
||||
0.10062746966933134,
|
||||
0.09938275862858448,
|
||||
0.10004382368984566,
|
||||
0.1001213027289063,
|
||||
];
|
||||
|
||||
let mut all_gaps = Vec::new();
|
||||
|
||||
// Measure gaps for all unique values
|
||||
for category_pointer in 0..samples.len() - 1 {
|
||||
for i in category_pointer + 1..samples.len() {
|
||||
if samples[i] == samples[category_pointer] {
|
||||
let gap = i - category_pointer - 1;
|
||||
all_gaps.push(gap as u32);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if all_gaps.is_empty() {
|
||||
return 1.0; // No gaps found, assume randomness
|
||||
}
|
||||
|
||||
// Create histogram of observed gaps
|
||||
let mut observed = vec![0; bin_edges.len() - 1];
|
||||
for &gap in &all_gaps {
|
||||
for i in 0..bin_edges.len() - 1 {
|
||||
if (gap as u32) <= bin_edges[i + 1] {
|
||||
observed[i] += 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Calculate expected counts for each bin
|
||||
let total_gaps = all_gaps.len() as f64;
|
||||
let expected: Vec<f64> = expected_frequencies
|
||||
.iter()
|
||||
.map(|&freq| freq * total_gaps)
|
||||
.collect();
|
||||
|
||||
// Perform chi-square test
|
||||
let chi_squared_stat: f64 = observed
|
||||
.iter()
|
||||
.zip(expected.iter())
|
||||
.map(|(&o, &e)| if e > 0.0 { (o as f64 - e).powi(2) / e } else { 0.0 })
|
||||
.sum();
|
||||
|
||||
let degrees_of_freedom = bin_edges.len() as f64 - 2.0; // Number of bins - 1
|
||||
let chi_squared_dist = ChiSquared::new(degrees_of_freedom).expect("Failed to create ChiSquared distribution");
|
||||
let p_value = 1.0 - chi_squared_dist.cdf(chi_squared_stat);
|
||||
|
||||
p_value
|
||||
}
|
||||
|
||||
}
|
||||
40
src/uniformity/ks.rs
Normal file
40
src/uniformity/ks.rs
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
use kolmogorov_smirnov::test_f64;
|
||||
|
||||
use crate::Onod;
|
||||
|
||||
impl Onod {
|
||||
|
||||
/// KS randomness test
|
||||
/// Performs the Kolmogorov-Smirnov test to evaluate uniformity of data distribution and returns a p-value.
|
||||
// Note: This implementation of the Kolmogorov-Smirnov (KS) test differs slightly from the
|
||||
// Java implementation due to differences in library behavior and floating-point handling.
|
||||
// The Java implementation (Apache Commons Math) adds random jitter to handle ties in small
|
||||
// datasets, uses strict inequality for small sample sizes, and applies specific precision
|
||||
// rules based on the IEEE 754 standard for `double`. The Rust implementation, using the
|
||||
// `kolmogorov_smirnov` crate, does not add jitter or handle ties in the same way, and
|
||||
// adheres to the crate's internal handling of floating-point comparisons. These differences
|
||||
// may result in slight variations in p-values or KS statistics between the two versions.
|
||||
pub fn ks(samples: &[u8]) -> f64 {
|
||||
|
||||
if samples.is_empty() {
|
||||
return 0.0; // empty data
|
||||
}
|
||||
|
||||
// Normalize the input samples to [0, 1] range
|
||||
let normalized_samples: Vec<f64> = samples.iter().map(|&x| x as f64 / 255.0).collect();
|
||||
|
||||
// Generate a uniform distribution for comparison
|
||||
let uniform_distribution: Vec<f64> = (0..normalized_samples.len())
|
||||
.map(|i| i as f64 / (normalized_samples.len() as f64 - 1.0))
|
||||
.collect();
|
||||
|
||||
// Perform the Kolmogorov-Smirnov test
|
||||
let confidence = 0.01; // Significance level
|
||||
let result = test_f64(&normalized_samples, &uniform_distribution, confidence);
|
||||
|
||||
// Extract p-value from the result
|
||||
let p_value = 1.0 - result.reject_probability;
|
||||
|
||||
p_value
|
||||
}
|
||||
}
|
||||
34
src/uniformity/mean_byte.rs
Normal file
34
src/uniformity/mean_byte.rs
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
use statrs::distribution::{Normal, ContinuousCDF};
|
||||
|
||||
use crate::Onod;
|
||||
|
||||
impl Onod {
|
||||
|
||||
/// Mean randomness test
|
||||
/// Calculates the p-value for the mean of the byte slice compared to expected mean.
|
||||
pub fn mean_byte(samples: &[u8]) -> f64 {
|
||||
|
||||
let len = samples.len() as f64;
|
||||
if len == 0.0 {
|
||||
return 0.0;
|
||||
}
|
||||
|
||||
// Calculate observed mean
|
||||
let observed_mean: f64 = samples.iter().map(|&x| x as f64).sum::<f64>() / len;
|
||||
|
||||
// Expected mean for uniform distribution
|
||||
let expected_mean = 127.5;
|
||||
|
||||
// Calculate standard deviation of the mean
|
||||
let std_dev_mean = ((256.0 * 256.0 - 1.0) / (12.0 * len)).sqrt();
|
||||
|
||||
// Calculate the z-score
|
||||
let z_score = (observed_mean - expected_mean) / std_dev_mean;
|
||||
|
||||
// Use normal distribution to calculate p-value
|
||||
let normal_dist = Normal::new(0.0, 1.0).expect("Failed to create Normal distribution");
|
||||
let p_value = 2.0 * (1.0 - normal_dist.cdf(z_score.abs())); // Two-tailed test
|
||||
|
||||
p_value
|
||||
}
|
||||
}
|
||||
16
src/uniformity/mod.rs
Normal file
16
src/uniformity/mod.rs
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
pub mod avalanche;
|
||||
pub mod chi_bit;
|
||||
pub mod chi_byte;
|
||||
pub mod compression;
|
||||
pub mod gaps;
|
||||
pub mod ks;
|
||||
pub mod mean_byte;
|
||||
pub mod monobit;
|
||||
pub mod pi;
|
||||
pub mod prediction;
|
||||
pub mod runs;
|
||||
pub mod runups;
|
||||
pub mod sanity;
|
||||
pub mod shannon;
|
||||
pub mod shells;
|
||||
pub mod uncorrelation;
|
||||
40
src/uniformity/monobit.rs
Normal file
40
src/uniformity/monobit.rs
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
use statrs::distribution::{Normal, ContinuousCDF};
|
||||
|
||||
use crate::Onod;
|
||||
|
||||
impl Onod {
|
||||
|
||||
/// Monobit randomness test
|
||||
/// Evaluates the balance of 0s and 1s in the binary representation of the data and returns a p-value.
|
||||
pub fn monobit(samples: &[u8]) -> f64 {
|
||||
|
||||
if samples.is_empty() {
|
||||
return 0.0; // Default to perfect randomness for empty data
|
||||
}
|
||||
|
||||
// Count the total number of 1s in the dataset
|
||||
let mut total_ones = 0;
|
||||
let mut total_bits = 0;
|
||||
|
||||
for &byte in samples {
|
||||
total_ones += byte.count_ones() as usize;
|
||||
total_bits += 8;
|
||||
}
|
||||
|
||||
// Calculate the observed proportion of 1s
|
||||
let observed_proportion = total_ones as f64 / total_bits as f64;
|
||||
|
||||
// Expected proportion for random data
|
||||
let expected_proportion = 0.5;
|
||||
let std_dev = (0.5 * 0.5 / total_bits as f64).sqrt(); // Standard deviation for a binomial distribution
|
||||
|
||||
// Calculate the z-score
|
||||
let z_score = (observed_proportion - expected_proportion) / std_dev;
|
||||
|
||||
// Use normal distribution to calculate p-value
|
||||
let normal_dist = Normal::new(0.0, 1.0).expect("Failed to create Normal distribution");
|
||||
let p_value = 2.0 * (1.0 - normal_dist.cdf(z_score.abs()));
|
||||
|
||||
p_value
|
||||
}
|
||||
}
|
||||
67
src/uniformity/pi.rs
Normal file
67
src/uniformity/pi.rs
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
use statrs::distribution::{Normal, ContinuousCDF};
|
||||
|
||||
use crate::Onod;
|
||||
|
||||
impl Onod {
|
||||
|
||||
/// Pi randomness test
|
||||
/// Uses a Monte Carlo simulation to estimate randomness by calculating the approximation of Pi.
|
||||
/// This implementation of the Pi randomness test closely follows the logic of the original Java implementation.
|
||||
/// However, minor differences in the results may arise due to the following reasons:
|
||||
///
|
||||
/// 1. **Floating-Point Precision**:
|
||||
/// Rust and Java both use 64-bit floating-point numbers (`double` in Java, `f64` in Rust), but slight differences
|
||||
/// in their implementations (e.g., rounding modes, intermediate representations) can lead to small deviations.
|
||||
///
|
||||
/// 2. **Math Libraries**:
|
||||
/// Java uses Apache Commons Math for statistical computations, which may implement certain calculations
|
||||
/// (e.g., Z-scores and normal distribution CDFs) differently compared to the `statrs` crate used in Rust.
|
||||
///
|
||||
/// 3. **Bit Accuracy**:
|
||||
/// The Java implementation notes the significance of bit accuracy in floating-point computations,
|
||||
/// as defined in the IEEE 754 standard. Differences in handling edge cases (e.g., subnormal values,
|
||||
/// precision limits) could lead to slight variations.
|
||||
///
|
||||
/// These differences are generally negligible for practical purposes and do not affect the overall functionality or
|
||||
/// statistical significance of the test.
|
||||
pub fn pi(samples: &[u8]) -> f64 {
|
||||
|
||||
if samples.is_empty() {
|
||||
return 0.0;
|
||||
}
|
||||
|
||||
// Normalize samples to [0.0, 1.0)
|
||||
let normalized_samples: Vec<f64> = samples.iter().map(|&x| x as f64 / 255.0).collect();
|
||||
|
||||
// Initialize variables for summary statistics
|
||||
let mut sum_y = 0.0;
|
||||
let mut count = 0.0;
|
||||
|
||||
// Compute y-values (sqrt(1 - x^2)) and update summary statistics
|
||||
for &x in &normalized_samples {
|
||||
let y = (1.0 - x * x).sqrt();
|
||||
sum_y += y;
|
||||
count += 1.0;
|
||||
}
|
||||
|
||||
// Calculate mean of y-values
|
||||
let mean_y = sum_y / count;
|
||||
|
||||
// Calculate the test statistic
|
||||
let test_statistic = 4.0 * mean_y;
|
||||
|
||||
// Calculate variance and standard deviation
|
||||
let variance = (16.0 / count) * ((2.0 / 3.0) - (std::f64::consts::PI / 4.0).powi(2));
|
||||
let std_dev = variance.sqrt();
|
||||
|
||||
// Calculate Z-score
|
||||
let z_score = (test_statistic - std::f64::consts::PI) / std_dev;
|
||||
|
||||
// Use normal distribution to calculate p-value
|
||||
let normal_dist = Normal::new(0.0, 1.0).expect("Failed to create Normal distribution");
|
||||
let p_value = 2.0 * (1.0 - normal_dist.cdf(z_score.abs()));
|
||||
|
||||
p_value
|
||||
}
|
||||
|
||||
}
|
||||
51
src/uniformity/prediction.rs
Normal file
51
src/uniformity/prediction.rs
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
use statrs::distribution::{Normal, ContinuousCDF};
|
||||
|
||||
use crate::Onod;
|
||||
|
||||
impl Onod {
|
||||
|
||||
/// Prediction randomness test
|
||||
/// Evaluates the predictability of the next bit based on current data and returns a p-value.
|
||||
pub fn prediction(samples: &[u8]) -> f64 {
|
||||
|
||||
if samples.is_empty() {
|
||||
return 0.0;
|
||||
}
|
||||
|
||||
let mut correct_predictions = 0;
|
||||
let mut total_predictions = 0;
|
||||
|
||||
for window in samples.windows(2) {
|
||||
if let [current, next] = window {
|
||||
let predicted = if current & 0x01 == 0 { 0 } else { 1 }; // Predict next bit based on LSB
|
||||
let actual = next & 0x01; // Check LSB of the next byte
|
||||
|
||||
if predicted == actual {
|
||||
correct_predictions += 1;
|
||||
}
|
||||
|
||||
total_predictions += 1;
|
||||
}
|
||||
}
|
||||
|
||||
if total_predictions == 0 {
|
||||
return 0.0; // No predictions possible
|
||||
}
|
||||
|
||||
// Calculate observed proportion of correct predictions
|
||||
let observed_proportion = correct_predictions as f64 / total_predictions as f64;
|
||||
|
||||
// Expected proportion for random data
|
||||
let expected_proportion = 0.5;
|
||||
let std_dev = (0.5 * 0.5 / total_predictions as f64).sqrt(); // Standard deviation for a binomial distribution
|
||||
|
||||
// Calculate the z-score
|
||||
let z_score = (observed_proportion - expected_proportion) / std_dev;
|
||||
|
||||
// Use normal distribution to calculate p-value
|
||||
let normal_dist = Normal::new(0.0, 1.0).expect("Failed to create Normal distribution");
|
||||
let p_value = 2.0 * (1.0 - normal_dist.cdf(z_score.abs()));
|
||||
|
||||
p_value
|
||||
}
|
||||
}
|
||||
74
src/uniformity/runs.rs
Normal file
74
src/uniformity/runs.rs
Normal file
|
|
@ -0,0 +1,74 @@
|
|||
use statrs::distribution::{Normal, ContinuousCDF};
|
||||
|
||||
use crate::Onod;
|
||||
|
||||
impl Onod {
|
||||
|
||||
/// Runs randomness test
|
||||
/// Evaluates the randomness by counting both increasing and decreasing runs and returns a p-value.
|
||||
pub fn runs(samples: &[u8]) -> f64 {
|
||||
|
||||
if samples.is_empty() {
|
||||
return 0.0;
|
||||
}
|
||||
|
||||
// Clone the samples to avoid modifying the original input
|
||||
let samples = samples.to_vec();
|
||||
let median = calculate_median(&samples);
|
||||
|
||||
let mut above = 0;
|
||||
let mut below = 0;
|
||||
|
||||
// Transform the data into a dichotomous vector and count above/below values
|
||||
let mut purged_samples = Vec::new();
|
||||
for &sample in &samples {
|
||||
if (sample as f64) > median {
|
||||
purged_samples.push(1); // Mark as above
|
||||
above += 1;
|
||||
} else if (sample as f64) < median {
|
||||
purged_samples.push(0); // Mark as below
|
||||
below += 1;
|
||||
}
|
||||
}
|
||||
|
||||
// Count runs
|
||||
let mut runs_observed = 1; // At least one run exists
|
||||
for window in purged_samples.windows(2) {
|
||||
if window[0] != window[1] {
|
||||
runs_observed += 1;
|
||||
}
|
||||
}
|
||||
|
||||
// Calculate expected runs and standard deviation
|
||||
let runs_expected = ((2.0 * above as f64 * below as f64) / (above + below) as f64) + 1.0;
|
||||
let std_dev = ((2.0 * above as f64 * below as f64 * (2.0 * above as f64 * below as f64 - above as f64 - below as f64))
|
||||
/ (((above + below) as f64).powi(2) * (above + below - 1) as f64))
|
||||
.sqrt();
|
||||
|
||||
// Calculate Z-score
|
||||
let z_score = (runs_observed as f64 - runs_expected) / std_dev;
|
||||
|
||||
// Use normal distribution to calculate p-value
|
||||
let normal_dist = Normal::new(0.0, 1.0).expect("Failed to create Normal distribution");
|
||||
let p_value = 2.0 * (1.0 - normal_dist.cdf(z_score.abs()));
|
||||
|
||||
if p_value.is_nan() {
|
||||
return 0.0; // Return 0 if p-value is NaN
|
||||
}
|
||||
|
||||
p_value
|
||||
}
|
||||
}
|
||||
|
||||
/// Helper function to calculate the median of a dataset
|
||||
fn calculate_median(samples: &[u8]) -> f64 {
|
||||
let mut sorted_samples = samples.to_vec();
|
||||
sorted_samples.sort_unstable();
|
||||
|
||||
let len = sorted_samples.len();
|
||||
if len % 2 == 0 {
|
||||
(sorted_samples[len / 2 - 1] as f64 + sorted_samples[len / 2] as f64) / 2.0
|
||||
} else {
|
||||
sorted_samples[len / 2] as f64
|
||||
}
|
||||
}
|
||||
44
src/uniformity/runups.rs
Normal file
44
src/uniformity/runups.rs
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
use statrs::distribution::{ChiSquared, ContinuousCDF};
|
||||
|
||||
use crate::Onod;
|
||||
|
||||
impl Onod {
|
||||
|
||||
/// RunUps randomness test
|
||||
/// Evaluates the number of four-byte run-ups in the data and returns a p-value.
|
||||
pub fn run_ups(input: &[u8]) -> f64 {
|
||||
|
||||
let samples = input.iter().map(|&x| x as u32).collect::<Vec<u32>>();
|
||||
|
||||
if samples.len() < 4 {
|
||||
return 0.0; // Not enough data for meaningful calculation
|
||||
}
|
||||
|
||||
let mut test_statistic = 0;
|
||||
|
||||
for chunk in samples.chunks(4) {
|
||||
if let [first, second, third, fourth] = chunk {
|
||||
if first < second && second < third && third < fourth {
|
||||
test_statistic += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let total_chunks = samples.len() / 4;
|
||||
let no_expected = 2_731_135.0 / 67_108_864.0 * total_chunks as f64;
|
||||
|
||||
let observed = [test_statistic as f64, total_chunks as f64 - test_statistic as f64];
|
||||
let expected = [no_expected, total_chunks as f64 - no_expected];
|
||||
|
||||
// Use chi-squared test to calculate p-value
|
||||
let chi_squared_dist = ChiSquared::new(1.0).expect("Failed to create ChiSquared distribution");
|
||||
let chi_squared_stat: f64 = observed.iter()
|
||||
.zip(expected.iter())
|
||||
.map(|(o, e)| (o - e).powi(2) / e)
|
||||
.sum();
|
||||
|
||||
let p_value = 1.0 - chi_squared_dist.cdf(chi_squared_stat);
|
||||
|
||||
p_value
|
||||
}
|
||||
}
|
||||
33
src/uniformity/sanity.rs
Normal file
33
src/uniformity/sanity.rs
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
use crate::Onod;
|
||||
|
||||
impl Onod {
|
||||
|
||||
/// Sanity randomness test
|
||||
/// Checks for basic properties of randomness and returns a p-value.
|
||||
pub fn sanity(samples: &[u8]) -> f64 {
|
||||
if samples.is_empty() {
|
||||
return 0.0;
|
||||
}
|
||||
|
||||
// Check the proportion of ones and zeros in the byte data
|
||||
let mut one_bits = 0;
|
||||
let mut total_bits = 0;
|
||||
|
||||
for &byte in samples {
|
||||
one_bits += byte.count_ones();
|
||||
total_bits += 8;
|
||||
}
|
||||
|
||||
let observed_ratio = one_bits as f64 / total_bits as f64;
|
||||
let expected_ratio = 0.5; // Expected for a truly random sequence
|
||||
let std_dev = (0.5 * 0.5 / total_bits as f64).sqrt(); // Standard deviation for binomial distribution
|
||||
|
||||
// Use normal distribution to calculate p-value
|
||||
use statrs::distribution::{Normal, ContinuousCDF};
|
||||
let normal_dist = Normal::new(expected_ratio, std_dev).expect("Failed to create Normal distribution");
|
||||
let p_value = 1.0 - normal_dist.cdf(observed_ratio);
|
||||
|
||||
p_value
|
||||
}
|
||||
|
||||
}
|
||||
43
src/uniformity/shannon.rs
Normal file
43
src/uniformity/shannon.rs
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
use statrs::distribution::{Normal, ContinuousCDF};
|
||||
|
||||
use crate::Onod;
|
||||
|
||||
impl Onod {
|
||||
/// Entropy randomness test
|
||||
/// Calculates the Shannon entropy of a byte slice and outputs a p-value.
|
||||
pub fn shannon(samples: &[u8]) -> f64 {
|
||||
|
||||
let len = samples.len() as f64;
|
||||
if len == 0.0 {
|
||||
return 0.0;
|
||||
}
|
||||
|
||||
// Count occurrences of each byte
|
||||
let mut counts = [0usize; 256];
|
||||
for &byte in samples {
|
||||
counts[byte as usize] += 1;
|
||||
}
|
||||
|
||||
// Calculate Shannon entropy
|
||||
let entropy: f64 = counts.iter()
|
||||
.filter(|&&count| count > 0)
|
||||
.map(|&count| {
|
||||
let p = count as f64 / len;
|
||||
-p * p.log2()
|
||||
})
|
||||
.sum();
|
||||
|
||||
// Expected entropy for a uniform distribution
|
||||
let expected_entropy = 8.0;
|
||||
|
||||
// Calculate Z statistic
|
||||
let std_dev = (0.833_f64).sqrt();
|
||||
let z_score = (entropy - expected_entropy) * len.sqrt() / std_dev;
|
||||
|
||||
// Calculate p-value from Z score
|
||||
let normal_dist = Normal::new(0.0, 1.0).expect("Failed to create Normal distribution");
|
||||
let p_value = 2.0 * (1.0 - normal_dist.cdf(z_score.abs())); // Two-tailed test
|
||||
|
||||
p_value
|
||||
}
|
||||
}
|
||||
96
src/uniformity/shells.rs
Normal file
96
src/uniformity/shells.rs
Normal file
|
|
@ -0,0 +1,96 @@
|
|||
use statrs::distribution::{ChiSquared, ContinuousCDF};
|
||||
|
||||
use crate::Onod;
|
||||
|
||||
impl Onod {
|
||||
|
||||
/// Shells randomness test
|
||||
/// Evaluates the uniformity of distances between identical byte values and returns a p-value.
|
||||
pub fn shells(input: &[u8]) -> f64 {
|
||||
|
||||
// Define shell radii (precomputed to ensure equal volumes)
|
||||
const SHELL_RADII: [f64; 35] = [
|
||||
1., 0.990384019787941, 0.980577593308067, 0.970571001281035, 0.960353705642329,
|
||||
0.949914251592996, 0.939240154232372, 0.928317766722556, 0.91713212619864,
|
||||
0.905666772691187, 0.893903535096568, 0.881822276616739, 0.869400589952457,
|
||||
0.856613429672063, 0.843432665301749, 0.829826533366243, 0.815758959214771,
|
||||
0.801188709029197, 0.786068317431936, 0.770342714221672, 0.753947441129154,
|
||||
0.736806299728077, 0.718828193851318, 0.699902804775202, 0.67989452969576,
|
||||
0.65863375600835, 0.635903899768996, 0.61142141746576, 0.584803547642573,
|
||||
0.555513224287824, 0.52275795857471, 0.485285500640517, 0.440911138308369,
|
||||
0.385171357110836, 0.30571070873288
|
||||
];
|
||||
|
||||
let samples = convert_to_3d_points(input);
|
||||
|
||||
if samples.len() < 25000 {
|
||||
eprintln!("Shells test requires at least 25,000 points for statistical validity.");
|
||||
return 0.0; // Skip the test for small datasets
|
||||
}
|
||||
|
||||
let sphere_radius = SHELL_RADII[0];
|
||||
let no_shells = SHELL_RADII.len();
|
||||
|
||||
// Calculate sphere and cube volume proportions
|
||||
let cube_side = 2.0 * sphere_radius;
|
||||
let cube_volume = cube_side.powi(3);
|
||||
let sphere_volume = (4.0 / 3.0) * std::f64::consts::PI * sphere_radius.powi(3);
|
||||
let sphere_proportion = sphere_volume / cube_volume; // Theoretical value: π/6
|
||||
|
||||
let no_points = samples.len() as f64;
|
||||
let no_points_per_shell = sphere_proportion * no_points / no_shells as f64;
|
||||
|
||||
let mut observed = vec![0u64; no_shells];
|
||||
let expected: Vec<f64> = vec![no_points_per_shell; no_shells];
|
||||
|
||||
for (x, y, z) in samples {
|
||||
// Compute radius from origin
|
||||
let radius = (x.powi(2) + y.powi(2) + z.powi(2)).sqrt();
|
||||
|
||||
// Ignore points outside the sphere
|
||||
if radius > sphere_radius {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Assign to the correct shell
|
||||
for j in 1..SHELL_RADII.len() {
|
||||
if radius > SHELL_RADII[j] {
|
||||
observed[j - 1] += 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if radius < SHELL_RADII[no_shells - 1] {
|
||||
observed[no_shells - 1] += 1;
|
||||
}
|
||||
}
|
||||
|
||||
// Perform Chi-Square Test
|
||||
let chi_squared_stat: f64 = observed.iter()
|
||||
.zip(expected.iter())
|
||||
.map(|(&o, &e)| (o as f64 - e).powi(2) / e)
|
||||
.sum();
|
||||
|
||||
let degrees_of_freedom = no_shells as f64 - 1.0;
|
||||
let chi_squared_dist = ChiSquared::new(degrees_of_freedom).expect("Failed to create ChiSquared distribution");
|
||||
let p_value = 1.0 - chi_squared_dist.cdf(chi_squared_stat);
|
||||
|
||||
p_value
|
||||
}
|
||||
}
|
||||
|
||||
fn convert_to_3d_points(data: &[u8]) -> Vec<(f64, f64, f64)> {
|
||||
let mut points = Vec::new();
|
||||
|
||||
for chunk in data.chunks(3) {
|
||||
if chunk.len() == 3 {
|
||||
// Normalize the bytes to [0.0, 1.0) range
|
||||
let x = chunk[0] as f64 / 255.0;
|
||||
let y = chunk[1] as f64 / 255.0;
|
||||
let z = chunk[2] as f64 / 255.0;
|
||||
points.push((x, y, z));
|
||||
}
|
||||
}
|
||||
|
||||
points
|
||||
}
|
||||
57
src/uniformity/uncorrelation.rs
Normal file
57
src/uniformity/uncorrelation.rs
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
use statrs::distribution::{Normal, ContinuousCDF};
|
||||
|
||||
use crate::Onod;
|
||||
|
||||
impl Onod {
|
||||
|
||||
/// UnCorrelation randomness test
|
||||
/// Computes the Pearson correlation between the sequence and its shifted version, returning a p-value.
|
||||
pub fn un_correlation(input: &[u8]) -> f64 {
|
||||
|
||||
let samples = input.iter().map(|&x| x as i32).collect::<Vec<i32>>();
|
||||
|
||||
if samples.len() < 2 {
|
||||
return 0.0; // Default to perfect randomness for insufficient data
|
||||
}
|
||||
|
||||
// Convert samples to f64 for correlation computation
|
||||
let samples_a: Vec<f64> = samples.iter().map(|&x| x as f64).collect();
|
||||
|
||||
// Create a shifted version of the sequence
|
||||
let mut samples_b = vec![0.0; samples.len()];
|
||||
samples_b[0] = samples_a[samples.len() - 1]; // Wrap around
|
||||
for i in 1..samples.len() {
|
||||
samples_b[i] = samples_a[i - 1];
|
||||
}
|
||||
|
||||
// Calculate mean of both sequences
|
||||
let mean_a = samples_a.iter().sum::<f64>() / samples_a.len() as f64;
|
||||
let mean_b = samples_b.iter().sum::<f64>() / samples_b.len() as f64;
|
||||
|
||||
// Compute Pearson correlation coefficient
|
||||
let mut numerator = 0.0;
|
||||
let mut denominator_a = 0.0;
|
||||
let mut denominator_b = 0.0;
|
||||
|
||||
for i in 0..samples.len() {
|
||||
let diff_a = samples_a[i] - mean_a;
|
||||
let diff_b = samples_b[i] - mean_b;
|
||||
numerator += diff_a * diff_b;
|
||||
denominator_a += diff_a.powi(2);
|
||||
denominator_b += diff_b.powi(2);
|
||||
}
|
||||
|
||||
let correlation = numerator / (denominator_a.sqrt() * denominator_b.sqrt());
|
||||
|
||||
// Calculate p-value for null hypothesis of zero correlation
|
||||
let n = samples.len() as f64;
|
||||
let t_stat = correlation * ((n - 2.0) / (1.0 - correlation.powi(2))).sqrt();
|
||||
|
||||
// Use t-distribution approximation for large n
|
||||
let normal_dist = Normal::new(0.0, 1.0).expect("Failed to create Normal distribution");
|
||||
let p_value = 2.0 * (1.0 - normal_dist.cdf(t_stat.abs()));
|
||||
|
||||
p_value
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue