diff --git a/examples/measure_gemm_perf.rs b/examples/measure_gemm_perf.rs index fff356c..93a106b 100644 --- a/examples/measure_gemm_perf.rs +++ b/examples/measure_gemm_perf.rs @@ -15,7 +15,10 @@ fn main() { let c = a.product(&b); let end = start.elapsed().as_secs_f64(); - println!("Size: {n}x{n} Perf: {:.2} GFLOP/S", (2 * n.pow(3)) as f64 / end / 1000000000.0); + println!( + "Size: {n}x{n} Perf: {:.2} GFLOP/S", + (2 * n.pow(3)) as f64 / end / 1000000000.0 + ); } } diff --git a/examples/mnist.rs b/examples/mnist.rs index 543c88c..e7fbe57 100644 --- a/examples/mnist.rs +++ b/examples/mnist.rs @@ -1,4 +1,8 @@ -use std::{fs::File, io::{self, BufReader, Read, Write}, time::Instant}; +use std::{ + fs::File, + io::{self, BufReader, Read, Write}, + time::Instant, +}; use rand::seq::SliceRandom; @@ -6,17 +10,53 @@ use neural::nn::NeuralNetwork; macro_rules! verify_img_header { ($n:expr, $buf:expr) => { - assert_eq!((($buf[0] as i32) << 24)+ (($buf[1] as i32) << 16)+ (($buf[2] as i32) << 8)+ ($buf[3] as i32), 0x00000803); - assert_eq!((($buf[4] as i32) << 24)+ (($buf[5] as i32) << 16)+ (($buf[6] as i32) << 8)+ ($buf[7] as i32), $n); - assert_eq!((($buf[8] as i32) << 24)+ (($buf[9] as i32) << 16)+ (($buf[10] as i32) << 8)+ ($buf[11] as i32), 28); - assert_eq!((($buf[12] as i32) << 24)+ (($buf[13] as i32) << 16)+ (($buf[14] as i32) << 8)+ ($buf[15] as i32), 28); + assert_eq!( + (($buf[0] as i32) << 24) + + (($buf[1] as i32) << 16) + + (($buf[2] as i32) << 8) + + ($buf[3] as i32), + 0x00000803 + ); + assert_eq!( + (($buf[4] as i32) << 24) + + (($buf[5] as i32) << 16) + + (($buf[6] as i32) << 8) + + ($buf[7] as i32), + $n + ); + assert_eq!( + (($buf[8] as i32) << 24) + + (($buf[9] as i32) << 16) + + (($buf[10] as i32) << 8) + + ($buf[11] as i32), + 28 + ); + assert_eq!( + (($buf[12] as i32) << 24) + + (($buf[13] as i32) << 16) + + (($buf[14] as i32) << 8) + + ($buf[15] as i32), + 28 + ); }; } macro_rules! verify_labels_header { ($n:expr, $buf:expr) => { - assert_eq!((($buf[0] as i32) << 24)+ (($buf[1] as i32) << 16)+ (($buf[2] as i32) << 8)+ ($buf[3] as i32), 0x00000801); - assert_eq!((($buf[4] as i32) << 24)+ (($buf[5] as i32) << 16)+ (($buf[6] as i32) << 8)+ ($buf[7] as i32), $n); + assert_eq!( + (($buf[0] as i32) << 24) + + (($buf[1] as i32) << 16) + + (($buf[2] as i32) << 8) + + ($buf[3] as i32), + 0x00000801 + ); + assert_eq!( + (($buf[4] as i32) << 24) + + (($buf[5] as i32) << 16) + + (($buf[6] as i32) << 8) + + ($buf[7] as i32), + $n + ); }; } @@ -67,12 +107,15 @@ fn parse_training_images() -> Vec { assert!(label_buf[0] < 10); let mut label = [0.0; 10]; label[label_buf[0] as usize] = 1.0; - images.push( - Image { - data: img_buf.iter().map(|v| *v as f32 / 255.0).collect::>().try_into().unwrap(), - label, - } - ); + images.push(Image { + data: img_buf + .iter() + .map(|v| *v as f32 / 255.0) + .collect::>() + .try_into() + .unwrap(), + label, + }); } images @@ -101,12 +144,15 @@ fn parse_test_images() -> Vec { assert!(label_buf[0] < 10); let mut label = [0.0; 10]; label[label_buf[0] as usize] = 1.0; - images.push( - Image { - data: img_buf.iter().map(|v| *v as f32 / 255.0).collect::>().try_into().unwrap(), - label, - } - ); + images.push(Image { + data: img_buf + .iter() + .map(|v| *v as f32 / 255.0) + .collect::>() + .try_into() + .unwrap(), + label, + }); } images @@ -155,7 +201,6 @@ fn main() { println!("Accuracy: {:.20}%", before); } - let start = Instant::now(); let mut last = 0; // const TRAINING_ITERATIONS: usize = 100000; @@ -168,13 +213,20 @@ fn main() { nn.train(&training_img.data, &training_img.label); let elapsed_secs = start.elapsed().as_secs(); if elapsed_secs - last > 0 { - print!("\r\x1B[0JTraining... Elapsed time: {} [{index}/{TRAINING_ITERATIONS} {:.2}%]", secs_to_human(elapsed_secs), index as f32 / TRAINING_ITERATIONS as f32 * 100.0); + print!( + "\r\x1B[0JTraining... Elapsed time: {} [{index}/{TRAINING_ITERATIONS} {:.2}%]", + secs_to_human(elapsed_secs), + index as f32 / TRAINING_ITERATIONS as f32 * 100.0 + ); io::stdout().flush().unwrap(); last = elapsed_secs; } } - println!("\r\x1B[0JTrained on {TRAINING_ITERATIONS} images in {}", secs_to_human(start.elapsed().as_secs())); + println!( + "\r\x1B[0JTrained on {TRAINING_ITERATIONS} images in {}", + secs_to_human(start.elapsed().as_secs()) + ); println!("Testing..."); diff --git a/examples/xor.rs b/examples/xor.rs index eaf6131..acd2c8d 100644 --- a/examples/xor.rs +++ b/examples/xor.rs @@ -1,5 +1,5 @@ -use rand::seq::SliceRandom; use neural::nn::NeuralNetwork; +use rand::seq::SliceRandom; fn main() { let training_data = vec![ @@ -13,10 +13,22 @@ fn main() { nn.set_learning_rate(0.01); println!("Before training:"); - println!(" NN says: {:?} (should be ~1.0)", nn.feedforward(vec![1.0, 0.0])); - println!(" NN says: {:?} (should be ~1.0)", nn.feedforward(vec![0.0, 1.0])); - println!(" NN says: {:?} (should be ~0.0)", nn.feedforward(vec![0.0, 0.0])); - println!(" NN says: {:?} (should be ~0.0)", nn.feedforward(vec![1.0, 1.0])); + println!( + " NN says: {:?} (should be ~1.0)", + nn.feedforward(vec![1.0, 0.0]) + ); + println!( + " NN says: {:?} (should be ~1.0)", + nn.feedforward(vec![0.0, 1.0]) + ); + println!( + " NN says: {:?} (should be ~0.0)", + nn.feedforward(vec![0.0, 0.0]) + ); + println!( + " NN says: {:?} (should be ~0.0)", + nn.feedforward(vec![1.0, 1.0]) + ); for _ in 0..1000000 { let (inputs, target) = training_data.choose(&mut rand::thread_rng()).unwrap(); @@ -25,10 +37,22 @@ fn main() { println!("########################################"); println!("After training:"); - println!(" NN says: {:?} (should be ~1.0)", nn.feedforward(vec![1.0, 0.0])); - println!(" NN says: {:?} (should be ~1.0)", nn.feedforward(vec![0.0, 1.0])); - println!(" NN says: {:?} (should be ~0.0)", nn.feedforward(vec![0.0, 0.0])); - println!(" NN says: {:?} (should be ~0.0)", nn.feedforward(vec![1.0, 1.0])); + println!( + " NN says: {:?} (should be ~1.0)", + nn.feedforward(vec![1.0, 0.0]) + ); + println!( + " NN says: {:?} (should be ~1.0)", + nn.feedforward(vec![0.0, 1.0]) + ); + println!( + " NN says: {:?} (should be ~0.0)", + nn.feedforward(vec![0.0, 0.0]) + ); + println!( + " NN says: {:?} (should be ~0.0)", + nn.feedforward(vec![1.0, 1.0]) + ); // (Correct) Example output: // Before training: diff --git a/src/lib.rs b/src/lib.rs index a2fd881..c83aade 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,2 +1,2 @@ -pub mod nn; pub mod matrix; +pub mod nn; diff --git a/src/matrix.rs b/src/matrix.rs index 08b6495..57785ff 100644 --- a/src/matrix.rs +++ b/src/matrix.rs @@ -85,7 +85,7 @@ impl Matrix { // TODO: Do tilewise for better cache friendliness #[inline(always)] - pub fn product(&self, b: &Self ) -> Self { + pub fn product(&self, b: &Self) -> Self { #[cfg(debug_assertions)] assert_eq!(self.columns, b.size().0); @@ -107,10 +107,7 @@ impl Matrix { let mut sum = 0.0; for k in 0..self_cols { unsafe { - sum += - *a.offset(k + self_cols * i) - * - *btm.offset(k + b_t_cols * j); + sum += *a.offset(k + self_cols * i) * *btm.offset(k + b_t_cols * j); } } unsafe { @@ -144,10 +141,7 @@ impl Matrix { let mut sum = 0.0; for k in 0..self_cols { unsafe { - sum += - *a.offset(k + self_cols * i) - * - *btm.offset(k + b_t_cols * j); + sum += *a.offset(k + self_cols * i) * *btm.offset(k + b_t_cols * j); } } unsafe { @@ -233,34 +227,47 @@ mod tests { macro_rules! indx { ($a:expr, $cc:expr, $r:expr, $c:expr) => { $a[$c + $cc * $r] - } + }; } #[test] fn matrix_product() { let mut a = Matrix::new(2, 3); - indx!(a.data,a.columns,0,0)=3.0;indx!(a.data, a.columns,0,1)=6.0;indx!(a.data,a.columns,0,2)=7.0; - indx!(a.data,a.columns,1,0)=13.0;indx!(a.data, a.columns,1,1)=16.0;indx!(a.data,a.columns,1,2)=17.0; + indx!(a.data, a.columns, 0, 0) = 3.0; + indx!(a.data, a.columns, 0, 1) = 6.0; + indx!(a.data, a.columns, 0, 2) = 7.0; + indx!(a.data, a.columns, 1, 0) = 13.0; + indx!(a.data, a.columns, 1, 1) = 16.0; + indx!(a.data, a.columns, 1, 2) = 17.0; let mut b = Matrix::new(3, 2); - indx!(b.data,b.columns,0,0)=30.0;indx!(b.data, b.columns,0,1)=60.0; - indx!(b.data,b.columns,1,0)=130.0;indx!(b.data, b.columns,1,1)=160.0; - indx!(b.data,b.columns,2,0)=130.0;indx!(b.data, b.columns,2,1)=160.0; + indx!(b.data, b.columns, 0, 0) = 30.0; + indx!(b.data, b.columns, 0, 1) = 60.0; + indx!(b.data, b.columns, 1, 0) = 130.0; + indx!(b.data, b.columns, 1, 1) = 160.0; + indx!(b.data, b.columns, 2, 0) = 130.0; + indx!(b.data, b.columns, 2, 1) = 160.0; let res = a.product(&b); let mut exp = Matrix::new(2, 2); - indx!(exp.data,exp.columns,0,0)=1780.0;indx!(exp.data, exp.columns,0,1)=2260.0; - indx!(exp.data,exp.columns,1,0)=4680.0;indx!(exp.data, exp.columns,1,1)=6060.0; + indx!(exp.data, exp.columns, 0, 0) = 1780.0; + indx!(exp.data, exp.columns, 0, 1) = 2260.0; + indx!(exp.data, exp.columns, 1, 0) = 4680.0; + indx!(exp.data, exp.columns, 1, 1) = 6060.0; assert_eq!(res.data, exp.data); } #[test] fn matrix_map() { let mut a = Matrix::new(2, 2); - indx!(a.data,a.columns,0,0)=1.0;indx!(a.data, a.columns,0,1)=2.0; - indx!(a.data,a.columns,1,0)=3.0;indx!(a.data, a.columns,1,1)=4.0; + indx!(a.data, a.columns, 0, 0) = 1.0; + indx!(a.data, a.columns, 0, 1) = 2.0; + indx!(a.data, a.columns, 1, 0) = 3.0; + indx!(a.data, a.columns, 1, 1) = 4.0; a.map(&|v| v * 2.0); let mut exp = Matrix::new(2, 2); - indx!(exp.data,exp.columns,0,0)=2.0;indx!(exp.data, exp.columns,0,1)=4.0; - indx!(exp.data,exp.columns,1,0)=6.0;indx!(exp.data, exp.columns,1,1)=8.0; + indx!(exp.data, exp.columns, 0, 0) = 2.0; + indx!(exp.data, exp.columns, 0, 1) = 4.0; + indx!(exp.data, exp.columns, 1, 0) = 6.0; + indx!(exp.data, exp.columns, 1, 1) = 8.0; assert_eq!(a.data, exp.data); } @@ -268,19 +275,22 @@ mod tests { fn matrix_from_vec() { let a = Matrix::from_slice(&[1.0, 2.0, 3.0, 4.0]); let mut exp = Matrix::new(4, 1); - indx!(exp.data,exp.columns,0,0)=1.0; - indx!(exp.data, exp.columns,1,0)=2.0; - indx!(exp.data,exp.columns,2,0)=3.0; - indx!(exp.data, exp.columns,3,0)=4.0; + indx!(exp.data, exp.columns, 0, 0) = 1.0; + indx!(exp.data, exp.columns, 1, 0) = 2.0; + indx!(exp.data, exp.columns, 2, 0) = 3.0; + indx!(exp.data, exp.columns, 3, 0) = 4.0; assert_eq!(a.data, exp.data); } #[test] fn matrix_to_vec() { let mut a = Matrix::new(2, 3); - indx!(a.data,a.columns,0,0)=3.0;indx!(a.data, a.columns,0,1)=6.0;indx!(a.data,a.columns,0,2)=7.0; - indx!(a.data,a.columns,1,0)=13.0;indx!(a.data, a.columns,1,1)=16.0;indx!(a.data,a.columns,1,2)=17.0; + indx!(a.data, a.columns, 0, 0) = 3.0; + indx!(a.data, a.columns, 0, 1) = 6.0; + indx!(a.data, a.columns, 0, 2) = 7.0; + indx!(a.data, a.columns, 1, 0) = 13.0; + indx!(a.data, a.columns, 1, 1) = 16.0; + indx!(a.data, a.columns, 1, 2) = 17.0; assert_eq!(a.to_vec(), vec![3.0, 6.0, 7.0, 13.0, 16.0, 17.0]); } } - diff --git a/src/nn.rs b/src/nn.rs index 213cb5e..8863686 100644 --- a/src/nn.rs +++ b/src/nn.rs @@ -47,16 +47,14 @@ impl NeuralNetwork { let mut bias = Matrix::new(neuron_count, 1); bias.randomize(); - layers.push( - Layer { - weights, - bias, - gradients: Matrix::new(1, 1), - transposed: Matrix::new(1, 1), - weights_t: Matrix::new(input_weights_count, neuron_count), - weights_deltas: Matrix::new(1, 1), - } - ); + layers.push(Layer { + weights, + bias, + gradients: Matrix::new(1, 1), + transposed: Matrix::new(1, 1), + weights_t: Matrix::new(input_weights_count, neuron_count), + weights_deltas: Matrix::new(1, 1), + }); input_weights_count = neuron_count; } @@ -85,17 +83,16 @@ impl NeuralNetwork { pub fn feedforward(&mut self, input: Vec) -> Vec { let inputs = Matrix::from_slice(&input); - self.layers[0].weights.product_into( - &inputs, - &mut self.results[0] - ); + self.layers[0] + .weights + .product_into(&inputs, &mut self.results[0]); self.results[0].add_matrix(&self.layers[0].bias); self.results[0].map(&sigmoid); for (index, layer) in self.layers.iter().enumerate().skip(1) { layer.weights.product_into( - unsafe { &*self.results.as_ptr().offset(index as isize - 1)}, - &mut self.results[index] + unsafe { &*self.results.as_ptr().offset(index as isize - 1) }, + &mut self.results[index], ); self.results[index].add_matrix(&layer.bias); self.results[index].map(&sigmoid); @@ -108,17 +105,16 @@ impl NeuralNetwork { let inputs = Matrix::from_slice(inputs); let orig_inputs = inputs.clone(); - self.layers[0].weights.product_into( - &inputs, - &mut self.results[0] - ); + self.layers[0] + .weights + .product_into(&inputs, &mut self.results[0]); self.results[0].add_matrix(&self.layers[0].bias); self.results[0].map(&sigmoid); for (index, layer) in self.layers.iter().enumerate().skip(1) { layer.weights.product_into( - unsafe { &*self.results.as_ptr().offset(index as isize - 1)}, - &mut self.results[index] + unsafe { &*self.results.as_ptr().offset(index as isize - 1) }, + &mut self.results[index], ); self.results[index].add_matrix(&layer.bias); self.results[index].map(&sigmoid); @@ -136,7 +132,9 @@ impl NeuralNetwork { self.results[index - 1].transpose_into(&mut layer.transposed); - layer.gradients.product_into(&layer.transposed, &mut layer.weights_deltas); + layer + .gradients + .product_into(&layer.transposed, &mut layer.weights_deltas); layer.weights.add_matrix(&layer.weights_deltas); layer.bias.add_matrix(&layer.gradients); @@ -151,7 +149,9 @@ impl NeuralNetwork { layer.gradients.multiply_scalar(self.learning_rate); orig_inputs.transpose_into(&mut layer.transposed); - layer.gradients.product_into(&layer.transposed, &mut layer.weights_deltas); + layer + .gradients + .product_into(&layer.transposed, &mut layer.weights_deltas); layer.weights.add_matrix(&layer.weights_deltas); layer.bias.add_matrix(&layer.gradients);