@inproceedings {Santa Clara, CA, title = {BNN Pruning: Pruning Binary Neural Network Guided by Weight Flipping Frequency}, year = {2020}, month = {03/2020}, address = {Santa Clara, CA}, abstract = {
A binary neural network (BNN) is a compact form of neural network. Both the weights and activations in BNNs can be binary values, which leads to a significant reduction in both parameter size and computational complexity compared to their full-precision counterparts. Such reductions can directly translate into reduced memory footprint and computation cost in hardware, making BNNs highly suitable for a wide range of hardware accelerators. However, it is unclear whether and how a BNN can be further pruned for ultimate compactness. As both 0s and 1s are non-trivial in BNNs, it is not proper to adopt any existing pruning method of full- precision networks that interprets 0s as trivial. In this paper, we present a pruning method tailored to BNNs and illustrate that BNNs can be further pruned by using weight flipping frequency as an indicator of sensitivity to accuracy. The experiments performed on the binary versions of a 9- layer Network-in-Network (NIN) and the AlexNet with the CIFAR-10 dataset show that the proposed BNN-pruning method can achieve 20-40\% reduction in binary operations with 0.5-1.0\% accuracy drop, which leads to a 15-40\% run- time speedup on a TitanX GPU.
}, keywords = {psclab}, author = {YIxing Li and Fengbo Ren} } @article {359, title = {Build a Compact Binary Neural Network through Bit-level Sensitivity and Data Pruning}, journal = { Neurocomputing}, volume = {398}, year = {2020}, month = {07/2020}, pages = {45-54}, abstract = {

Due to the high computational complexity and memory storage requirement, it is hard to directly de- ploy a full-precision convolutional neural network (CNN) on embedded devices. The hardware-friendly designs are needed for resource-limited and energy-constrained embedded devices. Emerging solutions are adopted for the neural network compression, e.g., binary/ternary weight network, pruned network and quantized network. Among them, binary neural network (BNN) is believed to be the most hardware- friendly framework due to its small network size and low computational complexity. No existing work has further shrunk the size of BNN. In this work, we explore the redundancy in BNN and build a com- pact BNN (CBNN) based on the bit-level sensitivity analysis and bit-level data pruning. The input data is converted to a high dimensional bit-sliced format. In the post-training stage, we analyze the impact of different bit slices to the accuracy. By pruning the redundant input bit slices and shrinking the network size, we are able to build a more compact BNN. Our result shows that we can further scale down the net- work size of the BNN up to 3.9x with no more than 1\% accuracy drop. The actual runtime can be reduced up to 2x and 9.9x compared with the baseline BNN and its full-precision counterpart, respectively.

}, keywords = {psclab}, author = {YIxing Li and Shuai Zhang and Xichuan Zhou and Fengbo Ren} } @inproceedings {hubara2016binarized, title = {Binarized neural networks}, year = {2016}, pages = {4107{\textendash}4115}, keywords = {Deep Learning, ref}, author = {Hubara, Itay and Courbariaux, Matthieu and Soudry, Daniel and El-Yaniv, Ran and Bengio, Yoshua} } @article {courbariaux2016binarized, title = {Binarized neural networks: Training deep neural networks with weights and activations constrained to+ 1 or-1}, journal = {arXiv preprint arXiv:1602.02830}, year = {2016}, keywords = {Deep Learning, ref}, author = {Courbariaux, Matthieu and Hubara, Itay and Soudry, Daniel and El-Yaniv, Ran and Bengio, Yoshua} } @inproceedings {2016, title = {A Binary Convolutional Encoder-decoder Network for Real-time Natural Scene Text Processing}, year = {2016}, month = {12/2016}, abstract = {

In this paper, we develop a binary convolutional encoder-decoder network (B-CEDNet) for natural scene text processing (NSTP). It converts a text image to a class-distinguished salience map that reveals the categorical, spatial and morphological information of characters. The existing solutions are either memory consuming or run-time consuming that cannot be applied to real-time applications on resource-constrained devices such as advanced driver assistance systems. The developed network can process multiple regions containing characters by one-off forward operation, and is trained to have binary weights and binary feature maps, which lead to both remarkable inference run-time speedup and memory usage reduction. By training with over 200, 000 synthesis scene text images (size of 32 \× 128), it can achieve 90\% and 91\% pixel-wise accuracy on ICDAR-03 and ICDAR-13 datasets. It only consumes 4.59 ms inference run-time realized on GPU with a small network size of 2.14 MB, which is up to 8\× faster and 96\% smaller than it full-precision version.\ 

}, keywords = {psclab}, author = {Liu, Zichuan and Yixing Li and Ren, Fengbo and Yu, Hao} } @article {kim2016bitwise, title = {Bitwise neural networks}, journal = {arXiv preprint arXiv:1601.06071}, year = {2016}, keywords = {Deep Learning, ref}, author = {Kim, Minje and Smaragdis, Paris} } @inproceedings {courbariaux2015binaryconnect, title = {Binaryconnect: Training deep neural networks with binary weights during propagations}, year = {2015}, pages = {3123{\textendash}3131}, keywords = {Deep Learning, ref}, author = {Courbariaux, Matthieu and Bengio, Yoshua and David, Jean-Pierre} } @article {32, title = {Body Voltage Sensing Based Short Pulse Reading Circuit}, number = {PCT/US2012/056136}, year = {2014}, month = {Mar 28, 2013}, chapter = {WO2013043738A1}, abstract = {

As memory geometries continue to scale down, current density of magnetic tunnel junctions (MTJs) make conventional low current reading scheme problematic with regard to performance and reliability. A body-voltage sense circuit (BVSC) short pulse reading (SPR) circuit is described using body connected load transistors and a novel sensing circuit with second stage amplifier which allows for very short read pulses providing much higher read margins, less sensing time, and shorter sensing current pulses. Simulation results (using 65-nm CMOS model SPICE simulations) show that our technique can achieve 550 mV of read margin at 1 ns performance under a 1 V supply voltage, which is greater than reference designs achieve at 5 ns performance.

}, keywords = {psclab}, issn = {US8917562B2}, url = {http://www.google.com/patents/WO2013043738A1}, author = {Kang-Lung Wang and Chih-Kong K. Yang and Dejan Markovic and Fengbo Ren} } @inproceedings {28, title = {A Body-voltage-sensing-based Short Pulse Reading Circuit for Spin-torque Transfer RAMs (STT-RAMs)}, year = {2012}, month = {03/2012}, pages = {275-282}, publisher = {IEEE}, abstract = {

With scaling of CMOS and Magnetic Tunnel Junction (MTJ) devices, conventional low-current reading techniques for STT-RAMs face challenges in achieving reliability and performance improvements that are expected from scaled devices. The challenges arise from the increasing variability of the CMOS sensing current and the reduction in MTJ switching current. This paper proposes a short-pulse reading circuit, based on a body-voltage sensing scheme to mitigate the scaling issues. Compared to existing sensing techniques, our technique shows substantially higher read margin (RM) despite a much shorter sensing time. A narrow current pulse applied to an MTJ significantly reduces the probability of read disturbance. The RM analysis is validated by Monte-Carlo simulations in a 65-nm CMOS technology with both CMOS and MTJ variations considered. Simulation results show that our technique is able to provide over 300 mV RM at a GHz frequency across process-voltage-temperature (PVT) variations, while the reference designs require 4.3 ns and 2.3 ns sensing time for a 200 mV RM, respectively. The effective read energy per bit required by the proposed sensing circuit is around 195 ft in the nominal case.

}, keywords = {psclab}, url = {http://ieeexplore.ieee.org/xpls/abs_all.jsp?arnumber=6187506}, author = {Ren, Fengbo and Park, Henry and Dorrance, Richard and Toriyama, Yuta and Yang, C-KK and Markovi{\'c}, Dejan} }