1 package SmartLightsController;
4 import iotcode.annotation.*;
5 import iotcode.interfaces.*;
8 import boofcv.alg.background.BackgroundModelStationary;
9 import boofcv.factory.background.ConfigBackgroundGaussian;
10 import boofcv.factory.background.FactoryBackgroundModel;
11 import boofcv.gui.binary.VisualizeBinaryData;
12 import boofcv.gui.image.ImageGridPanel;
13 import boofcv.gui.image.ShowImages;
14 import boofcv.io.MediaManager;
15 import boofcv.io.UtilIO;
16 import boofcv.io.image.SimpleImageSequence;
17 import boofcv.io.image.ConvertBufferedImage;
18 import boofcv.io.wrapper.DefaultMediaManager;
19 import boofcv.struct.image.ImageBase;
20 import boofcv.struct.image.ImageFloat32;
21 import boofcv.struct.image.ImageType;
22 import boofcv.struct.image.ImageUInt8;
23 import boofcv.alg.filter.blur.BlurImageOps;
26 // Standard Java Packages
27 import java.awt.image.BufferedImage;
28 import java.util.Date;
29 import java.util.concurrent.locks.Lock;
30 import java.util.concurrent.locks.ReadWriteLock;
31 import java.util.concurrent.locks.ReentrantReadWriteLock;
32 import java.util.concurrent.atomic.AtomicBoolean;
33 import java.awt.image.ColorModel;
34 import java.awt.image.WritableRaster;
35 import java.util.List;
36 import java.util.ArrayList;
37 import java.io.ByteArrayInputStream;
38 import java.io.ByteArrayOutputStream;
40 import java.io.IOException;
41 import java.io.InputStream;
42 import javax.imageio.ImageIO;
45 import java.rmi.RemoteException;
50 import java.rmi.Remote;
51 import java.rmi.RemoteException;
53 import java.util.Iterator;
55 // Checker annotations
56 //import iotchecker.qual.*;
58 /** Class MotionDetection to do motion detection using images
61 * @author Ali Younis <ayounis @ uci.edu>
65 class MotionDetection implements CameraCallback {
67 // Define Like variables
68 private static boolean DO_GRAPHICAL_USER_INTERFACE = false;
70 /*******************************************************************************************************************************************
74 *******************************************************************************************************************************************/
75 private final float MOTION_DETECTED_THRESHOLD_PERCENTAGE = 15;
78 /*******************************************************************************************************************************************
82 *******************************************************************************************************************************************/
84 // Timestamp buffer and locks needed for that safety on that buffer
85 // This is the buffer for post-detection algorithm use
86 private Date timestampOfLastMotion = null;
87 private ReadWriteLock timestampReadWriteLock = new ReentrantReadWriteLock();
88 private Lock timestampReadLock = timestampReadWriteLock.readLock();
89 private Lock timestampWriteLock = timestampReadWriteLock.writeLock();
91 // Flag for when new data is available and ready for processing
92 private AtomicBoolean newFrameAvailable = new AtomicBoolean(false);
94 // Flag for determining if motion has been detected and therefore
95 // the callbacks should be issued
96 private AtomicBoolean motionDetected = new AtomicBoolean(false);
98 // Image and timestamp buffers and locks needed for that safety on those buffers
99 // Timestamp buffer for pre-detection algorithm use
100 private BufferedImage latestImage = null;
101 private Date possibleDate = null;
102 private ReadWriteLock imageReadWriteLock = new ReentrantReadWriteLock();
103 private Lock imageReadLock = imageReadWriteLock.readLock();
104 private Lock imageWriteLock = imageReadWriteLock.writeLock();
106 // List of objects wishing to receive callbacks from this class.
107 private List<MotionDetectionCallback>
108 callbackList = new ArrayList<MotionDetectionCallback>();
110 // Variables to help with motion detection
111 private ConfigBackgroundGaussian configGaussian = null;
112 private BackgroundModelStationary backgroundDetector = null;
113 private ImageUInt8 segmented = null;
114 private ImageFloat32 newFrameFloat = null;
116 // counts the number of frames since a background image is added to algorithm
117 private int frameCounter = 0;
119 private CameraSmart _camera;
121 /*******************************************************************************************************************************************
125 *******************************************************************************************************************************************/
126 private Thread workThread = null;
127 private Thread callBackThread = null;
129 /*******************************************************************************************************************************************
131 ** GUI Stuff (Used Only for Testing)
133 *******************************************************************************************************************************************/
138 * @param _threshold [float], Variable for gaussian background detector.
139 * @param _learnSpeed [float], Variable for gaussian background detector.
140 * @param _initialVariance [float], Variable for gaussian background detector.
141 * @param _minDifference [float], Variable for gaussian background detector.
144 public MotionDetection(float _threshold, float _learnSpeed, float _initialVariance, float _minDifference)
145 throws RemoteException {
147 // Configure the Gaussian model used for background detection
148 configGaussian = new ConfigBackgroundGaussian(_threshold, _learnSpeed);
149 configGaussian.initialVariance = _initialVariance;
150 configGaussian.minimumDifference = _minDifference;
152 // setup the background detector
153 ImageType imageType = ImageType.single(ImageFloat32.class);
154 backgroundDetector = FactoryBackgroundModel.stationaryGaussian(configGaussian, imageType);
156 // setup the gui if we are going to use it
157 if (DO_GRAPHICAL_USER_INTERFACE) {
159 // create an image grid for images to place on, tile fashion
160 gui = new ImageGridPanel(1, 2);
162 // make the window large so we dont have to manually resize with the mouse
163 gui.setSize(1920, 1080);
165 // Make the window visible and set the title
166 ShowImages.showWindow(gui, "Static Scene: Background Segmentation", true);
169 // Launch the worker thread
170 workThread = new Thread(new Runnable() {
174 runMotionDetection();
181 // Launch the callback thread
182 callBackThread = new Thread(new Runnable() {
190 callBackThread.start();
194 /*******************************************************************************************************************************************
198 *******************************************************************************************************************************************/
200 /** Method to add a new frame to the motion detector.
202 * @param _newFrame [byte[]], Frame data of new frame.
203 * @param _timestamp [Date] , Timestamp of new frame.
205 * @return [void] None.
207 public void addFrame(byte[] _newFrame, Date _timestamp) {
208 BufferedImage img = null;
211 // Parse the byte array into a Buffered Image
212 InputStream in = new ByteArrayInputStream(_newFrame);
213 img = ImageIO.read(in);
215 } catch (Exception e) {
220 // Save the image and timestamp for use later
221 imageWriteLock.lock();// lock the image and timestamp buffers since multithread
222 latestImage = img;// image into image buffer
223 possibleDate = _timestamp;// timestamp into timestamp buffer
224 imageWriteLock.unlock();// Never forget to unlock
226 // flag the worker thread that there is new data ready for processing
227 newFrameAvailable.set(true);
230 /** Method to get the timestamp of the last time motion was detected
232 * @return [Date] timestamp of last motion or null if no motion was ever detected.
234 public Date getTimestampOfLastMotion() {
237 // Be safe because multithread
238 timestampReadLock.lock();
240 // Checks if there was ever motion, if not then timestampOfLastMotion
242 if (timestampOfLastMotion != null) {
243 // Clone since we don't know what the other person is going to do
244 // with the timestamp
245 ret = (Date)timestampOfLastMotion.clone();
248 timestampReadLock.unlock();
254 /** Method to add a new frame to the motion detector from a camera
256 * @param _camera [Camera], Camera that has the new data.
258 * @return [void] None.
260 public void newCameraFrameAvailable(byte[] latestFrame, long timeStamp) {
261 //public void newCameraFrameAvailable(CameraSmart _camera) {
262 BufferedImage img = null;
264 //byte[] newImg = _camera.getLatestFrame();
266 //InputStream in = new ByteArrayInputStream(_camera.getLatestFrame());
267 //InputStream in = new ByteArrayInputStream(newImg);
268 // Parse the byte array into a Buffered Image
269 InputStream in = new ByteArrayInputStream(latestFrame);
270 img = ImageIO.read(in);
272 } catch (RemoteException e) {
276 } catch (Exception e) {
282 // Save the image and timestamp for use later
283 imageWriteLock.lock(); // lock the image and timestamp buffers since multithread
284 latestImage = img; // image into image buffer
286 // timestamp from camera into timestamp buffer
287 long dateLong = timeStamp;
288 //long dateLong = _camera.getTimestamp();
289 //System.out.println("DEBUG: New image at time: " + dateLong);
290 possibleDate = new Date(dateLong);
292 imageWriteLock.unlock(); // Never forget to unlock
294 // flag the worker thread that there is new data ready for processing
295 newFrameAvailable.set(true);
298 /** Method to register an object to recieve callbacks from this motion detector
300 * @param _mdc [MotionDetectionCallback], object to recieve callbacks.
302 * @return [void] None.
304 public void registerCallback(MotionDetectionCallback _mdc) {
305 callbackList.add(_mdc);
308 /*******************************************************************************************************************************************
312 *******************************************************************************************************************************************/
314 /** Method that constantly loops checking if new data is available. If there is
315 * new data, it is processed.
316 * This method should be run on a separate thread.
318 * @return [void] None.
320 private void runMotionDetection() {
322 // check if there is a new frame availble, only runs detection if there is new data to save
324 if (!newFrameAvailable.get()) {
328 // Lock since we are accessing the data buffers
329 imageReadLock.lock();
331 // processing data so now the buffered data is old
332 newFrameAvailable.set(false);
334 // copy from buffer to local for processing
335 Date tmpDate = possibleDate;
337 // Allocate space for the segmented image based on the first image we received
338 // cannot pre-allocate this since we do not know what the size of the images is
339 // before the first image arrives
340 if (segmented == null) {
341 segmented = new ImageUInt8(latestImage.getWidth(), latestImage.getHeight());
344 // copy from data buffers and convert into correct data type for BoofCv libraries
345 newFrameFloat = ConvertBufferedImage.convertFrom(latestImage, newFrameFloat);
347 // All done accessing the data buffers
348 imageReadLock.unlock();
350 // Run background detection
351 backgroundDetector.segment(newFrameFloat, segmented);
353 // Update the background baseline every 10 frames, helps the algorithm
355 if (frameCounter > 10) {
356 backgroundDetector.updateBackground(newFrameFloat);
360 // get the raw pixel data, gray-scale image
361 byte[] frameData = segmented.getData();
363 // count the number of pixels of the image that was deemed as "motion"
365 double countMotion = 0;
366 for (byte b : frameData) {
373 // calculate the percentage of the image that was in motion
374 double percentMotion = (countMotion / count) * 100.0;
375 //System.out.println("countMotion: " + countMotion);
376 //System.out.println("count: " + count);
377 //System.out.println("Percent motion: " + percentMotion);
379 // Check if a high enough percentage of the image was in motion to say that there was motion in this frame of data
380 if (percentMotion > MOTION_DETECTED_THRESHOLD_PERCENTAGE) {
382 // Motion detected so save timestamp of this frame to another buffer
383 timestampWriteLock.lock();
384 timestampOfLastMotion = (Date)tmpDate.clone();// clone to a different buffer
385 timestampWriteLock.unlock();
387 System.out.println("Motion Detected (with percentage: " + Double.toString(percentMotion) + "%)");
390 // Do output to the screen if we are using gui mode
391 if (DO_GRAPHICAL_USER_INTERFACE) {
393 // change image data unto correct type for rendering
394 BufferedImage visualized1 = new BufferedImage(segmented.width, segmented.height, BufferedImage.TYPE_INT_RGB);
395 VisualizeBinaryData.renderBinary(segmented, false, visualized1);
397 // change image data unto correct type for rendering
398 BufferedImage visualized2 = null;
399 visualized2 = ConvertBufferedImage.convertTo(newFrameFloat, visualized2, true);
401 // place the images into the image grid
402 gui.setImage(0, 1, visualized1);
403 gui.setImage(0, 2, visualized2);
410 /** Method that constantly loops checking if the callbacks should be issues and
411 * issues the callbacks if they should be issues.
412 * This method should be run on a separate thread.
414 * @return [void] None.
416 private void doCallbacks() {
418 // Keep looping forever for callback
421 // If motion detected
422 if (motionDetected.compareAndSet(true, false)) {
424 // Motion was detected so issue callbacks to all objects that registered
425 // to receive callback from this class.
426 for (MotionDetectionCallback c : callbackList) {
428 c.motionDetected(this);
429 //} catch (RemoteException re) {
435 // Sleep for 15 millisec to give time for new frame to arrive
438 } catch (InterruptedException ie) {
447 // /*******************************************************************************************************************************************
448 // ** Main Method used for testing
449 // *******************************************************************************************************************************************/
450 // public static void main(String[] args) {
451 // MotionDetection mo = new MotionDetection(12, 0.5f, 10, 10);
453 // AmcrestCamera cam = null;
456 // InetAddress addr = InetAddress.getByName("192.168.1.29");
457 // cam = new AmcrestCamera(addr, "admin", "55779CatSoundz32");
458 // cam.registerCallback(mo);
460 // // cam.streamDisconnect();
462 // } catch (Exception e) {