1 package IrrigationController;
3 import iotcode.interfaces.*;
6 import boofcv.alg.background.BackgroundModelStationary;
7 import boofcv.factory.background.ConfigBackgroundGaussian;
8 import boofcv.factory.background.FactoryBackgroundModel;
9 import boofcv.gui.binary.VisualizeBinaryData;
10 import boofcv.gui.image.ImageGridPanel;
11 import boofcv.gui.image.ShowImages;
12 import boofcv.io.MediaManager;
13 import boofcv.io.UtilIO;
14 import boofcv.io.image.SimpleImageSequence;
15 import boofcv.io.image.ConvertBufferedImage;
16 import boofcv.io.wrapper.DefaultMediaManager;
17 import boofcv.struct.image.ImageBase;
18 import boofcv.struct.image.ImageFloat32;
19 import boofcv.struct.image.ImageType;
20 import boofcv.struct.image.ImageUInt8;
21 import boofcv.alg.filter.blur.BlurImageOps;
24 // Standard Java Packages
25 import java.awt.image.BufferedImage;
26 import java.util.Date;
27 import java.util.concurrent.locks.Lock;
28 import java.util.concurrent.locks.ReadWriteLock;
29 import java.util.concurrent.locks.ReentrantReadWriteLock;
30 import java.util.concurrent.atomic.AtomicBoolean;
31 import java.awt.image.ColorModel;
32 import java.awt.image.WritableRaster;
33 import java.util.List;
34 import java.util.ArrayList;
35 import java.io.ByteArrayInputStream;
36 import java.io.ByteArrayOutputStream;
38 import java.io.IOException;
39 import java.io.InputStream;
40 import javax.imageio.ImageIO;
43 import java.rmi.RemoteException;
44 import java.rmi.server.UnicastRemoteObject;
46 // Checker annotations
47 //import iotchecker.qual.*;
51 /** Class MotionDetection to do motion detection using images
54 * @author Ali Younis <ayounis @ uci.edu>
58 class MotionDetection extends UnicastRemoteObject implements CameraCallback {
60 // Define Like variables
61 private static boolean DO_GRAPHICAL_USER_INTERFACE = false;
63 /*******************************************************************************************************************************************
67 *******************************************************************************************************************************************/
68 private final float MOTION_DETECTED_THRESHOLD_PERCENTAGE = 5;
71 /*******************************************************************************************************************************************
75 *******************************************************************************************************************************************/
77 // Timestamp buffer and locks needed for that safety on that buffer
78 // This is the buffer for post-detection algorithm use
79 private Date timestampOfLastMotion = null;
80 private ReadWriteLock timestampReadWriteLock = new ReentrantReadWriteLock();
81 private Lock timestampReadLock = timestampReadWriteLock.readLock();
82 private Lock timestampWriteLock = timestampReadWriteLock.writeLock();
84 // Flag for when new data is available and ready for processing
85 private AtomicBoolean newFrameAvailable = new AtomicBoolean(false);
87 // Flag for determining if motion has been detected and therefore
88 // the callbacks should be issued
89 private AtomicBoolean motionDetected = new AtomicBoolean(false);
91 // Image and timestamp buffers and locks needed for that safety on those buffers
92 // Timestamp buffer for pre-detection algorithm use
93 private BufferedImage latestImage = null;
94 private Date possibleDate = null;
95 private ReadWriteLock imageReadWriteLock = new ReentrantReadWriteLock();
96 private Lock imageReadLock = imageReadWriteLock.readLock();
97 private Lock imageWriteLock = imageReadWriteLock.writeLock();
99 // List of objects wishing to receive callbacks from this class.
100 private List<MotionDetectionCallback> callbackList = new ArrayList<MotionDetectionCallback>();
102 // Variables to help with motion detection
103 private ConfigBackgroundGaussian configGaussian = null;
104 private BackgroundModelStationary backgroundDetector = null;
105 private ImageUInt8 segmented = null;
106 private ImageFloat32 newFrameFloat = null;
108 // counts the number of frames since a background image is added to algorithm
109 private int frameCounter = 0;
113 /*******************************************************************************************************************************************
117 *******************************************************************************************************************************************/
118 private Thread workThread = null;
119 private Thread callBackThread = null;
121 /*******************************************************************************************************************************************
123 ** GUI Stuff (Used Only for Testing)
125 *******************************************************************************************************************************************/
132 * @param _threshold [float], Variable for gaussian background detector.
133 * @param _learnSpeed [float], Variable for gaussian background detector.
134 * @param _initialVariance [float], Variable for gaussian background detector.
135 * @param _minDifference [float], Variable for gaussian background detector.
138 public MotionDetection(float _threshold, float _learnSpeed, float _initialVariance, float _minDifference) throws RemoteException {
140 // Configure the Gaussian model used for background detection
141 configGaussian = new ConfigBackgroundGaussian(_threshold, _learnSpeed);
142 configGaussian.initialVariance = _initialVariance;
143 configGaussian.minimumDifference = _minDifference;
145 // setup the background detector
146 ImageType imageType = ImageType.single(ImageFloat32.class);
147 backgroundDetector = FactoryBackgroundModel.stationaryGaussian(configGaussian, imageType);
149 // setup the gui if we are going to use it
150 if (DO_GRAPHICAL_USER_INTERFACE) {
152 // create an image grid for images to place on, tile fashion
153 gui = new ImageGridPanel(1, 2);
155 // make the window large so we dont have to manually resize with the mouse
156 gui.setSize(1920, 1080);
158 // Make the window visible and set the title
159 ShowImages.showWindow(gui, "Static Scene: Background Segmentation", true);
162 // Launch the worker thread
163 workThread = new Thread(new Runnable() {
167 runMotionDetection();
174 // Launch the callback thread
175 callBackThread = new Thread(new Runnable() {
183 callBackThread.start();
187 /*******************************************************************************************************************************************
191 *******************************************************************************************************************************************/
193 /** Method to add a new frame to the motion detector.
195 * @param _newFrame [byte[]], Frame data of new frame.
196 * @param _timestamp [Date] , Timestamp of new frame.
198 * @return [void] None.
200 public void addFrame(byte[] _newFrame, Date _timestamp) {
201 BufferedImage img = null;
204 // Parse the byte array into a Buffered Image
205 InputStream in = new ByteArrayInputStream(_newFrame);
206 img = ImageIO.read(in);
208 } catch (Exception e) {
213 // Save the image and timestamp for use later
214 imageWriteLock.lock(); // lock the image and timestamp buffers since multithread
215 latestImage = img; // image into image buffer
216 possibleDate = _timestamp; // timestamp into timestamp buffer
217 imageWriteLock.unlock(); // Never forget to unlock
219 // flag the worker thread that there is new data ready for processing
220 newFrameAvailable.set(true);
223 /** Method to get the timestamp of the last time motion was detected
225 * @return [Date] timestamp of last motion or null if no motion was ever detected.
227 public long getTimestampOfLastMotion() {
230 // Be safe because multithread
231 timestampReadLock.lock();
233 // Checks if there was ever motion, if not then timestampOfLastMotion
235 if (timestampOfLastMotion != null) {
236 // Clone since we don't know what the other person is going to do
237 // with the timestamp
238 ret = (Date)timestampOfLastMotion.clone();
241 timestampReadLock.unlock();
242 long retLong = ret.getTime();
248 /** Method to add a new frame to the motion detector from a camera
250 * @param _camera [Camera], Camera that has the new data.
252 * @return [void] None.
254 //public void newCameraFrameAvailable(@NonLocalRemote Camera _camera) {
255 public void newCameraFrameAvailable(byte[] latestFrame, long timeStamp) {
256 BufferedImage img = null;
259 // Parse the byte array into a Buffered Image
260 //InputStream in = new ByteArrayInputStream(_camera.getLatestFrame());
261 InputStream in = new ByteArrayInputStream(latestFrame);
262 img = ImageIO.read(in);
264 } catch (RemoteException e) {
268 } catch (Exception e) {
274 // Save the image and timestamp for use later
275 imageWriteLock.lock(); // lock the image and timestamp buffers since multithread
276 latestImage = img; // image into image buffer
278 // timestamp from camera into timestamo buffer
280 //long dateLong = _camera.getTimestamp();
281 long dateLong = timeStamp;
282 possibleDate = new Date(dateLong);
283 //} catch (RemoteException e) {
284 // e.printStackTrace();
287 imageWriteLock.unlock(); // Never forget to unlock
289 // flag the worker thread that there is new data ready for processing
290 newFrameAvailable.set(true);
293 /** Method to register an object to recieve callbacks from this motion detector
295 * @param _mdc [MotionDetectionCallback], object to recieve callbacks.
297 * @return [void] None.
299 public void registerCallback(MotionDetectionCallback _mdc) {
300 callbackList.add(_mdc);
303 /*******************************************************************************************************************************************
307 *******************************************************************************************************************************************/
309 /** Method that constantly loops checking if new data is available. If there is
310 * new data, it is processed.
311 * This method should be run on a separate thread.
313 * @return [void] None.
315 private void runMotionDetection() {
317 // check if there is a new frame availble, only runs detection if there is new data to save
319 if (!newFrameAvailable.get()) {
323 // Lock since we are accessing the data buffers
324 imageReadLock.lock();
326 // processing data so now the buffered data is old
327 newFrameAvailable.set(false);
329 // copy from buffer to local for processing
330 Date tmpDate = possibleDate;
332 // Allocate space for the segmented image based on the first image we received
333 // cannot pre-allocate this since we do not know what the size of the images is
334 // before the first image arrives
335 if (segmented == null) {
336 segmented = new ImageUInt8(latestImage.getWidth(), latestImage.getHeight());
339 // copy from data buffers and convert into correct data type for BoofCv libraries
340 newFrameFloat = ConvertBufferedImage.convertFrom(latestImage, newFrameFloat);
342 // All done accessing the data buffers
343 imageReadLock.unlock();
345 // Run background detection
346 backgroundDetector.segment(newFrameFloat, segmented);
348 // Update the background baseline every 10 frames, helps the algorithm
350 if (frameCounter > 10) {
351 backgroundDetector.updateBackground(newFrameFloat);
355 // get the raw pixel data, gray-scale image
356 byte[] frameData = segmented.getData();
358 // count the number of pixels of the image that was deemed as "motion"
360 double countMotion = 0;
361 for (byte b : frameData) {
368 // calculate the percentage of the image that was in motion
369 double percentMotion = (countMotion / count) * 100.0;
371 // Check if a high enough percentage of the image was in motion to say that there was motion in this frame of data
372 if (percentMotion > MOTION_DETECTED_THRESHOLD_PERCENTAGE) {
374 // Motion detected so save timestamp of this frame to another buffer
375 timestampWriteLock.lock();
376 timestampOfLastMotion = (Date)tmpDate.clone(); // clone to a different buffer
377 timestampWriteLock.unlock();
379 System.out.println("Motion Detected (with percentage: " + Double.toString(percentMotion) + "%)");
382 // Do output to the screen if we are using gui mode
383 if (DO_GRAPHICAL_USER_INTERFACE) {
385 // change image data unto correct type for rendering
386 BufferedImage visualized1 = new BufferedImage(segmented.width, segmented.height, BufferedImage.TYPE_INT_RGB);
387 VisualizeBinaryData.renderBinary(segmented, false, visualized1);
389 // change image data unto correct type for rendering
390 BufferedImage visualized2 = null;
391 visualized2 = ConvertBufferedImage.convertTo(newFrameFloat, visualized2, true);
393 // place the images into the image grid
394 gui.setImage(0, 1, visualized1);
395 gui.setImage(0, 2, visualized2);
402 /** Method that constantly loops checking if the callbacks should be issues and
403 * issues the callbacks if they should be issues.
404 * This method should be run on a separate thread.
406 * @return [void] None.
408 private void doCallbacks() {
410 // Keep looping forever for callback
413 // If motion detected
414 if (motionDetected.compareAndSet(true, false)) {
416 // Motion was detected so issue callbacks to all objects that registered
417 // to receive callback from this class.
418 for (MotionDetectionCallback c : callbackList) {
420 c.motionDetected(this.getTimestampOfLastMotion());
421 //} catch (RemoteException re) {
427 // Sleep for 15 millisec to give time for new frame to arrive
430 } catch (InterruptedException ie) {