001package org.openimaj.image;
002
003import java.awt.image.BufferedImage;
004import java.nio.ByteBuffer;
005import java.nio.ByteOrder;
006import java.nio.FloatBuffer;
007import java.util.ArrayList;
008import java.util.List;
009
010import org.openimaj.image.colour.ColourSpace;
011
012import com.nativelibs4java.opencl.CLContext;
013import com.nativelibs4java.opencl.CLEvent;
014import com.nativelibs4java.opencl.CLImage2D;
015import com.nativelibs4java.opencl.CLImageFormat;
016import com.nativelibs4java.opencl.CLImageFormat.ChannelDataType;
017import com.nativelibs4java.opencl.CLImageFormat.ChannelOrder;
018import com.nativelibs4java.opencl.CLMem;
019import com.nativelibs4java.opencl.CLQueue;
020
021/**
022 * Utility methods for converting between OpenIMAJ {@link Image} types
023 * and {@link CLImage2D}s for GPGPU acceleration.
024 * 
025 * @author Jonathon Hare (jsh2@ecs.soton.ac.uk)
026 */
027public class CLImageConversion {
028        private static final int FLOAT_SIZE = Float.SIZE / 8;
029        
030        /**
031         * The optimal channel orders for FImages. Only RGBA is guaranteed to 
032         * be available. 
033         */
034        private static final ChannelOrder[] bestFImageChannels = {
035                ChannelOrder.INTENSITY, ChannelOrder.LUMINANCE, ChannelOrder.R, ChannelOrder.RGBA
036        };
037        
038        private CLImageConversion() {}
039        
040        private static FloatBuffer convertToFBDirect(MBFImage image, ByteOrder byteOrder) {
041                final int width = image.getWidth();
042                final int height = image.getHeight();
043
044                final ByteBuffer bb = ByteBuffer.allocateDirect(width * height * 4 * FLOAT_SIZE);
045                bb.order(byteOrder);
046
047                final FloatBuffer fb = bb.asFloatBuffer();
048
049                final FImage[] bands = image.bands.toArray(new FImage[image.numBands()]);
050
051                final int nbands = Math.min(4, image.numBands());
052                final int extraBands = 4 - nbands;
053                
054                for (int y=0; y<height; y++) {
055                        for (int x=0; x<width; x++) {
056                                for (int b=0; b<nbands; b++) { 
057                                        fb.put(bands[b].pixels[y][x]);
058                                }
059                                for (int b=0; b<extraBands; b++) { 
060                                        fb.put(1);
061                                }
062                        }
063                }
064
065                return fb;
066        }
067
068        private static FloatBuffer convertToFBDirect(FImage image, ByteOrder byteOrder) {
069                final int width = image.getWidth();
070                final int height = image.getHeight();
071
072                final ByteBuffer bb = ByteBuffer.allocateDirect(width * height * FLOAT_SIZE);
073                bb.order(byteOrder);
074
075                final FloatBuffer fb = bb.asFloatBuffer();
076                final float[][] pix = image.pixels;
077                
078                for (int y=0; y<height; y++) {
079                        for (int x=0; x<width; x++) {
080                                fb.put( pix[y][x] );
081                        }
082                }
083
084                return fb;
085        }
086        
087        /**
088         * Converts an {@link MBFImage} to a {@link FloatBuffer} containing packed
089         * RGBA pixels. If the byte order is BIG_ENDIAN then the data is
090         * copied to a buffer on the Java heap which is then wrapped
091         * by a {@link FloatBuffer}. If the byte order is LITTLE_ENDIAN
092         * then we allocate a direct buffer in system RAM and copy
093         * the pixel values to that directly.
094         * 
095         * @param image The image to convert
096         * @param byteOrder the required byte order of the data
097         * @return a {@link FloatBuffer} containing packed RGBA pixels
098         */
099        public static FloatBuffer convertToFB(MBFImage image, ByteOrder byteOrder) {
100                if (byteOrder != ByteOrder.BIG_ENDIAN)
101                        return convertToFBDirect(image, byteOrder);
102
103                final int width = image.getWidth();
104                final int height = image.getHeight();
105                final float[] data = new float[width * height * 4];
106
107                final FImage[] bands = image.bands.toArray(new FImage[image.numBands()]);
108
109                final int nbands = Math.min(4, bands.length);
110                final int extraBands = 4 - nbands;
111                
112                for (int y=0, i=0; y<height; y++) {
113                        for (int x=0; x<width; x++) {
114                                for (int b=0; b<nbands; b++, i++) { 
115                                        data[i] = bands[b].pixels[y][x];
116                                }
117                                for (int b=0; b<extraBands; b++, i++) { 
118                                        data[i] = 1;
119                                }
120                        }
121                }
122
123                return FloatBuffer.wrap(data);
124        }
125        
126        /**
127         * Converts an FImage to a {@link FloatBuffer} containing packed
128         * intensity pixels. If the byte order is BIG_ENDIAN then the data is
129         * copied to a buffer on the Java heap which is then wrapped
130         * by a {@link FloatBuffer}. If the byte order is LITTLE_ENDIAN
131         * then we allocate a direct buffer in system RAM and copy
132         * the pixel values to that directly.
133         * 
134         * @param image The image to convert
135         * @param byteOrder the required byte order of the data
136         * @return a {@link FloatBuffer} containing packed intensity pixels
137         */
138        public static FloatBuffer convertToFB(FImage image, ByteOrder byteOrder) {
139                if (byteOrder != ByteOrder.BIG_ENDIAN)
140                        return convertToFBDirect(image, byteOrder);
141
142                final int width = image.getWidth();
143                final int height = image.getHeight();
144                
145                final float[] data = new float[width * height];
146                final float[][] pix = image.pixels;
147
148                for (int y=0, i=0; y<height; y++) {
149                        for (int x=0; x<width; x++, i++) {
150                                data[i] = pix[y][x];
151                        }
152                }
153
154                return FloatBuffer.wrap(data);
155        }
156
157        /**
158         * Convert an {@link MBFImage} to an RGBA {@link CLImage2D}.
159         * @param context the OpenCL context
160         * @param image the image to convert
161         * @return an OpenCL image
162         */
163        public static CLImage2D convert(CLContext context, MBFImage image) {
164                CLImageFormat format = new CLImageFormat(ChannelOrder.RGBA, ChannelDataType.Float);
165
166                FloatBuffer cvt = convertToFB(image, context.getByteOrder());
167                
168                return context.createImage2D(CLMem.Usage.InputOutput, 
169                                format, image.getWidth(), image.getHeight(), 
170                                4 * image.getWidth() * FLOAT_SIZE, cvt, false);
171        }
172        
173        private static ChannelOrder getBestFImageChannelOrder(CLContext context) {
174                CLImageFormat[] formats = context.getSupportedImageFormats(CLMem.Flags.ReadWrite, CLMem.ObjectType.Image2D);
175                List<ChannelOrder> found = new ArrayList<ChannelOrder>();
176                
177                for (CLImageFormat fmt : formats) {
178                        if (fmt.getChannelDataType() == ChannelDataType.Float)
179                                found.add(fmt.getChannelOrder());
180                }
181                
182                for (ChannelOrder co : bestFImageChannels) {
183                        if (found.contains(co))
184                                return co;
185                }
186                return ChannelOrder.RGBA;
187        }
188        
189        /**
190         * Convert an {@link FImage} to {@link CLImage2D}.
191         * @param context the OpenCL context
192         * @param image the image to convert
193         * @return an OpenCL image
194         */
195        public static CLImage2D convert(CLContext context, FImage image) {
196                ChannelOrder order = getBestFImageChannelOrder(context);
197                CLImageFormat format = new CLImageFormat(order, ChannelDataType.Float);
198
199                if (order == ChannelOrder.RGBA) {
200                        return convert(context, new MBFImage(image, image, image));
201                } else {
202                        FloatBuffer cvt = convertToFB(image, context.getByteOrder());
203                
204                        return context.createImage2D(CLMem.Usage.InputOutput, 
205                                        format, image.getWidth(), image.getHeight(), 
206                                        image.getWidth() * FLOAT_SIZE, cvt, false);
207                }
208        }
209
210        /**
211         * Convert an {@link Image} to {@link CLImage2D}. {@link MBFImage}s will
212         * be converted to floating-point {@link ChannelOrder#RGBA} {@link CLImage2D}s. 
213         * {@link FImage}s will be converted to either single band or RGBA floating-point
214         * {@link CLImage2D}s depending on the hardware. All other types of
215         * image are converted by first converting to {@link BufferedImage}s, and 
216         * will have 1-byte per band pixels (ARGB). 
217         * 
218         * @param <I> The type of image being converted 
219         * @param context the OpenCL context
220         * @param image the image to convert
221         * @return an OpenCL image
222         */
223        public static <I extends Image<?, I>> CLImage2D convert(CLContext context, I image) {
224                if (((Object)image) instanceof MBFImage)
225                        return convert(context, (MBFImage) ((Object)image));
226                if (((Object)image) instanceof FImage)
227                        return convert(context, (FImage) ((Object)image));
228                
229                BufferedImage bi = ImageUtilities.createBufferedImage(image);
230                
231                return context.createImage2D(CLMem.Usage.InputOutput, bi, true);
232        }
233        
234        /**
235         * Convert packed RGBA pixels in a {@link FloatBuffer} back into an
236         * {@link MBFImage}. The method takes the {@link MBFImage} as an argument, and
237         * will fill it accordingly. If the image argument is null, a new
238         * {@link MBFImage} with the RGBA {@link ColourSpace} will be created. 
239         * If the given image is the wrong size, it will be resized. If the given
240         * image has less than four bands, then only these bands will be filled. 
241         * Any bands above the fourth will be ignored.  
242         * 
243         * @param fb the {@link FloatBuffer} containing the pixels 
244         * @param width the width
245         * @param height the height
246         * @param image the image to write to or null
247         * @return the image
248         */
249        public static MBFImage convertFromFB(FloatBuffer fb, int width, int height, MBFImage image) {
250                if (image == null)
251                        image = new MBFImage(width, height, ColourSpace.RGBA);
252                
253                if (image.getWidth() != width || image.getHeight() != height)
254                        image.internalAssign(image.newInstance(width, height));
255                
256                final FImage[] bands = image.bands.toArray(new FImage[image.numBands()]);
257
258                final int nbands = Math.min(4, image.numBands());
259                final int extraBands = 4 - nbands;
260                
261                for (int y=0; y<height; y++) {
262                        for (int x=0; x<width; x++) {
263                                for (int b=0; b<nbands; b++) { 
264                                        bands[b].pixels[y][x] = fb.get();
265                                }
266                                for (int b=0; b<extraBands; b++) {
267                                        fb.get();
268                                }
269                        }
270                }
271
272                return image;
273        }
274        
275        /**
276         * Convert packed intensity pixels in a {@link FloatBuffer} back into an
277         * {@link FImage}. The method takes the {@link FImage} as an argument, and
278         * will fill it accordingly. If the image argument is null, a new
279         * {@link FImage} will be created.  If the given image is the wrong size, 
280         * it will be resized.   
281         * 
282         * @param fb the {@link FloatBuffer} containing the pixels 
283         * @param width the width
284         * @param height the height
285         * @param image the image to write to or null
286         * @return the image
287         */
288        public static FImage convertFromFB(FloatBuffer fb, int width, int height, FImage image) {
289                if (image == null)
290                        image = new FImage(width, height);
291                
292                if (image.getWidth() != width || image.getHeight() != height)
293                        image.internalAssign(image.newInstance(width, height));
294                
295                final float[][] pix = image.pixels;
296                
297                for (int y=0; y<height; y++) {
298                        for (int x=0; x<width; x++) {
299                                pix[y][x] = fb.get();
300                        }
301                }
302
303                return image;
304        }
305        
306        /**
307         * Convert a {@link CLImage2D} to an {@link MBFImage}.
308         * 
309         * @param queue the {@link CLQueue}
310         * @param evt event to wait for
311         * @param clImage the image to convert
312         * @param oiImage the output image (or null)
313         * @return the output image
314         */
315        public static MBFImage convert(CLQueue queue, CLEvent evt, CLImage2D clImage, MBFImage oiImage) {
316                final int width = (int) clImage.getWidth();
317                final int height = (int) clImage.getHeight();
318                
319                final ByteBuffer bb = ByteBuffer.allocateDirect(width * height * 4 * FLOAT_SIZE);
320                bb.order(clImage.getContext().getByteOrder());
321                
322                final FloatBuffer fb = bb.asFloatBuffer();
323
324                clImage.read(queue, 0, 0, width, height, clImage.getRowPitch(), fb, true, evt);
325
326                return convertFromFB(fb, width, height, oiImage);
327        }
328        
329        /**
330         * Convert a {@link CLImage2D} to an {@link FImage}.
331         * 
332         * @param queue the {@link CLQueue}
333         * @param evt event to wait for
334         * @param clImage the image to convert
335         * @param oiImage the output image (or null)
336         * @return the output image
337         */
338        public static FImage convert(CLQueue queue, CLEvent evt, CLImage2D clImage, FImage oiImage) {
339                if (clImage.getFormat().getChannelOrder() == ChannelOrder.RGBA) {
340                        return convert(queue, evt, clImage, oiImage == null ? null : new MBFImage(oiImage)).getBand(0);
341                }
342                
343                final int width = (int) clImage.getWidth();
344                final int height = (int) clImage.getHeight();
345                
346                final ByteBuffer bb = ByteBuffer.allocateDirect(width * height * FLOAT_SIZE);
347                bb.order(clImage.getContext().getByteOrder());
348                
349                final FloatBuffer fb = bb.asFloatBuffer();
350
351                clImage.read(queue, 0, 0, width, height, clImage.getRowPitch(), fb, true, evt);
352
353                return convertFromFB(fb, width, height, oiImage);
354        }
355        
356        /**
357         * Convert a {@link CLImage2D} to an {@link Image}.
358         * 
359         * @param <I> Type of image 
360         * @param queue the {@link CLQueue}
361         * @param evt event to wait for
362         * @param clImage the image to convert
363         * @param oiImage the output image (or null)
364         * @return the output image
365         */
366        @SuppressWarnings("unchecked")
367        public static <I extends Image<?, I>> I convert(CLQueue queue, CLEvent evt, CLImage2D clImage, I oiImage) {
368                if (oiImage == null)
369                        throw new IllegalArgumentException("Output image cannot be null");
370                
371                if (((Object)oiImage) instanceof MBFImage) {
372                        MBFImage i = (MBFImage) ((Object)oiImage);
373                        return (I) (Object)convert(queue, evt, clImage, i);
374                }
375                
376                if (((Object)oiImage) instanceof FImage) {
377                        FImage i = (FImage) ((Object)oiImage);
378                        return (I) (Object)convert(queue, evt, clImage, i);
379                }
380                
381                
382                BufferedImage bimg = clImage.read(queue, evt);
383                
384                return ImageUtilities.assignBufferedImage(bimg, oiImage);
385        }
386}