1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30 package org.openimaj.tools.imagecollection.processor;
31
32 import java.io.ByteArrayOutputStream;
33 import java.io.IOException;
34 import java.net.URI;
35
36 import org.apache.hadoop.conf.Configuration;
37 import org.apache.hadoop.fs.FileSystem;
38 import org.apache.hadoop.fs.LocalFileSystem;
39 import org.apache.hadoop.fs.Path;
40 import org.apache.hadoop.io.BytesWritable;
41 import org.apache.hadoop.io.Text;
42 import org.openimaj.hadoop.sequencefile.SequenceFileUtility;
43 import org.openimaj.hadoop.sequencefile.TextBytesSequenceFileUtility;
44 import org.openimaj.image.Image;
45 import org.openimaj.image.ImageUtilities;
46 import org.openimaj.tools.imagecollection.collection.ImageCollectionEntry;
47
48 public class SequenceFileProcessor<T extends Image<?, T>> extends ImageCollectionProcessor<T> {
49
50 String sequenceFile = "output.seq";
51 boolean force = true;
52 private TextBytesSequenceFileUtility utility;
53 private int seen = 0;
54
55 public SequenceFileProcessor(String output, boolean force) {
56 this.sequenceFile = output;
57 this.force = force;
58 }
59
60 public static FileSystem getFileSystem(URI uri) throws IOException {
61 Configuration config = new Configuration();
62 FileSystem fs = FileSystem.get(uri, config);
63 if (fs instanceof LocalFileSystem) fs = ((LocalFileSystem)fs).getRaw();
64 return fs;
65 }
66
67 @Override
68 public void start() throws IOException{
69 if(force)
70 {
71 URI outuri = SequenceFileUtility.convertToURI(sequenceFile);
72 FileSystem fs = getFileSystem(outuri);
73 fs.delete(new Path(outuri.toString()), true);
74 }
75
76 utility = new TextBytesSequenceFileUtility(sequenceFile, false);
77 }
78
79 @Override
80 public String process(ImageCollectionEntry<T> image) throws Exception {
81 ByteArrayOutputStream bos = new ByteArrayOutputStream();
82 ImageUtilities.write(image.image, "png", bos);
83 BytesWritable bw = new BytesWritable(bos.toByteArray());
84 String imageName = "" + this.seen ;
85 utility.appendData(new Text(imageName), bw);
86 this.seen ++;
87 return imageName;
88 }
89
90 @Override
91 public void end() throws IOException{
92 utility.close();
93 }
94
95 }