Skip to content

Commit dec2a6a

Browse files
mfl28mfl28
committed
Add torchserve client and basic bounding box prediction functionality.
1 parent a5d9756 commit dec2a6a

17 files changed

+653
-49
lines changed

‎build.gradle‎

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,21 @@ dependencies {
7777

7878
// Google GSON https://github.com/google/gson
7979
implementation 'com.google.code.gson:gson:2.8.6'
80+
81+
// Jersey REST client https://mvnrepository.com/artifact/org.glassfish.jersey.core/jersey-client
82+
implementation 'org.glassfish.jersey.core:jersey-client:2.32'
83+
84+
// https://mvnrepository.com/artifact/javax.xml.bind/jaxb-api
85+
//implementation 'javax.xml.bind:jaxb-api:2.3.1'
86+
implementation 'org.glassfish.jersey.inject:jersey-hk2:2.32'
87+
// https://mvnrepository.com/artifact/org.glassfish.jersey.media/jersey-media-json-jackson
88+
implementation 'org.glassfish.jersey.media:jersey-media-json-jackson:2.32'
89+
90+
// https://mvnrepository.com/artifact/org.glassfish.jersey.media/jersey-media-multipart
91+
implementation 'org.glassfish.jersey.media:jersey-media-multipart:2.32'
92+
93+
// https://mvnrepository.com/artifact/org.jvnet.mimepull/mimepull
94+
implementation 'org.jvnet.mimepull:mimepull:1.9.13'
8095
}
8196

8297
javafx {
@@ -102,7 +117,7 @@ ci {
102117
githubactions {
103118
test {
104119
systemProperty "fullScreenTests", false
105-
jvmArgs = ['-Dprism.verbose=true']
120+
jvmArgs = ['-Dprism.verbose=true', '-Djava.net.preferIPv6Addresses=system']
106121

107122
if(org.gradle.internal.os.OperatingSystem.current().macOsX) {
108123
// Currently there is no support for running UI tests on macOS in a VM via github-actions.
@@ -138,7 +153,7 @@ compileJava.finalizedBy('compileLibSass')
138153
application {
139154
mainModule = "com.github.mfl28.boundingboxeditor"
140155
mainClass = "com.github.mfl28.boundingboxeditor.BoundingBoxEditorApp"
141-
applicationDefaultJvmArgs = ['-Dprism.forceGPU=true']
156+
applicationDefaultJvmArgs = ['-Dprism.forceGPU=true', '-Djava.net.preferIPv6Addresses=system']
142157
}
143158

144159
java {
@@ -149,7 +164,7 @@ jlink {
149164
options = ['--strip-debug', '--compress', '2', '--no-header-files', '--no-man-pages']
150165
launcher {
151166
name = 'BoundingBoxEditor'
152-
jvmArgs = ['-Dprism.forceGPU=true']
167+
jvmArgs = ['-Dprism.forceGPU=true', '-Djava.net.preferIPv6Addresses=system']
153168
}
154169

155170
imageZip = project.file("${buildDir}/distributions/boundingboxeditor-${javafx.platform.classifier}.zip")

‎src/main/java/com/github/mfl28/boundingboxeditor/controller/Controller.java‎

Lines changed: 76 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -19,20 +19,18 @@
1919
package com.github.mfl28.boundingboxeditor.controller;
2020

2121
import com.github.mfl28.boundingboxeditor.model.Model;
22-
import com.github.mfl28.boundingboxeditor.model.data.ImageAnnotation;
23-
import com.github.mfl28.boundingboxeditor.model.data.ImageMetaData;
24-
import com.github.mfl28.boundingboxeditor.model.data.IoMetaData;
25-
import com.github.mfl28.boundingboxeditor.model.data.ObjectCategory;
26-
import com.github.mfl28.boundingboxeditor.model.io.FileChangeWatcher;
27-
import com.github.mfl28.boundingboxeditor.model.io.ImageAnnotationLoadStrategy;
28-
import com.github.mfl28.boundingboxeditor.model.io.ImageAnnotationSaveStrategy;
22+
import com.github.mfl28.boundingboxeditor.model.data.*;
23+
import com.github.mfl28.boundingboxeditor.model.io.*;
24+
import com.github.mfl28.boundingboxeditor.model.io.results.BoundingBoxPredictionResult;
2925
import com.github.mfl28.boundingboxeditor.model.io.results.IOResult;
3026
import com.github.mfl28.boundingboxeditor.model.io.results.ImageAnnotationImportResult;
3127
import com.github.mfl28.boundingboxeditor.model.io.results.ImageMetaDataLoadingResult;
28+
import com.github.mfl28.boundingboxeditor.model.io.services.BoundingBoxPredictorService;
3229
import com.github.mfl28.boundingboxeditor.model.io.services.ImageAnnotationExportService;
3330
import com.github.mfl28.boundingboxeditor.model.io.services.ImageAnnotationImportService;
3431
import com.github.mfl28.boundingboxeditor.model.io.services.ImageMetaDataLoadingService;
3532
import com.github.mfl28.boundingboxeditor.ui.*;
33+
import com.github.mfl28.boundingboxeditor.ui.statusevents.BoundingBoxPredictionSuccessfulEvent;
3634
import com.github.mfl28.boundingboxeditor.ui.statusevents.ImageAnnotationsImportingSuccessfulEvent;
3735
import com.github.mfl28.boundingboxeditor.ui.statusevents.ImageAnnotationsSavingSuccessfulEvent;
3836
import com.github.mfl28.boundingboxeditor.ui.statusevents.ImageFilesLoadingSuccessfulEvent;
@@ -99,13 +97,9 @@ public class Controller {
9997
private static final String SAVE_IMAGE_ANNOTATIONS_ERROR_DIALOG_TITLE = "Save Error";
10098
private static final String NO_IMAGE_ANNOTATIONS_TO_SAVE_ERROR_DIALOG_CONTENT =
10199
"There are no image annotations to save.";
102-
private static final String SAVING_ANNOTATIONS_PROGRESS_DIALOG_TITLE = "Saving Annotations";
103-
private static final String SAVING_ANNOTATIONS_PROGRESS_DIALOGUE_HEADER = "Saving in progress...";
104100
private static final String ANNOTATION_IMPORT_ERROR_TITLE = "Annotation Import Error";
105101
private static final String ANNOTATION_IMPORT_ERROR_NO_VALID_FILES_CONTENT =
106102
"The source does not contain any valid annotations.";
107-
private static final String LOADING_ANNOTATIONS_DIALOG_TITLE = "Loading";
108-
private static final String LOADING_ANNOTATIONS_DIALOG_HEADER = "Loading annotations...";
109103
private static final String OPEN_IMAGE_FOLDER_OPTION_DIALOG_TITLE = "Open image folder";
110104
private static final String OPEN_IMAGE_FOLDER_OPTION_DIALOG_CONTENT =
111105
"Opening a new image folder will remove any existing annotation data. " +
@@ -136,15 +130,15 @@ public class Controller {
136130
private static final String IMAGE_IMPORT_ERROR_ALERT_TITLE = "Image Import Error";
137131
private static final String IMAGE_IMPORT_ERROR_ALERT_CONTENT =
138132
"The folder does not contain any valid image files.";
139-
private static final String IMAGE_FILES_LOADING_DIALOG_TITLE = "Loading images";
140-
private static final String IMAGE_FILES_LOADING_DIALOG_HEADER = "Loading image meta-data";
141133
private static final String IMAGE_FILES_CHANGED_ERROR_TITLE = "Image files changed";
142134
private static final String IMAGE_FILES_CHANGED_ERROR_CONTENT =
143135
"Image files were changed externally, will reload folder.";
144136
private static final String IMAGE_FILE_CHANGE_WATCHER_THREAD_NAME = "ImageFileChangeWatcher";
137+
145138
private final ImageAnnotationExportService annotationExportService = new ImageAnnotationExportService();
146139
private final ImageAnnotationImportService annotationImportService = new ImageAnnotationImportService();
147140
private final ImageMetaDataLoadingService imageMetaDataLoadingService = new ImageMetaDataLoadingService();
141+
private final BoundingBoxPredictorService boundingBoxPredictorService = new BoundingBoxPredictorService();
148142
private final Stage stage;
149143
private final MainView view = new MainView();
150144
private final Model model = new Model();
@@ -179,6 +173,15 @@ public Controller(final Stage mainStage) {
179173
view.connectToController(this);
180174
setUpModelListeners();
181175
setUpServices();
176+
connectServicesToView();
177+
}
178+
179+
private void connectServicesToView() {
180+
view.connectAnnotationImportService(annotationImportService);
181+
view.connectAnnotationExportService(annotationExportService);
182+
view.connectImageMetaDataLoadingService(imageMetaDataLoadingService);
183+
view.connectBoundingBoxPredictorService(boundingBoxPredictorService);
184+
view.setUpProgressDialogs();
182185
}
183186

184187
/**
@@ -194,6 +197,14 @@ public void onRegisterOpenImageFolderAction() {
194197
}
195198
}
196199

200+
public void onRegisterPerformCurrentImageBoundingBoxPredictionAction() {
201+
if(model.containsImageFiles()) {
202+
initiateBoundingBoxPrediction(model.getCurrentImageFile());
203+
} else {
204+
// TODO: Error handling
205+
}
206+
}
207+
197208
/**
198209
* Initiates the loading of image files from a provided folder.
199210
*
@@ -305,6 +316,11 @@ public void initiateAnnotationImport(File source, ImageAnnotationLoadStrategy.Ty
305316
startAnnotationImportService(source, loadFormat);
306317
}
307318

319+
public void initiateBoundingBoxPrediction(File imageFile) {
320+
updateModelFromView();
321+
startBoundingBoxPredictionService(imageFile);
322+
}
323+
308324
/**
309325
* Handles the event of the user adding a new object category.
310326
*/
@@ -555,6 +571,35 @@ public Model getModel() {
555571
return model;
556572
}
557573

574+
private void onBoundingBoxPredictionSucceeded(WorkerStateEvent event) {
575+
final BoundingBoxPredictionResult predictionResult = boundingBoxPredictorService.getValue();
576+
577+
if(predictionResult.getNrSuccessfullyProcessedItems() != 0) {
578+
model.updateFromImageAnnotationData(predictionResult.getImageAnnotationData());
579+
view.getStatusBar().setStatusEvent(new BoundingBoxPredictionSuccessfulEvent(predictionResult));
580+
}
581+
582+
updateViewFileExplorerFileInfoElements();
583+
584+
final ImageAnnotation annotation = model.getCurrentImageAnnotation();
585+
586+
if(annotation != null) {
587+
view.getObjectTree().reset();
588+
view.getCurrentBoundingShapes().removeListener(boundingShapeCountPerCategoryListener);
589+
view.loadBoundingShapeViewsFromAnnotation(annotation);
590+
view.getCurrentBoundingShapes().addListener(boundingShapeCountPerCategoryListener);
591+
view.getObjectCategoryTable().refresh();
592+
view.getObjectTree().refresh();
593+
}
594+
595+
if(!predictionResult.getErrorTableEntries().isEmpty()) {
596+
MainView.displayIOResultErrorInfoAlert(predictionResult);
597+
} else if(predictionResult.getNrSuccessfullyProcessedItems() == 0) {
598+
MainView.displayErrorAlert("Bounding Box Prediction Error",
599+
"Could not predict any bounding boxes.");
600+
}
601+
}
602+
558603
IoMetaData getIoMetaData() {
559604
return ioMetaData;
560605
}
@@ -578,13 +623,11 @@ Stage getStage() {
578623
void initiateAnnotationExport(File destination,
579624
ImageAnnotationSaveStrategy.Type exportFormat,
580625
Runnable chainedOperation) {
626+
annotationExportService.reset();
581627
annotationExportService.setDestination(destination);
582628
annotationExportService.setExportFormat(exportFormat);
583629
annotationExportService.setAnnotationData(model.createImageAnnotationData());
584630
annotationExportService.setChainedOperation(chainedOperation);
585-
annotationExportService.reset();
586-
MainView.displayServiceProgressDialog(annotationExportService, SAVING_ANNOTATIONS_PROGRESS_DIALOG_TITLE,
587-
SAVING_ANNOTATIONS_PROGRESS_DIALOGUE_HEADER);
588631
annotationExportService.restart();
589632
}
590633

@@ -598,10 +641,6 @@ private void startAnnotationImportService(File source, ImageAnnotationLoadStrate
598641
annotationImportService.setImportFormat(importFormat);
599642
annotationImportService.setImportableFileNames(model.getImageFileNameSet());
600643
annotationImportService.setCategoryNameToCategoryMap(model.getCategoryNameToCategoryMap());
601-
602-
MainView.displayServiceProgressDialog(annotationImportService, LOADING_ANNOTATIONS_DIALOG_TITLE,
603-
LOADING_ANNOTATIONS_DIALOG_HEADER);
604-
605644
annotationImportService.restart();
606645
}
607646

@@ -610,12 +649,23 @@ private void startImageMetaDataLoadingService(File source, List<File> imageFiles
610649
imageMetaDataLoadingService.setSource(source);
611650
imageMetaDataLoadingService.setImageFiles(imageFiles);
612651
imageMetaDataLoadingService.setReload(reload);
613-
MainView.displayServiceProgressDialog(imageMetaDataLoadingService, IMAGE_FILES_LOADING_DIALOG_TITLE,
614-
IMAGE_FILES_LOADING_DIALOG_HEADER);
615-
616652
imageMetaDataLoadingService.restart();
617653
}
618654

655+
private void startBoundingBoxPredictionService(File imageFile) {
656+
boundingBoxPredictorService.reset();
657+
boundingBoxPredictorService.setImageFile(imageFile);
658+
boundingBoxPredictorService.setCategoryNameToCategoryMap(model.getCategoryNameToCategoryMap());
659+
boundingBoxPredictorService.setImageMetaData(model.getImageFileNameToMetaDataMap().get(imageFile.getName()));
660+
boundingBoxPredictorService.setBoundingBoxPredictorConfig(model.getBoundingBoxPredictorConfig());
661+
662+
model.getBoundingBoxPredictorClientConfig().setInferenceModelName("fastrcnn");
663+
664+
boundingBoxPredictorService.setPredictorClient(BoundingBoxPredictorClient.create(model.getBoundingBoxPredictorClientConfig()));
665+
666+
boundingBoxPredictorService.restart();
667+
}
668+
619669
private void setUpServices() {
620670
annotationExportService.setOnSucceeded(this::onAnnotationExportSucceeded);
621671
annotationExportService.setOnFailed(this::onIoServiceFailed);
@@ -625,6 +675,9 @@ private void setUpServices() {
625675

626676
imageMetaDataLoadingService.setOnSucceeded(this::onImageMetaDataLoadingSucceeded);
627677
imageMetaDataLoadingService.setOnFailed(this::onIoServiceFailed);
678+
679+
boundingBoxPredictorService.setOnSucceeded(this::onBoundingBoxPredictionSucceeded);
680+
boundingBoxPredictorService.setOnFailed(this::onIoServiceFailed);
628681
}
629682

630683
private void onImageMetaDataLoadingSucceeded(WorkerStateEvent workerStateEvent) {

‎src/main/java/com/github/mfl28/boundingboxeditor/model/Model.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@
1919
package com.github.mfl28.boundingboxeditor.model;
2020

2121
import com.github.mfl28.boundingboxeditor.model.data.*;
22+
import com.github.mfl28.boundingboxeditor.model.io.BoundingBoxPredictorConfig;
23+
import com.github.mfl28.boundingboxeditor.model.io.BoundingBoxPredictorClientConfig;
2224
import javafx.beans.property.BooleanProperty;
2325
import javafx.beans.property.IntegerProperty;
2426
import javafx.beans.property.SimpleBooleanProperty;
@@ -80,6 +82,11 @@ public class Model {
8082
private final BooleanProperty nextImageFileExists = new SimpleBooleanProperty(false);
8183
private final BooleanProperty previousImageFileExists = new SimpleBooleanProperty(false);
8284
private final BooleanProperty saved = new SimpleBooleanProperty(true);
85+
86+
private final BoundingBoxPredictorClientConfig
87+
boundingBoxPredictorClientConfig = new BoundingBoxPredictorClientConfig();
88+
private final BoundingBoxPredictorConfig boundingBoxPredictorConfig = new BoundingBoxPredictorConfig();
89+
8390
/**
8491
* Maps the filenames of the currently loaded image-files onto the corresponding {@link File} objects. A
8592
* {@link ListOrderedMap} data-structure is used to preserve an order (in this case the input order) to
@@ -266,6 +273,14 @@ public int getCurrentFileIndex() {
266273
return fileIndex.get();
267274
}
268275

276+
public BoundingBoxPredictorClientConfig getBoundingBoxPredictorClientConfig() {
277+
return boundingBoxPredictorClientConfig;
278+
}
279+
280+
public BoundingBoxPredictorConfig getBoundingBoxPredictorConfig() {
281+
return boundingBoxPredictorConfig;
282+
}
283+
269284
/**
270285
* Returns the currently existing image-annotation data.
271286
*

‎src/main/java/com/github/mfl28/boundingboxeditor/model/io/BoundingBoxPredictionEntry.java‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
package com.github.mfl28.boundingboxeditor.model.io;
2+
3+
import java.util.List;
4+
import java.util.Map;
5+
6+
public class BoundingBoxPredictionEntry {
7+
private final Map<String, List<Double>> categoryToBoundingBoxes;
8+
private final Double score;
9+
10+
public BoundingBoxPredictionEntry(Map<String, List<Double>> categoryToBoundingBoxes, Double score) {
11+
this.categoryToBoundingBoxes = categoryToBoundingBoxes;
12+
this.score = score;
13+
}
14+
15+
public Map<String, List<Double>> getCategoryToBoundingBoxes() {
16+
return categoryToBoundingBoxes;
17+
}
18+
19+
public Double getScore() {
20+
return score;
21+
}
22+
}

‎src/main/java/com/github/mfl28/boundingboxeditor/model/io/BoundingBoxPredictor.java‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
package com.github.mfl28.boundingboxeditor.model.io;
2+
3+
import com.github.mfl28.boundingboxeditor.model.data.*;
4+
import com.github.mfl28.boundingboxeditor.model.io.results.BoundingBoxPredictionResult;
5+
import com.github.mfl28.boundingboxeditor.model.io.results.IOErrorInfoEntry;
6+
import com.github.mfl28.boundingboxeditor.utils.ColorUtils;
7+
8+
import java.io.File;
9+
import java.util.ArrayList;
10+
import java.util.HashMap;
11+
import java.util.List;
12+
import java.util.Map;
13+
14+
public class BoundingBoxPredictor {
15+
private final BoundingBoxPredictorClient client;
16+
private final BoundingBoxPredictorConfig predictorConfig;
17+
18+
public BoundingBoxPredictor(BoundingBoxPredictorClient client, BoundingBoxPredictorConfig predictorConfig) {
19+
this.client = client;
20+
this.predictorConfig = predictorConfig;
21+
}
22+
23+
public BoundingBoxPredictionResult predict(File imageFile, ImageMetaData imageMetaData,
24+
Map<String, ObjectCategory> existingCategoryNameToCategoryMap)
25+
throws Exception {
26+
return IOOperationTimer.time(() -> {
27+
final List<IOErrorInfoEntry> errorInfoEntries = new ArrayList<>();
28+
29+
final List<BoundingBoxPredictionEntry> boundingBoxPredictions;
30+
31+
try {
32+
boundingBoxPredictions = client.predict(imageFile);
33+
} catch(Exception e) {
34+
errorInfoEntries.add(new IOErrorInfoEntry("Torch serve model", e.getMessage()));
35+
return new BoundingBoxPredictionResult(
36+
0,
37+
errorInfoEntries,
38+
ImageAnnotationData.empty());
39+
}
40+
41+
if(boundingBoxPredictions == null || boundingBoxPredictions.isEmpty()) {
42+
errorInfoEntries.add(new IOErrorInfoEntry("Torch serve model",
43+
"No bounding boxes predicted for image " +
44+
imageFile.getName()));
45+
46+
return new BoundingBoxPredictionResult(
47+
0,
48+
errorInfoEntries,
49+
ImageAnnotationData.empty());
50+
}
51+
52+
final Map<String, Integer> categoryToCount = new HashMap<>();
53+
final ImageAnnotation imageAnnotation = new ImageAnnotation(imageMetaData);
54+
55+
for(BoundingBoxPredictionEntry entry : boundingBoxPredictions) {
56+
// TODO: nicer
57+
58+
if(entry.getScore() < predictorConfig.getMinimumScore()) {
59+
continue;
60+
}
61+
62+
var boundingbox = entry.getCategoryToBoundingBoxes().entrySet().iterator().next();
63+
ObjectCategory objectCategory = existingCategoryNameToCategoryMap.computeIfAbsent(boundingbox.getKey(),
64+
key -> new ObjectCategory(
65+
boundingbox
66+
.getKey(),
67+
ColorUtils
68+
.createRandomColor()));
69+
double xMin = boundingbox.getValue().get(0) / imageAnnotation.getImageWidth();
70+
double yMin = boundingbox.getValue().get(1) / imageAnnotation.getImageHeight();
71+
double xMax = boundingbox.getValue().get(2) / imageAnnotation.getImageWidth();
72+
double yMax = boundingbox.getValue().get(3) / imageAnnotation.getImageHeight();
73+
74+
imageAnnotation.getBoundingShapeData().add(new BoundingBoxData(objectCategory, xMin, yMin, xMax, yMax,
75+
new ArrayList<>()));
76+
categoryToCount.merge(objectCategory.getName(), 1, Integer::sum);
77+
}
78+
79+
return new BoundingBoxPredictionResult(1, errorInfoEntries,
80+
new ImageAnnotationData(List.of(imageAnnotation), categoryToCount,
81+
existingCategoryNameToCategoryMap));
82+
});
83+
}
84+
}

0 commit comments

Comments
 (0)