Skip to content

Commit d041d90

Browse files
mfriesenmfriesen
authored
v1.12.2 (#183)
* Added OCR support for DOC, DOCX, PDF
1 parent 4d5c0dd commit d041d90

File tree

46 files changed

+1271
-289
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

46 files changed

+1271
-289
lines changed

‎actions/src/main/java/com/formkiq/module/actions/services/ActionsService.java‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -74,15 +74,16 @@ public interface ActionsService {
7474
boolean hasActions(String siteId, String documentId);
7575

7676
/**
77-
* Save {@link List} {@link Action}.
77+
* Insert {@link Action}.
7878
*
7979
* @param siteId {@link String}
8080
* @param documentId {@link String}
8181
* @param actions {@link List} {@link Action}
82-
* @return {@link List} {@link Map}
82+
* @param currentAction {@link Action}
83+
* @param insertedAction {@link Action}
8384
*/
84-
List<Map<String, AttributeValue>> saveActions(String siteId, String documentId,
85-
List<Action> actions);
85+
void insertBeforeAction(String siteId, String documentId, List<Action> actions,
86+
Action currentAction, Action insertedAction);
8687

8788
/**
8889
* Save {@link Action}.
@@ -94,6 +95,17 @@ List<Map<String, AttributeValue>> saveActions(String siteId, String documentId,
9495
*/
9596
void saveAction(String siteId, String documentId, Action action, int index);
9697

98+
/**
99+
* Save {@link List} {@link Action}.
100+
*
101+
* @param siteId {@link String}
102+
* @param documentId {@link String}
103+
* @param actions {@link List} {@link Action}
104+
* @return {@link List} {@link Map}
105+
*/
106+
List<Map<String, AttributeValue>> saveActions(String siteId, String documentId,
107+
List<Action> actions);
108+
97109
/**
98110
* Update {@link Action} {@link ActionStatus}.
99111
*

‎actions/src/main/java/com/formkiq/module/actions/services/ActionsServiceDynamoDb.java‎

Lines changed: 32 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,19 @@ private Map<String, AttributeValue> buildValueMap(final String siteId, final Str
106106
return valueMap;
107107
}
108108

109+
private void deleteAction(final String siteId, final String documentId, final Action action,
110+
final int index) {
111+
112+
String pk = getPk(siteId, documentId);
113+
String sk = getSk(action, index);
114+
115+
Map<String, AttributeValue> key = Map.of(PK, AttributeValue.builder().s(pk).build(), SK,
116+
AttributeValue.builder().s(sk).build());
117+
118+
this.dbClient
119+
.deleteItem(DeleteItemRequest.builder().tableName(this.documentTableName).key(key).build());
120+
}
121+
109122
@Override
110123
public void deleteActions(final String siteId, final String documentId) {
111124

@@ -114,14 +127,7 @@ public void deleteActions(final String siteId, final String documentId) {
114127
int index = 0;
115128
for (Action action : actions) {
116129

117-
String pk = getPk(siteId, documentId);
118-
String sk = getSk(action, index);
119-
120-
Map<String, AttributeValue> key = Map.of(PK, AttributeValue.builder().s(pk).build(), SK,
121-
AttributeValue.builder().s(sk).build());
122-
123-
this.dbClient.deleteItem(
124-
DeleteItemRequest.builder().tableName(this.documentTableName).key(key).build());
130+
deleteAction(siteId, documentId, action, index);
125131

126132
index++;
127133
}
@@ -171,6 +177,24 @@ public boolean hasActions(final String siteId, final String documentId) {
171177
return !actions.isEmpty();
172178
}
173179

180+
@Override
181+
public void insertBeforeAction(final String siteId, final String documentId,
182+
final List<Action> actions, final Action currentAction, final Action insertedAction) {
183+
184+
int pos = actions.indexOf(currentAction);
185+
186+
saveAction(siteId, documentId, insertedAction, pos);
187+
188+
for (int i = pos; i < actions.size(); i++) {
189+
190+
Action action = actions.get(i);
191+
deleteAction(siteId, documentId, action, i);
192+
193+
pos++;
194+
saveAction(siteId, documentId, action, pos);
195+
}
196+
}
197+
174198
/**
175199
* Query Document Actions.
176200
*

‎actions/src/test/java/com/formkiq/module/actions/ActionsServiceDynamoDbTest.java‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,12 +28,15 @@
2828
import static org.junit.jupiter.api.Assertions.assertFalse;
2929
import static org.junit.jupiter.api.Assertions.assertNull;
3030
import static org.junit.jupiter.api.Assertions.assertTrue;
31+
import java.util.ArrayList;
3132
import java.util.Arrays;
3233
import java.util.Date;
34+
import java.util.Iterator;
3335
import java.util.List;
3436
import java.util.Map;
3537
import java.util.UUID;
3638
import org.junit.jupiter.api.BeforeAll;
39+
import org.junit.jupiter.api.Disabled;
3740
import org.junit.jupiter.api.Test;
3841
import org.junit.jupiter.api.extension.ExtendWith;
3942
import com.formkiq.aws.dynamodb.DynamoDbConnectionBuilder;
@@ -141,6 +144,47 @@ public void testDeleteDocumentActions() {
141144
}
142145
}
143146

147+
/**
148+
* Test Inserting OCR into fulltext list.
149+
*/
150+
@Test
151+
public void testInsertAction01() {
152+
// given
153+
String documentId = UUID.randomUUID().toString();
154+
String user = "joe";
155+
156+
for (String siteId : Arrays.asList(null, UUID.randomUUID().toString())) {
157+
158+
Action action0 = new Action().type(ActionType.DOCUMENTTAGGING).userId(user)
159+
.parameters(Map.of("tags", "type")).status(ActionStatus.COMPLETE);
160+
161+
Action action1 = new Action().type(ActionType.FULLTEXT).userId(user);
162+
Action insertedAction = new Action().type(ActionType.OCR).userId(user);
163+
164+
List<Action> actions = Arrays.asList(action0, action1);
165+
service.saveActions(siteId, documentId, actions);
166+
assertEquals(2, service.getActions(siteId, documentId).size());
167+
168+
// when
169+
service.insertBeforeAction(siteId, documentId, actions, action1, insertedAction);
170+
171+
// then
172+
final int expected = 3;
173+
List<Action> list = service.getActions(siteId, documentId);
174+
assertEquals(expected, list.size());
175+
176+
int i = 0;
177+
assertEquals(ActionType.DOCUMENTTAGGING, list.get(i).type());
178+
assertEquals(ActionStatus.COMPLETE, list.get(i++).status());
179+
180+
assertEquals(ActionType.OCR, list.get(i).type());
181+
assertEquals(ActionStatus.PENDING, list.get(i++).status());
182+
183+
assertEquals(ActionType.FULLTEXT, list.get(i).type());
184+
assertEquals(ActionStatus.PENDING, list.get(i++).status());
185+
}
186+
}
187+
144188
/**
145189
* Test Action.
146190
*
@@ -204,6 +248,38 @@ public void testSave01() throws Exception {
204248
}
205249
}
206250

251+
/**
252+
* Test save more than 10 actions. TODO enable in 1.13
253+
*/
254+
@Test
255+
@Disabled
256+
public void testSave02() {
257+
// given
258+
final int numberOfActions = 15;
259+
String documentId = UUID.randomUUID().toString();
260+
String user = "joe";
261+
for (String siteId : Arrays.asList(null, UUID.randomUUID().toString())) {
262+
263+
List<Action> actions = new ArrayList<>();
264+
for (int i = 0; i < numberOfActions; i++) {
265+
actions.add(new Action().type(ActionType.DOCUMENTTAGGING).userId(user)
266+
.parameters(Map.of("tags", "" + i)));
267+
}
268+
269+
// when
270+
service.saveActions(siteId, documentId, actions);
271+
272+
// then
273+
int i = 0;
274+
List<Action> list = service.getActions(siteId, documentId);
275+
Iterator<Action> itr = list.iterator();
276+
while (itr.hasNext()) {
277+
assertEquals("" + i, itr.next().parameters().get("tags"));
278+
i++;
279+
}
280+
}
281+
}
282+
207283
/**
208284
* Update Action Status.
209285
*/

‎aws-dynamodb/src/main/java/com/formkiq/aws/dynamodb/model/DocumentMapToDocument.java‎

Lines changed: 44 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,6 @@
3030
import java.util.Map;
3131
import java.util.function.Function;
3232
import java.util.stream.Collectors;
33-
import com.formkiq.aws.dynamodb.objects.Strings;
3433
import software.amazon.awssdk.services.dynamodb.model.AttributeValue;
3534

3635
/**
@@ -42,35 +41,46 @@ public class DocumentMapToDocument
4241
implements Function<Map<String, ? extends Object>, Map<String, Object>> {
4342

4443
/** Fields to Process. */
45-
private static final List<String> FIELDS = Arrays.asList("documentId", "path", "userId");
44+
private static final List<String> FIELDS =
45+
Arrays.asList("documentId", "path", "content", "contentType");
4646

4747
@SuppressWarnings("unchecked")
48-
@Override
49-
public Map<String, Object> apply(final Map<String, ? extends Object> data) {
48+
private String getValue(final Object obj) {
49+
String value = null;
5050

51-
Map<String, Object> document = new HashMap<>();
51+
if (obj != null) {
5252

53-
for (String field : FIELDS) {
53+
if (obj instanceof AttributeValue) {
5454

55-
Object ob = data.get(field);
55+
AttributeValue av = (AttributeValue) obj;
56+
value = av.s();
57+
58+
} else if (obj instanceof Map) {
5659

57-
if (ob != null) {
60+
Map<String, Object> values = (Map<String, Object>) obj;
61+
value = values.get("S").toString();
62+
63+
} else if (obj instanceof String) {
64+
value = obj.toString();
65+
}
66+
}
67+
68+
return value;
69+
}
5870

59-
String value = null;
71+
@SuppressWarnings("unchecked")
72+
@Override
73+
public Map<String, Object> apply(final Map<String, ? extends Object> data) {
6074

61-
if (ob instanceof AttributeValue) {
75+
Map<String, Object> document = new HashMap<>();
6276

63-
AttributeValue av = (AttributeValue) ob;
64-
value = av.s();
77+
for (String field : FIELDS) {
6578

66-
} else {
67-
Map<String, Object> values = (Map<String, Object>) data.get(field);
68-
value = values.get("S").toString();
69-
}
79+
Object ob = data.get(field);
80+
String value = getValue(ob);
7081

71-
if (!Strings.isEmpty(value)) {
72-
document.put(field, value);
73-
}
82+
if (value != null) {
83+
document.put(field, value);
7484
}
7585
}
7686

@@ -79,14 +89,24 @@ public Map<String, Object> apply(final Map<String, ? extends Object> data) {
7989
.map(e -> e.getKey()).collect(Collectors.toList());
8090

8191
metadata.forEach(m -> {
82-
Map<String, Object> values = (Map<String, Object>) data.get(m);
8392

84-
if (values.containsKey("S")) {
85-
String value = values.get("S").toString();
86-
document.put(m, value);
87-
}
93+
Map<String, Object> obj = (Map<String, Object>) data.get(m);
94+
String value = getValue(obj);
95+
String key = m.replaceAll("md#", "metadata#");
96+
document.put(key, value);
8897
});
8998

99+
if (data.containsKey("metadata")) {
100+
List<Map<String, String>> list = (List<Map<String, String>>) data.get("metadata");
101+
for (Map<String, String> map : list) {
102+
String key = map.get("key");
103+
String value = map.get("value");
104+
document.put("metadata#" + key, value != null ? value : "");
105+
}
106+
}
107+
108+
document.put("metadata#", "");
109+
90110
return document;
91111
}
92112
}

‎aws-dynamodb/src/main/java/com/formkiq/aws/dynamodb/model/DocumentToFulltextDocument.java‎

Lines changed: 0 additions & 83 deletions
This file was deleted.

‎aws-dynamodb/src/main/java/com/formkiq/aws/dynamodb/objects/MimeType.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@ public enum MimeType {
3131

3232
/** image/bmp. */
3333
MIME_BMP("image/bmp"),
34+
/** application/msword. */
35+
MIME_DOC("application/msword"),
3436
/** application/vnd.openxmlformats-officedocument.wordprocessingml.document. */
3537
MIME_DOCX("application/vnd.openxmlformats-officedocument.wordprocessingml.document"),
3638
/** image/gif. */

0 commit comments

Comments
 (0)