Merge pull request #6 from livetheoogway/candidate/1.0.8

Handling bugs on indexing null values
livetheoogway · Feb 24, 2023 · 67a6aec · 67a6aec
2 parents 2d013f4 + 1109644
commit 67a6aec
Show file tree

Hide file tree

Showing 27 changed files with 331 additions and 47 deletions.
diff --git a/forage-core/pom.xml b/forage-core/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <artifactId>forage</artifactId>
         <groupId>com.livetheoogway.forage</groupId>
-        <version>1.0.7</version>
+        <version>1.0.8</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/forage-core/src/main/java/com/livetheoogway/forage/core/AsyncQueuedConsumer.java b/forage-core/src/main/java/com/livetheoogway/forage/core/AsyncQueuedConsumer.java
@@ -141,6 +141,7 @@ private void finishListener() {
             consumer.finish();
         } catch (Exception e) {
             log.error("[forage] Error while finishing the listener", e);
+            itemConsumptionErrorHandler.handleError(null, e);
         }
     }
 

diff --git a/forage-core/src/main/java/com/livetheoogway/forage/core/UpdateEngine.java b/forage-core/src/main/java/com/livetheoogway/forage/core/UpdateEngine.java
@@ -17,6 +17,8 @@
 import com.livetheoogway.forage.models.DataId;
 import lombok.extern.slf4j.Slf4j;
 
+import java.util.concurrent.atomic.AtomicBoolean;
+
 /**
  * This is the main update engine that is responsible for exposing the boostrap function
  *
@@ -28,30 +30,41 @@ public abstract class UpdateEngine<D extends DataId> {
     private final Bootstrapper<D> bootstrapper;
     private final ItemConsumer<D> itemConsumer;
     private final ErrorHandler<D> errorHandler;
+    private final AtomicBoolean alreadyRunning;
 
     protected UpdateEngine(final Bootstrapper<D> bootstrapper,
                            final ItemConsumer<D> itemConsumer,
                            final ErrorHandler<D> errorHandler) {
         this.bootstrapper = bootstrapper;
         this.itemConsumer = itemConsumer;
         this.errorHandler = errorHandler;
+        this.alreadyRunning = new AtomicBoolean(false);
     }
 
     /**
      * the primary function that is supposed to bootstrap all items into the consumer
      */
     public void bootstrap() throws Exception {
-        log.info("[forage] Bootstrapping forage ...");
-        itemConsumer.init();
-        bootstrapper.bootstrap(item -> {
-            try {
-                itemConsumer.consume(item);
-            } catch (Exception e) {
-                errorHandler.handleError(item, e);
-            }
-        });
-        itemConsumer.finish();
-        log.info("[forage] ... Bootstrapping forage done");
+        if (alreadyRunning.get()) {
+            log.warn("A bootstrap is already running, not bootstrapping again..");
+            return;
+        }
+        alreadyRunning.set(true);
+        try {
+            log.info("[forage] Bootstrapping forage ...");
+            itemConsumer.init();
+            bootstrapper.bootstrap(item -> {
+                try {
+                    itemConsumer.consume(item);
+                } catch (Exception e) {
+                    errorHandler.handleError(item, e);
+                }
+            });
+            itemConsumer.finish();
+            log.info("[forage] ... Bootstrapping forage done");
+        } finally {
+            alreadyRunning.set(false);
+        }
     }
 
     /**

diff --git a/forage-dropwizard-bundle/pom.xml b/forage-dropwizard-bundle/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <artifactId>forage</artifactId>
         <groupId>com.livetheoogway.forage</groupId>
-        <version>1.0.7</version>
+        <version>1.0.8</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/...pwizard-bundle/src/main/java/com/livetheoogway/forage/dropwizard/bundle/ForageBundle.java b/...pwizard-bundle/src/main/java/com/livetheoogway/forage/dropwizard/bundle/ForageBundle.java
@@ -97,7 +97,8 @@ public void start() {
                 updateEngineRef.set(updateEngine);
 
                 updateEngineRef.get().start();
-                log.info("[forage][startup] .. Done starting engine and setting up periodic updates");
+                log.info("[forage][startup] .. Done starting engine and setting up periodic updates, every {}s",
+                         forageConfiguration(configuration).getRefreshIntervalInSeconds());
             }
 
             @Override

diff --git a/...ard-bundle/src/test/java/com/livetheoogway/forage/dropwizard/bundle/ForageBundleTest.java b/...ard-bundle/src/test/java/com/livetheoogway/forage/dropwizard/bundle/ForageBundleTest.java
@@ -19,10 +19,10 @@
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.livetheoogway.forage.core.Bootstrapper;
 import com.livetheoogway.forage.dropwizard.bundle.model.Book;
+import com.livetheoogway.forage.models.query.util.QueryBuilder;
 import com.livetheoogway.forage.models.result.ForageQueryResult;
 import com.livetheoogway.forage.search.engine.exception.ForageErrorCode;
 import com.livetheoogway.forage.search.engine.exception.ForageSearchError;
-import com.livetheoogway.forage.models.query.util.QueryBuilder;
 import com.livetheoogway.forage.search.engine.model.index.ForageDocument;
 import com.livetheoogway.forage.search.engine.model.index.IndexableDocument;
 import com.livetheoogway.forage.search.engine.store.Store;
@@ -33,6 +33,7 @@
 import io.dropwizard.lifecycle.setup.LifecycleEnvironment;
 import io.dropwizard.setup.AdminEnvironment;
 import io.dropwizard.setup.Environment;
+import lombok.extern.slf4j.Slf4j;
 import org.awaitility.Awaitility;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.BeforeEach;
@@ -43,11 +44,14 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
 import java.util.function.Consumer;
 
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
+@Slf4j
 class ForageBundleTest {
     private final HealthCheckRegistry healthChecks = mock(HealthCheckRegistry.class);
     private final MetricRegistry metricRegistry = new MetricRegistry();
@@ -56,6 +60,8 @@ class ForageBundleTest {
     private final Environment environment = mock(Environment.class);
     private final AdminEnvironment adminEnvironment = mock(AdminEnvironment.class);
 
+    private final CountDownLatch countDownLatch = new CountDownLatch(1);
+
     @BeforeEach
     public void setUp() {
         when(jerseyEnvironment.getResourceConfig()).thenReturn(new DropwizardResourceConfig());
@@ -68,7 +74,7 @@ public void setUp() {
     }
 
     @Test
-    void testBundleExecution() throws ForageSearchError {
+    void testBundleExecution() throws ForageSearchError, InterruptedException {
         final BookStore store = new BookStore();
         final ForageBundle<SampleConfig, Book> bundle = new ForageBundle<>() {
 
@@ -119,6 +125,14 @@ public ForageConfiguration forageConfiguration(final SampleConfig configuration)
                         return false;
                     }
                 });
+
+        log.info("Pausing for 3 seconds...");
+        final boolean await = countDownLatch.await(3, TimeUnit.SECONDS);
+        if (await) {
+            Assertions.fail();
+        }
+
+        /* query should work even after a few more refreshes */
         final ForageQueryResult<Book> results = bundle.searchEngine().search(
                 QueryBuilder.matchQuery("author", "rowling").buildForageQuery());
         Assertions.assertEquals("id1", results.getMatchingResults().get(0).getId());

diff --git a/forage-models/pom.xml b/forage-models/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <artifactId>forage</artifactId>
         <groupId>com.livetheoogway.forage</groupId>
-        <version>1.0.7</version>
+        <version>1.0.8</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/forage-models/src/main/java/com/livetheoogway/forage/models/result/field/FloatField.java b/forage-models/src/main/java/com/livetheoogway/forage/models/result/field/FloatField.java
@@ -18,6 +18,8 @@
 import lombok.ToString;
 import lombok.Value;
 
+import java.util.Arrays;
+
 @Value
 @EqualsAndHashCode(callSuper = true)
 @ToString(callSuper = true)
@@ -35,4 +37,14 @@ public FloatField(final String name, final float[] points) {
     public <T> T accept(final FieldVisitor<T> fieldVisitor) {
         return fieldVisitor.visit(this);
     }
+
+    @Override
+    public String toString() {
+        return new StringBuilder().append("FLOAT[")
+                .append(name)
+                .append(":")
+                .append(Arrays.toString(points))
+                .append("]")
+                .toString();
+    }
 }
diff --git a/forage-models/src/main/java/com/livetheoogway/forage/models/result/field/IntField.java b/forage-models/src/main/java/com/livetheoogway/forage/models/result/field/IntField.java
@@ -15,13 +15,13 @@
 package com.livetheoogway.forage.models.result.field;
 
 import lombok.EqualsAndHashCode;
-import lombok.ToString;
 import lombok.Value;
 
+import java.util.Arrays;
+
 @Value
 @EqualsAndHashCode(callSuper = true)
-@ToString(callSuper = true)
-public class IntField extends Field{
+public class IntField extends Field {
     String name;
     int[] points;
 
@@ -35,4 +35,10 @@ public IntField(final String name, final int[] points) {
     public <T> T accept(final FieldVisitor<T> fieldVisitor) {
         return fieldVisitor.visit(this);
     }
+
+    @Override
+    public String toString() {
+        return new StringBuilder().append("INT[").append(name)
+                .append(":").append(Arrays.toString(points)).append("]").toString();
+    }
 }
diff --git a/forage-models/src/main/java/com/livetheoogway/forage/models/result/field/StringField.java b/forage-models/src/main/java/com/livetheoogway/forage/models/result/field/StringField.java
@@ -15,15 +15,13 @@
 package com.livetheoogway.forage.models.result.field;
 
 import lombok.EqualsAndHashCode;
-import lombok.ToString;
 import lombok.Value;
 
 /**
  * Use this field to store fields verbatim (without analysis)
  */
 @Value
 @EqualsAndHashCode(callSuper = true)
-@ToString(callSuper = true)
 public class StringField extends Field {
     String name;
     String value;
@@ -38,4 +36,9 @@ public StringField(final String name, final String value) {
     public <T> T accept(final FieldVisitor<T> fieldVisitor) {
         return fieldVisitor.visit(this);
     }
+
+    @Override
+    public String toString() {
+        return new StringBuilder().append("STRING[").append(name).append(":").append(value).append("]").toString();
+    }
 }
diff --git a/forage-models/src/main/java/com/livetheoogway/forage/models/result/field/TextField.java b/forage-models/src/main/java/com/livetheoogway/forage/models/result/field/TextField.java
@@ -15,15 +15,13 @@
 package com.livetheoogway.forage.models.result.field;
 
 import lombok.EqualsAndHashCode;
-import lombok.ToString;
 import lombok.Value;
 
 /**
  * Use this for indexing fields for full text search
  */
 @Value
 @EqualsAndHashCode(callSuper = true)
-@ToString(callSuper = true)
 public class TextField extends Field {
     String name;
     String value;
@@ -38,4 +36,9 @@ public TextField(String name, String value) {
     public <T> T accept(FieldVisitor<T> fieldVisitor) {
         return fieldVisitor.visit(this);
     }
+
+    @Override
+    public String toString() {
+        return new StringBuilder().append("TEXT[").append(name).append(":").append(value).append("]").toString();
+    }
 }
diff --git a/forage-search-engine/pom.xml b/forage-search-engine/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <artifactId>forage</artifactId>
         <groupId>com.livetheoogway.forage</groupId>
-        <version>1.0.7</version>
+        <version>1.0.8</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/...in/java/com/livetheoogway/forage/search/engine/core/SearchEngineSwapReferenceHandler.java b/...in/java/com/livetheoogway/forage/search/engine/core/SearchEngineSwapReferenceHandler.java
@@ -17,6 +17,7 @@
 import com.livetheoogway.forage.core.ItemConsumer;
 import com.livetheoogway.forage.search.engine.DocumentIndexer;
 import com.livetheoogway.forage.search.engine.exception.ForageSearchError;
+import com.livetheoogway.forage.search.engine.util.Utils;
 import lombok.extern.slf4j.Slf4j;
 import lombok.val;
 
@@ -84,10 +85,12 @@ public void consume(final T indexableDocument) throws Exception {
     @Override
     public void finish() throws IOException, ForageSearchError {
         val writeStamp = lock.writeLock();
-        try (DocumentIndexer<T> ignored = liveReference.get()) {
+        try {
+            final DocumentIndexer<T> referenceToBeSwapped = liveReference.get();
             newReferenceBeingBuilt.get().flush();
             liveReference.set(newReferenceBeingBuilt.get());
             log.info("[forage] reference successfully swapped. Indexed: {}", counter.get());
+            Utils.closeSafe(referenceToBeSwapped, "referenceToBeSwapped");
         } finally {
             newReferenceBeingBuilt.set(null);
             lock.unlockWrite(writeStamp);

diff --git a/...ne/src/main/java/com/livetheoogway/forage/search/engine/lucene/LuceneDocumentHandler.java b/...ne/src/main/java/com/livetheoogway/forage/search/engine/lucene/LuceneDocumentHandler.java
@@ -14,8 +14,9 @@
 
 package com.livetheoogway.forage.search.engine.lucene;
 
-import com.livetheoogway.forage.search.engine.model.index.DocumentVisitor;
 import com.livetheoogway.forage.search.engine.lucene.field.LuceneFieldHandler;
+import com.livetheoogway.forage.search.engine.lucene.field.LuceneFieldValidator;
+import com.livetheoogway.forage.search.engine.model.index.DocumentVisitor;
 import com.livetheoogway.forage.search.engine.model.index.ForageDocument;
 import com.livetheoogway.forage.search.engine.model.index.LuceneDocument;
 import lombok.val;
@@ -26,12 +27,14 @@
 public class LuceneDocumentHandler implements DocumentVisitor<Document> {
     private static final String ID = "__ID__";
     private final LuceneFieldHandler fieldGenerator = new LuceneFieldHandler();
+    private final LuceneFieldValidator fieldValidator = new LuceneFieldValidator();
 
     @Override
     public Document visit(final ForageDocument forageDocument) {
         val document = new Document();
         forageDocument.getFields()
                 .stream()
+                .filter(field -> field.accept(fieldValidator))
                 .map(field -> field.accept(fieldGenerator))
                 .forEach(document::add);
         document.add(new StringField(ID, forageDocument.id(), Field.Store.YES));

diff --git a/...earch-engine/src/main/java/com/livetheoogway/forage/search/engine/lucene/LuceneIndex.java b/...earch-engine/src/main/java/com/livetheoogway/forage/search/engine/lucene/LuceneIndex.java
@@ -28,6 +28,4 @@ public interface LuceneIndex extends Closeable {
     void flush() throws ForageSearchError;
 
     DocRetriever docRetriever();
-
-    LuceneIndex freshIndex() throws ForageSearchError;
 }