Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
  • Loading branch information
djtfmartin committed Jun 27, 2024
1 parent d342fa7 commit 84a32fc
Show file tree
Hide file tree
Showing 38 changed files with 1,103,685 additions and 235 deletions.
32 changes: 21 additions & 11 deletions matching-ws/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -9,17 +9,33 @@ ARG CLB_URL=""
ARG CLB_USER=""
ARG CLB_PASSWORD=""
ENV CLB_API_URL="https://api.checklistbank.org"
ENV BACKEND_BRANCH="matching-ws"
ARG GIT_BRANCH=""

# Clone the backend repository
RUN rm -Rf backend
RUN git clone https://github.com/CatalogueOfLife/backend.git
# Local builds - run from root of backend
COPY .. /app/backend
WORKDIR /app/backend
RUN git checkout $BACKEND_BRANCH

# Run a script to validate the arguments
#RUN if [ -z "$GIT_BRANCH" ]; then \
# echo "Error: GIT_BRANCH is not set." >&2; \
# exit 1; \
# fi

# Clone the backend repository
#RUN rm -Rf backend
#RUN git clone https://github.com/CatalogueOfLife/backend.git
#WORKDIR /app/backend
#RUN git checkout $GIT_BRANCH

# Build all the CLB modules
RUN mvn clean install package -DskipTests

# Build the Maven project and create a exec file
WORKDIR /app/backend/matching-ws

# Run tests - full backend tests require additional services (e.g. ES)
RUN mvn clean install package

# Store git commit id and log
RUN curl -o /app/backend/git.json -H "Accept: application/vnd.github+json" "https://api.github.com/repos/catalogueoflife/backend/commits/$(git rev-parse HEAD)"

Expand All @@ -38,12 +54,6 @@ RUN if [ -n "$CLB_DATASET_ID" ]; then \
echo "{}" > /app/backend/dataset.json; \
fi

# Build the Maven project and create a exec file
WORKDIR /app/backend/matching-ws

# Run tests - full backend tests require additional services (e.g. ES)
RUN mvn verify

# Copy the executable JAR file from the builder image to the new image
FROM openjdk:11

Expand Down
13 changes: 10 additions & 3 deletions matching-ws/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -58,10 +58,9 @@ mvn clean install spring-boot:repackage
### Software only
To build a software only docker image, run the following command from the root directory of this module (backend/matching-ws):
```bash
docker buildx build \
--platform linux/amd64 . -t matching-ws:v1
docker buildx build --platform linux/amd64 . -t matching-ws:1.0-SNAPSHOT
```
For this image to be of use, an lucene index must be pre-generated and mounted as a docker
For this image to be of use, a lucene index must be pre-generated and mounted as a docker
volume with the docker container.


Expand All @@ -87,6 +86,14 @@ docker buildx build \
--build-arg CLB_PASSWORD=****
```

### Local docker builds for development purposes

For local development, the following command can be used to build a docker image.
Run this from the root of the cloned copy of the `backend` repository:
```bash
docker build -f matching-ws/Dockerfile . -t matching-local-build
```

## Running docker image

Pre-built Docker images can be pulled from the Docker repository at docker.gbif.org and run as follows:
Expand Down
25 changes: 21 additions & 4 deletions matching-ws/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,7 @@
<spring-boot.version>2.7.18</spring-boot.version>
<lombok.version>1.18.22</lombok.version>
<spotless-maven-plugin.version>2.43.0</spotless-maven-plugin.version>
<gbif-api.version>1.6.0</gbif-api.version>
<logback.version>1.2.11</logback.version>
<logback.version>1.4.5</logback.version>
<mybatis-spring-boot.version>2.3.1</mybatis-spring-boot.version>
<gbif-common.version>0.60</gbif-common.version>
<jackson.version>2.17.0</jackson.version>
Expand Down Expand Up @@ -79,6 +78,10 @@
<artifactId>spring-boot-starter</artifactId>
<version>${spring-boot.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-to-slf4j</artifactId>
</exclusion>
<exclusion>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
Expand Down Expand Up @@ -146,22 +149,34 @@
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.36</version>
</dependency>
<!-- <dependency>-->
<!-- <groupId>org.slf4j</groupId>-->
<!-- <artifactId>slf4j-simple</artifactId>-->
<!-- <version>2.0.12</version>-->
<!-- </dependency>-->
<!-- <dependency>-->
<!-- <groupId>org.slf4j</groupId>-->
<!-- <artifactId>jul-to-slf4j</artifactId>-->
<!-- <version>2.0.6</version>-->
<!-- </dependency>-->
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-classic</artifactId>
<version>${logback.version}</version>
<version>1.2.13</version>
</dependency>
<dependency>
<groupId>ch.qos.logback</groupId>
<artifactId>logback-core</artifactId>
<version>${logback.version}</version>
<version>1.2.13</version>
</dependency>
<dependency>
<groupId>org.mybatis.spring.boot</groupId>
<artifactId>mybatis-spring-boot-starter</artifactId>
<version>${mybatis-spring-boot.version}</version>
</dependency>
<!-- enable for dev -->
<!-- <dependency>-->
<!-- <groupId>org.springframework.boot</groupId>-->
<!-- <artifactId>spring-boot-devtools</artifactId>-->
Expand Down Expand Up @@ -291,6 +306,8 @@
</configuration>
<executions>
<execution>
<id>repackage</id>
<phase>package</phase>
<goals>
<goal>repackage</goal>
</goals>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import org.springframework.context.annotation.Profile;

import java.text.NumberFormat;
import java.util.Optional;

/**
* Main application class for the matching-ws module.
Expand All @@ -41,36 +42,46 @@ public MatchingApplication(MatchingService matchingService) {

@Override
public void run(ApplicationArguments args) {
APIMetadata metadata = matchingService.getIndexMetadata();
LOG.info("Web services started. Index size: {} taxa, size on disk: {}",
NumberFormat.getInstance().format(metadata.getMainIndex().getNameUsageCount()),
metadata.getMainIndex().getSizeInMB() > 0 ? NumberFormat.getInstance().format(metadata.getMainIndex().getSizeInMB()) + "MB" : "unknown"
);

Optional<APIMetadata> metadata = matchingService.getIndexMetadata();
if (metadata.isEmpty()) {
LOG.error("No main index found. Cannot start web services");
return;
}

metadata.ifPresent(m -> {
LOG.info("Web services started. Index size: {} taxa, size on disk: {}",
NumberFormat.getInstance().format(
m.getMainIndex().getNameUsageCount()),
m.getMainIndex().getSizeInMB() > 0 ? NumberFormat.getInstance().format(m.getMainIndex().getSizeInMB()) + "MB" : "unknown");
});
}

@Bean
public OpenAPI customOpenAPI() {
APIMetadata metadata = matchingService.getIndexMetadata();
String title = metadata.getMainIndex().getDatasetTitle() != null ?
metadata.getMainIndex().getDatasetTitle() + " Matching Service API" :
"COL Matching Service API";
String description = "API for matching scientific names to taxa in the checklist" +
(metadata.getMainIndex().getDatasetTitle() != null ? " " + metadata.getMainIndex().getDatasetTitle() : "");

OpenAPI openAPI = new OpenAPI()
.info(new Info()
.title(title)
.description(description)
.version(version)
.license(new License()
.name(licence)
.url(licenceUrl)));

if (metadata.getMainIndex().getDatasetKey() != null) {
openAPI.externalDocs(new ExternalDocumentation()
.description(metadata.getMainIndex().getDatasetTitle())
.url("https://checklistbank.org/dataset/" + metadata.getMainIndex().getDatasetKey()));
}
Optional<APIMetadata> metadata = matchingService.getIndexMetadata();
OpenAPI openAPI = new OpenAPI();
metadata.ifPresent(m -> {
String title = m.getMainIndex().getDatasetTitle() != null ?
m.getMainIndex().getDatasetTitle() + " Matching Service API" :
"COL Matching Service API";
String description = "API for matching scientific names to taxa in the checklist" +
(m.getMainIndex().getDatasetTitle() != null ? " " + m.getMainIndex().getDatasetTitle() : "");

openAPI.info(new Info()
.title(title)
.description(description)
.version(version)
.license(new License()
.name(licence)
.url(licenceUrl)));

if (m.getMainIndex().getDatasetKey() != null) {
openAPI.externalDocs(new ExternalDocumentation()
.description(m.getMainIndex().getDatasetTitle())
.url("https://checklistbank.org/dataset/" + m.getMainIndex().getDatasetKey()));
}
});

return openAPI;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,11 @@
@RestController
public class IDController {

@Autowired
MatchingService matchingService;
final MatchingService matchingService;

public IDController(MatchingService matchingService) {
this.matchingService = matchingService;
}

@Operation(
operationId = "idLookupByDataset",
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package life.catalogue.matching.controller;

import static life.catalogue.matching.util.CleanupUtils.clean;
import static life.catalogue.matching.service.MatchingService.first;
import static life.catalogue.matching.util.CleanupUtils.*;

import io.swagger.v3.oas.annotations.Hidden;
import io.swagger.v3.oas.annotations.Operation;
Expand All @@ -23,7 +23,7 @@
import life.catalogue.matching.service.MatchingService;
import life.catalogue.parser.RankParser;
import life.catalogue.parser.UnparsableException;
import org.apache.commons.lang3.StringUtils;

import org.gbif.nameparser.api.Rank;

import org.springdoc.api.annotations.ParameterObject;
Expand All @@ -46,9 +46,7 @@
@RestController
public class MatchController implements ErrorController {

@Autowired
MatchingService matchingService;

private final MatchingService matchingService;
private final ErrorAttributes errorAttributes;

@Value("${v1.enabled:false}")
Expand All @@ -72,14 +70,14 @@ public Map<String, Object> error(WebRequest request) {
}

private boolean isTraceRequested(String traceRequested) {
return StringUtils.isNotBlank(traceRequested) &&
(traceRequested.equalsIgnoreCase("true") || traceRequested.equalsIgnoreCase("on"));
return "true".equalsIgnoreCase(traceRequested) || "on".equalsIgnoreCase(traceRequested);
}

@Autowired
public MatchController(ErrorAttributes errorAttributes) {
public MatchController(ErrorAttributes errorAttributes, MatchingService matchingService) {
Assert.notNull(errorAttributes, "ErrorAttributes must not be null");
this.errorAttributes = errorAttributes;
this.matchingService = matchingService;
}

private Map<String, Object> getErrorAttributes(WebRequest request) {
Expand Down Expand Up @@ -112,7 +110,7 @@ private Map<String, Object> getErrorAttributes(WebRequest request) {
@GetMapping(
value = {"v2/metadata"},
produces = "application/json")
public APIMetadata metadata(){
public Optional<APIMetadata> metadata(){
return matchingService.getIndexMetadata();
}

Expand Down Expand Up @@ -397,6 +395,9 @@ public NameUsageMatch matchV2(
produces = "application/json")
public Object matchFlatV1(
@RequestParam(value = "usageKey", required = false) String usageKey,
@RequestParam(value = "taxonID", required = false) String taxonID,
@RequestParam(value = "taxonConceptID", required = false) String taxonConceptID,
@RequestParam(value = "scientificNameID", required = false) String scientificNameID,
@RequestParam(value = "name", required = false) String scientificName2,
@RequestParam(value = "scientificName", required = false) String scientificName,
@RequestParam(value = "authorship", required = false) String authorship2,
Expand All @@ -419,9 +420,9 @@ public Object matchFlatV1(
Optional<NameUsageMatchFlatV1> optionalNameUsageMatchV1 = NameUsageMatchFlatV1.createFrom(
matchingService.match(
removeNulls(usageKey),
null,
null,
null,
removeNulls(taxonID),
removeNulls(taxonConceptID),
removeNulls(scientificNameID),
first(removeNulls(scientificName), removeNulls(scientificName2)),
first(removeNulls(authorship), removeNulls(authorship2)),
removeNulls(genericName),
Expand Down Expand Up @@ -538,6 +539,9 @@ public Object matchFlatV1(
produces = "application/json")
public Object matchV1(
@RequestParam(value = "usageKey", required = false) String usageKey,
@RequestParam(value = "taxonID", required = false) String taxonID,
@RequestParam(value = "taxonConceptID", required = false) String taxonConceptID,
@RequestParam(value = "scientificNameID", required = false) String scientificNameID,
@RequestParam(value = "name", required = false) String scientificName2,
@RequestParam(value = "scientificName", required = false) String scientificName,
@RequestParam(value = "authorship", required = false) String authorship2,
Expand All @@ -559,10 +563,11 @@ public Object matchV1(
classification.setClazz(response.getParameter("class"));
Optional<NameUsageMatchV1> optionalNameUsageMatchV1 = NameUsageMatchV1.createFrom(
matchingService.match(

removeNulls(usageKey),
null,
null,
null,
removeNulls(taxonID),
removeNulls(taxonConceptID),
removeNulls(scientificNameID),
first(removeNulls(scientificName), removeNulls(scientificName2)),
first(removeNulls(authorship), removeNulls(authorship2)),
removeNulls(genericName),
Expand All @@ -581,17 +586,6 @@ public Object matchV1(
}
}

public static String removeNulls(String value) {
if (value != null) {
value = StringUtils.trimToEmpty(value);
if (value.equalsIgnoreCase("null")) {
return null;
}
return value;
}
return null;
}

private Rank parseRank(String value) {
try {
if (!Objects.isNull(value) && !value.isEmpty()) {
Expand All @@ -605,8 +599,4 @@ private Rank parseRank(String value) {
}
return null;
}

private boolean bool(Boolean bool) {
return bool != null && bool;
}
}
Loading

0 comments on commit 84a32fc

Please sign in to comment.