Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

EA-40999 Fix AnnoSync (add 1 to page number since Annotation API page… #142

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/maven.yml
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ jobs:
repositories: '[{ "id": "europeana-libs-release", "url": "https://artifactory.eanadev.org/artifactory/libs-release"}, { "id": "europeana-libs-snapshot", "url": "https://artifactory.eanadev.org/artifactory/libs-snapshot", "snapshots": { "enabled": "true" }}]'
plugin_repositories: '[{ "id": "europeana-plugins-release", "url": "https://artifactory.eanadev.org/artifactory/plugins-release" }, { "id": "europeana-plugins-snapshot", "url": "https://artifactory.eanadev.org/artifactory/plugins-snapshot", "snapshots": { "enabled": "true" }}]'
- name: Cache Maven packages
uses: actions/cache@v2
uses: actions/cache@v4
with:
path: ~/.m2
key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,12 @@
import eu.europeana.fulltext.service.AnnotationApiRestService;
import eu.europeana.fulltext.subtitles.external.AnnotationItem;

/**
* Reads pages of items from Annotation API based on "from" and "to" date
*/
public class AnnotationItemReader extends AbstractPaginatedDataItemReader<AnnotationItem> {

private static final Logger logger = LogManager.getLogger(AnnotationItemReader.class);
private static final Logger LOG = LogManager.getLogger(AnnotationItemReader.class);

private final AnnotationApiRestService annotationsRestService;
private final Instant from;
Expand All @@ -34,16 +37,16 @@ String getClassName() {

@Override
protected Iterator<AnnotationItem> doPageRead() {
// page is incremented in parent class every time this method is invoked
List<AnnotationItem> searchResponse =
annotationsRestService.getAnnotations(page, pageSize, from, to);
// Page is incremented in parent class every time this method is invoked. By default it starts with 0 but in
// Annotation API it starts with 1 so we add 1
List<AnnotationItem> searchResponse = annotationsRestService.getAnnotations(page + 1, pageSize, from, to);

if (searchResponse == null || searchResponse.isEmpty()) {
logger.info("No results found in page:{} , pageSize:{}, from: {}, to: {} ", page, pageSize, from, to);
LOG.info("No results found in page:{} , pageSize:{}, from: {}, to: {} ", page + 1, pageSize, from, to);
return null;
}

logger.info("Fetched Annotations ids - {} ", searchResponse.size());
LOG.info("Fetched Annotations ids - {} ", searchResponse.size());

return searchResponse.iterator();
}
Expand Down
11 changes: 11 additions & 0 deletions api/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
<apache-commons-text.version>1.12.0</apache-commons-text.version>
<spring-boot-starter-data-solr.version>2.4.13</spring-boot-starter-data-solr.version>
<maven-surefire.version>2.22.2</maven-surefire.version>
<maven.resource.version>3.1.0</maven.resource.version>
</properties>

<dependencies>
Expand Down Expand Up @@ -300,6 +301,16 @@
<version>${maven-surefire.version}</version>
</plugin>

<!-- For version above Spring-boot 2.4.x - counter to error
Failed to execute goal org.apache.maven.plugins:maven-resources-plugin:3.2.0:resources (default-resources)
on project application Input length = 1
See : https://stackoverflow.com/questions/65910112/maven-clean-install-failed-to-execute-goal-org-apache-maven-pluginsmaven-resou-->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-resources-plugin</artifactId>
<version>${maven.resource.version}</version>
</plugin>

<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
Expand Down
7 changes: 7 additions & 0 deletions common/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -124,6 +124,13 @@
<version>4.5.14</version>
</dependency>

<!-- https://mvnrepository.com/artifact/javax.servlet/javax.servlet-api -->
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
<version>4.0.1</version>
</dependency>

<!-- Saxon-HE for Page Xml parser-->
<dependency>
<groupId>net.sf.saxon</groupId>
Expand Down
3 changes: 3 additions & 0 deletions k8s/overlays/cloud/cronjob.yaml.template
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ spec:
concurrencyPolicy: Forbid
jobTemplate:
spec:
backoffLimit: 1
ttlSecondsAfterFinished: ${CRON_TTL_AFTER_FINISHED}
template:
metadata:
Expand Down Expand Up @@ -61,6 +62,7 @@ spec:
concurrencyPolicy: Forbid
jobTemplate:
spec:
backoffLimit: 1
ttlSecondsAfterFinished: ${CRON_TTL_AFTER_FINISHED}
template:
metadata:
Expand Down Expand Up @@ -119,6 +121,7 @@ spec:
concurrencyPolicy: Forbid
jobTemplate:
spec:
backoffLimit: 1
ttlSecondsAfterFinished: ${CRON_TTL_AFTER_FINISHED}
template:
metadata:
Expand Down
Loading