-
-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Added scraping an entire series [#1]
- Loading branch information
Showing
12 changed files
with
413 additions
and
54 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
102 changes: 102 additions & 0 deletions
102
src/main/java/org/comixedproject/metadata/marvel/actions/MarvelGetAllIssuesAction.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,102 @@ | ||
| /* | ||
| * ComiXed - A digital comic book library management application. | ||
| * Copyright (C) 2024, The ComiXed Project | ||
| * | ||
| * This program is free software: you can redistribute it and/or modify | ||
| * it under the terms of the GNU General Public License as published by | ||
| * the Free Software Foundation, either version 3 of the License, or | ||
| * (at your option) any later version. | ||
| * | ||
| * This program is distributed in the hope that it will be useful, | ||
| * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| * GNU General Public License for more details. | ||
| * | ||
| * You should have received a copy of the GNU General Public License | ||
| * along with this program. If not, see <http://www.gnu.org/licenses> | ||
| */ | ||
|
|
||
| package org.comixedproject.metadata.marvel.actions; | ||
|
|
||
| import static org.comixedproject.metadata.marvel.MarvelMetadataAdaptor.PUBLISHER_NAME; | ||
|
|
||
| import java.util.ArrayList; | ||
| import java.util.List; | ||
| import lombok.Getter; | ||
| import lombok.Setter; | ||
| import lombok.extern.log4j.Log4j2; | ||
| import org.apache.commons.lang3.StringUtils; | ||
| import org.comixedproject.metadata.MetadataException; | ||
| import org.comixedproject.metadata.marvel.models.MarvelGetAllIssuesQueryResponse; | ||
| import org.comixedproject.metadata.model.IssueDetailsMetadata; | ||
| import org.springframework.web.reactive.function.client.WebClient; | ||
| import reactor.core.publisher.Mono; | ||
|
|
||
| /** | ||
| * <code>MarvelGetAllIssuesAction</code> retrieves the metadata for all comics for a given volume. | ||
| * | ||
| * @author Darryl L. Pierce | ||
| */ | ||
| @Log4j2 | ||
| public class MarvelGetAllIssuesAction | ||
| extends AbstractMarvelScrapingAction<List<IssueDetailsMetadata>> { | ||
| // URL: | ||
| // https://gateway.marvel.com:443/v1/public/series/2069/comics?noVariants=true&apikey=763df8a7c3c0f6d3bb7fcf088bbf6ee1 | ||
|
|
||
| @Getter @Setter private String seriesId; | ||
|
|
||
| @Override | ||
| public List<IssueDetailsMetadata> execute() throws MetadataException { | ||
| this.doCheckSetup(); | ||
|
|
||
| if (StringUtils.isBlank(this.seriesId)) throw new MetadataException("Missing series id"); | ||
|
|
||
| final List<IssueDetailsMetadata> result = new ArrayList<>(); | ||
| boolean done = false; | ||
|
|
||
| while (!done) { | ||
| log.trace("Generating request URL: series id={}", this.seriesId); | ||
| final String url = | ||
| this.doCreateUrl(String.format("series/%s/comics", this.seriesId), "noVariants=true"); | ||
| final WebClient client = this.createWebClient(url); | ||
| final Mono<MarvelGetAllIssuesQueryResponse> request = | ||
| client.get().uri(url).retrieve().bodyToMono(MarvelGetAllIssuesQueryResponse.class); | ||
| MarvelGetAllIssuesQueryResponse response = null; | ||
|
|
||
| try { | ||
| response = request.block(); | ||
| } catch (Exception error) { | ||
| throw new MetadataException("Failed to get response", error); | ||
| } | ||
|
|
||
| if (response == null) { | ||
| throw new MetadataException("Failed to receive response"); | ||
| } | ||
|
|
||
| log.debug("Received: {} volume(s)", response.getData().getResults().size()); | ||
| response | ||
| .getData() | ||
| .getResults() | ||
| .forEach( | ||
| issue -> { | ||
| log.trace("Processing volume record: {} name={}", issue.getId(), issue.getTitle()); | ||
| final IssueDetailsMetadata entry = new IssueDetailsMetadata(); | ||
| entry.setSourceId(issue.getId()); | ||
| entry.setPublisher(PUBLISHER_NAME); | ||
| entry.setSeries(issue.getSeries().getName()); | ||
| entry.setIssueNumber(issue.getIssueNumber()); | ||
| // TODO get the start year | ||
| // entry.setVolume(issue.getStartYear()); | ||
| entry.setVolume(""); | ||
| entry.setTitle(issue.getTitle()); | ||
| entry.setCoverDate(this.getCoverDate(issue.getDates())); | ||
| entry.setStoreDate(this.getStoreDate(issue.getDates())); | ||
| result.add(entry); | ||
| }); | ||
| done = isDone(response); | ||
| } | ||
|
|
||
| log.debug("Returning {} volume(s)", result.size()); | ||
| return result; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
28 changes: 28 additions & 0 deletions
28
src/main/java/org/comixedproject/metadata/marvel/models/MarvelGetAllIssuesQueryResponse.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| /* | ||
| * ComiXed - A digital comic book library management application. | ||
| * Copyright (C) 2024, The ComiXed Project | ||
| * | ||
| * This program is free software: you can redistribute it and/or modify | ||
| * it under the terms of the GNU General Public License as published by | ||
| * the Free Software Foundation, either version 3 of the License, or | ||
| * (at your option) any later version. | ||
| * | ||
| * This program is distributed in the hope that it will be useful, | ||
| * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| * GNU General Public License for more details. | ||
| * | ||
| * You should have received a copy of the GNU General Public License | ||
| * along with this program. If not, see <http://www.gnu.org/licenses> | ||
| */ | ||
|
|
||
| package org.comixedproject.metadata.marvel.models; | ||
|
|
||
| /** | ||
| * <code>MarvelGetAllIssuesQueryResponse</code> represents the response body when retrieving all | ||
| * issues for a series. | ||
| * | ||
| * @author Darryl L. Pierce | ||
| */ | ||
| public class MarvelGetAllIssuesQueryResponse | ||
| extends BaseMarvelResponse<MarvelSeriesIssueDetailRecord> {} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
43 changes: 43 additions & 0 deletions
43
src/main/java/org/comixedproject/metadata/marvel/models/MarvelSeriesIssueCreators.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,43 @@ | ||
| /* | ||
| * ComiXed - A digital comic book library management application. | ||
| * Copyright (C) 2024, The ComiXed Project | ||
| * | ||
| * This program is free software: you can redistribute it and/or modify | ||
| * it under the terms of the GNU General Public License as published by | ||
| * the Free Software Foundation, either version 3 of the License, or | ||
| * (at your option) any later version. | ||
| * | ||
| * This program is distributed in the hope that it will be useful, | ||
| * but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| * GNU General Public License for more details. | ||
| * | ||
| * You should have received a copy of the GNU General Public License | ||
| * along with this program. If not, see <http://www.gnu.org/licenses> | ||
| */ | ||
|
|
||
| package org.comixedproject.metadata.marvel.models; | ||
|
|
||
| import com.fasterxml.jackson.annotation.JsonProperty; | ||
| import java.util.List; | ||
| import lombok.Getter; | ||
|
|
||
| /** | ||
| * <code>MarvelSeriesIssueCreators</code> represents all credits for a single issue when scraping a | ||
| * series. | ||
| * | ||
| * @author Darryl L. Pierc | ||
| */ | ||
| public class MarvelSeriesIssueCreators { | ||
| @JsonProperty("available") | ||
| @Getter | ||
| private int available; | ||
|
|
||
| @JsonProperty("collectionURI") | ||
| @Getter | ||
| private String collectionURI; | ||
|
|
||
| @JsonProperty("items") | ||
| @Getter | ||
| private List<MarvelCreditEntry> items; | ||
| } |
Oops, something went wrong.