Skip to content

Commit

Permalink
ATLAS-4968: checkstyle compliance updates - tools module (#281)
Browse files Browse the repository at this point in the history 
(cherry picked from commit c482cc3)
  • Loading branch information
@kumaab @mneethiraj
kumaab authored and mneethiraj committed Feb 11, 2025
1 parent 25f3717 commit 3fa48f5
Show file tree
Hide file tree
Showing 6 changed files with 114 additions and 133 deletions.
13 changes: 4 additions & 9 deletions tools/atlas-index-repair/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,40 +18,37 @@
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>

<parent>
<groupId>org.apache.atlas</groupId>
<artifactId>apache-atlas</artifactId>
<version>2.5.0-SNAPSHOT</version>
<relativePath>../../</relativePath>
</parent>

<artifactId>atlas-index-repair-tool</artifactId>
<packaging>jar</packaging>

<name>Apache Atlas index repair tool</name>
<description>Apache Atlas index repair Module</description>
<properties>
<checkstyle.failOnViolation>true</checkstyle.failOnViolation>
<checkstyle.skip>false</checkstyle.skip>
</properties>

<dependencies>

<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-client-v2</artifactId>
<version>${project.version}</version>
</dependency>

<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-graphdb-janus</artifactId>
<version>${project.version}</version>
</dependency>

<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-repository</artifactId>
<version>${project.version}</version>
</dependency>

<dependency>
<groupId>org.janusgraph</groupId>
<artifactId>janusgraph-core</artifactId>
Expand All @@ -67,7 +64,5 @@
</exclusion>
</exclusions>
</dependency>

</dependencies>

</project>
77 changes: 36 additions & 41 deletions tools/atlas-index-repair/src/main/java/org/apache/atlas/tools/RepairIndex.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.graphdb.janus.AtlasJanusGraphDatabase;
import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
import org.apache.atlas.security.SecurityProperties;
import org.apache.atlas.utils.AuthenticationUtil;
import org.apache.atlas.utils.SSLUtil;
import org.apache.commons.cli.CommandLine;
Expand Down Expand Up @@ -57,29 +56,28 @@
public class RepairIndex {
private static final Logger LOG = LoggerFactory.getLogger(RepairIndex.class);

private static final int EXIT_CODE_SUCCESS = 0;
private static final int EXIT_CODE_FAILED = 1;
private static final int MAX_TRIES_ON_FAILURE = 3;

private static final String INDEX_NAME_VERTEX_INDEX = "vertex_index";
private static final String INDEX_NAME_FULLTEXT_INDEX = "fulltext_index";
private static final String INDEX_NAME_EDGE_INDEX = "edge_index";
private static final String DEFAULT_ATLAS_URL = "http://localhost:21000/";
private static final int EXIT_CODE_SUCCESS = 0;
private static final int EXIT_CODE_FAILED = 1;
private static final int MAX_TRIES_ON_FAILURE = 3;
private static final String INDEX_NAME_VERTEX_INDEX = "vertex_index";
private static final String INDEX_NAME_FULLTEXT_INDEX = "fulltext_index";
private static final String INDEX_NAME_EDGE_INDEX = "edge_index";
private static final String DEFAULT_ATLAS_URL = "http://localhost:21000/";
private static final String APPLICATION_PROPERTY_ATLAS_ENDPOINT = "atlas.rest.address";

private static JanusGraph graph;
private static JanusGraph graph;
private static AtlasClientV2 atlasClientV2;
private static boolean isSelectiveRestore;
private static boolean isSelectiveRestore;

public static void main(String[] args) {
int exitCode = EXIT_CODE_FAILED;
LOG.info("Started index repair");

try {
CommandLine cmd = getCommandLine(args);
String guid = cmd.getOptionValue("g");
CommandLine cmd = getCommandLine(args);
String guid = cmd.getOptionValue("g");

if(guid != null && !guid.isEmpty()){
if (guid != null && !guid.isEmpty()) {
isSelectiveRestore = true;
String uid = cmd.getOptionValue("u");
String pwd = cmd.getOptionValue("p");
Expand Down Expand Up @@ -108,7 +106,7 @@ private static void process(String guid) throws Exception {

if (isSelectiveRestore) {
repairIndex.restoreSelective(guid);
}else{
} else {
repairIndex.restoreAll();
}

Expand All @@ -131,24 +129,24 @@ private static void setupGraph() {
}

private static String[] getIndexes() {
return new String[]{ INDEX_NAME_VERTEX_INDEX, INDEX_NAME_EDGE_INDEX, INDEX_NAME_FULLTEXT_INDEX};
return new String[] {INDEX_NAME_VERTEX_INDEX, INDEX_NAME_EDGE_INDEX, INDEX_NAME_FULLTEXT_INDEX};
}

private static void setupAtlasClient(String uid, String pwd) throws AtlasException {
String[] atlasEndpoint = getAtlasRESTUrl();
if (atlasEndpoint == null || atlasEndpoint.length == 0) {
atlasEndpoint = new String[]{DEFAULT_ATLAS_URL};
atlasEndpoint = new String[] {DEFAULT_ATLAS_URL};
}
atlasClientV2 = getAtlasClientV2(atlasEndpoint, new String[]{uid, pwd});
atlasClientV2 = getAtlasClientV2(atlasEndpoint, new String[] {uid, pwd});
}

private void restoreAll() throws Exception {
for (String indexName : getIndexes()){
for (String indexName : getIndexes()) {
displayCrlf("Restoring: " + indexName);
long startTime = System.currentTimeMillis();

ManagementSystem mgmt = (ManagementSystem) graph.openManagement();
JanusGraphIndex index = mgmt.getGraphIndex(indexName);
ManagementSystem mgmt = (ManagementSystem) graph.openManagement();
JanusGraphIndex index = mgmt.getGraphIndex(indexName);
mgmt.updateIndex(index, SchemaAction.REINDEX).get();
mgmt.commit();

Expand All @@ -159,15 +157,14 @@ private void restoreAll() throws Exception {
}
}


private void restoreSelective(String guid) throws Exception {
private void restoreSelective(String guid) throws Exception {
Set<String> referencedGUIDs = new HashSet<>(getEntityAndReferenceGuids(guid));
displayCrlf("processing referencedGuids => "+ referencedGUIDs);
displayCrlf("processing referencedGuids => " + referencedGUIDs);

StandardJanusGraph janusGraph = (StandardJanusGraph) graph;
IndexSerializer indexSerializer = janusGraph.getIndexSerializer();
StandardJanusGraph janusGraph = (StandardJanusGraph) graph;
IndexSerializer indexSerializer = janusGraph.getIndexSerializer();

for (String indexName : getIndexes()){
for (String indexName : getIndexes()) {
displayCrlf("Restoring: " + indexName);
long startTime = System.currentTimeMillis();
reindexVertex(indexName, indexSerializer, referencedGUIDs);
Expand All @@ -179,19 +176,19 @@ private void restoreSelective(String guid) throws Exception {

private static void reindexVertex(String indexName, IndexSerializer indexSerializer, Set<String> entityGUIDs) throws Exception {
Map<String, Map<String, List<IndexEntry>>> documentsPerStore = new java.util.HashMap<>();
ManagementSystem mgmt = (ManagementSystem) graph.openManagement();
StandardJanusGraphTx tx = mgmt.getWrappedTx();
BackendTransaction mutator = tx.getTxHandle();
JanusGraphIndex index = mgmt.getGraphIndex(indexName);
MixedIndexType indexType = (MixedIndexType) mgmt.getSchemaVertex(index).asIndexType();
ManagementSystem mgmt = (ManagementSystem) graph.openManagement();
StandardJanusGraphTx tx = mgmt.getWrappedTx();
BackendTransaction mutator = tx.getTxHandle();
JanusGraphIndex index = mgmt.getGraphIndex(indexName);
MixedIndexType indexType = (MixedIndexType) mgmt.getSchemaVertex(index).asIndexType();

for (String entityGuid : entityGUIDs){
for (String entityGuid : entityGUIDs) {
for (int attemptCount = 1; attemptCount <= MAX_TRIES_ON_FAILURE; attemptCount++) {
AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(entityGuid);
try {
indexSerializer.reindexElement(vertex.getWrappedElement(), indexType, documentsPerStore);
break;
}catch (Exception e){
} catch (Exception e) {
displayCrlf("Exception: " + e.getMessage());
displayCrlf("Pausing before retry..");
Thread.sleep(2000 * attemptCount);
Expand All @@ -204,8 +201,8 @@ private static void reindexVertex(String indexName, IndexSerializer indexSeriali
private static Set<String> getEntityAndReferenceGuids(String guid) throws Exception {
Set<String> set = new HashSet<>();
set.add(guid);
AtlasEntityWithExtInfo entity = atlasClientV2.getEntityByGuid(guid);
Map<String, AtlasEntity> map = entity.getReferredEntities();
AtlasEntityWithExtInfo entity = atlasClientV2.getEntityByGuid(guid);
Map<String, AtlasEntity> map = entity.getReferredEntities();
if (map == null || map.isEmpty()) {
return set;
}
Expand All @@ -230,21 +227,19 @@ private static void displayFn(Consumer<String> fn, String... formatMessage) {
}

private static String[] getAtlasRESTUrl() {
Configuration atlasConf = null;
Configuration atlasConf;
try {
atlasConf = ApplicationProperties.get();
return atlasConf.getStringArray(APPLICATION_PROPERTY_ATLAS_ENDPOINT);
} catch (AtlasException e) {
return new String[]{DEFAULT_ATLAS_URL};
return new String[] {DEFAULT_ATLAS_URL};
}
}

private static AtlasClientV2 getAtlasClientV2(String[] atlasEndpoint, String[] uidPwdFromCommandLine) throws AtlasException {
AtlasClientV2 atlasClientV2;
if (!AuthenticationUtil.isKerberosAuthenticationEnabled()) {
String[] uidPwd = (uidPwdFromCommandLine[0] == null || uidPwdFromCommandLine[1] == null)
? AuthenticationUtil.getBasicAuthenticationInput()
: uidPwdFromCommandLine;
String[] uidPwd = (uidPwdFromCommandLine[0] == null || uidPwdFromCommandLine[1] == null) ? AuthenticationUtil.getBasicAuthenticationInput() : uidPwdFromCommandLine;

atlasClientV2 = new AtlasClientV2(atlasEndpoint, uidPwd);
} else {
Expand Down
16 changes: 2 additions & 14 deletions tools/classification-updater/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,50 +18,41 @@
-->
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>

<parent>
<groupId>org.apache.atlas</groupId>
<artifactId>apache-atlas</artifactId>
<version>2.5.0-SNAPSHOT</version>
<relativePath>../../</relativePath>
</parent>

<artifactId>atlas-classification-updater</artifactId>
<packaging>jar</packaging>

<name>Apache Atlas classification updater</name>
<description>Apache Atlas classification updater Module</description>

<properties>
<calcite.version>0.9.2-incubating</calcite.version>
<checkstyle.failOnViolation>true</checkstyle.failOnViolation>
<checkstyle.skip>false</checkstyle.skip>
</properties>

<dependencies>

<!-- to bring up atlas server for integration tests -->

<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>${jackson.databind.version}</version>
</dependency>

<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-client-v1</artifactId>
</dependency>

<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-client-v2</artifactId>
<version>${project.version}</version>
</dependency>

<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-notification</artifactId>
</dependency>

<dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
Expand All @@ -71,17 +62,14 @@
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>

<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</dependency>

<dependency>
<groupId>org.testng</groupId>
<artifactId>testng</artifactId>
</dependency>

</dependencies>

<build>
Expand Down
Loading

0 comments on commit 3fa48f5

Please sign in to comment.