Skip to content

Commit

Permalink
0.77
Browse files Browse the repository at this point in the history 
Much faster catalog loading
  • Loading branch information
@zsvoboda
zsvoboda committed Jan 31, 2021
1 parent 388e660 commit fc9482c
Show file tree
Hide file tree
Showing 8 changed files with 146 additions and 118 deletions.
3 changes: 0 additions & 3 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,6 @@ and it expects the following JDBC URL format

``` jdbc:gd://<your-gooddata-domain-name>/gdc/projects/<your-gooddata-project-id> ```

Please be patient during the first use. The initial load can take 1 or 2 minutes on top of workspaces
with large LDM. Subsequent starts are much faster (seconds).

### Supported features
- You don't use FROM clause. Just list of columns in the ```SELECT <column-list> ```
and ```WHERE <conditions> ```
Expand Down
2 changes: 1 addition & 1 deletion pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
<artifactId>gooddata-jdbc</artifactId>
<name>gooddata-jdbc</name>
<description>GoodData workspace JDBC driver</description>
<version>0.77-SNAPSHOT</version>
<version>0.77</version>
<url>https://github.com/zsvoboda/gooddata-jdbc</url>

<licenses>
Expand Down
156 changes: 103 additions & 53 deletions src/main/java/com/gooddata/jdbc/catalog/Catalog.java
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.gooddata.jdbc.catalog;

import com.gooddata.jdbc.driver.AfmDriver;
import com.gooddata.jdbc.parser.DataTypeParser;
import com.gooddata.jdbc.parser.SQLParser;
import com.gooddata.jdbc.rest.GoodDataRestConnection;
Expand Down Expand Up @@ -31,13 +32,14 @@
public class Catalog implements Serializable {

private final static Logger LOGGER = Logger.getLogger(Catalog.class.getName());
// Caching catalogs by schema
private static final Map<String, Map<String, CatalogEntry>> entriesCache = new HashMap<>();
private static final String SERIALIZATION_EXTENSION = ".gdcat";
private final Comparator<CatalogEntry> CatalogEntryComparator = Comparator.comparing(CatalogEntry::getTitle);

/**
* AFM objects (displayForms, and metrics)
*/
private Map<String, CatalogEntry> entries = new HashMap<>();
private final GoodData gd;
private final GoodDataRestConnection gdRest;

private final Comparator<CatalogEntry> CatalogEntryComparator = Comparator.comparing(CatalogEntry::getTitle);
private final int[] ATTRIBUTE_FILTER_OPERATORS = new int[]{
SQLParser.ParsedSQL.FilterExpression.OPERATOR_EQUAL,
SQLParser.ParsedSQL.FilterExpression.OPERATOR_NOT_EQUAL,
Expand All @@ -56,25 +58,49 @@ public class Catalog implements Serializable {
SQLParser.ParsedSQL.FilterExpression.OPERATOR_NOT_BETWEEN
};

/**
* AFM objects (displayForms, and metrics)
*/
private Map<String, CatalogEntry> entries = new HashMap<>();
// Is catalog populated?
private boolean isCatalogPopulated = true;
private boolean isCatalogPopulated = false;

/**
* Constructor
*/
public Catalog(GoodData gd, GoodDataRestConnection gdRest, Schema schema) throws SQLException {
try {
LOGGER.info(String.format("Trying to deserialize catalog for workspace '%s'.",
this.gd = gd;
this.gdRest = gdRest;
LOGGER.info(String.format("Getting cached catalog entries for schema '%s'.",
schema.getSchemaUri()));
/*
this.isCatalogPopulated = false;
this.entries = entriesCache.get(schema.getSchemaUri());
if (this.entries == null) {
LOGGER.info(String.format("Cached catalog entries for schema '%s' not found.",
schema.getSchemaUri()));
this.entries = this.deserialize(TextUtil
.extractWorkspaceIdFromWorkspaceUri(schema.getSchemaUri()));
LOGGER.info("Catalog successfully deserialized.");
} catch (IOException e) {
LOGGER.info("Catalog deserialization failed. Creating new one.");
this.populate(gd, gdRest, schema.getSchemaUri());
} catch (ClassNotFoundException | TextUtil.InvalidFormatException e) {
throw new SQLException(e);
LOGGER.info(String.format("Trying to deserialize catalog entries from permanent cache for workspace '%s'.",
schema.getSchemaUri()));
try {
this.entries = this.deserialize(TextUtil
.extractWorkspaceIdFromWorkspaceUri(schema.getSchemaUri()));
this.isCatalogPopulated = true;
LOGGER.info("Catalog successfully deserialized from permanent cache.");
} catch (IOException e) {
// do nothing and continue to population of ne
LOGGER.info("Catalog entries deserialization failed. Populating new one.");
} catch (ClassNotFoundException | TextUtil.InvalidFormatException e) {
throw new SQLException(e);
}
// Either no cached entries found and populate from scratch
// or deserialized catalog asynchronous refresh
this.populateSync(gd, gdRest, schema.getSchemaUri());
} else {
this.isCatalogPopulated = true;
LOGGER.info(String.format("Cached catalog for schema '%s' found.", schema.getSchemaUri()));
}
*/
this.populate(schema.getSchemaUri());
}

/**
Expand All @@ -98,7 +124,7 @@ private void addAttribute(Attribute attribute, Map<String, CatalogEntry> c) {
LOGGER.info(String.format("Default display form title='%s'", displayForm.getTitle()));
CatalogEntry e = new CatalogEntry(attribute.getUri(),
attribute.getTitle(), attribute.getCategory(), attribute.getIdentifier(),
new UriObjQualifier(attribute.getUri()), new UriObjQualifier(displayForm.getUri()));
new UriObjQualifier(attribute.getUri()), new UriObjQualifier(displayForm.getUri()));
//TODO getting default display form only
e.setDataType(CatalogEntry.DEFAULT_ATTRIBUTE_DATATYPE);
c.put(attribute.getUri(), e);
Expand All @@ -107,13 +133,12 @@ private void addAttribute(Attribute attribute, Map<String, CatalogEntry> c) {
}
}


/**
* Adds metric to catalog
*
* @param metric metric to add
*/
private void addMetric(Entry metric, Map<String,CatalogEntry> c) {
private void addMetric(Entry metric, Map<String, CatalogEntry> c) {
CatalogEntry e = new CatalogEntry(metric.getUri(),
metric.getTitle(), metric.getCategory(), metric.getIdentifier(),
new UriObjQualifier(metric.getUri()));
Expand Down Expand Up @@ -147,7 +172,7 @@ private void addFact(Entry fact, Map<String, CatalogEntry> c) {
}

public synchronized void waitForCatalogPopulationFinished() {
while (!this.isCatalogPopulated) {
while (!this.isCatalogPopulated()) {
try {
wait();
} catch (InterruptedException e) {
Expand All @@ -160,58 +185,65 @@ public synchronized void waitForCatalogPopulationFinished() {
/**
* Populates the catalog of attributes and metrics
*
* @param gd Gooddata reference
* @param gdRest Gooddata REST connection
* @param workspaceUri GoodData workspace URI
*
*/
public void populate(GoodData gd, GoodDataRestConnection gdRest, String workspaceUri) {
CatalogRefreshExecutor exec = new CatalogRefreshExecutor(this, gd, gdRest, workspaceUri);
public void populate(String workspaceUri) {
CatalogRefreshExecutor exec = new CatalogRefreshExecutor(this, workspaceUri);
exec.start();
}

public synchronized boolean isCatalogPopulated() {
return this.isCatalogPopulated;
}

public synchronized void setIsCatalogPopulated(boolean b) {
this.isCatalogPopulated = b;
}

public synchronized void setEntries(Map<String,CatalogEntry> m) {
this.entries = m;
}

/**
* Populates the catalog of attributes and metrics
*
* @param gd Gooddata reference
* @param gdRest Gooddata REST connection
* @param workspaceUri GoodData workspace URI
* @throws SQLException generic issue
*/
public synchronized void populateSync(GoodData gd, GoodDataRestConnection gdRest, String workspaceUri) throws SQLException {
protected synchronized void populateSync(String workspaceUri) throws SQLException {
LOGGER.info(String.format("Populating catalog for schema '%s'", workspaceUri));
if(this.entries.size() == 0) {
if (this.entries.size() == 0) {
// Must block all queries if the catalog is empty
LOGGER.info("Catalog lock acquired.");
this.isCatalogPopulated = false;
this.setIsCatalogPopulated(false);
} else {
LOGGER.info("Populating new catalog without locking. " +
"Queries will run against old catalog.");
// if there is something in the catalog, the queries can use the old content
this.isCatalogPopulated = true;
this.setIsCatalogPopulated(true);
}
this.entries = this.populateCatalogEntries(gd, gdRest, workspaceUri);
this.isCatalogPopulated = true;
this.setEntries(this.populateCatalogEntries(workspaceUri));
this.setIsCatalogPopulated(true);
notifyAll();
LOGGER.info("Catalog lock released");
}

/**
* Populate new entries map
*
* @param gd Gooddata reference
* @param gdRest Gooddata REST connection
* @param workspaceUri GoodData workspace URI
* @return new entries Map
* @throws SQLException generic issue
*/
private Map<String,CatalogEntry> populateCatalogEntries(GoodData gd, GoodDataRestConnection gdRest, String workspaceUri) throws SQLException {
private Map<String, CatalogEntry> populateCatalogEntries(String workspaceUri) throws SQLException {
LOGGER.info(String.format("Refreshing catalog for workspace uri '%s'", workspaceUri));
Map<String,CatalogEntry> newCatalog = new HashMap<>();
Map<String, CatalogEntry> newCatalog = new HashMap<>();

Project workspace = gd.getProjectService().getProjectByUri(workspaceUri);
Project workspace = this.gd.getProjectService().getProjectByUri(workspaceUri);
if (workspace == null) {
throw new SQLException(String.format("Workspace '%s' doesn't exist.", workspaceUri));
}
MetadataService gdMeta = gd.getMetadataService();
MetadataService gdMeta = this.gd.getMetadataService();
LOGGER.info("Fetching metrics.");
Collection<Entry> metricEntries = gdMeta.find(workspace, Metric.class);

Expand All @@ -221,8 +253,28 @@ private Map<String,CatalogEntry> populateCatalogEntries(GoodData gd, GoodDataRes

LOGGER.info("Fetching attributes.");
Collection<Entry> attributeEntries = gdMeta.find(workspace, Attribute.class);
for (Entry attribute : attributeEntries) {
this.addAttribute(gdMeta.getObjByUri(attribute.getUri(), Attribute.class), newCatalog);
List<String> attributeUris = attributeEntries.stream().map(a->a.getUri())
.collect(Collectors.toList());
Collection<Obj> attributes = new ArrayList<>();
List<String> uriBatch = new ArrayList<>();
for(String attributeUri: attributeUris) {
if(uriBatch.size()<50) {
uriBatch.add(attributeUri);
} else {
Collection<Obj> batchResult = gdMeta.getObjsByUris(workspace, uriBatch);
attributes.addAll(batchResult);
uriBatch.clear();
}
}
if(uriBatch.size()>0) {
Collection<Obj> batchResult = gdMeta.getObjsByUris(workspace, uriBatch);
attributes.addAll(batchResult);
uriBatch.clear();
}

for (Obj obj : attributes) {
Attribute attribute = (Attribute)obj;
this.addAttribute(attribute, newCatalog);
}

LOGGER.info("Fetching facts.");
Expand All @@ -232,36 +284,34 @@ private Map<String,CatalogEntry> populateCatalogEntries(GoodData gd, GoodDataRes
}

LOGGER.info("Fetching variables.");
List<CatalogEntry> variableEntries = gdRest.getVariables(workspaceUri);
List<CatalogEntry> variableEntries = this.gdRest.getVariables(workspaceUri);
for (CatalogEntry variable : variableEntries) {
newCatalog.put(variable.getUri(), variable);
}
/*
LOGGER.info(String.format("Catalog refresh finished. Fetched '%d' objects.", newCatalog.size()));

LOGGER.info(String.format("Serializing catalog for workspace '%s'.", workspaceUri));
try {
this.serialize(TextUtil.extractWorkspaceIdFromWorkspaceUri(workspaceUri), newCatalog);
} catch (TextUtil.InvalidFormatException | IOException e) {
throw new SQLException(e);
}
LOGGER.info(String.format("Catalog serialization finished for workspace '%s'.", workspaceUri));
*/
return newCatalog;
}

private static final String SERIALIZATION_DIR = ".gdjdbc";
private static final String SERIALIZATION_EXTENSION = ".gdcat";

private void ensureSerializationDirectory() throws IOException {
Files.createDirectories(Paths.get(String.format("%s/%s",
System.getProperty("user.home"), SERIALIZATION_DIR)));
System.getProperty("user.home"), AfmDriver.GDJDBC_DIR)));
}

private void serialize(String schema, Map<String, CatalogEntry> c) throws IOException {
LOGGER.info(String.format("Serializing catalog '%s'", schema));
ensureSerializationDirectory();

FileOutputStream fileOut = new FileOutputStream(String.format("%s/%s/%s.%s",
System.getProperty("user.home"), SERIALIZATION_DIR, schema, SERIALIZATION_EXTENSION));
FileOutputStream fileOut = new FileOutputStream(String.format("%s/%s.%s",
AfmDriver.GDJDBC_DIR, schema, SERIALIZATION_EXTENSION));
ObjectOutputStream out = new ObjectOutputStream(fileOut);
out.writeObject(c);
out.close();
Expand All @@ -273,14 +323,14 @@ private void serialize(String schema, Map<String, CatalogEntry> c) throws IOExce
private Map<String, CatalogEntry> deserialize(String schema) throws IOException, ClassNotFoundException {
LOGGER.info(String.format("Deserializing catalog '%s'", schema));
ensureSerializationDirectory();
FileInputStream fileIn = new FileInputStream(String.format("%s/%s/%s.%s",
System.getProperty("user.home"), SERIALIZATION_DIR, schema, SERIALIZATION_EXTENSION));
FileInputStream fileIn = new FileInputStream(String.format("%s/%s.%s",
AfmDriver.GDJDBC_DIR, schema, SERIALIZATION_EXTENSION));
ObjectInputStream in = new ObjectInputStream(fileIn);
Object o = in.readObject();
in.close();
fileIn.close();
LOGGER.info(String.format("Catalog '%s' deserialized successfully.", schema));
return (Map<String, CatalogEntry>)o;
return (Map<String, CatalogEntry>) o;
}

/**
Expand Down Expand Up @@ -550,7 +600,7 @@ public String getMetricPrettyPrint(MetadataService gdMeta, GoodDataRestConnectio
* @return metric MAQL with resolved URIs
*/
public String getVariablePrettyPrint(GoodDataRestConnection gdRest, String uri)
throws CatalogEntryNotFoundException, TextUtil.InvalidFormatException {
throws CatalogEntryNotFoundException, TextUtil.InvalidFormatException{
CatalogEntry e = this.entries.get(uri);
if (!e.getType().equalsIgnoreCase("prompt")) {
throw new CatalogEntryNotFoundException(String.format("Variable with uri '%s' not found.", uri));
Expand Down
30 changes: 16 additions & 14 deletions src/main/java/com/gooddata/jdbc/catalog/CatalogEntry.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,10 @@ public class CatalogEntry implements Serializable {
/**
* Constructor
*
* @param uri Catalog object URI
* @param title Catalog object title
* @param type Catalog object type
* @param identifier Catalog object identifier
* @param uri Catalog object URI
* @param title Catalog object title
* @param type Catalog object type
* @param identifier Catalog object identifier
* @param defaultDisplayForm default display form for attributes
*/
public CatalogEntry(String uri, String title, String type, String identifier, ObjQualifier gdObject,
Expand Down Expand Up @@ -52,6 +52,7 @@ public CatalogEntry(String uri, String title, String type, String identifier, Ob

/**
* Constructor
*
* @param uri Catalog object URI
* @param title Catalog object title
* @param type Catalog object type
Expand All @@ -75,15 +76,16 @@ public CatalogEntry(String uri, String title, String type, String identifier, Ob

/**
* Constructor
* @param uri Catalog object URI
* @param title Catalog object title
* @param type Catalog object type
* @param identifier Catalog object identifier
* @param gdObject Original gd object
*
* @param uri Catalog object URI
* @param title Catalog object title
* @param type Catalog object type
* @param identifier Catalog object identifier
* @param gdObject Original gd object
* @param defaultDisplayForm default display form for attributes
* @param dataType Datatype
* @param size Datatype size
* @param precision Datatype precision
* @param dataType Datatype
* @param size Datatype size
* @param precision Datatype precision
*/
public CatalogEntry(String uri, String title, String type, String identifier, ObjQualifier gdObject,
ObjQualifier defaultDisplayForm, String dataType, int size, int precision) {
Expand All @@ -101,6 +103,7 @@ public CatalogEntry(String uri, String title, String type, String identifier, Ob

/**
* Clone catalog object entry (when it needs to be modified externally)
*
* @return catalog entry clone
*/
public CatalogEntry cloneEntry() {
Expand Down Expand Up @@ -172,7 +175,7 @@ public void setPrecision(int precision) {
}

public ObjQualifier getDefaultDisplayForm() {
return defaultDisplayForm;
return this.defaultDisplayForm;
}

private String identifier;
Expand All @@ -185,5 +188,4 @@ public ObjQualifier getDefaultDisplayForm() {
private int precision;
private final ObjQualifier gdObject;


}
Loading

0 comments on commit fc9482c

Please sign in to comment.