Skip to content

Commit

Permalink
Added ability to GET lineage from Fusion Object.
Browse files Browse the repository at this point in the history
  • Loading branch information
knighto82 committed Dec 3, 2024
1 parent 41694e0 commit 2a80821
Show file tree
Hide file tree
Showing 7 changed files with 213 additions and 12 deletions.
Original file line number Diff line number Diff line change
@@ -1,11 +1,21 @@
package io.github.jpmorganchase.fusion.packaging;

import com.github.tomakehurst.wiremock.client.WireMock;
import io.github.jpmorganchase.fusion.model.Application;
import io.github.jpmorganchase.fusion.model.Catalog;
import io.github.jpmorganchase.fusion.model.DataDictionaryAttribute;
import io.github.jpmorganchase.fusion.model.DataDictionaryAttributeLineage;
import io.github.jpmorganchase.fusion.test.TestUtils;
import org.hamcrest.MatcherAssert;
import org.hamcrest.Matchers;
import org.junit.jupiter.api.Test;

import java.util.HashMap;

import static com.github.tomakehurst.wiremock.client.WireMock.equalToJson;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.is;

public class DataDictionaryAttributeLineageOperationsIT extends BaseOperationsIT {

Expand Down Expand Up @@ -82,13 +92,37 @@ public void testDeleteDataDictionaryAttributeLineage(){


@Test
public void testUpdateDataDictionaryAttributeLineageRetrievedFromListFunction(){
public void testUpdateDataDictionaryAttributeLineageRetrievedFromGetFunction(){
// Given

wireMockRule.stubFor(WireMock.get(WireMock.urlEqualTo("/catalogs/base/attributes/BA0001/lineage"))
.willReturn(WireMock.aResponse()
.withBody(TestUtils.loadJsonForIt("data-dictionary-lineage/lineage-BA0001-get-response.json"))
.withHeader("Content-Type", "application/json")
.withStatus(200)));

// When
DataDictionaryAttributeLineage actual = getSdk().dataDictionaryAttributeLineage("base", "BA0001");

// Then Verify the response
assertThat(actual.getIdentifier(), is(equalTo("DR0001")));
assertThat(actual.getDescription(), is(equalTo("Derived Attribute Description")));
assertThat(actual.getTitle(), is(equalTo("Derived Attribute Title")));
assertThat(actual.getLinkedEntity(), is(equalTo("lineage/")));
assertThat(actual.getCatalogIdentifier(), is(equalTo("derived")));
assertThat(actual.getApplicationId(), is(equalTo(Application.builder().sealId("12345").build())));
assertThat(actual.getApiManager(), is(Matchers.notNullValue()));
assertThat(actual.getRootUrl(), is(equalTo(getSdk().getRootURL())));
assertThat(actual.getBaseCatalogIdentifier(), is(equalTo("base")));
assertThat(actual.getBaseIdentifier(), is(equalTo("BA0001")));
assertThat(actual.getCatalog(), is(equalTo(Catalog.builder()
.identifier("derived")
.description("Derived Catalog Description")
.title("Derived Catalog Title")
.linkedEntity("derived/")
.build())));



}


Expand Down
15 changes: 15 additions & 0 deletions src/main/java/io/github/jpmorganchase/fusion/Fusion.java
Original file line number Diff line number Diff line change
Expand Up @@ -226,6 +226,21 @@ public Map<String, Map<String, Object>> dataDictionaryAttributeResources(String
return this.callForMap(url);
}

/**
* Get the metadata for a data dictionary attribute lineage, using the specified catalog
*
* @param catalogName identifier of the catalog to be queried
* @param attributeIdentifier identifier of the base attribute
* @throws APICallException if the call to the Fusion API fails
* @throws ParsingException if the response from Fusion could not be parsed successfully
* @throws OAuthException if a token could not be retrieved for authentication
*/
public DataDictionaryAttributeLineage dataDictionaryAttributeLineage(String catalogName, String attributeIdentifier) {
String url = String.format("%1scatalogs/%2s/attributes/%3s/lineage", this.rootURL, catalogName, attributeIdentifier);
String json = this.api.callAPI(url);
return responseParser.parseDataDictionaryAttributeLineageResponse(json, catalogName, attributeIdentifier);
}

/**
* Get a filtered list of the datasets in the specified catalog
* <p>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,16 @@ public interface APIResponseParser {
Map<String, Attribute> parseAttributeResponse(String json, String dataset);

Map<String, DataDictionaryAttribute> parseDataDictionaryAttributeResponse(String json);

DataDictionaryAttributeLineage parseDataDictionaryAttributeLineageResponse(String json, String baseCatalogIdentifier, String baseIdentifier);
Map<String, DataProduct> parseDataProductResponse(String json);

Map<String, DatasetSeries> parseDatasetSeriesResponse(String json);

Map<String, Distribution> parseDistributionResponse(String json);

<T extends CatalogResource> T parseResourceFromResponse(
String json, Class<T> resourceClass, ResourceMutationFactory<T> mutator);

<T extends CatalogResource> Map<String, T> parseResourcesFromResponse(String json, Class<T> resourceClass);

<T extends CatalogResource> Map<String, T> parseResourcesWithVarArgsFromResponse(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,20 @@ public Map<String, DataDictionaryAttribute> parseDataDictionaryAttributeResponse
.build());
}

@Override
public DataDictionaryAttributeLineage parseDataDictionaryAttributeLineageResponse(String json, String baseCatalogIdentifier, String baseIdentifier) {
return parseResourceFromResponse(
json, DataDictionaryAttributeLineage.class, (resource, mc) -> resource.toBuilder()
.varArgs(mc.getVarArgs())
.apiManager(mc.getApiContext().getApiManager())
.rootUrl(mc.getApiContext().getRootUrl())
.catalogIdentifier(resource.getCatalog().getIdentifier())
.baseCatalogIdentifier(baseCatalogIdentifier)
.baseIdentifier(baseIdentifier)
.build());
}


@Override
public Map<String, Attribute> parseAttributeResponse(String json, String dataset) {
return parseResourcesWithVarArgsFromResponse(json, Attribute.class, (resource, mc) -> resource.toBuilder()
Expand Down Expand Up @@ -105,6 +119,17 @@ public UploadedPart parseUploadPartResponse(String json) {
return new GsonBuilder().create().fromJson(json, UploadedPart.class);
}

@Override
public <T extends CatalogResource> T parseResourceFromResponse(
String json, Class<T> resourceClass, ResourceMutationFactory<T> mutator) {

Map<String, Object> responseMap = getMapFromJsonResponse(json);
T obj = gson.fromJson(json, resourceClass);

Set<String> excludes = varArgsExclusions(resourceClass);
return parseResourceWithVarArgs(excludes, obj, responseMap, mutator);
}

@Override
public <T extends CatalogResource> Map<String, T> parseResourcesWithVarArgsFromResponse(
String json, Class<T> resourceClass, ResourceMutationFactory<T> mutator) {
Expand All @@ -115,7 +140,9 @@ public <T extends CatalogResource> Map<String, T> parseResourcesWithVarArgsFromR
Set<String> excludes = varArgsExclusions(resourceClass);
List<T> resourceList = new ArrayList<>();
for (JsonElement element : resources) {
resourceList.add(parseResourceWithVarArgs(resourceClass, excludes, element, untypedResources, mutator));
T obj = gson.fromJson(element, resourceClass);
Map<String, Object> untypedResource = untypedResources.get(obj.getIdentifier());
resourceList.add(parseResourceWithVarArgs(excludes, obj, untypedResource, mutator));
}

return collectMapOfUniqueResources(resourceList);
Expand All @@ -134,8 +161,7 @@ public <T extends CatalogResource> Map<String, T> parseResourcesFromResponse(Str

@Override
public Map<String, Map<String, Object>> parseResourcesUntyped(String json) {
Type mapType = new TypeToken<Map<String, Object>>() {}.getType();
Map<String, Object> responseMap = gson.fromJson(json, mapType);
Map<String, Object> responseMap = getMapFromJsonResponse(json);

Object resources = responseMap.get("resources");
if (resources instanceof List) {
Expand Down Expand Up @@ -173,14 +199,12 @@ private ParsingException generateNoResourceException() {
}

private <T extends CatalogResource> T parseResourceWithVarArgs(
Class<T> resourceClass,
Set<String> excludes,
JsonElement element,
Map<String, Map<String, Object>> untypedResources,
T obj,
Map<String, Object> untypedResource,
ResourceMutationFactory<T> mutator) {
T obj = gson.fromJson(element, resourceClass);

Map<String, Object> varArgs = getVarArgsToInclude(untypedResources.get(obj.getIdentifier()), excludes);
Map<String, Object> varArgs = getVarArgsToInclude(untypedResource, excludes);
return mutator.mutate(
obj,
MutationContext.builder()
Expand All @@ -196,7 +220,7 @@ private Map<String, Object> getVarArgsToInclude(Map<String, Object> untypedResou
}

private static <T extends CatalogResource> Set<String> varArgsExclusions(Class<T> resourceClass) {
// TODO :: Should this be returned by the Model Object ? It Should
// TODO :: Should this be returned by the Model Object ? My Thinking is yes.
Set<String> excludes = new HashSet<>();
Set<String> excludeFromType = Arrays.stream(resourceClass.getDeclaredFields())
.map(Field::getName)
Expand All @@ -207,9 +231,16 @@ private static <T extends CatalogResource> Set<String> varArgsExclusions(Class<T
excludes.addAll(excludeFromType);
excludes.addAll(excludeFromCatalogResource);
excludes.add("@id");
excludes.add("@context");
excludes.add("@base");
return excludes;
}

private Map<String, Object> getMapFromJsonResponse(String json) {
Type mapType = new TypeToken<Map<String, Object>>() {}.getType();
return gson.fromJson(json, mapType);
}

private static <T extends CatalogResource> Map<String, T> collectMapOfUniqueResources(List<T> resourceList) {
return resourceList.stream()
.collect(Collectors.toMap(
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
package io.github.jpmorganchase.fusion.parsing;

import io.github.jpmorganchase.fusion.api.APIManager;
import io.github.jpmorganchase.fusion.api.context.APIContext;
import io.github.jpmorganchase.fusion.model.Application;
import io.github.jpmorganchase.fusion.model.Catalog;
import io.github.jpmorganchase.fusion.model.DataDictionaryAttribute;
import io.github.jpmorganchase.fusion.model.DataDictionaryAttributeLineage;
import org.hamcrest.Matchers;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;

import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.Map;

import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.*;

public class GsonAPIResponseParserDataDictionaryAttributeLineageTest {

private static final String singleDataDictionaryAttributeLineageJson =
loadTestResource("single-dd-attribute-lineage-response.json");

private final DataDictionaryAttributeLineage lineage = DataDictionaryAttributeLineage.builder()
.identifier("DR0001")
.description("Derived Attribute Description")
.title("Derived Attribute Title")
.linkedEntity("lineage/")
.catalogIdentifier(apiContext.getDefaultCatalog())
.applicationId(Application.builder().sealId("12345").build())
.apiManager(apiContext.getApiManager())
.rootUrl(apiContext.getRootUrl())
.catalogIdentifier("derived")
.catalog(Catalog.builder()
.identifier("derived")
.description("Derived Catalog Description")
.title("Derived Catalog Title")
.linkedEntity("derived/")
.build())
.varArgs(new HashMap<>())
.baseCatalogIdentifier("base")
.baseIdentifier("BA0001")
.build();


private static final APIContext apiContext = APIContext.builder()
.apiManager(Mockito.mock(APIManager.class))
.rootUrl("http://foobar/api/v1/")
.defaultCatalog("foobar")
.build();

private static final APIResponseParser responseParser = new GsonAPIResponseParser(apiContext);

@Test
public void singleDataDictionaryAttributeResourceIsParsedCorrectly() {
DataDictionaryAttributeLineage l =
responseParser.parseDataDictionaryAttributeLineageResponse(singleDataDictionaryAttributeLineageJson, "base", "BA0001");
assertThat(l, is(notNullValue()));

assertThat(l, is(equalTo(lineage)));
}


private static String loadTestResource(String resourceName) {
URL url = GsonAPIResponseParser.class.getResource(resourceName);
try {
Path path = Paths.get(url.toURI());
return new String(Files.readAllBytes(path), StandardCharsets.UTF_8);
} catch (Exception e) {
throw new RuntimeException("Failed to load test data", e);
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
{
"@context": {
"@vocab": "https://www.w3.org/ns/dcat3.jsonld",
"@base": "https://fusion-api.jpmorgan.com/fusion/v1/catalogs/base/attributes/BA001/lineage"
},
"@id": "lineage/",
"description": "Derived Attribute Description",
"identifier": "DR0001",
"title": "Derived Attribute Title",
"applicationId": {
"id": "12345",
"idType": "SEAL"
},
"catalog": {
"@id": "derived/",
"description": "Derived Catalog Description",
"identifier": "derived",
"title": "Derived Catalog Title"
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
{
"@context": {
"@vocab": "https://www.w3.org/ns/dcat3.jsonld",
"@base": "https://fusion-api.jpmorgan.com/fusion/v1/catalogs/base/attributes/BA001/lineage"
},
"@id": "lineage/",
"description": "Derived Attribute Description",
"identifier": "DR0001",
"title": "Derived Attribute Title",
"applicationId": {
"id": "12345",
"idType": "SEAL"
},
"catalog": {
"@id": "derived/",
"description": "Derived Catalog Description",
"identifier": "derived",
"title": "Derived Catalog Title"
}
}

0 comments on commit 2a80821

Please sign in to comment.