From fbf8033b88255aea0cb125bb28a6c10ab93699e3 Mon Sep 17 00:00:00 2001 From: Umesh Patil Date: Sat, 27 Dec 2025 19:47:44 +0530 Subject: [PATCH 1/2] ATLAS_4988: Resolve slow deletion issue of Business Metadata with array-type attributes. --- .../v2/AtlasBusinessMetadataDefStoreV2.java | 64 ++++++++++++++----- 1 file changed, 48 insertions(+), 16 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasBusinessMetadataDefStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasBusinessMetadataDefStoreV2.java index 5511935f918..8bceb102d8c 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasBusinessMetadataDefStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasBusinessMetadataDefStoreV2.java @@ -27,10 +27,12 @@ import org.apache.atlas.model.TypeCategory; import org.apache.atlas.model.discovery.AtlasSearchResult; import org.apache.atlas.model.discovery.SearchParameters; +import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.typedef.AtlasBaseTypeDef; import org.apache.atlas.model.typedef.AtlasBusinessMetadataDef; import org.apache.atlas.model.typedef.AtlasStructDef; import org.apache.atlas.repository.Constants; +import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.type.AtlasBusinessMetadataType; import org.apache.atlas.type.AtlasStructType; @@ -58,6 +60,9 @@ public class AtlasBusinessMetadataDefStoreV2 extends AtlasAbstractDefStoreV2 attributeDefs = businessMetadataDef.getAttributeDefs(); + if (businessMetadataDef == null || CollectionUtils.isEmpty(businessMetadataDef.getAttributeDefs())) { + return; + } - for (AtlasStructDef.AtlasAttributeDef attributeDef : attributeDefs) { - String qualifiedName = AtlasStructType.AtlasAttribute.getQualifiedAttributeName(businessMetadataDef, attributeDef.getName()); - String vertexPropertyName = AtlasStructType.AtlasAttribute.generateVertexPropertyName(businessMetadataDef, attributeDef, qualifiedName); - Set applicableTypes = AtlasJson.fromJson(attributeDef.getOption(AtlasBusinessMetadataDef.ATTR_OPTION_APPLICABLE_ENTITY_TYPES), Set.class); + for (AtlasStructDef.AtlasAttributeDef attributeDef : businessMetadataDef.getAttributeDefs()) { + String applicableTypesStr = attributeDef.getOption(ATTR_OPTION_APPLICABLE_ENTITY_TYPES); + Set applicableTypes = StringUtils.isBlank(applicableTypesStr) ? null : AtlasJson.fromJson(applicableTypesStr, Set.class); - if (CollectionUtils.isNotEmpty(applicableTypes) && isBusinessAttributePresent(vertexPropertyName, applicableTypes)) { - throw new AtlasBaseException(AtlasErrorCode.TYPE_HAS_REFERENCES, typeName); - } + if (CollectionUtils.isEmpty(applicableTypes)) { + continue; + } + + String qualifiedName = AtlasStructType.AtlasAttribute.getQualifiedAttributeName(businessMetadataDef, attributeDef.getName()); + String vertexPropertyName = AtlasStructType.AtlasAttribute.generateVertexPropertyName(businessMetadataDef, attributeDef, qualifiedName); + + boolean isPresent; + + if (Boolean.TRUE.equals(attributeDef.getIsIndexable())) { + isPresent = isBusinessAttributePresent(vertexPropertyName, applicableTypes); + } else { + isPresent = isBusinessAttributePresentInGraph(vertexPropertyName, applicableTypes); + } + + if (isPresent) { + throw new AtlasBaseException(AtlasErrorCode.TYPE_HAS_REFERENCES, typeName); } } } private boolean isBusinessAttributePresent(String attrName, Set applicableTypes) throws AtlasBaseException { SearchParameters.FilterCriteria criteria = new SearchParameters.FilterCriteria(); - criteria.setAttributeName(attrName); - criteria.setOperator(SearchParameters.Operator.NOT_EMPTY); + criteria.setOperator(SearchParameters.Operator.NOT_NULL); SearchParameters.FilterCriteria entityFilters = new SearchParameters.FilterCriteria(); - entityFilters.setCondition(SearchParameters.FilterCriteria.Condition.OR); entityFilters.setCriterion(Collections.singletonList(criteria)); SearchParameters searchParameters = new SearchParameters(); - searchParameters.setTypeName(String.join(SearchContext.TYPENAME_DELIMITER, applicableTypes)); searchParameters.setExcludeDeletedEntities(true); searchParameters.setIncludeSubClassifications(false); searchParameters.setEntityFilters(entityFilters); - searchParameters.setAttributes(Collections.singleton(attrName)); - searchParameters.setOffset(0); + searchParameters.setAttributes(Collections.emptySet()); searchParameters.setLimit(1); AtlasSearchResult atlasSearchResult = entityDiscoveryService.searchWithParameters(searchParameters); + return atlasSearchResult != null && CollectionUtils.isNotEmpty(atlasSearchResult.getEntities()); + } + + private boolean isBusinessAttributePresentInGraph(String vertexPropertyName, Set applicableTypes) { + for (String typeName : applicableTypes) { + + Iterable vertices = graph.getVertices(Constants.ENTITY_TYPE_PROPERTY_KEY, typeName); - return CollectionUtils.isNotEmpty(atlasSearchResult.getEntities()); + if (vertices != null) { + for (AtlasVertex vertex : vertices) { + + if (vertex.getProperty(vertexPropertyName, Object.class) != null && + AtlasGraphUtilsV2.getState(vertex) == AtlasEntity.Status.ACTIVE) { + return true; + } + } + } + } + return false; } } From 5bf649f7baf7ab23d3c31e405115f79e6c488ea8 Mon Sep 17 00:00:00 2001 From: Umesh Patil Date: Tue, 30 Dec 2025 10:43:35 +0530 Subject: [PATCH 2/2] ATLAS-4988 : Replaced manual iteration with graph query, added error handling and logging. --- .../v2/AtlasBusinessMetadataDefStoreV2.java | 67 ++++++++++++------- 1 file changed, 44 insertions(+), 23 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasBusinessMetadataDefStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasBusinessMetadataDefStoreV2.java index 8bceb102d8c..d22f4eac7cc 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasBusinessMetadataDefStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasBusinessMetadataDefStoreV2.java @@ -33,6 +33,7 @@ import org.apache.atlas.model.typedef.AtlasStructDef; import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.repository.graphdb.AtlasGraphQuery; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.type.AtlasBusinessMetadataType; import org.apache.atlas.type.AtlasStructType; @@ -373,27 +374,37 @@ private void checkBusinessMetadataRef(String typeName) throws AtlasBaseException } for (AtlasStructDef.AtlasAttributeDef attributeDef : businessMetadataDef.getAttributeDefs()) { - String applicableTypesStr = attributeDef.getOption(ATTR_OPTION_APPLICABLE_ENTITY_TYPES); - Set applicableTypes = StringUtils.isBlank(applicableTypesStr) ? null : AtlasJson.fromJson(applicableTypesStr, Set.class); + validateAttributeReferences(businessMetadataDef, attributeDef); + } + } - if (CollectionUtils.isEmpty(applicableTypes)) { - continue; - } + private void validateAttributeReferences(AtlasBusinessMetadataDef bmDef, AtlasStructDef.AtlasAttributeDef attributeDef) throws AtlasBaseException { + String applicableTypesStr = attributeDef.getOption(ATTR_OPTION_APPLICABLE_ENTITY_TYPES); + Set applicableTypes = StringUtils.isBlank(applicableTypesStr) ? null : AtlasJson.fromJson(applicableTypesStr, Set.class); - String qualifiedName = AtlasStructType.AtlasAttribute.getQualifiedAttributeName(businessMetadataDef, attributeDef.getName()); - String vertexPropertyName = AtlasStructType.AtlasAttribute.generateVertexPropertyName(businessMetadataDef, attributeDef, qualifiedName); + if (CollectionUtils.isEmpty(applicableTypes)) { + return; + } - boolean isPresent; + String qualifiedName = AtlasStructType.AtlasAttribute.getQualifiedAttributeName(bmDef, attributeDef.getName()); + String vertexPropertyName = AtlasStructType.AtlasAttribute.generateVertexPropertyName(bmDef, attributeDef, qualifiedName); - if (Boolean.TRUE.equals(attributeDef.getIsIndexable())) { - isPresent = isBusinessAttributePresent(vertexPropertyName, applicableTypes); - } else { - isPresent = isBusinessAttributePresentInGraph(vertexPropertyName, applicableTypes); - } + boolean isPresent; + long startTime = System.currentTimeMillis(); - if (isPresent) { - throw new AtlasBaseException(AtlasErrorCode.TYPE_HAS_REFERENCES, typeName); - } + if (Boolean.TRUE.equals(attributeDef.getIsIndexable())) { + isPresent = isBusinessAttributePresent(vertexPropertyName, applicableTypes); + } else { + isPresent = isBusinessAttributePresentInGraph(vertexPropertyName, applicableTypes); + } + + if (LOG.isDebugEnabled()) { + LOG.info("Reference check for attribute {} took {} ms. Found: {}", + attributeDef.getName(), (System.currentTimeMillis() - startTime), isPresent); + } + + if (isPresent) { + throw new AtlasBaseException(AtlasErrorCode.TYPE_HAS_REFERENCES, bmDef.getName()); } } @@ -419,19 +430,29 @@ private boolean isBusinessAttributePresent(String attrName, Set applicab } private boolean isBusinessAttributePresentInGraph(String vertexPropertyName, Set applicableTypes) { - for (String typeName : applicableTypes) { + if (graph == null || CollectionUtils.isEmpty(applicableTypes)) { + return false; + } - Iterable vertices = graph.getVertices(Constants.ENTITY_TYPE_PROPERTY_KEY, typeName); + try { + for (String typeName : applicableTypes) { - if (vertices != null) { - for (AtlasVertex vertex : vertices) { + Iterable vertices = graph.query() + .has(Constants.ENTITY_TYPE_PROPERTY_KEY, typeName) + .has(vertexPropertyName, AtlasGraphQuery.ComparisionOperator.NOT_EQUAL, (Object) null) + .has(Constants.STATE_PROPERTY_KEY, AtlasEntity.Status.ACTIVE.name()) + .vertices(); - if (vertex.getProperty(vertexPropertyName, Object.class) != null && - AtlasGraphUtilsV2.getState(vertex) == AtlasEntity.Status.ACTIVE) { - return true; + if (vertices != null && vertices.iterator().hasNext()) { + if (LOG.isDebugEnabled()) { + LOG.info("Found at least one reference for {} on type {}", vertexPropertyName, typeName); } + return true; } } + } catch (Exception e) { + LOG.error("Error occurred while querying graph for references of property: {}", vertexPropertyName, e); + return true; } return false; }