Skip to content

Commit

Permalink
RANGER-4380: updated purge API to support purging of x_policy_export_…
Browse files Browse the repository at this point in the history
…audit records
  • Loading branch information
mneethiraj committed Aug 27, 2023
1 parent 66bce09 commit b876ae7
Show file tree
Hide file tree
Showing 9 changed files with 259 additions and 45 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

package org.apache.ranger.plugin.util;

import org.codehaus.jackson.annotate.JsonAutoDetect;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.annotate.JsonSerialize;


@JsonAutoDetect(getterVisibility= JsonAutoDetect.Visibility.NONE, setterVisibility= JsonAutoDetect.Visibility.NONE, fieldVisibility= JsonAutoDetect.Visibility.ANY)
@JsonSerialize(include=JsonSerialize.Inclusion.NON_EMPTY )
@JsonIgnoreProperties(ignoreUnknown=true)
public class RangerPurgeResult {
private String recordType;
private Long totalRecordCount;
private Long purgedRecordCount;

public RangerPurgeResult() { }

public RangerPurgeResult(String recordType, Long totalRecordCount, Long purgedRecordCount) {
this.recordType = recordType;
this.totalRecordCount = totalRecordCount;
this.purgedRecordCount = purgedRecordCount;
}

public String getRecordType() {
return recordType;
}

public void setRecordType(String recordType) {
this.recordType = recordType;
}

public Long getTotalRecordCount() {
return totalRecordCount;
}

public void setTotalRecordCount(Long totalRecordCount) {
this.totalRecordCount = totalRecordCount;
}

public Long getPurgedRecordCount() {
return purgedRecordCount;
}

public void setPurgedRecordCount(Long purgedRecordCount) {
this.purgedRecordCount = purgedRecordCount;
}

@Override
public String toString( ) {
StringBuilder sb = new StringBuilder();

toString(sb);

return sb.toString();
}

public StringBuilder toString(StringBuilder sb) {
sb.append("RangerPurgeResult={");

sb.append("recordType={").append(recordType).append("} ");
sb.append("totalRecordCount={").append(totalRecordCount).append("} ");
sb.append("purgedRecordCount={").append(purgedRecordCount).append("} ");

sb.append("}");

return sb;
}

}
14 changes: 14 additions & 0 deletions intg/src/main/java/org/apache/ranger/RangerClient.java
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.ranger.audit.provider.MiscUtil;
import org.apache.ranger.authorization.hadoop.config.RangerPluginConfig;
import org.apache.ranger.plugin.util.RangerPurgeResult;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.sun.jersey.api.client.ClientResponse;
Expand Down Expand Up @@ -49,6 +50,8 @@ public class RangerClient {
private static final String PARAM_POLICY_NAME = "policyname";
private static final String PARAM_SERVICE_NAME = "serviceName";
private static final String PARAM_ZONE_NAME = "zoneName";
private static final String PARAM_PURGE_RECORD_TYPE = "type";
private static final String PARAM_PURGE_RETENTION_DAYS = "retentionDays";

private static final String PARAM_RELOAD_SERVICE_POLICIES_CACHE = "reloadServicePoliciesCache";

Expand Down Expand Up @@ -87,6 +90,7 @@ public class RangerClient {
private static final String URI_SERVICE_TAGS = URI_SERVICE + "/%s/tags";
private static final String URI_PLUGIN_INFO = URI_BASE + "/plugins/info";
private static final String URI_POLICY_DELTAS = URI_BASE + "/server/policydeltas";
private static final String URI_PURGE_RECORDS = URI_BASE + "/server/purge/records";


// APIs
Expand Down Expand Up @@ -146,6 +150,7 @@ public class RangerClient {
public static final API GET_SERVICE_TAGS = new API(URI_SERVICE_TAGS, HttpMethod.GET, Response.Status.OK);
public static final API GET_PLUGIN_INFO = new API(URI_PLUGIN_INFO, HttpMethod.GET, Response.Status.OK);
public static final API DELETE_POLICY_DELTAS = new API(URI_POLICY_DELTAS, HttpMethod.DELETE, Response.Status.NO_CONTENT);
public static final API PURGE_RECORDS = new API(URI_PURGE_RECORDS, HttpMethod.DELETE, Response.Status.OK);


private final RangerRESTClient restClient;
Expand Down Expand Up @@ -456,6 +461,15 @@ public void deletePolicyDeltas(int days, boolean reloadServicePoliciesCache) thr
callAPI(DELETE_POLICY_DELTAS, queryParams);
}

public List<RangerPurgeResult> purgeRecords(String recordType, int retentionDays) throws RangerServiceException {
Map<String, String> queryParams = new HashMap<>();

queryParams.put(PARAM_PURGE_RECORD_TYPE, recordType);
queryParams.put(PARAM_PURGE_RETENTION_DAYS, String.valueOf(retentionDays));

return callAPI(PURGE_RECORDS, queryParams, null, new GenericType<List<RangerPurgeResult>>(){});
}

private ClientResponse invokeREST(API api, Map<String, String> params, Object request) throws RangerServiceException {
final ClientResponse clientResponse;
try {
Expand Down
7 changes: 7 additions & 0 deletions intg/src/main/python/apache_ranger/client/ranger_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -299,6 +299,9 @@ def get_service_tags(self, serviceName):
def delete_policy_deltas(self, days, reloadServicePoliciesCache):
self.client_http.call_api(RangerClient.DELETE_POLICY_DELTAS, { 'days': days, 'reloadServicePoliciesCache': reloadServicePoliciesCache})

def purge_records(self, record_type, retention_days):
return self.client_http.call_api(RangerClient.PURGE_RECORDS, { 'type': record_type, 'retentionDays': retention_days})




Expand Down Expand Up @@ -338,6 +341,7 @@ def delete_policy_deltas(self, days, reloadServicePoliciesCache):
URI_SERVICE_TAGS = URI_SERVICE + "/{serviceName}/tags"
URI_PLUGIN_INFO = URI_BASE + "/plugins/info"
URI_POLICY_DELTAS = URI_BASE + "/server/policydeltas"
URI_PURGE_RECORDS = URI_BASE + "/server/purge/records"

# APIs
CREATE_SERVICEDEF = API(URI_SERVICEDEF, HttpMethod.POST, HTTPStatus.OK)
Expand Down Expand Up @@ -397,6 +401,7 @@ def delete_policy_deltas(self, days, reloadServicePoliciesCache):
GET_SERVICE_TAGS = API(URI_SERVICE_TAGS, HttpMethod.GET, HTTPStatus.OK)
GET_PLUGIN_INFO = API(URI_PLUGIN_INFO, HttpMethod.GET, HTTPStatus.OK)
DELETE_POLICY_DELTAS = API(URI_POLICY_DELTAS, HttpMethod.DELETE, HTTPStatus.NO_CONTENT)
PURGE_RECORDS = API(URI_PURGE_RECORDS, HttpMethod.DELETE, HTTPStatus.OK)



Expand Down Expand Up @@ -531,6 +536,8 @@ def call_api(self, api, query_params=None, request_data=None):
elif response.status_code == HTTPStatus.NOT_FOUND:
LOG.error("Not found. HTTP Status: %s", HTTPStatus.NOT_FOUND)

ret = None
elif response.status_code == HTTPStatus.NOT_MODIFIED:
ret = None
else:
raise RangerServiceException(api, response)
Expand Down
14 changes: 14 additions & 0 deletions intg/src/test/java/org/apache/ranger/TestRangerClient.java
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import org.apache.ranger.plugin.model.RangerSecurityZoneHeaderInfo;
import org.apache.ranger.plugin.model.RangerService;
import org.apache.ranger.plugin.model.RangerServiceHeaderInfo;
import org.apache.ranger.plugin.util.RangerPurgeResult;
import org.apache.ranger.plugin.util.RangerRESTClient;
import org.junit.Assert;
import org.junit.Test;
Expand Down Expand Up @@ -202,4 +203,17 @@ public void testFindSecurityZones() throws RangerServiceException {

Assert.assertEquals(Collections.emptyList(), securityZones);
}

@Test
public void testPurgeRecords() throws RangerServiceException {
RangerClient client = Mockito.mock(RangerClient.class);
String recordType = "login_records";
int retentionDays = 180;

when(client.purgeRecords(recordType, retentionDays)).thenReturn(Collections.emptyList());

List<RangerPurgeResult> purgeResults = client.purgeRecords(recordType, retentionDays);

Assert.assertEquals(Collections.emptyList(), purgeResults);
}
}
110 changes: 83 additions & 27 deletions security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
Original file line number Diff line number Diff line change
Expand Up @@ -67,11 +67,14 @@
import org.apache.ranger.common.MessageEnums;
import org.apache.ranger.common.RangerCommonEnums;
import org.apache.ranger.common.db.RangerTransactionSynchronizationAdapter;
import org.apache.ranger.db.XXAuthSessionDao;
import org.apache.ranger.db.XXGlobalStateDao;
import org.apache.ranger.db.XXPolicyDao;
import org.apache.ranger.db.XXTrxLogDao;
import org.apache.ranger.entity.XXTagChangeLog;
import org.apache.ranger.plugin.model.RangerSecurityZone;
import org.apache.ranger.plugin.util.RangerCommonConstants;
import org.apache.ranger.plugin.util.RangerPurgeResult;
import org.apache.ranger.plugin.util.ServiceTags;
import org.apache.ranger.plugin.model.validation.RangerServiceDefValidator;
import org.apache.ranger.plugin.model.validation.RangerValidator;
Expand Down Expand Up @@ -102,6 +105,7 @@
import org.apache.ranger.db.XXEnumDefDao;
import org.apache.ranger.db.XXEnumElementDefDao;
import org.apache.ranger.db.XXPolicyConditionDefDao;
import org.apache.ranger.db.XXPolicyExportAuditDao;
import org.apache.ranger.db.XXPolicyLabelMapDao;
import org.apache.ranger.db.XXResourceDefDao;
import org.apache.ranger.db.XXServiceConfigDefDao;
Expand Down Expand Up @@ -255,6 +259,8 @@ public class ServiceDBStore extends AbstractServiceStore {
public static Integer LOGIN_RECORDS_RETENTION_PERIOD_IN_DAYS = 0;
public static boolean SUPPORTS_PURGE_TRANSACTION_RECORDS = false;
public static Integer TRANSACTION_RECORDS_RETENTION_PERIOD_IN_DAYS = 0;
public static boolean SUPPORTS_PURGE_POLICY_EXPORT_LOGS = false;
public static Integer POLICY_EXPORT_LOGS_RETENTION_PERIOD_IN_DAYS = 0;

private static final String RANGER_PLUGIN_CONFIG_PREFIX = "ranger.plugin.";
public static final String RANGER_PLUGIN_AUDIT_FILTERS = "ranger.plugin.audit.filters";
Expand Down Expand Up @@ -398,10 +404,12 @@ public void initStore() {
RETENTION_PERIOD_IN_DAYS = config.getInt("ranger.admin.delta.retention.time.in.days", 7);
TAG_RETENTION_PERIOD_IN_DAYS = config.getInt("ranger.admin.tag.delta.retention.time.in.days", 3);

SUPPORTS_PURGE_LOGIN_RECORDS = config.getBoolean("ranger.admin.init.purge.login_records", false);
LOGIN_RECORDS_RETENTION_PERIOD_IN_DAYS = config.getInt("ranger.admin.init.purge.login_records.retention.days", 0);
SUPPORTS_PURGE_LOGIN_RECORDS = config.getBoolean("ranger.admin.init.purge.login_records", false);
SUPPORTS_PURGE_TRANSACTION_RECORDS = config.getBoolean("ranger.admin.init.purge.transaction_records", false);
SUPPORTS_PURGE_POLICY_EXPORT_LOGS = config.getBoolean("ranger.admin.init.purge.policy_export_logs", false);
LOGIN_RECORDS_RETENTION_PERIOD_IN_DAYS = config.getInt("ranger.admin.init.purge.login_records.retention.days", 0);
TRANSACTION_RECORDS_RETENTION_PERIOD_IN_DAYS = config.getInt("ranger.admin.init.purge.transaction_records.retention.days", 0);
POLICY_EXPORT_LOGS_RETENTION_PERIOD_IN_DAYS = config.getInt("ranger.admin.init.purge.policy_export_logs.retention.days", 0);

isRolesDownloadedByService = config.getBoolean("ranger.support.for.service.specific.role.download", false);
SUPPORTS_IN_PLACE_POLICY_UPDATES = SUPPORTS_POLICY_DELTAS && config.getBoolean("ranger.admin" + RangerCommonConstants.RANGER_ADMIN_SUFFIX_IN_PLACE_POLICY_UPDATES, RangerCommonConstants.RANGER_ADMIN_SUFFIX_IN_PLACE_POLICY_UPDATES_DEFAULT);
Expand All @@ -413,6 +421,8 @@ public void initStore() {
LOG.info("LOGIN_RECORDS_RETENTION_PERIOD_IN_DAYS=" + LOGIN_RECORDS_RETENTION_PERIOD_IN_DAYS);
LOG.info("SUPPORTS_PURGE_TRANSACTION_RECORDS=" + SUPPORTS_PURGE_TRANSACTION_RECORDS);
LOG.info("TRANSACTION_RECORDS_RETENTION_PERIOD_IN_DAYS=" + TRANSACTION_RECORDS_RETENTION_PERIOD_IN_DAYS);
LOG.info("SUPPORTS_PURGE_POLICY_EXPORT_LOGS=" + SUPPORTS_PURGE_POLICY_EXPORT_LOGS);
LOG.info("POLICY_EXPORT_LOGS_RETENTION_PERIOD_IN_DAYS=" + POLICY_EXPORT_LOGS_RETENTION_PERIOD_IN_DAYS);
LOG.info("isRolesDownloadedByService=" + isRolesDownloadedByService);
LOG.info("SUPPORTS_IN_PLACE_POLICY_UPDATES=" + SUPPORTS_IN_PLACE_POLICY_UPDATES);

Expand All @@ -430,12 +440,21 @@ public Object doInTransaction(TransactionStatus status) {
createGenericUsers();
resetPolicyUpdateLog(RETENTION_PERIOD_IN_DAYS, RangerPolicyDelta.CHANGE_TYPE_RANGER_ADMIN_START);
resetTagUpdateLog(TAG_RETENTION_PERIOD_IN_DAYS, ServiceTags.TagsChangeType.RANGER_ADMIN_START);

List<RangerPurgeResult> purgeResults = new ArrayList<>();

if (SUPPORTS_PURGE_LOGIN_RECORDS) {
removeAuthSessions(LOGIN_RECORDS_RETENTION_PERIOD_IN_DAYS);
removeAuthSessions(LOGIN_RECORDS_RETENTION_PERIOD_IN_DAYS, purgeResults);
}

if (SUPPORTS_PURGE_TRANSACTION_RECORDS) {
removeTransactionLogs(TRANSACTION_RECORDS_RETENTION_PERIOD_IN_DAYS);
removeTransactionLogs(TRANSACTION_RECORDS_RETENTION_PERIOD_IN_DAYS, purgeResults);
}

if (SUPPORTS_PURGE_POLICY_EXPORT_LOGS) {
removePolicyExportLogs(POLICY_EXPORT_LOGS_RETENTION_PERIOD_IN_DAYS, purgeResults);
}

//createUnzonedSecurityZone();
initRMSDaos();
return null;
Expand Down Expand Up @@ -5326,53 +5345,90 @@ public void resetTagUpdateLog(int retentionInDays, ServiceTags.TagsChangeType ta
}
}

public void removeAuthSessions(int retentionInDays) {
public void removeAuthSessions(int retentionInDays, List<RangerPurgeResult> result) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> removeAuthSessions(" + retentionInDays + ")");
}

if (retentionInDays > 0) {
long rowsCount = daoMgr.getXXAuthSession().getAllCount();
long rowsDeleted = daoMgr.getXXAuthSession().deleteOlderThan(retentionInDays);
XXAuthSessionDao dao = daoMgr.getXXAuthSession();
long rowsCount = dao.getAllCount();
long rowsDeleted = dao.deleteOlderThan(retentionInDays);

LOG.info("Deleted " + rowsDeleted + " records from x_auth_sess that are older than " + retentionInDays + " days");
List<XXTrxLog> trxLogList = new ArrayList<XXTrxLog>();
XXTrxLog xxTrxLog = new XXTrxLog();
xxTrxLog.setAction("Deleted Auth Session records");
xxTrxLog.setObjectClassType(AppConstants.CLASS_TYPE_AUTH_SESS);
xxTrxLog.setPreviousValue("Total Records : "+rowsCount);
xxTrxLog.setNewValue("Deleted Records : "+rowsDeleted);
trxLogList.add(xxTrxLog);
bizUtil.createTrxLog(trxLogList);

XXTrxLog trxLog = new XXTrxLog();

trxLog.setAction("Deleted Auth Session records");
trxLog.setObjectClassType(AppConstants.CLASS_TYPE_AUTH_SESS);
trxLog.setPreviousValue("Total Records : " + rowsCount);
trxLog.setNewValue("Deleted Records : " + rowsDeleted);

bizUtil.createTrxLog(Collections.singletonList(trxLog));

result.add(new RangerPurgeResult(ServiceREST.PURGE_RECORD_TYPE_LOGIN_LOGS, rowsCount, rowsDeleted));
}

if (LOG.isDebugEnabled()) {
LOG.debug("<== removeAuthSessions(" + retentionInDays + ")");

}
}

public void removeTransactionLogs(int retentionInDays) {
public void removeTransactionLogs(int retentionInDays, List<RangerPurgeResult> result) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> removeTransactionLogs(" + retentionInDays + ")");
}

if (retentionInDays > 0) {
long rowsCount = daoMgr.getXXTrxLog().getAllCount();
long rowsDeleted = daoMgr.getXXTrxLog().deleteOlderThan(retentionInDays);
XXTrxLogDao dao = daoMgr.getXXTrxLog();
long rowsCount = dao.getAllCount();
long rowsDeleted = dao.deleteOlderThan(retentionInDays);

LOG.info("Deleted " + rowsDeleted + " records from x_trx_log that are older than " + retentionInDays + " days");
List<XXTrxLog> trxLogList = new ArrayList<XXTrxLog>();
XXTrxLog xxTrxLog = new XXTrxLog();
xxTrxLog.setAction("Deleted Transaction records");
xxTrxLog.setObjectClassType(AppConstants.CLASS_TYPE_AUTH_SESS);
xxTrxLog.setPreviousValue("Total Records : "+rowsCount);
xxTrxLog.setNewValue("Deleted Records : "+rowsDeleted);
trxLogList.add(xxTrxLog);
bizUtil.createTrxLog(trxLogList);

XXTrxLog trxLog = new XXTrxLog();

trxLog.setAction("Deleted Transaction records");
trxLog.setObjectClassType(AppConstants.CLASS_TYPE_TRX_LOG);
trxLog.setPreviousValue("Total Records : " + rowsCount);
trxLog.setNewValue("Deleted Records : " + rowsDeleted);

bizUtil.createTrxLog(Collections.singletonList(trxLog));

result.add(new RangerPurgeResult(ServiceREST.PURGE_RECORD_TYPE_TRX_LOGS, rowsCount, rowsDeleted));
}

if (LOG.isDebugEnabled()) {
LOG.debug("<== removeTransactionLogs(" + retentionInDays + ")");
}
}

public void removePolicyExportLogs(int retentionInDays, List<RangerPurgeResult> result) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> removePolicyExportLogs({})", retentionInDays);
}

if (retentionInDays > 0) {
XXPolicyExportAuditDao dao = daoMgr.getXXPolicyExportAudit();
long rowsCount = dao.getAllCount();
long rowsDeleted = dao.deleteOlderThan(retentionInDays);

LOG.info("Deleted {} records from x_policy_export_audit that are older than {} days", rowsDeleted, retentionInDays);

XXTrxLog trxLog = new XXTrxLog();

trxLog.setAction("Deleted policy export audit records");
trxLog.setObjectClassType(AppConstants.CLASS_TYPE_XA_POLICY_EXPORT_AUDIT);
trxLog.setPreviousValue("Total Records: " + rowsCount);
trxLog.setNewValue("Deleted Records: " + rowsDeleted);

bizUtil.createTrxLog(Collections.singletonList(trxLog));

result.add(new RangerPurgeResult(ServiceREST.PURGE_RECORD_TYPE_POLICY_EXPORT_LOGS, rowsCount, rowsDeleted));
}

if (LOG.isDebugEnabled()) {
LOG.debug("<== removePolicyExportLogs({})", retentionInDays);
}
}

Expand Down
Loading

0 comments on commit b876ae7

Please sign in to comment.