001package com.identityworksllc.iiq.common.task.export; 002 003import com.identityworksllc.iiq.common.request.SailPointWorkerExecutor; 004import com.identityworksllc.iiq.common.threads.SailPointWorker; 005import sailpoint.api.SailPointContext; 006import sailpoint.object.Attributes; 007import sailpoint.object.Request; 008import sailpoint.object.RequestDefinition; 009import sailpoint.object.TaskResult; 010import sailpoint.object.TaskSchedule; 011import sailpoint.task.AbstractTaskExecutor; 012import sailpoint.tools.GeneralException; 013import sailpoint.tools.Message; 014import sailpoint.tools.Util; 015 016import java.sql.Connection; 017import java.sql.PreparedStatement; 018import java.sql.ResultSet; 019import java.sql.SQLException; 020import java.util.ArrayList; 021import java.util.Date; 022import java.util.List; 023import java.util.Objects; 024import java.util.concurrent.atomic.AtomicBoolean; 025 026/** 027 * A partitioned task for handling data exports. The task can be provided multiple filters 028 * that should cover the entire set of desired export users. 029 * 030 * The partitions will run in three phases: the actual export, then a cleanup of any Links 031 * no longer in IIQ, then a finalization step that sets the last run date. 032 */ 033public class IDWDataExporter extends AbstractTaskExecutor { 034 private final AtomicBoolean stopped; 035 036 public IDWDataExporter() { 037 this.stopped = new AtomicBoolean(); 038 } 039 /** 040 * @see sailpoint.object.TaskExecutor#execute(SailPointContext, TaskSchedule, TaskResult, Attributes) 041 */ 042 @Override 043 public void execute(SailPointContext context, TaskSchedule taskSchedule, TaskResult taskResult, Attributes<String, Object> attributes) throws Exception { 044 String requestDefinitionName = attributes.getString("requestDefinitionName"); 045 if (Util.isNullOrEmpty(requestDefinitionName)) { 046 requestDefinitionName = SailPointWorkerExecutor.REQUEST_DEFINITION; 047 } 048 RequestDefinition requestDefinition = context.getObjectByName(RequestDefinition.class, requestDefinitionName); 049 if (requestDefinition == null) { 050 throw new IllegalArgumentException("Request definition called " + requestDefinitionName + " does not exist; do you need to import it?"); 051 } 052 053 taskResult.addMessage("Partitions will execute using the Request Definition called " + requestDefinitionName); 054 055 List<ExportPartition> clusters = getPartitions(context, taskResult, attributes); 056 057 List<Request> partitionRequests = new ArrayList<>(); 058 for(ExportPartition partition : clusters) { 059 List<SailPointWorker> workerCluster = new ArrayList<>(); 060 workerCluster.add(partition); 061 // Serializes the SailPointWorker object so that it can be saved 062 Request partitionRequest = SailPointWorker.toRequest(requestDefinition, workerCluster); 063 partitionRequest.setName(partition.getName()); 064 partitionRequests.add(partitionRequest); 065 } 066 067 taskResult.addMessage(Message.info("Launching " + partitionRequests.size() + " partitions")); 068 069 launchPartitions(context, taskResult, partitionRequests); 070 } 071 072 /** 073 * Gets the list of partitions for the export operation. These will each have their 'phase' 074 * attribute set so that they run in order. 075 * 076 * @param context The context 077 * @param taskResult The task result for the parent task 078 * @param attributes The attributes of the task execution 079 * @return The resulting list of partitions to launch 080 * @throws GeneralException if any failures occur 081 */ 082 public List<ExportPartition> getPartitions(SailPointContext context, TaskResult taskResult, Attributes<String, Object> attributes) throws GeneralException { 083 long now = System.currentTimeMillis(); 084 085 List<ExportPartition> partitions = new ArrayList<>(); 086 String configurationName = attributes.getString("configurationName"); 087 if (Util.isNullOrEmpty(configurationName)) { 088 throw new GeneralException("A configurationName setting is required for the data export job"); 089 } 090 091 List<String> identityFilters = attributes.getStringList("identityFilters"); 092 List<String> linkFilters = attributes.getStringList("linkFilters"); 093 List<String> linkFilters2 = attributes.getStringList("linkFilters2"); 094 095 // Everything in one giant partition by default 096 if (identityFilters == null || identityFilters.isEmpty()) { 097 identityFilters = new ArrayList<>(); 098 identityFilters.add("id.notNull()"); 099 } 100 101 // Everything in one giant partition by default 102 if (linkFilters == null || linkFilters.isEmpty()) { 103 linkFilters = new ArrayList<>(); 104 linkFilters.add("id.notNull()"); 105 } 106 107 if (linkFilters2 == null || linkFilters2.isEmpty()) { 108 linkFilters2 = new ArrayList<>(); 109 linkFilters2.add(null); 110 } 111 112 boolean doLinkCleanup = attributes.getBoolean("linkCleanup", true); 113 114 String driver = attributes.getString("driver"); 115 String url = attributes.getString("url"); 116 String username = attributes.getString("username"); 117 String password = attributes.getString("password"); 118 119 ExportConnectionInfo connectionInfo = new ExportConnectionInfo(url, username, password); 120 connectionInfo.setDriver(driver); 121 122 long cutoffDate = 0L; 123 String configHash = String.valueOf(Objects.hash(doLinkCleanup, linkFilters, linkFilters2, identityFilters, configurationName, connectionInfo)); 124 125 try (Connection connection = ExportPartition.openConnection(context, connectionInfo)) { 126 try (PreparedStatement statement = connection.prepareStatement("select last_start_time, config_hash from de_runs order by last_start_time desc")) { 127 try (ResultSet results = statement.executeQuery()) { 128 if (results.next()) { 129 String configHashString = results.getString(2); 130 taskResult.addMessage(Message.info("New config hash = " + configHash + ", old config hash = " + configHashString)); 131 if (Util.nullSafeEq(configHashString, configHash)) { 132 cutoffDate = results.getLong(1); 133 } else { 134 taskResult.addMessage(Message.warn("Configuration has changed; forcing a full export")); 135 } 136 } 137 } 138 } 139 } catch(SQLException e) { 140 throw new GeneralException(e); 141 } 142 143 taskResult.addMessage(Message.info("Cutoff timestamp is: " + new Date(cutoffDate))); 144 145 int count = 1; 146 for(String filter : Util.safeIterable(identityFilters)) { 147 ExportIdentitiesPartition eip = new ExportIdentitiesPartition(); 148 eip.setName("Identity export partition " + count++); 149 eip.setPhase(1); 150 eip.setExportTimestamp(now); 151 eip.setCutoffDate(cutoffDate); 152 eip.setFilterString(filter); 153 eip.setConnectionInfo(connectionInfo); 154 eip.setConfigurationName(configurationName); 155 156 partitions.add(eip); 157 } 158 159 count = 1; 160 for(String filter : Util.safeIterable(linkFilters)) { 161 for(String filter2 : Util.safeIterable(linkFilters2)) { 162 ExportLinksPartition eip = new ExportLinksPartition(); 163 eip.setName("Link export partition " + count++); 164 eip.setPhase(2); 165 eip.setDependentPhase(1); 166 eip.setExportTimestamp(now); 167 eip.setCutoffDate(cutoffDate); 168 eip.setFilterString(filter); 169 eip.setFilterString2(filter2); 170 eip.setConnectionInfo(connectionInfo); 171 eip.setConfigurationName(configurationName); 172 173 partitions.add(eip); 174 } 175 } 176 177 if (doLinkCleanup) { 178 CleanupLinksPartition clp = new CleanupLinksPartition(); 179 clp.setPhase(3); 180 clp.setDependentPhase(2); 181 clp.setName("Clean up deleted Links"); 182 clp.setConnectionInfo(connectionInfo); 183 partitions.add(clp); 184 } 185 186 ExportFinishPartition efp = new ExportFinishPartition(); 187 efp.setExportTimestamp(now); 188 efp.setCutoffDate(cutoffDate); 189 efp.setName("Finalize export"); 190 efp.setPhase(4); 191 efp.setDependentPhase(2); 192 efp.setConfigurationName(configurationName); 193 efp.setConfigHash(configHash); 194 partitions.add(efp); 195 196 return partitions; 197 } 198 199 @Override 200 public boolean terminate() { 201 this.stopped.set(true); 202 return true; 203 } 204}