001package com.identityworksllc.iiq.common.task.export;
002
003import com.identityworksllc.iiq.common.request.SailPointWorkerExecutor;
004import com.identityworksllc.iiq.common.threads.SailPointWorker;
005import sailpoint.api.SailPointContext;
006import sailpoint.object.Attributes;
007import sailpoint.object.Request;
008import sailpoint.object.RequestDefinition;
009import sailpoint.object.TaskResult;
010import sailpoint.object.TaskSchedule;
011import sailpoint.task.AbstractTaskExecutor;
012import sailpoint.tools.GeneralException;
013import sailpoint.tools.Message;
014import sailpoint.tools.Util;
015
016import java.sql.Connection;
017import java.sql.PreparedStatement;
018import java.sql.ResultSet;
019import java.sql.SQLException;
020import java.util.ArrayList;
021import java.util.Date;
022import java.util.List;
023import java.util.Objects;
024import java.util.concurrent.atomic.AtomicBoolean;
025
026/**
027 * A partitioned task for handling data exports. The task can be provided multiple filters
028 * that should cover the entire set of desired export users.
029 *
030 * The partitions will run in three phases: the actual export, then a cleanup of any Links
031 * no longer in IIQ, then a finalization step that sets the last run date.
032 */
033public class IDWDataExporter extends AbstractTaskExecutor {
034    private final AtomicBoolean stopped;
035
036    public IDWDataExporter() {
037        this.stopped = new AtomicBoolean();
038    }
039    /**
040     * @see sailpoint.object.TaskExecutor#execute(SailPointContext, TaskSchedule, TaskResult, Attributes)
041     */
042    @Override
043    public void execute(SailPointContext context, TaskSchedule taskSchedule, TaskResult taskResult, Attributes<String, Object> attributes) throws Exception {
044        String requestDefinitionName = attributes.getString("requestDefinitionName");
045        if (Util.isNullOrEmpty(requestDefinitionName)) {
046            requestDefinitionName = SailPointWorkerExecutor.REQUEST_DEFINITION;
047        }
048        RequestDefinition requestDefinition = context.getObjectByName(RequestDefinition.class, requestDefinitionName);
049        if (requestDefinition == null) {
050            throw new IllegalArgumentException("Request definition called " + requestDefinitionName + " does not exist; do you need to import it?");
051        }
052
053        taskResult.addMessage("Partitions will execute using the Request Definition called " + requestDefinitionName);
054
055        List<ExportPartition> clusters = getPartitions(context, taskResult, attributes);
056
057        List<Request> partitionRequests = new ArrayList<>();
058        for(ExportPartition partition : clusters) {
059            List<SailPointWorker> workerCluster = new ArrayList<>();
060            workerCluster.add(partition);
061            // Serializes the SailPointWorker object so that it can be saved
062            Request partitionRequest = SailPointWorker.toRequest(requestDefinition, workerCluster);
063            partitionRequest.setName(partition.getName());
064            partitionRequests.add(partitionRequest);
065        }
066
067        taskResult.addMessage(Message.info("Launching " + partitionRequests.size() + " partitions"));
068
069        launchPartitions(context, taskResult, partitionRequests);
070    }
071
072    /**
073     * Gets the list of partitions for the export operation. These will each have their 'phase'
074     * attribute set so that they run in order.
075     *
076     * @param context The context
077     * @param taskResult The task result for the parent task
078     * @param attributes The attributes of the task execution
079     * @return The resulting list of partitions to launch
080     * @throws GeneralException if any failures occur
081     */
082    public List<ExportPartition> getPartitions(SailPointContext context, TaskResult taskResult, Attributes<String, Object> attributes) throws GeneralException {
083        long now = System.currentTimeMillis();
084
085        List<ExportPartition> partitions = new ArrayList<>();
086        String configurationName = attributes.getString("configurationName");
087        if (Util.isNullOrEmpty(configurationName)) {
088            throw new GeneralException("A configurationName setting is required for the data export job");
089        }
090
091        List<String> identityFilters = attributes.getStringList("identityFilters");
092        List<String> linkFilters = attributes.getStringList("linkFilters");
093        List<String> linkFilters2 = attributes.getStringList("linkFilters2");
094
095        // Everything in one giant partition by default
096        if (identityFilters == null || identityFilters.isEmpty()) {
097            identityFilters = new ArrayList<>();
098            identityFilters.add("id.notNull()");
099        }
100
101        // Everything in one giant partition by default
102        if (linkFilters == null || linkFilters.isEmpty()) {
103            linkFilters = new ArrayList<>();
104            linkFilters.add("id.notNull()");
105        }
106
107        if (linkFilters2 == null || linkFilters2.isEmpty()) {
108            linkFilters2 = new ArrayList<>();
109            linkFilters2.add(null);
110        }
111
112        boolean doLinkCleanup = attributes.getBoolean("linkCleanup", true);
113
114        String driver = attributes.getString("driver");
115        String url = attributes.getString("url");
116        String username = attributes.getString("username");
117        String password = attributes.getString("password");
118
119        ExportConnectionInfo connectionInfo = new ExportConnectionInfo(url, username, password);
120        connectionInfo.setDriver(driver);
121
122        long cutoffDate = 0L;
123        String configHash = String.valueOf(Objects.hash(doLinkCleanup, linkFilters, linkFilters2, identityFilters, configurationName, connectionInfo));
124
125        try (Connection connection = ExportPartition.openConnection(context, connectionInfo)) {
126            try (PreparedStatement statement = connection.prepareStatement("select last_start_time, config_hash from de_runs order by last_start_time desc")) {
127                try (ResultSet results = statement.executeQuery()) {
128                    if (results.next()) {
129                        String configHashString = results.getString(2);
130                        taskResult.addMessage(Message.info("New config hash = " + configHash + ", old config hash = " + configHashString));
131                        if (Util.nullSafeEq(configHashString, configHash)) {
132                            cutoffDate = results.getLong(1);
133                        } else {
134                            taskResult.addMessage(Message.warn("Configuration has changed; forcing a full export"));
135                        }
136                    }
137                }
138            }
139        } catch(SQLException e) {
140            throw new GeneralException(e);
141        }
142
143        taskResult.addMessage(Message.info("Cutoff timestamp is: " + new Date(cutoffDate)));
144
145        int count = 1;
146        for(String filter : Util.safeIterable(identityFilters)) {
147            ExportIdentitiesPartition eip = new ExportIdentitiesPartition();
148            eip.setName("Identity export partition " + count++);
149            eip.setPhase(1);
150            eip.setExportTimestamp(now);
151            eip.setCutoffDate(cutoffDate);
152            eip.setFilterString(filter);
153            eip.setConnectionInfo(connectionInfo);
154            eip.setConfigurationName(configurationName);
155
156            partitions.add(eip);
157        }
158
159        count = 1;
160        for(String filter : Util.safeIterable(linkFilters)) {
161            for(String filter2 : Util.safeIterable(linkFilters2)) {
162                ExportLinksPartition eip = new ExportLinksPartition();
163                eip.setName("Link export partition " + count++);
164                eip.setPhase(2);
165                eip.setDependentPhase(1);
166                eip.setExportTimestamp(now);
167                eip.setCutoffDate(cutoffDate);
168                eip.setFilterString(filter);
169                eip.setFilterString2(filter2);
170                eip.setConnectionInfo(connectionInfo);
171                eip.setConfigurationName(configurationName);
172
173                partitions.add(eip);
174            }
175        }
176
177        if (doLinkCleanup) {
178            CleanupLinksPartition clp = new CleanupLinksPartition();
179            clp.setPhase(3);
180            clp.setDependentPhase(2);
181            clp.setName("Clean up deleted Links");
182            clp.setConnectionInfo(connectionInfo);
183            partitions.add(clp);
184        }
185
186        ExportFinishPartition efp = new ExportFinishPartition();
187        efp.setExportTimestamp(now);
188        efp.setCutoffDate(cutoffDate);
189        efp.setName("Finalize export");
190        efp.setPhase(4);
191        efp.setDependentPhase(2);
192        efp.setConfigurationName(configurationName);
193        efp.setConfigHash(configHash);
194        partitions.add(efp);
195
196        return partitions;
197    }
198
199    @Override
200    public boolean terminate() {
201        this.stopped.set(true);
202        return true;
203    }
204}