Source code

001package com.identityworksllc.iiq.common.task.export;
002
003import com.identityworksllc.iiq.common.Metered;
004import com.identityworksllc.iiq.common.TaskUtil;
005import com.identityworksllc.iiq.common.Utilities;
006import com.identityworksllc.iiq.common.query.NamedParameterStatement;
007import org.apache.commons.logging.Log;
008import org.apache.commons.logging.LogFactory;
009import sailpoint.api.IncrementalObjectIterator;
010import sailpoint.api.IncrementalProjectionIterator;
011import sailpoint.api.Meter;
012import sailpoint.api.SailPointContext;
013import sailpoint.object.*;
014import sailpoint.tools.GeneralException;
015import sailpoint.tools.Util;
016
017import java.io.UnsupportedEncodingException;
018import java.nio.charset.StandardCharsets;
019import java.sql.Connection;
020import java.sql.SQLException;
021import java.util.ArrayList;
022import java.util.Date;
023import java.util.HashMap;
024import java.util.HashSet;
025import java.util.Iterator;
026import java.util.List;
027import java.util.Map;
028import java.util.Set;
029import java.util.concurrent.atomic.AtomicInteger;
030
031public class ExportLinksPartition extends ExportPartition {
032
033    protected static final String ATTRIBUTE_VALUE_FIELD = "attributeValue";
034
035    public static final String DELETE_LINK =
036            "delete from de_link where id = :id";
037
038    public static final String DELETE_LINK_ATTRS =
039            "delete from de_link_attr where id = :id";
040
041    /**
042     * The application name used to specify fields we do not want to export on EVERY type of account
043     */
044    protected static final String GLOBAL_SETTING = "global";
045
046    private static final String INSERT_LINK =
047            "insert into de_link " +
048                    "( id, identity_id, application, native_identity, created, modified, last_refresh, de_timestamp ) " +
049                    "values ( :id, :identityId, :application, :nativeIdentity, :created, :modified, :lastRefresh, :now )";
050
051    private static final String INSERT_LINK_ATTR =
052            "insert into de_link_attr ( id, attribute_name, attribute_value ) values ( :id, :attributeName, :attributeValue )";
053
054    private static final String METER_FETCH = "ExportLinkPartition.fetch";
055    private static final String METER_LINK = "ExportLinkPartition.link";
056    private static final String METER_STORE = "ExportLinkPartition.store";
057
058    private static final String REGEX_PREFIX = "regex:";
059
060    private final Log logger;
061
062    public ExportLinksPartition() {
063        this.logger = LogFactory.getLog(ExportLinksPartition.class);
064    }
065
066    /**
067     * Exports the identified Link objects to the export table
068     * @param context The context
069     * @param connection The connection to the target database
070     * @param _logger The logger attached to the {@link com.identityworksllc.iiq.common.threads.SailPointWorker}
071     * @throws GeneralException if there are any failures
072     */
073    @Override
074    protected void export(SailPointContext context, Connection connection, Log _logger) throws GeneralException {
075        Integer linkBatchSize = configuration.getInteger("linkBatchSize");
076        if (linkBatchSize == null || linkBatchSize < 1) {
077            linkBatchSize = getBatchSize();
078        }
079
080        logger.info("Partition batch size is " + getBatchSize());
081
082        // Mapped from Application name to a set of column names
083        Map<String, Set<String>> excludedByApplication = getExcludedColumnsByApplication(context);
084
085        Date exportDate = new Date(exportTimestamp);
086
087        QueryOptions qo = new QueryOptions();
088        qo.addFilter(Filter.compile(filterString));
089        if (Util.isNotNullOrEmpty(filterString2)) {
090            qo.addFilter(Filter.compile(filterString2));
091        }
092        qo.addFilter(Filter.or(Filter.gt("created", new Date(cutoffDate)), Filter.gt("modified", new Date(cutoffDate))));
093        qo.setCacheResults(false);
094        qo.setTransactionLock(false);
095
096        TaskUtil.withLockedPartitionResult(monitor, (partitionResult) -> {
097            monitor.updateProgress(partitionResult, "Executing query", -1);
098        });
099
100        List<String> projectionFields = new ArrayList<>();
101        projectionFields.add("id");                 // 0
102        projectionFields.add("nativeIdentity");
103        projectionFields.add("application.name");
104        projectionFields.add("identity.id");
105        projectionFields.add("displayName");
106        projectionFields.add("created");            // 5
107        projectionFields.add("modified");
108        projectionFields.add("lastRefresh");
109        projectionFields.add("attributes");
110
111        long count = context.countObjects(Link.class, qo);
112
113        IncrementalProjectionIterator links = new IncrementalProjectionIterator(context, Link.class, qo, projectionFields);
114
115        AtomicInteger totalCount = new AtomicInteger();
116
117        Map<String, Schema> schemaMap = new HashMap<>();
118
119        List<String> linksInBatch = new ArrayList<>();
120
121        ObjectConfig linkConfig = Link.getObjectConfig();
122
123        try (NamedParameterStatement deleteAttrs = new NamedParameterStatement(connection, DELETE_LINK_ATTRS); NamedParameterStatement deleteLink = new NamedParameterStatement(connection, DELETE_LINK); NamedParameterStatement insertLink = new NamedParameterStatement(connection, INSERT_LINK); NamedParameterStatement insertAttribute = new NamedParameterStatement(connection, INSERT_LINK_ATTR)) {
124            int batchCount = 0;
125
126            while (links.hasNext()) {
127                if (isTerminated()) {
128                    logger.info("Thread has been terminated; exiting cleanly");
129                    break;
130                }
131                Meter.enterByName(METER_LINK);
132                try {
133                    Meter.enterByName(METER_FETCH);
134                    Object[] link = links.next();
135                    Meter.exitByName(METER_FETCH);
136
137                    String linkId = Util.otoa(link[0]);
138                    String nativeIdentity = Util.otoa(link[1]);
139                    String applicationName = Util.otoa(link[2]);
140                    String identityId = Util.otoa(link[3]);
141                    String displayName = Util.otoa(link[4]);
142
143                    Date created = (Date) link[5];
144                    Date modified = (Date) link[6];
145                    Date lastRefresh = (Date) link[7];
146
147                    // Skip Links created after the job began; they'll be captured on the next run
148                    if (created != null && created.after(exportDate)) {
149                        continue;
150                    }
151
152                    @SuppressWarnings("unchecked")
153                    Attributes<String, Object> attributes = (Attributes<String, Object>) link[8];
154
155                    if (logger.isTraceEnabled()) {
156                        logger.trace("Exporting Link " + linkId + ": " + applicationName + " " + nativeIdentity);
157                    }
158
159                    if (isDeleteEnabled()) {
160                        deleteLink.setString("id", linkId);
161                        deleteLink.addBatch();
162
163                        deleteAttrs.setString("id", linkId);
164                        deleteAttrs.addBatch();
165                    }
166
167                    insertLink.setString("id", linkId);
168                    if (identityId != null) {
169                        insertLink.setString("identityId", identityId);
170                    } else {
171                        logger.warn("Link with ID " + linkId + " is orphaned and does not have an Identity");
172                        continue;
173                    }
174                    insertLink.setString("application", applicationName);
175                    insertLink.setString("nativeIdentity", nativeIdentity);
176
177                    addCommonDateFields(insertLink, exportDate, created, modified, lastRefresh);
178
179                    insertLink.addBatch();
180
181                    linksInBatch.add(applicationName + ": " + nativeIdentity);
182
183                    if (!schemaMap.containsKey(applicationName)) {
184                        Application application = context.getObjectByName(Application.class, applicationName);
185                        schemaMap.put(applicationName, application.getAccountSchema());
186                    }
187
188                    Schema schema = schemaMap.get(applicationName);
189
190                    Set<String> excludedColumns = excludedByApplication.get(applicationName);
191
192                    for (ObjectAttribute attribute : linkConfig.getObjectAttributes()) {
193                        if (attribute.isSystem() || attribute.isStandard()) {
194                            continue;
195                        }
196
197                        String attrName = attribute.getName();
198
199                        if (excludedColumns != null) {
200                            boolean excluded = excludedColumns.contains(attrName);
201                            if (excluded) {
202                                continue;
203                            }
204                        }
205
206                        Object value = attributes.get(attrName);
207                        if (!Utilities.isNothing(value)) {
208                            insertAttribute.setString("id", linkId);
209                            insertAttribute.setString("attributeName", attrName);
210
211                            if (attribute.isMulti()) {
212                                for (String val : Util.otol(value)) {
213                                    insertAttribute.setString(ATTRIBUTE_VALUE_FIELD, Util.truncate(val, 3996));
214                                    insertAttribute.addBatch();
215                                }
216                            } else {
217                                insertAttribute.setString(ATTRIBUTE_VALUE_FIELD, Util.truncate(Util.otoa(value), 3996));
218                                insertAttribute.addBatch();
219                            }
220                        }
221                    }
222
223                    for (AttributeDefinition attribute : schema.getAttributes()) {
224                        String attrName = attribute.getName();
225
226                        if (excludedColumns != null) {
227                            boolean excluded = excludedColumns.contains(attrName);
228                            if (excluded) {
229                                continue;
230                            }
231                        }
232
233                        Object value = attributes.get(attrName);
234                        if (!Utilities.isNothing(value)) {
235                            insertAttribute.setString("id", linkId);
236                            insertAttribute.setString("attributeName", attrName);
237
238                            if (attribute.isMulti()) {
239                                for (String val : Util.otol(value)) {
240                                    insertAttribute.setString(ATTRIBUTE_VALUE_FIELD, Utilities.truncateStringToBytes(val, 4000, StandardCharsets.UTF_8));
241                                    insertAttribute.addBatch();
242                                }
243                            } else {
244                                insertAttribute.setString(ATTRIBUTE_VALUE_FIELD, Utilities.truncateStringToBytes(Util.otoa(value), 4000, StandardCharsets.UTF_8));
245                                insertAttribute.addBatch();
246                            }
247                        }
248                    }
249
250                    if (batchCount++ > linkBatchSize) {
251                        Meter.enterByName(METER_STORE);
252                        try {
253                            if (isDeleteEnabled()) {
254                                deleteLink.executeBatch();
255                                deleteAttrs.executeBatch();
256                            }
257                            insertLink.executeBatch();
258                            insertAttribute.executeBatch();
259
260                            connection.commit();
261                        } catch(SQLException e) {
262                            logger.error("Caught an error committing a batch containing these accounts: " + linksInBatch, e);
263                            throw e;
264                        } finally {
265                            linksInBatch.clear();
266                            Meter.exitByName(METER_STORE);
267                        }
268                        batchCount = 0;
269                    }
270
271                    int currentCount = totalCount.incrementAndGet();
272                    if ((currentCount % 100) == 0) {
273                        TaskUtil.withLockedPartitionResult(monitor, (partitionResult) -> {
274                            monitor.updateProgress(partitionResult, "Processed " + currentCount + " of " + count + " links", -1);
275                            partitionResult.setInt("exportedLinks", currentCount);
276                        });
277                    }
278                } finally{
279                    Meter.exitByName(METER_LINK);
280                }
281            }
282
283            try {
284                deleteLink.executeBatch();
285                deleteAttrs.executeBatch();
286                insertLink.executeBatch();
287                insertAttribute.executeBatch();
288
289                connection.commit();
290
291                int currentCount = totalCount.get();
292                TaskUtil.withLockedPartitionResult(monitor, (partitionResult) -> {
293                    monitor.updateProgress(partitionResult, "Processed " + currentCount + " of " + count + " links", -1);
294                    partitionResult.setInt("exportedLinks", currentCount);
295                });
296            } catch(SQLException e) {
297                logger.error("Caught an error committing a batch containing these accounts: " + linksInBatch, e);
298                throw e;
299            }
300        } catch(SQLException e) {
301            throw new GeneralException(e);
302        }
303    }
304
305    /**
306     * If the given timestamp is positive, a Date representing that timestamp
307     * will be returned. Otherwise, null will be returned.
308     *
309     * @param timestamp The timestamp
310     * @return The date, or null
311     */
312    private Date toDate(long timestamp) {
313        if (timestamp > 0) {
314            return new Date(timestamp);
315        } else {
316            return null;
317        }
318    }
319
320    /**
321     * Builds the map of excluded attributes by application. This can be mapped as a global
322     * list, as regular expressions on the name, as the name itself, or as the connector type.
323     *
324     * @param context The Sailpoint context for querying the DB
325     * @return The resulting set of exclusions
326     * @throws GeneralException if any failures occur reading the Application objects
327     */
328    private Map<String, Set<String>> getExcludedColumnsByApplication(SailPointContext context) throws GeneralException {
329        Map<String, Set<String>> excludeLinkColumns = new HashMap<>();
330
331        if (configuration.containsAttribute("excludeLinkColumns")) {
332            Object config = configuration.get("excludeLinkColumns");
333            if (config instanceof Map) {
334                Map<String, Object> mapConfig = (Map<String, Object>) config;
335
336                List<Application> allApplications = context.getObjects(Application.class);
337                for(Application a : allApplications) {
338                    Set<String> mergedExclude = getMergedExcludeSet(a, mapConfig);
339                    if (!mergedExclude.isEmpty()) {
340                        excludeLinkColumns.put(a.getName(), mergedExclude);
341                    }
342                }
343            } else {
344                throw new GeneralException("Invalid configuration: excludeLinkColumns must be an instance of a Map");
345            }
346        }
347
348        return excludeLinkColumns;
349    }
350
351    /**
352     * Merges together the various exclusion lists that apply to this application.
353     *
354     * @param application The application to find exclusion lists for
355     * @param excludeLinkColumns The resulting merged set of exclusions
356     * @return if any failures occur
357     */
358    private Set<String> getMergedExcludeSet(Application application, Map<String, Object> excludeLinkColumns) {
359        List<String> globalSet = Util.otol(excludeLinkColumns.get(GLOBAL_SETTING));
360        List<String> typeSpecific = Util.otol(excludeLinkColumns.get("connector:" + application.getType()));
361
362
363        Set<String> merged = new HashSet<>();
364        if (globalSet != null) {
365            merged.addAll(globalSet);
366        }
367
368        if (typeSpecific != null) {
369            merged.addAll(typeSpecific);
370        }
371
372        String applicationName = application.getName();
373
374        for(String key : excludeLinkColumns.keySet()) {
375            List<String> colsForKey = Util.otol(excludeLinkColumns.get(key));
376
377            if (key.startsWith(REGEX_PREFIX)) {
378                String expression = key.substring(REGEX_PREFIX.length());
379                if (Util.isNotNullOrEmpty(expression) && applicationName.matches(expression)) {
380                    merged.addAll(colsForKey);
381                }
382            } else if (applicationName.equalsIgnoreCase(key)) {
383                merged.addAll(colsForKey);
384            }
385        }
386
387        return merged;
388    }
389}