Source code

001package com.identityworksllc.iiq.common.task.export;
002
003import com.identityworksllc.iiq.common.Metered;
004import com.identityworksllc.iiq.common.TaskUtil;
005import com.identityworksllc.iiq.common.Utilities;
006import com.identityworksllc.iiq.common.query.NamedParameterStatement;
007import org.apache.commons.logging.Log;
008import org.apache.commons.logging.LogFactory;
009import sailpoint.api.IncrementalObjectIterator;
010import sailpoint.api.IncrementalProjectionIterator;
011import sailpoint.api.Meter;
012import sailpoint.api.SailPointContext;
013import sailpoint.object.*;
014import sailpoint.tools.GeneralException;
015import sailpoint.tools.Util;
016
017import java.io.UnsupportedEncodingException;
018import java.nio.charset.StandardCharsets;
019import java.sql.Connection;
020import java.sql.SQLException;
021import java.util.*;
022import java.util.concurrent.atomic.AtomicInteger;
023
024public class ExportLinksPartition extends ExportPartition {
025
026    protected static final String ATTRIBUTE_VALUE_FIELD = "attributeValue";
027
028    public static final String DELETE_LINK =
029            "delete from de_link where id = :id";
030
031    public static final String DELETE_LINK_ATTRS =
032            "delete from de_link_attr where id = :id";
033
034    /**
035     * The application name used to specify fields we do not want to export on EVERY type of account
036     */
037    protected static final String GLOBAL_SETTING = "global";
038
039    private static final String INSERT_LINK =
040            "insert into de_link " +
041                    "( id, identity_id, application, native_identity, created, modified, last_refresh, de_timestamp ) " +
042                    "values ( :id, :identityId, :application, :nativeIdentity, :created, :modified, :lastRefresh, :now )";
043
044    private static final String INSERT_LINK_ATTR =
045            "insert into de_link_attr ( id, attribute_name, attribute_value ) values ( :id, :attributeName, :attributeValue )";
046
047    private static final String METER_FETCH = "ExportLinkPartition.fetch";
048    private static final String METER_LINK = "ExportLinkPartition.link";
049    private static final String METER_STORE = "ExportLinkPartition.store";
050
051    private static final String REGEX_PREFIX = "regex:";
052
053    private final Log logger;
054
055    /**
056     * Constructs a new partition for exporting link objects
057     */
058    public ExportLinksPartition() {
059        this.logger = LogFactory.getLog(ExportLinksPartition.class);
060    }
061
062    /**
063     * Exports the identified Link objects to the export table
064     * @param context The context
065     * @param connection The connection to the target database
066     * @param _logger The logger attached to the {@link com.identityworksllc.iiq.common.threads.SailPointWorker}
067     * @throws GeneralException if there are any failures
068     */
069    @Override
070    public void export(SailPointContext context, Connection connection, Log _logger) throws GeneralException {
071        Integer linkBatchSize = configuration.getInteger("linkBatchSize");
072        if (linkBatchSize == null || linkBatchSize < 1) {
073            linkBatchSize = getBatchSize();
074        }
075
076        logger.info("Partition batch size is " + getBatchSize());
077
078        // Mapped from Application name to a set of column names
079        Map<String, Set<String>> excludedByApplication = getExcludedColumnsByApplication(context);
080
081        Date exportDate = new Date(exportTimestamp);
082
083        QueryOptions qo = new QueryOptions();
084        qo.addFilter(Filter.compile(filterString));
085        if (Util.isNotNullOrEmpty(filterString2)) {
086            qo.addFilter(Filter.compile(filterString2));
087        }
088        qo.addFilter(Filter.or(Filter.gt("created", new Date(cutoffDate)), Filter.gt("modified", new Date(cutoffDate))));
089        qo.setCacheResults(false);
090        qo.setTransactionLock(false);
091
092        TaskUtil.withLockedPartitionResult(monitor, (partitionResult) -> {
093            monitor.updateProgress(partitionResult, "Executing query", -1);
094        });
095
096        List<String> projectionFields = new ArrayList<>();
097        projectionFields.add("id");                 // 0
098        projectionFields.add("nativeIdentity");
099        projectionFields.add("application.name");
100        projectionFields.add("identity.id");
101        projectionFields.add("displayName");
102        projectionFields.add("created");            // 5
103        projectionFields.add("modified");
104        projectionFields.add("lastRefresh");
105        projectionFields.add("attributes");
106
107        long count = context.countObjects(Link.class, qo);
108
109        IncrementalProjectionIterator links = new IncrementalProjectionIterator(context, Link.class, qo, projectionFields);
110
111        AtomicInteger totalCount = new AtomicInteger();
112
113        Map<String, Schema> schemaMap = new HashMap<>();
114
115        List<String> linksInBatch = new ArrayList<>();
116
117        ObjectConfig linkConfig = Link.getObjectConfig();
118
119        try (NamedParameterStatement deleteAttrs = new NamedParameterStatement(connection, DELETE_LINK_ATTRS); NamedParameterStatement deleteLink = new NamedParameterStatement(connection, DELETE_LINK); NamedParameterStatement insertLink = new NamedParameterStatement(connection, INSERT_LINK); NamedParameterStatement insertAttribute = new NamedParameterStatement(connection, INSERT_LINK_ATTR)) {
120            int batchCount = 0;
121
122            while (links.hasNext()) {
123                if (isTerminated()) {
124                    logger.info("Thread has been terminated; exiting cleanly");
125                    break;
126                }
127                Meter.enterByName(METER_LINK);
128                try {
129                    Meter.enterByName(METER_FETCH);
130                    Object[] link = links.next();
131                    Meter.exitByName(METER_FETCH);
132
133                    String linkId = Util.otoa(link[0]);
134                    String nativeIdentity = Util.otoa(link[1]);
135                    String applicationName = Util.otoa(link[2]);
136                    String identityId = Util.otoa(link[3]);
137                    String displayName = Util.otoa(link[4]);
138
139                    Date created = (Date) link[5];
140                    Date modified = (Date) link[6];
141                    Date lastRefresh = (Date) link[7];
142
143                    // Skip Links created after the job began; they'll be captured on the next run
144                    if (created != null && created.after(exportDate)) {
145                        continue;
146                    }
147
148                    @SuppressWarnings("unchecked")
149                    Attributes<String, Object> attributes = (Attributes<String, Object>) link[8];
150
151                    if (logger.isTraceEnabled()) {
152                        logger.trace("Exporting Link " + linkId + ": " + applicationName + " " + nativeIdentity);
153                        logger.trace("Link attributes map: " + attributes);
154                    }
155
156                    if (isDeleteEnabled()) {
157                        deleteLink.setString("id", linkId);
158                        deleteLink.addBatch();
159
160                        if (logger.isTraceEnabled()) {
161                            logger.trace("Deleting Link " + linkId + ": " + applicationName + " " + nativeIdentity);
162                        }
163
164                        deleteAttrs.setString("id", linkId);
165                        deleteAttrs.addBatch();
166
167                        if (logger.isTraceEnabled()) {
168                            logger.trace("Deleting Link Attributes for " + linkId + ": " + applicationName + " " + nativeIdentity);
169                        }
170                    }
171
172                    insertLink.setString("id", linkId);
173                    if (identityId != null) {
174                        insertLink.setString("identityId", identityId);
175                    } else {
176                        logger.warn("Link with ID " + linkId + " is orphaned and does not have an Identity");
177                        continue;
178                    }
179                    insertLink.setString("application", applicationName);
180                    insertLink.setString("nativeIdentity", nativeIdentity);
181
182                    addCommonDateFields(insertLink, exportDate, created, modified, lastRefresh);
183
184                    insertLink.addBatch();
185
186                    linksInBatch.add(applicationName + ": " + nativeIdentity);
187
188                    if (!schemaMap.containsKey(applicationName)) {
189                        if (logger.isTraceEnabled()) {
190                            logger.trace("Fetching schema for " + applicationName);
191                        }
192
193                        Application application = context.getObjectByName(Application.class, applicationName);
194                        schemaMap.put(applicationName, application.getAccountSchema());
195                    }
196
197                    Schema schema = schemaMap.get(applicationName);
198
199                    Set<String> excludedColumns = excludedByApplication.get(applicationName);
200
201                    for (ObjectAttribute attribute : linkConfig.getObjectAttributes()) {
202                        if (logger.isTraceEnabled()) {
203                            logger.trace("Examining Link extension attribute " + attribute.getName() + ": " + linkId);
204                        }
205
206                        if (attribute.isSystem() || attribute.isStandard()) {
207                            if (logger.isTraceEnabled()) {
208                                logger.trace("Skipping system or standard attribute " + attribute.getName() + ": " + linkId);
209                            }
210                            continue;
211                        }
212
213                        String attrName = attribute.getName();
214
215                        if (excludedColumns != null) {
216                            boolean excluded = excludedColumns.contains(attrName);
217                            if (excluded) {
218                                if (logger.isTraceEnabled()) {
219                                    logger.trace("Skipping excluded attribute " + attrName + ": " + linkId);
220                                }
221                                continue;
222                            }
223                        }
224
225                        Object value = attributes.get(attrName);
226                        if (!Utilities.isNothing(value)) {
227                            insertAttribute.setString("id", linkId);
228                            insertAttribute.setString("attributeName", attrName);
229
230                            if (attribute.isMulti()) {
231                                for (String val : Util.otol(value)) {
232                                    String truncatedValue = Utilities.truncateStringToBytes(val, 4000, StandardCharsets.UTF_8);
233                                    if (logger.isTraceEnabled()) {
234                                        logger.trace("Inserting attribute value " + attrName + ": " + truncatedValue);
235                                    }
236                                    insertAttribute.setString(ATTRIBUTE_VALUE_FIELD, truncatedValue);
237                                    insertAttribute.addBatch();
238                                }
239                            } else {
240                                String truncatedValue = Utilities.truncateStringToBytes(Util.otoa(value), 4000, StandardCharsets.UTF_8);
241                                if (logger.isTraceEnabled()) {
242                                    logger.trace("Inserting attribute value " + attrName + ": " + truncatedValue);
243                                }
244                                insertAttribute.setString(ATTRIBUTE_VALUE_FIELD, truncatedValue);
245                                insertAttribute.addBatch();
246                            }
247                        }
248                    }
249
250                    boolean excludeDisabled = (excludedColumns != null && excludedColumns.contains("IIQDisabled"));
251                    boolean excludeLocked = (excludedColumns != null && excludedColumns.contains("IIQLocked"));
252
253                    if (!excludeDisabled) {
254                        boolean disabled = Util.otob(attributes.get("IIQDisabled"));
255                        if (logger.isTraceEnabled()) {
256                            logger.trace("Inserting IIQDisabled attribute: " + disabled);
257                        }
258                        insertAttribute.setString("id", linkId);
259                        insertAttribute.setString("attributeName", "IIQDisabled");
260                        insertAttribute.setString(ATTRIBUTE_VALUE_FIELD, String.valueOf(disabled));
261                        insertAttribute.addBatch();
262                    } else {
263                        if (logger.isTraceEnabled()) {
264                            logger.trace("Skipping IIQDisabled attribute: " + linkId);
265                        }
266                    }
267
268                    if (!excludeLocked) {
269                        boolean locked = Util.otob(attributes.get("IIQLocked"));
270                        if (logger.isTraceEnabled()) {
271                            logger.trace("Inserting IIQLocked attribute: " + locked);
272                        }
273                        insertAttribute.setString("id", linkId);
274                        insertAttribute.setString("attributeName", "IIQLocked");
275                        insertAttribute.setString(ATTRIBUTE_VALUE_FIELD, String.valueOf(locked));
276                        insertAttribute.addBatch();
277                    } else {
278                        if (logger.isTraceEnabled()) {
279                            logger.trace("Skipping IIQLocked attribute: " + linkId);
280                        }
281                    }
282
283                    for (AttributeDefinition attribute : schema.getAttributes()) {
284                        String attrName = attribute.getName();
285
286                        if (logger.isTraceEnabled()) {
287                            logger.trace("Examining schema attribute " + attrName + ": " + linkId);
288                        }
289
290                        if (excludedColumns != null) {
291                            boolean excluded = excludedColumns.contains(attrName);
292                            if (excluded) {
293                                if (logger.isTraceEnabled()) {
294                                    logger.trace("Skipping excluded attribute " + attrName + ": " + linkId);
295                                }
296                                continue;
297                            }
298                        }
299
300                        Object value = attributes.get(attrName);
301                        if (!Utilities.isNothing(value)) {
302                            insertAttribute.setString("id", linkId);
303                            insertAttribute.setString("attributeName", attrName);
304
305                            if (attribute.isMulti()) {
306                                for (String val : Util.otol(value)) {
307                                    String truncatedValue = Utilities.truncateStringToBytes(val, 4000, StandardCharsets.UTF_8);
308                                    if (logger.isTraceEnabled()) {
309                                        logger.trace("Inserting attribute value " + attrName + ": " + truncatedValue);
310                                    }
311                                    insertAttribute.setString(ATTRIBUTE_VALUE_FIELD, truncatedValue);
312                                    insertAttribute.addBatch();
313                                }
314                            } else {
315                                String truncatedValue = Utilities.truncateStringToBytes(Util.otoa(value), 4000, StandardCharsets.UTF_8);
316                                if (logger.isTraceEnabled()) {
317                                    logger.trace("Inserting attribute value " + attrName + ": " + truncatedValue);
318                                }
319                                insertAttribute.setString(ATTRIBUTE_VALUE_FIELD, truncatedValue);
320                                insertAttribute.addBatch();
321                            }
322                        }
323                    }
324
325                    if (batchCount++ > linkBatchSize) {
326                        Meter.enterByName(METER_STORE);
327                        try {
328                            if (isDeleteEnabled()) {
329                                if (logger.isTraceEnabled()) {
330                                    logger.trace("Executing batch of Link and Link Attribute deletes");
331                                }
332                                deleteLink.executeBatch();
333                                deleteAttrs.executeBatch();
334                            }
335
336                            if (logger.isTraceEnabled()) {
337                                logger.trace("Executing batch of Link and Link Attribute inserts");
338                            }
339
340                            insertLink.executeBatch();
341                            insertAttribute.executeBatch();
342
343                            connection.commit();
344                        } catch(SQLException e) {
345                            logger.error("Caught an error committing a batch containing these accounts: " + linksInBatch, e);
346                            throw e;
347                        } finally {
348                            linksInBatch.clear();
349                            Meter.exitByName(METER_STORE);
350                        }
351                        batchCount = 0;
352                    }
353
354                    int currentCount = totalCount.incrementAndGet();
355                    if ((currentCount % 100) == 0) {
356                        TaskUtil.withLockedPartitionResult(monitor, (partitionResult) -> {
357                            monitor.updateProgress(partitionResult, "Processed " + currentCount + " of " + count + " links", -1);
358                            partitionResult.setInt("exportedLinks", currentCount);
359                        });
360                    }
361                } finally{
362                    Meter.exitByName(METER_LINK);
363                }
364            }
365
366            try {
367                deleteLink.executeBatch();
368                deleteAttrs.executeBatch();
369                insertLink.executeBatch();
370                insertAttribute.executeBatch();
371
372                connection.commit();
373
374                int currentCount = totalCount.get();
375                TaskUtil.withLockedPartitionResult(monitor, (partitionResult) -> {
376                    monitor.updateProgress(partitionResult, "Processed " + currentCount + " of " + count + " links", -1);
377                    partitionResult.setInt("exportedLinks", currentCount);
378                });
379            } catch(SQLException e) {
380                logger.error("Caught an error committing a batch containing these accounts: " + linksInBatch, e);
381                throw e;
382            }
383        } catch(SQLException e) {
384            throw new GeneralException(e);
385        }
386    }
387
388    /**
389     * If the given timestamp is positive, a Date representing that timestamp
390     * will be returned. Otherwise, null will be returned.
391     *
392     * @param timestamp The timestamp
393     * @return The date, or null
394     */
395    private Date toDate(long timestamp) {
396        if (timestamp > 0) {
397            return new Date(timestamp);
398        } else {
399            return null;
400        }
401    }
402
403    /**
404     * Builds the map of excluded attributes by application. This can be mapped as a global
405     * list, as regular expressions on the name, as the name itself, or as the connector type.
406     *
407     * @param context The Sailpoint context for querying the DB
408     * @return The resulting set of exclusions
409     * @throws GeneralException if any failures occur reading the Application objects
410     */
411    private Map<String, Set<String>> getExcludedColumnsByApplication(SailPointContext context) throws GeneralException {
412        Map<String, Set<String>> excludeLinkColumns = new HashMap<>();
413
414        if (configuration.containsAttribute("excludeLinkColumns")) {
415            Object config = configuration.get("excludeLinkColumns");
416            if (config instanceof Map) {
417                Map<String, Object> mapConfig = (Map<String, Object>) config;
418
419                List<Application> allApplications = context.getObjects(Application.class);
420                for(Application a : allApplications) {
421                    Set<String> mergedExclude = getMergedExcludeSet(a, mapConfig);
422                    if (!mergedExclude.isEmpty()) {
423                        excludeLinkColumns.put(a.getName(), mergedExclude);
424                    }
425                }
426            } else {
427                throw new GeneralException("Invalid configuration: excludeLinkColumns must be an instance of a Map");
428            }
429        }
430
431        return excludeLinkColumns;
432    }
433
434    /**
435     * Merges together the various exclusion lists that apply to this application.
436     *
437     * @param application The application to find exclusion lists for
438     * @param excludeLinkColumns The resulting merged set of exclusions
439     * @return if any failures occur
440     */
441    private Set<String> getMergedExcludeSet(Application application, Map<String, Object> excludeLinkColumns) {
442        List<String> globalSet = Util.otol(excludeLinkColumns.get(GLOBAL_SETTING));
443        List<String> typeSpecific = Util.otol(excludeLinkColumns.get("connector:" + application.getType()));
444
445
446        Set<String> merged = new HashSet<>();
447        if (globalSet != null) {
448            merged.addAll(globalSet);
449        }
450
451        if (typeSpecific != null) {
452            merged.addAll(typeSpecific);
453        }
454
455        String applicationName = application.getName();
456
457        for(String key : excludeLinkColumns.keySet()) {
458            List<String> colsForKey = Util.otol(excludeLinkColumns.get(key));
459
460            if (key.startsWith(REGEX_PREFIX)) {
461                String expression = key.substring(REGEX_PREFIX.length());
462                if (Util.isNotNullOrEmpty(expression) && applicationName.matches(expression)) {
463                    merged.addAll(colsForKey);
464                }
465            } else if (applicationName.equalsIgnoreCase(key)) {
466                merged.addAll(colsForKey);
467            }
468        }
469
470        return merged;
471    }
472}