Source code

001package com.identityworksllc.iiq.common.task.export;
002
003import com.identityworksllc.iiq.common.Metered;
004import com.identityworksllc.iiq.common.TaskUtil;
005import com.identityworksllc.iiq.common.Utilities;
006import com.identityworksllc.iiq.common.query.NamedParameterStatement;
007import org.apache.commons.logging.Log;
008import org.apache.commons.logging.LogFactory;
009import sailpoint.api.IncrementalObjectIterator;
010import sailpoint.api.IncrementalProjectionIterator;
011import sailpoint.api.Meter;
012import sailpoint.api.SailPointContext;
013import sailpoint.object.*;
014import sailpoint.tools.GeneralException;
015import sailpoint.tools.Util;
016
017import java.io.UnsupportedEncodingException;
018import java.nio.charset.StandardCharsets;
019import java.sql.Connection;
020import java.sql.SQLException;
021import java.util.ArrayList;
022import java.util.Date;
023import java.util.HashMap;
024import java.util.HashSet;
025import java.util.Iterator;
026import java.util.List;
027import java.util.Map;
028import java.util.Set;
029import java.util.concurrent.atomic.AtomicInteger;
030
031public class ExportLinksPartition extends ExportPartition {
032
033    protected static final String ATTRIBUTE_VALUE_FIELD = "attributeValue";
034
035    public static final String DELETE_LINK =
036            "delete from de_link where id = :id";
037
038    public static final String DELETE_LINK_ATTRS =
039            "delete from de_link_attr where id = :id";
040
041    /**
042     * The application name used to specify fields we do not want to export on EVERY type of account
043     */
044    protected static final String GLOBAL_SETTING = "global";
045
046    private static final String INSERT_LINK =
047            "insert into de_link " +
048                    "( id, identity_id, application, native_identity, created, modified, last_refresh, de_timestamp ) " +
049                    "values ( :id, :identityId, :application, :nativeIdentity, :created, :modified, :lastRefresh, :now )";
050
051    private static final String INSERT_LINK_ATTR =
052            "insert into de_link_attr ( id, attribute_name, attribute_value ) values ( :id, :attributeName, :attributeValue )";
053
054    private static final String METER_FETCH = "ExportLinkPartition.fetch";
055    private static final String METER_LINK = "ExportLinkPartition.link";
056    private static final String METER_STORE = "ExportLinkPartition.store";
057
058    private static final String REGEX_PREFIX = "regex:";
059
060    private final Log logger;
061
062    public ExportLinksPartition() {
063        this.logger = LogFactory.getLog(ExportLinksPartition.class);
064    }
065
066    /**
067     * Exports the identified Link objects to the export table
068     * @param context The context
069     * @param connection The connection to the target database
070     * @param _logger The logger attached to the {@link com.identityworksllc.iiq.common.threads.SailPointWorker}
071     * @throws GeneralException if there are any failures
072     */
073    @Override
074    protected void export(SailPointContext context, Connection connection, Log _logger) throws GeneralException {
075        Integer linkBatchSize = configuration.getInteger("linkBatchSize");
076        if (linkBatchSize == null || linkBatchSize < 1) {
077            linkBatchSize = getBatchSize();
078        }
079
080        logger.info("Partition batch size is " + getBatchSize());
081
082        // Mapped from Application name to a set of column names
083        Map<String, Set<String>> excludedByApplication = getExcludedColumnsByApplication(context);
084
085        Date exportDate = new Date(exportTimestamp);
086
087        QueryOptions qo = new QueryOptions();
088        qo.addFilter(Filter.compile(filterString));
089        if (Util.isNotNullOrEmpty(filterString2)) {
090            qo.addFilter(Filter.compile(filterString2));
091        }
092        qo.addFilter(Filter.or(Filter.gt("created", new Date(cutoffDate)), Filter.gt("modified", new Date(cutoffDate))));
093        qo.setCacheResults(false);
094        qo.setTransactionLock(false);
095
096        TaskUtil.withLockedPartitionResult(monitor, (partitionResult) -> {
097            monitor.updateProgress(partitionResult, "Executing query", -1);
098        });
099
100        List<String> projectionFields = new ArrayList<>();
101        projectionFields.add("id");                 // 0
102        projectionFields.add("nativeIdentity");
103        projectionFields.add("application.name");
104        projectionFields.add("identity.id");
105        projectionFields.add("displayName");
106        projectionFields.add("created");            // 5
107        projectionFields.add("modified");
108        projectionFields.add("lastRefresh");
109        projectionFields.add("attributes");
110
111        long count = context.countObjects(Link.class, qo);
112
113        IncrementalProjectionIterator links = new IncrementalProjectionIterator(context, Link.class, qo, projectionFields);
114
115        AtomicInteger totalCount = new AtomicInteger();
116
117        Map<String, Schema> schemaMap = new HashMap<>();
118
119        List<String> linksInBatch = new ArrayList<>();
120
121        ObjectConfig linkConfig = Link.getObjectConfig();
122
123        try (NamedParameterStatement deleteAttrs = new NamedParameterStatement(connection, DELETE_LINK_ATTRS); NamedParameterStatement deleteLink = new NamedParameterStatement(connection, DELETE_LINK); NamedParameterStatement insertLink = new NamedParameterStatement(connection, INSERT_LINK); NamedParameterStatement insertAttribute = new NamedParameterStatement(connection, INSERT_LINK_ATTR)) {
124            int batchCount = 0;
125
126            while (links.hasNext()) {
127                if (isTerminated()) {
128                    logger.info("Thread has been terminated; exiting cleanly");
129                    break;
130                }
131                Meter.enterByName(METER_LINK);
132                try {
133                    Meter.enterByName(METER_FETCH);
134                    Object[] link = links.next();
135                    Meter.exitByName(METER_FETCH);
136
137                    String linkId = Util.otoa(link[0]);
138                    String nativeIdentity = Util.otoa(link[1]);
139                    String applicationName = Util.otoa(link[2]);
140                    String identityId = Util.otoa(link[3]);
141                    String displayName = Util.otoa(link[4]);
142
143                    Date created = (Date) link[5];
144                    Date modified = (Date) link[6];
145                    Date lastRefresh = (Date) link[7];
146
147                    // Skip Links created after the job began; they'll be captured on the next run
148                    if (created != null && created.after(exportDate)) {
149                        continue;
150                    }
151
152                    @SuppressWarnings("unchecked")
153                    Attributes<String, Object> attributes = (Attributes<String, Object>) link[8];
154
155                    if (logger.isTraceEnabled()) {
156                        logger.trace("Exporting Link " + linkId + ": " + applicationName + " " + nativeIdentity);
157                    }
158
159                    deleteLink.setString("id", linkId);
160                    deleteLink.addBatch();
161
162                    deleteAttrs.setString("id", linkId);
163                    deleteAttrs.addBatch();
164
165                    insertLink.setString("id", linkId);
166                    if (identityId != null) {
167                        insertLink.setString("identityId", identityId);
168                    } else {
169                        logger.warn("Link with ID " + linkId + " is orphaned and does not have an Identity");
170                        continue;
171                    }
172                    insertLink.setString("application", applicationName);
173                    insertLink.setString("nativeIdentity", nativeIdentity);
174
175                    addCommonDateFields(insertLink, exportDate, created, modified, lastRefresh);
176
177                    insertLink.addBatch();
178
179                    linksInBatch.add(applicationName + ": " + nativeIdentity);
180
181                    if (!schemaMap.containsKey(applicationName)) {
182                        Application application = context.getObjectByName(Application.class, applicationName);
183                        schemaMap.put(applicationName, application.getAccountSchema());
184                    }
185
186                    Schema schema = schemaMap.get(applicationName);
187
188                    Set<String> excludedColumns = excludedByApplication.get(applicationName);
189
190                    for (ObjectAttribute attribute : linkConfig.getObjectAttributes()) {
191                        if (attribute.isSystem() || attribute.isStandard()) {
192                            continue;
193                        }
194
195                        String attrName = attribute.getName();
196
197                        if (excludedColumns != null) {
198                            boolean excluded = excludedColumns.contains(attrName);
199                            if (excluded) {
200                                continue;
201                            }
202                        }
203
204                        Object value = attributes.get(attrName);
205                        if (!Utilities.isNothing(value)) {
206                            insertAttribute.setString("id", linkId);
207                            insertAttribute.setString("attributeName", attrName);
208
209                            if (attribute.isMulti()) {
210                                for (String val : Util.otol(value)) {
211                                    insertAttribute.setString(ATTRIBUTE_VALUE_FIELD, Util.truncate(val, 3996));
212                                    insertAttribute.addBatch();
213                                }
214                            } else {
215                                insertAttribute.setString(ATTRIBUTE_VALUE_FIELD, Util.truncate(Util.otoa(value), 3996));
216                                insertAttribute.addBatch();
217                            }
218                        }
219                    }
220
221                    for (AttributeDefinition attribute : schema.getAttributes()) {
222                        String attrName = attribute.getName();
223
224                        if (excludedColumns != null) {
225                            boolean excluded = excludedColumns.contains(attrName);
226                            if (excluded) {
227                                continue;
228                            }
229                        }
230
231                        Object value = attributes.get(attrName);
232                        if (!Utilities.isNothing(value)) {
233                            insertAttribute.setString("id", linkId);
234                            insertAttribute.setString("attributeName", attrName);
235
236                            if (attribute.isMulti()) {
237                                for (String val : Util.otol(value)) {
238                                    insertAttribute.setString(ATTRIBUTE_VALUE_FIELD, Utilities.truncateStringToBytes(val, 4000, StandardCharsets.UTF_8));
239                                    insertAttribute.addBatch();
240                                }
241                            } else {
242                                insertAttribute.setString(ATTRIBUTE_VALUE_FIELD, Utilities.truncateStringToBytes(Util.otoa(value), 4000, StandardCharsets.UTF_8));
243                                insertAttribute.addBatch();
244                            }
245                        }
246                    }
247
248                    if (batchCount++ > linkBatchSize) {
249                        Meter.enterByName(METER_STORE);
250                        try {
251                            deleteLink.executeBatch();
252                            deleteAttrs.executeBatch();
253                            insertLink.executeBatch();
254                            insertAttribute.executeBatch();
255
256                            connection.commit();
257                        } catch(SQLException e) {
258                            logger.error("Caught an error committing a batch containing these accounts: " + linksInBatch, e);
259                            throw e;
260                        } finally {
261                            linksInBatch.clear();
262                            Meter.exitByName(METER_STORE);
263                        }
264                        batchCount = 0;
265                    }
266
267                    int currentCount = totalCount.incrementAndGet();
268                    if ((currentCount % 100) == 0) {
269                        TaskUtil.withLockedPartitionResult(monitor, (partitionResult) -> {
270                            monitor.updateProgress(partitionResult, "Processed " + currentCount + " of " + count + " links", -1);
271                            partitionResult.setInt("exportedLinks", currentCount);
272                        });
273                    }
274                } finally{
275                    Meter.exitByName(METER_LINK);
276                }
277            }
278
279            try {
280                deleteLink.executeBatch();
281                deleteAttrs.executeBatch();
282                insertLink.executeBatch();
283                insertAttribute.executeBatch();
284
285                connection.commit();
286            } catch(SQLException e) {
287                logger.error("Caught an error committing a batch containing these accounts: " + linksInBatch, e);
288                throw e;
289            }
290        } catch(SQLException e) {
291            throw new GeneralException(e);
292        }
293    }
294
295    /**
296     * If the given timestamp is positive, a Date representing that timestamp
297     * will be returned. Otherwise, null will be returned.
298     *
299     * @param timestamp The timestamp
300     * @return The date, or null
301     */
302    private Date toDate(long timestamp) {
303        if (timestamp > 0) {
304            return new Date(timestamp);
305        } else {
306            return null;
307        }
308    }
309
310    /**
311     * Builds the map of excluded attributes by application. This can be mapped as a global
312     * list, as regular expressions on the name, as the name itself, or as the connector type.
313     *
314     * @param context The Sailpoint context for querying the DB
315     * @return The resulting set of exclusions
316     * @throws GeneralException if any failures occur reading the Application objects
317     */
318    private Map<String, Set<String>> getExcludedColumnsByApplication(SailPointContext context) throws GeneralException {
319        Map<String, Set<String>> excludeLinkColumns = new HashMap<>();
320
321        if (configuration.containsAttribute("excludeLinkColumns")) {
322            Object config = configuration.get("excludeLinkColumns");
323            if (config instanceof Map) {
324                Map<String, Object> mapConfig = (Map<String, Object>) config;
325
326                List<Application> allApplications = context.getObjects(Application.class);
327                for(Application a : allApplications) {
328                    Set<String> mergedExclude = getMergedExcludeSet(a, mapConfig);
329                    if (!mergedExclude.isEmpty()) {
330                        excludeLinkColumns.put(a.getName(), mergedExclude);
331                    }
332                }
333            } else {
334                throw new GeneralException("Invalid configuration: excludeLinkColumns must be an instance of a Map");
335            }
336        }
337
338        return excludeLinkColumns;
339    }
340
341    /**
342     * Merges together the various exclusion lists that apply to this application.
343     *
344     * @param application The application to find exclusion lists for
345     * @param excludeLinkColumns The resulting merged set of exclusions
346     * @return if any failures occur
347     */
348    private Set<String> getMergedExcludeSet(Application application, Map<String, Object> excludeLinkColumns) {
349        List<String> globalSet = Util.otol(excludeLinkColumns.get(GLOBAL_SETTING));
350        List<String> typeSpecific = Util.otol(excludeLinkColumns.get("connector:" + application.getType()));
351
352
353        Set<String> merged = new HashSet<>();
354        if (globalSet != null) {
355            merged.addAll(globalSet);
356        }
357
358        if (typeSpecific != null) {
359            merged.addAll(typeSpecific);
360        }
361
362        String applicationName = application.getName();
363
364        for(String key : excludeLinkColumns.keySet()) {
365            List<String> colsForKey = Util.otol(excludeLinkColumns.get(key));
366
367            if (key.startsWith(REGEX_PREFIX)) {
368                String expression = key.substring(REGEX_PREFIX.length());
369                if (Util.isNotNullOrEmpty(expression) && applicationName.matches(expression)) {
370                    merged.addAll(colsForKey);
371                }
372            } else if (applicationName.equalsIgnoreCase(key)) {
373                merged.addAll(colsForKey);
374            }
375        }
376
377        return merged;
378    }
379}