Source code

001package com.identityworksllc.iiq.common.task.export;
002
003import com.identityworksllc.iiq.common.Metered;
004import com.identityworksllc.iiq.common.TaskUtil;
005import com.identityworksllc.iiq.common.Utilities;
006import com.identityworksllc.iiq.common.query.NamedParameterStatement;
007import org.apache.commons.logging.Log;
008import sailpoint.api.IncrementalObjectIterator;
009import sailpoint.api.IncrementalProjectionIterator;
010import sailpoint.api.Meter;
011import sailpoint.api.SailPointContext;
012import sailpoint.object.*;
013import sailpoint.tools.GeneralException;
014import sailpoint.tools.Util;
015
016import java.sql.Connection;
017import java.sql.SQLException;
018import java.util.ArrayList;
019import java.util.Date;
020import java.util.HashMap;
021import java.util.HashSet;
022import java.util.Iterator;
023import java.util.List;
024import java.util.Map;
025import java.util.Set;
026import java.util.concurrent.atomic.AtomicInteger;
027
028public class ExportLinksPartition extends ExportPartition {
029
030    protected static final String ATTRIBUTE_VALUE_FIELD = "attributeValue";
031
032    public static final String DELETE_LINK =
033            "delete from de_link where id = :id";
034
035    public static final String DELETE_LINK_ATTRS =
036            "delete from de_link_attr where id = :id";
037
038    /**
039     * The application name used to specify fields we do not want to export on EVERY type of account
040     */
041    protected static final String GLOBAL_SETTING = "global";
042
043    private static final String INSERT_LINK =
044            "insert into de_link " +
045                    "( id, identity_id, application, native_identity, created, modified, last_refresh, de_timestamp ) " +
046                    "values ( :id, :identityId, :application, :nativeIdentity, :created, :modified, :lastRefresh, :now )";
047
048    private static final String INSERT_LINK_ATTR =
049            "insert into de_link_attr ( id, attribute_name, attribute_value ) values ( :id, :attributeName, :attributeValue )";
050
051    public static final int LINKS_BATCH_SIZE = 25;
052    public static final String METER_FETCH = "ExportLinkPartition.fetch";
053    public static final String METER_LINK = "ExportLinkPartition.link";
054    public static final String METER_STORE = "ExportLinkPartition.store";
055
056    public static final String REGEX_PREFIX = "regex:";
057
058    /**
059     * Exports the identified Link objects to the export table
060     * @param context The context
061     * @param connection The connection to the target database
062     * @param logger The logger
063     * @throws GeneralException if there are any failures
064     */
065    @Override
066    protected void export(SailPointContext context, Connection connection, Log logger) throws GeneralException {
067        // Mapped from Application name to a set of column names
068        Map<String, Set<String>> excludedByApplication = getExcludedColumnsByApplication(context);
069
070        Date exportDate = new Date(exportTimestamp);
071
072        QueryOptions qo = new QueryOptions();
073        qo.addFilter(Filter.compile(filterString));
074        if (Util.isNotNullOrEmpty(filterString2)) {
075            qo.addFilter(Filter.compile(filterString2));
076        }
077        qo.addFilter(Filter.or(Filter.gt("created", new Date(cutoffDate)), Filter.gt("modified", new Date(cutoffDate))));
078        qo.setCacheResults(false);
079        qo.setTransactionLock(false);
080
081        TaskUtil.withLockedPartitionResult(monitor, (partitionResult) -> {
082            monitor.updateProgress(partitionResult, "Executing query", -1);
083        });
084
085        List<String> projectionFields = new ArrayList<>();
086        projectionFields.add("id");                 // 0
087        projectionFields.add("nativeIdentity");
088        projectionFields.add("application.name");
089        projectionFields.add("identity.id");
090        projectionFields.add("displayName");
091        projectionFields.add("created");            // 5
092        projectionFields.add("modified");
093        projectionFields.add("lastRefresh");
094        projectionFields.add("attributes");
095
096        long count = context.countObjects(Link.class, qo);
097
098        IncrementalProjectionIterator links = new IncrementalProjectionIterator(context, Link.class, qo, projectionFields);
099
100        int batchCount = 0;
101        AtomicInteger totalCount = new AtomicInteger();
102
103        Map<String, Schema> schemaMap = new HashMap<>();
104
105        try (NamedParameterStatement deleteAttrs = new NamedParameterStatement(connection, DELETE_LINK_ATTRS); NamedParameterStatement deleteLink = new NamedParameterStatement(connection, DELETE_LINK); NamedParameterStatement insertLink = new NamedParameterStatement(connection, INSERT_LINK); NamedParameterStatement insertAttribute = new NamedParameterStatement(connection, INSERT_LINK_ATTR)) {
106            while (links.hasNext()) {
107                if (isTerminated()) {
108                    logger.info("Thread has been terminated; exiting cleanly");
109                    break;
110                }
111                Meter.enterByName(METER_LINK);
112                try {
113                    Meter.enterByName(METER_FETCH);
114                    Object[] link = links.next();
115                    Meter.exitByName(METER_FETCH);
116
117                    String linkId = Util.otoa(link[0]);
118                    String nativeIdentity = Util.otoa(link[1]);
119                    String applicationName = Util.otoa(link[2]);
120                    String identityId = Util.otoa(link[3]);
121                    String displayName = Util.otoa(link[4]);
122
123                    Date created = (Date) link[5];
124                    Date modified = (Date) link[6];
125                    Date lastRefresh = (Date) link[7];
126
127                    @SuppressWarnings("unchecked")
128                    Attributes<String, Object> attributes = (Attributes<String, Object>) link[8];
129
130                    if (logger.isTraceEnabled()) {
131                        logger.trace("Exporting Link " + linkId + ": " + applicationName + " " + nativeIdentity);
132                    }
133
134                    deleteLink.setString("id", linkId);
135                    deleteLink.addBatch();
136
137                    deleteAttrs.setString("id", linkId);
138                    deleteAttrs.addBatch();
139
140                    insertLink.setString("id", linkId);
141                    if (identityId != null) {
142                        insertLink.setString("identityId", identityId);
143                    } else {
144                        logger.warn("Link with ID " + linkId + " is orphaned and does not have an Identity");
145                        continue;
146                    }
147                    insertLink.setString("application", applicationName);
148                    insertLink.setString("nativeIdentity", nativeIdentity);
149
150                    addCommonDateFields(insertLink, exportDate, created, modified, lastRefresh);
151
152                    insertLink.addBatch();
153
154                    if (!schemaMap.containsKey(applicationName)) {
155                        Application application = context.getObjectByName(Application.class, applicationName);
156                        schemaMap.put(applicationName, application.getAccountSchema());
157                    }
158
159                    Schema schema = schemaMap.get(applicationName);
160
161                    Set<String> excludedColumns = excludedByApplication.get(applicationName);
162
163                    for (AttributeDefinition attribute : schema.getAttributes()) {
164                        String attrName = attribute.getName();
165
166                        if (excludedColumns != null) {
167                            boolean excluded = excludedColumns.contains(attrName);
168                            if (excluded) {
169                                continue;
170                            }
171                        }
172
173                        Object value = attributes.get(attrName);
174                        if (!Utilities.isNothing(value)) {
175                            insertAttribute.setString("id", linkId);
176                            insertAttribute.setString("attributeName", attrName);
177
178                            if (attribute.isMulti()) {
179                                for (String val : Util.otol(value)) {
180                                    insertAttribute.setString(ATTRIBUTE_VALUE_FIELD, Util.truncate(val, 4000));
181                                    insertAttribute.addBatch();
182                                }
183                            } else {
184                                insertAttribute.setString(ATTRIBUTE_VALUE_FIELD, Util.truncate(Util.otoa(value), 4000));
185                                insertAttribute.addBatch();
186                            }
187                        }
188                    }
189
190                    if (batchCount++ > LINKS_BATCH_SIZE) {
191                        Meter.enterByName(METER_STORE);
192                        try {
193                            deleteLink.executeBatch();
194                            deleteAttrs.executeBatch();
195                            insertLink.executeBatch();
196                            insertAttribute.executeBatch();
197
198                            connection.commit();
199                        } finally {
200                            Meter.exitByName(METER_STORE);
201                        }
202                        batchCount = 0;
203                    }
204
205                    int currentCount = totalCount.incrementAndGet();
206                    if ((currentCount % 100) == 0) {
207                        TaskUtil.withLockedPartitionResult(monitor, (partitionResult) -> {
208                            monitor.updateProgress(partitionResult, "Processed " + currentCount + " of " + count + " links", -1);
209                            partitionResult.setInt("exportedLinks", currentCount);
210                        });
211                    }
212                } finally{
213                    Meter.exitByName(METER_LINK);
214                }
215            }
216
217            deleteLink.executeBatch();
218            deleteAttrs.executeBatch();
219            insertLink.executeBatch();
220            insertAttribute.executeBatch();
221
222            connection.commit();
223        } catch(SQLException e) {
224            throw new GeneralException(e);
225        }
226    }
227
228    /**
229     * If the given timestamp is positive, a Date representing that timestamp
230     * will be returned. Otherwise, null will be returned.
231     *
232     * @param timestamp The timestamp
233     * @return The date, or null
234     */
235    private Date toDate(long timestamp) {
236        if (timestamp > 0) {
237            return new Date(timestamp);
238        } else {
239            return null;
240        }
241    }
242
243    /**
244     * Builds the map of excluded attributes by application. This can be mapped as a global
245     * list, as regular expressions on the name, as the name itself, or as the connector type.
246     *
247     * @param context The Sailpoint context for querying the DB
248     * @return The resulting set of exclusions
249     * @throws GeneralException if any failures occur reading the Application objects
250     */
251    private Map<String, Set<String>> getExcludedColumnsByApplication(SailPointContext context) throws GeneralException {
252        Map<String, Set<String>> excludeLinkColumns = new HashMap<>();
253
254        if (configuration.containsAttribute("excludeLinkColumns")) {
255            Object config = configuration.get("excludeLinkColumns");
256            if (config instanceof Map) {
257
258                Map<String, Object> mapConfig = (Map<String, Object>) config;
259
260                List<Application> allApplications = context.getObjects(Application.class);
261                for(Application a : allApplications) {
262                    Set<String> mergedExclude = getMergedExcludeSet(a, mapConfig);
263                    if (!mergedExclude.isEmpty()) {
264                        excludeLinkColumns.put(a.getName(), mergedExclude);
265                    }
266                }
267            } else {
268                throw new GeneralException("Invalid configuration: excludeLinkColumns must be an instance of a Map");
269            }
270        }
271
272        return excludeLinkColumns;
273    }
274
275    /**
276     * Merges together the various exclusion lists that apply to this application.
277     *
278     * @param application The application to find exclusion lists for
279     * @param excludeLinkColumns The resulting merged set of exclusions
280     * @return if any failures occur
281     */
282    private Set<String> getMergedExcludeSet(Application application, Map<String, Object> excludeLinkColumns) {
283        List<String> globalSet = Util.otol(excludeLinkColumns.get(GLOBAL_SETTING));
284        List<String> typeSpecific = Util.otol(excludeLinkColumns.get("connector:" + application.getType()));
285
286
287        Set<String> merged = new HashSet<>();
288        if (globalSet != null) {
289            merged.addAll(globalSet);
290        }
291
292        if (typeSpecific != null) {
293            merged.addAll(typeSpecific);
294        }
295
296        String applicationName = application.getName();
297
298        for(String key : excludeLinkColumns.keySet()) {
299            List<String> colsForKey = Util.otol(excludeLinkColumns.get(key));
300
301            if (key.startsWith(REGEX_PREFIX)) {
302                String expression = key.substring(REGEX_PREFIX.length());
303                if (Util.isNotNullOrEmpty(expression) && applicationName.matches(expression)) {
304                    merged.addAll(colsForKey);
305                }
306            } else if (applicationName.equalsIgnoreCase(key)) {
307                merged.addAll(colsForKey);
308            }
309        }
310
311        return merged;
312    }
313}