Source code

001package com.identityworksllc.iiq.common.task.export;
002
003import com.identityworksllc.iiq.common.TaskUtil;
004import com.identityworksllc.iiq.common.Utilities;
005import com.identityworksllc.iiq.common.query.NamedParameterStatement;
006import org.apache.commons.collections4.set.ListOrderedSet;
007import org.apache.commons.logging.Log;
008import sailpoint.api.IncrementalObjectIterator;
009import sailpoint.api.IncrementalProjectionIterator;
010import sailpoint.api.SailPointContext;
011import sailpoint.object.*;
012import sailpoint.tools.GeneralException;
013import sailpoint.tools.Util;
014
015import java.nio.charset.StandardCharsets;
016import java.sql.Connection;
017import java.sql.SQLException;
018import java.sql.Types;
019import java.util.*;
020import java.util.concurrent.atomic.AtomicInteger;
021
022public class ExportIdentitiesPartition extends ExportPartition {
023
024    private static final String DELETE_IDENTITY =
025            "delete from de_identity where id = :id";
026
027    private static final String DELETE_IDENTITY_ATTRS =
028            "delete from de_identity_attr where id = :id";
029
030    private static final String DELETE_IDENTITY_ROLES =
031            "delete from de_identity_roles where id = :id";
032
033    public static final int IDENTITY_BATCH_SIZE = 50;
034
035    private static final String INSERT_IDENTITY =
036            "insert into de_identity " +
037                    "( id, name, type, firstname, lastname, email, manager_id, administrator_id, created, modified, last_refresh, de_timestamp ) " +
038                    "values ( :id, :name, :type, :firstname, :lastname, :email, :managerId, :administratorId, :created, :modified, :lastRefresh, :now )";
039
040    private static final String INSERT_IDENTITY_ATTR =
041            "insert into de_identity_attr ( id, attribute_name, attribute_value ) values ( :id, :attributeName, :attributeValue )";
042
043    private static final String INSERT_IDENTITY_ROLES =
044            "insert into de_identity_roles ( id, role_name, role_type, role_date ) values ( :id, :roleName, :roleType, :assignedOn )";
045
046
047    /**
048     * Exports the identities identified by the filters
049     *
050     * @param context The context
051     * @param connection The connection to the target database
052     * @param logger The logger
053     * @throws GeneralException if anything fails during execution
054     */
055    protected void export(SailPointContext context, Connection connection, Log logger) throws GeneralException {
056        Set<String> excludeRoles = new HashSet<>();
057        if (configuration.containsAttribute("excludeRoles")) {
058            List<String> cols = configuration.getStringList("excludeRoles");
059            if (cols != null) {
060                excludeRoles.addAll(cols);
061            }
062        }
063
064        Set<String> excludeRoleTypes = new HashSet<>();
065        if (configuration.containsAttribute("excludeRoleTypes")) {
066            List<String> cols = configuration.getStringList("excludeRoleTypes");
067            if (cols != null) {
068                excludeRoleTypes.addAll(cols);
069            }
070        }
071
072        TaskUtil.withLockedPartitionResult(monitor, (partitionResult) -> {
073            monitor.updateProgress(partitionResult, "Caching role data", -1);
074        });
075
076        Map<String, Bundle> cachedRoles = new HashMap<>();
077
078        Iterator<Bundle> allRoles = context.search(Bundle.class, new QueryOptions());
079        while(allRoles.hasNext()) {
080            Bundle b = allRoles.next();
081            cachedRoles.put(b.getName(), Utilities.detach(context, b));
082        }
083
084        Set<String> excludeIdentityCols = new HashSet<>();
085        if (configuration.containsAttribute("excludeIdentityColumns")) {
086            List<String> cols = configuration.getStringList("excludeIdentityColumns");
087            if (cols != null) {
088                excludeIdentityCols.addAll(cols);
089            }
090        }
091
092        Date exportDate = new Date(exportTimestamp);
093
094        QueryOptions qo = new QueryOptions();
095        qo.addFilter(Filter.compile(filterString));
096        qo.addFilter(Filter.or(Filter.gt("created", new Date(cutoffDate)), Filter.gt("modified", new Date(cutoffDate))));
097        qo.setCacheResults(false);
098        qo.setTransactionLock(false);
099
100        TaskUtil.withLockedPartitionResult(monitor, (partitionResult) -> {
101            monitor.updateProgress(partitionResult, "Executing query", -1);
102        });
103
104        int batchCount = 0;
105        final AtomicInteger totalCount = new AtomicInteger();
106
107        ObjectConfig identityConfig = Identity.getObjectConfig();
108
109        List<String> fields = new ArrayList<>();
110        fields.add("id");               // 0
111        fields.add("name");
112        fields.add("type");
113        fields.add("firstname");
114        fields.add("lastname");
115        fields.add("displayName");      // 5
116        fields.add("email");
117        fields.add("manager.id");
118        fields.add("administrator.id");
119        fields.add("created");
120        fields.add("modified");         // 10
121        fields.add("lastRefresh");
122        fields.add("attributes");
123        fields.add("preferences");
124
125        long count = context.countObjects(Identity.class, qo);
126
127        IncrementalProjectionIterator identities = new IncrementalProjectionIterator(context, Identity.class, qo, fields);
128
129        try (NamedParameterStatement deleteAttrs = new NamedParameterStatement(connection, DELETE_IDENTITY_ATTRS); NamedParameterStatement deleteIdentity = new NamedParameterStatement(connection, DELETE_IDENTITY); NamedParameterStatement insertIdentityStatement = new NamedParameterStatement(connection, INSERT_IDENTITY); NamedParameterStatement insertAttributeStatement = new NamedParameterStatement(connection, INSERT_IDENTITY_ATTR); NamedParameterStatement deleteRolesStatement = new NamedParameterStatement(connection, DELETE_IDENTITY_ROLES); NamedParameterStatement insertRolesStatement = new NamedParameterStatement(connection, INSERT_IDENTITY_ROLES)) {
130            while (identities.hasNext()) {
131                if (isTerminated()) {
132                    logger.info("Thread has been terminated; exiting cleanly");
133                    break;
134                }
135
136                Object[] identity = identities.next();
137
138                String id = Util.otoa(identity[0]);
139                String name = Util.otoa(identity[1]);
140                String type = Util.otoa(identity[2]);
141                String firstname = Util.otoa(identity[3]);
142                String lastname = Util.otoa(identity[4]);
143                String displayName = Util.otoa(identity[5]);
144                String email = Util.otoa(identity[6]);
145                String managerId = Util.otoa(identity[7]);
146                String administratorId = Util.otoa(identity[8]);
147
148                Date created = (Date) identity[9];
149                Date modified = (Date) identity[10];
150                Date lastRefresh = (Date) identity[11];
151
152                Attributes<String, Object> attributes = (Attributes<String, Object>) identity[12];
153                Map<String, Object> preferences = (Map<String, Object>) identity[13];
154
155                if (logger.isTraceEnabled()) {
156                    logger.trace("Exporting Identity " + id + ": " +  displayName);
157                }
158
159                if (isDeleteEnabled()) {
160                    deleteIdentity.setString("id", id);
161                    deleteIdentity.addBatch();
162
163                    deleteAttrs.setString("id", id);
164                    deleteAttrs.addBatch();
165
166                    deleteRolesStatement.setString("id", id);
167                    deleteRolesStatement.addBatch();
168                }
169
170                insertIdentityStatement.setString("id", id);
171                insertIdentityStatement.setString("name", name);
172                insertIdentityStatement.setString("type", type);
173                insertIdentityStatement.setString("firstname", firstname);
174                insertIdentityStatement.setString("lastname", lastname);
175                insertIdentityStatement.setString("email", email);
176                if (managerId != null) {
177                    insertIdentityStatement.setString("managerId", managerId);
178                } else {
179                    insertIdentityStatement.setNull("managerId", Types.VARCHAR);
180                }
181                if (administratorId != null) {
182                    insertIdentityStatement.setString("administratorId", administratorId);
183                } else {
184                    insertIdentityStatement.setNull("administratorId", Types.VARCHAR);
185                }
186
187                addCommonDateFields(insertIdentityStatement, exportDate, created, modified, lastRefresh);
188
189                insertIdentityStatement.addBatch();
190
191                if (attributes != null) {
192                    for (ObjectAttribute attribute : identityConfig.getObjectAttributes()) {
193                        String attrName = attribute.getName();
194                        Object value = attributes.get(attrName);
195                        if (!Utilities.isNothing(value)) {
196                            // TODO - do we want to filter more precisely here? e.g., exclude SSN for certain people?
197                            if (!excludeIdentityCols.contains(attrName)) {
198                                insertAttributeStatement.setString("id", id);
199                                insertAttributeStatement.setString("attributeName", attrName);
200
201                                if (attribute.isMulti()) {
202                                    Set<String> uniqueValues = new ListOrderedSet<>();
203                                    uniqueValues.addAll(Util.otol(value));
204                                    for (String val : uniqueValues) {
205                                        insertAttributeStatement.setString("attributeValue", Utilities.truncateStringToBytes(val, 4000, StandardCharsets.UTF_8));
206                                        insertAttributeStatement.addBatch();
207                                    }
208                                } else {
209                                    insertAttributeStatement.setString("attributeValue", Utilities.truncateStringToBytes(Util.otoa(value), 4000, StandardCharsets.UTF_8));
210                                    insertAttributeStatement.addBatch();
211                                }
212                            }
213                        }
214                    }
215                }
216
217                if (preferences != null) {
218                    List<RoleAssignment> assignments = (List<RoleAssignment>) preferences.get("roleAssignments");
219                    for(RoleAssignment ra : Util.safeIterable(assignments)) {
220                        if (!excludeRoles.contains(ra.getRoleName())) {
221                            Bundle role = cachedRoles.get(ra.getRoleName());
222
223                            if (role == null) {
224                                logger.warn("Identity " + id + " appears to have non-real RoleAssignment to " + ra.getRoleName());
225                            } else if (!excludeRoleTypes.contains(role.getType())) {
226                                insertRolesStatement.setString("id", id);
227                                insertRolesStatement.setString("roleName", role.getName());
228                                insertRolesStatement.setString("roleType", role.getType());
229                                insertRolesStatement.setDate("assignedOn", ra.getDate());
230                                insertRolesStatement.addBatch();
231                            }
232                        }
233                    }
234
235                    List<RoleDetection> detections = (List<RoleDetection>) preferences.get("roleDetections");
236                    for(RoleDetection rd : Util.safeIterable(detections)) {
237                        if (!excludeRoles.contains(rd.getRoleName())) {
238                            Bundle role = cachedRoles.get(rd.getRoleName());
239
240                            if (role == null) {
241                                logger.warn("Identity " + id + " appears to have non-real RoleDetection of " + rd.getRoleName());
242                            } else if (!excludeRoleTypes.contains(role.getType())) {
243                                insertRolesStatement.setString("id", id);
244                                insertRolesStatement.setString("roleName", role.getName());
245                                insertRolesStatement.setString("roleType", role.getType());
246                                insertRolesStatement.setDate("assignedOn", rd.getDate());
247                                insertRolesStatement.addBatch();
248                            }
249                        }
250                    }
251                }
252
253                if (batchCount++ > IDENTITY_BATCH_SIZE) {
254                    if (isDeleteEnabled()) {
255                        deleteAttrs.executeBatch();
256                        deleteRolesStatement.executeBatch();
257                        deleteIdentity.executeBatch();
258                    }
259                    insertIdentityStatement.executeBatch();
260                    insertAttributeStatement.executeBatch();
261                    insertRolesStatement.executeBatch();
262
263                    connection.commit();
264                    batchCount = 0;
265                }
266
267                int currentCount = totalCount.incrementAndGet();
268                if ((currentCount % 100) == 0) {
269                    TaskUtil.withLockedPartitionResult(monitor, (partitionResult) -> {
270                        monitor.updateProgress(partitionResult, "Processed " + currentCount + " of " + count + " identities", -1);
271                        partitionResult.setInt("exportedIdentities", currentCount);
272                    });
273                }
274            }
275
276            deleteAttrs.executeBatch();
277            deleteRolesStatement.executeBatch();
278            deleteIdentity.executeBatch();
279            insertIdentityStatement.executeBatch();
280            insertAttributeStatement.executeBatch();
281            insertRolesStatement.executeBatch();
282
283            connection.commit();
284
285            int currentCount = totalCount.get();
286            TaskUtil.withLockedPartitionResult(monitor, (partitionResult) -> {
287                monitor.updateProgress(partitionResult, "Processed " + currentCount + " of " + count + " identities", -1);
288                partitionResult.setInt("exportedIdentities", currentCount);
289            });
290
291
292        } catch(SQLException e) {
293            throw new GeneralException(e);
294        }
295    }
296
297}