Source code

001package com.identityworksllc.iiq.common.task.export;
002
003import com.identityworksllc.iiq.common.TaskUtil;
004import com.identityworksllc.iiq.common.Utilities;
005import com.identityworksllc.iiq.common.query.NamedParameterStatement;
006import org.apache.commons.collections4.set.ListOrderedSet;
007import org.apache.commons.logging.Log;
008import sailpoint.api.IncrementalObjectIterator;
009import sailpoint.api.IncrementalProjectionIterator;
010import sailpoint.api.SailPointContext;
011import sailpoint.object.*;
012import sailpoint.tools.GeneralException;
013import sailpoint.tools.Util;
014
015import java.sql.Connection;
016import java.sql.SQLException;
017import java.sql.Types;
018import java.util.*;
019import java.util.concurrent.atomic.AtomicInteger;
020
021public class ExportIdentitiesPartition extends ExportPartition {
022
023    private static final String DELETE_IDENTITY =
024            "delete from de_identity where id = :id";
025
026    private static final String DELETE_IDENTITY_ATTRS =
027            "delete from de_identity_attr where id = :id";
028
029    private static final String DELETE_IDENTITY_ROLES =
030            "delete from de_identity_roles where id = :id";
031
032    public static final int IDENTITY_BATCH_SIZE = 50;
033
034    private static final String INSERT_IDENTITY =
035            "insert into de_identity " +
036                    "( id, name, type, firstname, lastname, email, manager_id, administrator_id, created, modified, last_refresh, de_timestamp ) " +
037                    "values ( :id, :name, :type, :firstname, :lastname, :email, :managerId, :administratorId, :created, :modified, :lastRefresh, :now )";
038
039    private static final String INSERT_IDENTITY_ATTR =
040            "insert into de_identity_attr ( id, attribute_name, attribute_value ) values ( :id, :attributeName, :attributeValue )";
041
042    private static final String INSERT_IDENTITY_ROLES =
043            "insert into de_identity_roles ( id, role_name, role_type, role_date ) values ( :id, :roleName, :roleType, :assignedOn )";
044
045
046    /**
047     * Exports the identities identified by the filters
048     *
049     * @param context The context
050     * @param connection The connection to the target database
051     * @param logger The logger
052     * @throws GeneralException if anything fails during execution
053     */
054    protected void export(SailPointContext context, Connection connection, Log logger) throws GeneralException {
055        Set<String> excludeRoles = new HashSet<>();
056        if (configuration.containsAttribute("excludeRoles")) {
057            List<String> cols = configuration.getStringList("excludeRoles");
058            if (cols != null) {
059                excludeRoles.addAll(cols);
060            }
061        }
062
063        Set<String> excludeRoleTypes = new HashSet<>();
064        if (configuration.containsAttribute("excludeRoleTypes")) {
065            List<String> cols = configuration.getStringList("excludeRoleTypes");
066            if (cols != null) {
067                excludeRoleTypes.addAll(cols);
068            }
069        }
070
071        TaskUtil.withLockedPartitionResult(monitor, (partitionResult) -> {
072            monitor.updateProgress(partitionResult, "Caching role data", -1);
073        });
074
075        Map<String, Bundle> cachedRoles = new HashMap<>();
076
077        Iterator<Bundle> allRoles = context.search(Bundle.class, new QueryOptions());
078        while(allRoles.hasNext()) {
079            Bundle b = allRoles.next();
080            cachedRoles.put(b.getName(), Utilities.detach(context, b));
081        }
082
083        Set<String> excludeIdentityCols = new HashSet<>();
084        if (configuration.containsAttribute("excludeIdentityColumns")) {
085            List<String> cols = configuration.getStringList("excludeIdentityColumns");
086            if (cols != null) {
087                excludeIdentityCols.addAll(cols);
088            }
089        }
090
091        Date exportDate = new Date(exportTimestamp);
092
093        QueryOptions qo = new QueryOptions();
094        qo.addFilter(Filter.compile(filterString));
095        qo.addFilter(Filter.or(Filter.gt("created", new Date(cutoffDate)), Filter.gt("modified", new Date(cutoffDate))));
096        qo.setCacheResults(false);
097        qo.setTransactionLock(false);
098
099        TaskUtil.withLockedPartitionResult(monitor, (partitionResult) -> {
100            monitor.updateProgress(partitionResult, "Executing query", -1);
101        });
102
103        int batchCount = 0;
104        final AtomicInteger totalCount = new AtomicInteger();
105
106        ObjectConfig identityConfig = Identity.getObjectConfig();
107
108        List<String> fields = new ArrayList<>();
109        fields.add("id");               // 0
110        fields.add("name");
111        fields.add("type");
112        fields.add("firstname");
113        fields.add("lastname");
114        fields.add("displayName");      // 5
115        fields.add("email");
116        fields.add("manager.id");
117        fields.add("administrator.id");
118        fields.add("created");
119        fields.add("modified");         // 10
120        fields.add("lastRefresh");
121        fields.add("attributes");
122        fields.add("preferences");
123
124        long count = context.countObjects(Identity.class, qo);
125
126        IncrementalProjectionIterator identities = new IncrementalProjectionIterator(context, Identity.class, qo, fields);
127
128        try (NamedParameterStatement deleteAttrs = new NamedParameterStatement(connection, DELETE_IDENTITY_ATTRS); NamedParameterStatement deleteIdentity = new NamedParameterStatement(connection, DELETE_IDENTITY); NamedParameterStatement insertIdentityStatement = new NamedParameterStatement(connection, INSERT_IDENTITY); NamedParameterStatement insertAttributeStatement = new NamedParameterStatement(connection, INSERT_IDENTITY_ATTR); NamedParameterStatement deleteRolesStatement = new NamedParameterStatement(connection, DELETE_IDENTITY_ROLES); NamedParameterStatement insertRolesStatement = new NamedParameterStatement(connection, INSERT_IDENTITY_ROLES)) {
129            while (identities.hasNext()) {
130                if (isTerminated()) {
131                    logger.info("Thread has been terminated; exiting cleanly");
132                    break;
133                }
134
135                Object[] identity = identities.next();
136
137                String id = Util.otoa(identity[0]);
138                String name = Util.otoa(identity[1]);
139                String type = Util.otoa(identity[2]);
140                String firstname = Util.otoa(identity[3]);
141                String lastname = Util.otoa(identity[4]);
142                String displayName = Util.otoa(identity[5]);
143                String email = Util.otoa(identity[6]);
144                String managerId = Util.otoa(identity[7]);
145                String administratorId = Util.otoa(identity[8]);
146
147                Date created = (Date) identity[9];
148                Date modified = (Date) identity[10];
149                Date lastRefresh = (Date) identity[11];
150
151                Attributes<String, Object> attributes = (Attributes<String, Object>) identity[12];
152                Map<String, Object> preferences = (Map<String, Object>) identity[13];
153
154                if (logger.isTraceEnabled()) {
155                    logger.trace("Exporting Identity " + id + ": " +  displayName);
156                }
157
158                deleteIdentity.setString("id", id);
159                deleteIdentity.addBatch();
160
161                deleteAttrs.setString("id", id);
162                deleteAttrs.addBatch();
163
164                deleteRolesStatement.setString("id", id);
165                deleteRolesStatement.addBatch();
166
167                insertIdentityStatement.setString("id", id);
168                insertIdentityStatement.setString("name", name);
169                insertIdentityStatement.setString("type", type);
170                insertIdentityStatement.setString("firstname", firstname);
171                insertIdentityStatement.setString("lastname", lastname);
172                insertIdentityStatement.setString("email", email);
173                if (managerId != null) {
174                    insertIdentityStatement.setString("managerId", managerId);
175                } else {
176                    insertIdentityStatement.setNull("managerId", Types.VARCHAR);
177                }
178                if (administratorId != null) {
179                    insertIdentityStatement.setString("administratorId", administratorId);
180                } else {
181                    insertIdentityStatement.setNull("administratorId", Types.VARCHAR);
182                }
183
184                addCommonDateFields(insertIdentityStatement, exportDate, created, modified, lastRefresh);
185
186                insertIdentityStatement.addBatch();
187
188                if (attributes != null) {
189                    for (ObjectAttribute attribute : identityConfig.getObjectAttributes()) {
190                        String attrName = attribute.getName();
191                        Object value = attributes.get(attrName);
192                        if (!Utilities.isNothing(value)) {
193                            // TODO - do we want to filter more precisely here? e.g., exclude SSN for certain people?
194                            if (!excludeIdentityCols.contains(attrName)) {
195                                insertAttributeStatement.setString("id", id);
196                                insertAttributeStatement.setString("attributeName", attrName);
197
198                                if (attribute.isMulti()) {
199                                    Set<String> uniqueValues = new ListOrderedSet<>();
200                                    uniqueValues.addAll(Util.otol(value));
201                                    for (String val : uniqueValues) {
202                                        insertAttributeStatement.setString("attributeValue", Util.truncate(val, 4000));
203                                        insertAttributeStatement.addBatch();
204                                    }
205                                } else {
206                                    insertAttributeStatement.setString("attributeValue", Util.truncate(Util.otoa(value), 4000));
207                                    insertAttributeStatement.addBatch();
208                                }
209                            }
210                        }
211                    }
212                }
213
214                if (preferences != null) {
215                    List<RoleAssignment> assignments = (List<RoleAssignment>) preferences.get("roleAssignments");
216                    for(RoleAssignment ra : Util.safeIterable(assignments)) {
217                        if (!excludeRoles.contains(ra.getRoleName())) {
218                            Bundle role = cachedRoles.get(ra.getRoleName());
219
220                            if (!excludeRoleTypes.contains(role.getType())) {
221                                insertRolesStatement.setString("id", id);
222                                insertRolesStatement.setString("roleName", role.getName());
223                                insertRolesStatement.setString("roleType", role.getType());
224                                insertRolesStatement.setDate("assignedOn", ra.getDate());
225                                insertRolesStatement.addBatch();
226                            }
227                        }
228                    }
229
230                    List<RoleDetection> detections = (List<RoleDetection>) preferences.get("roleDetections");
231                    for(RoleDetection rd : Util.safeIterable(detections)) {
232                        if (!excludeRoles.contains(rd.getRoleName())) {
233                            Bundle role = cachedRoles.get(rd.getRoleName());
234
235                            if (!excludeRoleTypes.contains(role.getType())) {
236                                insertRolesStatement.setString("id", id);
237                                insertRolesStatement.setString("roleName", role.getName());
238                                insertRolesStatement.setString("roleType", role.getType());
239                                insertRolesStatement.setDate("assignedOn", rd.getDate());
240                                insertRolesStatement.addBatch();
241                            }
242                        }
243                    }
244                }
245
246                if (batchCount++ > IDENTITY_BATCH_SIZE) {
247                    deleteAttrs.executeBatch();
248                    deleteRolesStatement.executeBatch();
249                    deleteIdentity.executeBatch();
250                    insertIdentityStatement.executeBatch();
251                    insertAttributeStatement.executeBatch();
252                    insertRolesStatement.executeBatch();
253
254                    connection.commit();
255                    batchCount = 0;
256                }
257
258                int currentCount = totalCount.incrementAndGet();
259                if ((currentCount % 100) == 0) {
260                    TaskUtil.withLockedPartitionResult(monitor, (partitionResult) -> {
261                        monitor.updateProgress(partitionResult, "Processed " + currentCount + " of " + count + " identities", -1);
262                        partitionResult.setInt("exportedIdentities", currentCount);
263                    });
264                }
265            }
266
267            deleteAttrs.executeBatch();
268            deleteRolesStatement.executeBatch();
269            deleteIdentity.executeBatch();
270            insertIdentityStatement.executeBatch();
271            insertAttributeStatement.executeBatch();
272            insertRolesStatement.executeBatch();
273
274            connection.commit();
275
276        } catch(SQLException e) {
277            throw new GeneralException(e);
278        }
279    }
280
281}