Source code

001package com.identityworksllc.iiq.common.task.export;
002
003import com.identityworksllc.iiq.common.TaskUtil;
004import com.identityworksllc.iiq.common.Utilities;
005import com.identityworksllc.iiq.common.query.NamedParameterStatement;
006import org.apache.commons.collections4.set.ListOrderedSet;
007import org.apache.commons.logging.Log;
008import sailpoint.api.IncrementalObjectIterator;
009import sailpoint.api.IncrementalProjectionIterator;
010import sailpoint.api.SailPointContext;
011import sailpoint.object.*;
012import sailpoint.tools.GeneralException;
013import sailpoint.tools.Util;
014
015import java.nio.charset.StandardCharsets;
016import java.sql.Connection;
017import java.sql.SQLException;
018import java.sql.Types;
019import java.util.*;
020import java.util.concurrent.atomic.AtomicInteger;
021
022public class ExportIdentitiesPartition extends ExportPartition {
023
024    private static final String DELETE_IDENTITY =
025            "delete from de_identity where id = :id";
026
027    private static final String DELETE_IDENTITY_ATTRS =
028            "delete from de_identity_attr where id = :id";
029
030    private static final String DELETE_IDENTITY_ROLES =
031            "delete from de_identity_roles where id = :id";
032
033    public static final int IDENTITY_BATCH_SIZE = 50;
034
035    private static final String INSERT_IDENTITY =
036            "insert into de_identity " +
037                    "( id, name, type, firstname, lastname, email, manager_id, administrator_id, created, modified, last_refresh, de_timestamp ) " +
038                    "values ( :id, :name, :type, :firstname, :lastname, :email, :managerId, :administratorId, :created, :modified, :lastRefresh, :now )";
039
040    private static final String INSERT_IDENTITY_ATTR =
041            "insert into de_identity_attr ( id, attribute_name, attribute_value ) values ( :id, :attributeName, :attributeValue )";
042
043    private static final String INSERT_IDENTITY_ROLES =
044            "insert into de_identity_roles ( id, role_name, role_type, role_date ) values ( :id, :roleName, :roleType, :assignedOn )";
045
046
047    /**
048     * Exports the identities identified by the filters
049     *
050     * @param context The context
051     * @param connection The connection to the target database
052     * @param logger The logger
053     * @throws GeneralException if anything fails during execution
054     */
055    protected void export(SailPointContext context, Connection connection, Log logger) throws GeneralException {
056        Set<String> excludeRoles = new HashSet<>();
057        if (configuration.containsAttribute("excludeRoles")) {
058            List<String> cols = configuration.getStringList("excludeRoles");
059            if (cols != null) {
060                excludeRoles.addAll(cols);
061            }
062        }
063
064        Set<String> excludeRoleTypes = new HashSet<>();
065        if (configuration.containsAttribute("excludeRoleTypes")) {
066            List<String> cols = configuration.getStringList("excludeRoleTypes");
067            if (cols != null) {
068                excludeRoleTypes.addAll(cols);
069            }
070        }
071
072        TaskUtil.withLockedPartitionResult(monitor, (partitionResult) -> {
073            monitor.updateProgress(partitionResult, "Caching role data", -1);
074        });
075
076        Map<String, Bundle> cachedRoles = new HashMap<>();
077
078        Iterator<Bundle> allRoles = context.search(Bundle.class, new QueryOptions());
079        while(allRoles.hasNext()) {
080            Bundle b = allRoles.next();
081            cachedRoles.put(b.getName(), Utilities.detach(context, b));
082        }
083
084        Set<String> excludeIdentityCols = new HashSet<>();
085        if (configuration.containsAttribute("excludeIdentityColumns")) {
086            List<String> cols = configuration.getStringList("excludeIdentityColumns");
087            if (cols != null) {
088                excludeIdentityCols.addAll(cols);
089            }
090        }
091
092        Date exportDate = new Date(exportTimestamp);
093
094        QueryOptions qo = new QueryOptions();
095        qo.addFilter(Filter.compile(filterString));
096        qo.addFilter(Filter.or(Filter.gt("created", new Date(cutoffDate)), Filter.gt("modified", new Date(cutoffDate))));
097        qo.setCacheResults(false);
098        qo.setTransactionLock(false);
099
100        TaskUtil.withLockedPartitionResult(monitor, (partitionResult) -> {
101            monitor.updateProgress(partitionResult, "Executing query", -1);
102        });
103
104        int batchCount = 0;
105        final AtomicInteger totalCount = new AtomicInteger();
106
107        ObjectConfig identityConfig = Identity.getObjectConfig();
108
109        List<String> fields = new ArrayList<>();
110        fields.add("id");               // 0
111        fields.add("name");
112        fields.add("type");
113        fields.add("firstname");
114        fields.add("lastname");
115        fields.add("displayName");      // 5
116        fields.add("email");
117        fields.add("manager.id");
118        fields.add("administrator.id");
119        fields.add("created");
120        fields.add("modified");         // 10
121        fields.add("lastRefresh");
122        fields.add("attributes");
123        fields.add("preferences");
124
125        long count = context.countObjects(Identity.class, qo);
126
127        IncrementalProjectionIterator identities = new IncrementalProjectionIterator(context, Identity.class, qo, fields);
128
129        try (NamedParameterStatement deleteAttrs = new NamedParameterStatement(connection, DELETE_IDENTITY_ATTRS); NamedParameterStatement deleteIdentity = new NamedParameterStatement(connection, DELETE_IDENTITY); NamedParameterStatement insertIdentityStatement = new NamedParameterStatement(connection, INSERT_IDENTITY); NamedParameterStatement insertAttributeStatement = new NamedParameterStatement(connection, INSERT_IDENTITY_ATTR); NamedParameterStatement deleteRolesStatement = new NamedParameterStatement(connection, DELETE_IDENTITY_ROLES); NamedParameterStatement insertRolesStatement = new NamedParameterStatement(connection, INSERT_IDENTITY_ROLES)) {
130            while (identities.hasNext()) {
131                if (isTerminated()) {
132                    logger.info("Thread has been terminated; exiting cleanly");
133                    break;
134                }
135
136                Object[] identity = identities.next();
137
138                String id = Util.otoa(identity[0]);
139                String name = Util.otoa(identity[1]);
140                String type = Util.otoa(identity[2]);
141                String firstname = Util.otoa(identity[3]);
142                String lastname = Util.otoa(identity[4]);
143                String displayName = Util.otoa(identity[5]);
144                String email = Util.otoa(identity[6]);
145                String managerId = Util.otoa(identity[7]);
146                String administratorId = Util.otoa(identity[8]);
147
148                Date created = (Date) identity[9];
149                Date modified = (Date) identity[10];
150                Date lastRefresh = (Date) identity[11];
151
152                Attributes<String, Object> attributes = (Attributes<String, Object>) identity[12];
153                Map<String, Object> preferences = (Map<String, Object>) identity[13];
154
155                if (logger.isTraceEnabled()) {
156                    logger.trace("Exporting Identity " + id + ": " +  displayName);
157                }
158
159                deleteIdentity.setString("id", id);
160                deleteIdentity.addBatch();
161
162                deleteAttrs.setString("id", id);
163                deleteAttrs.addBatch();
164
165                deleteRolesStatement.setString("id", id);
166                deleteRolesStatement.addBatch();
167
168                insertIdentityStatement.setString("id", id);
169                insertIdentityStatement.setString("name", name);
170                insertIdentityStatement.setString("type", type);
171                insertIdentityStatement.setString("firstname", firstname);
172                insertIdentityStatement.setString("lastname", lastname);
173                insertIdentityStatement.setString("email", email);
174                if (managerId != null) {
175                    insertIdentityStatement.setString("managerId", managerId);
176                } else {
177                    insertIdentityStatement.setNull("managerId", Types.VARCHAR);
178                }
179                if (administratorId != null) {
180                    insertIdentityStatement.setString("administratorId", administratorId);
181                } else {
182                    insertIdentityStatement.setNull("administratorId", Types.VARCHAR);
183                }
184
185                addCommonDateFields(insertIdentityStatement, exportDate, created, modified, lastRefresh);
186
187                insertIdentityStatement.addBatch();
188
189                if (attributes != null) {
190                    for (ObjectAttribute attribute : identityConfig.getObjectAttributes()) {
191                        String attrName = attribute.getName();
192                        Object value = attributes.get(attrName);
193                        if (!Utilities.isNothing(value)) {
194                            // TODO - do we want to filter more precisely here? e.g., exclude SSN for certain people?
195                            if (!excludeIdentityCols.contains(attrName)) {
196                                insertAttributeStatement.setString("id", id);
197                                insertAttributeStatement.setString("attributeName", attrName);
198
199                                if (attribute.isMulti()) {
200                                    Set<String> uniqueValues = new ListOrderedSet<>();
201                                    uniqueValues.addAll(Util.otol(value));
202                                    for (String val : uniqueValues) {
203                                        insertAttributeStatement.setString("attributeValue", Utilities.truncateStringToBytes(val, 4000, StandardCharsets.UTF_8));
204                                        insertAttributeStatement.addBatch();
205                                    }
206                                } else {
207                                    insertAttributeStatement.setString("attributeValue", Utilities.truncateStringToBytes(Util.otoa(value), 4000, StandardCharsets.UTF_8));
208                                    insertAttributeStatement.addBatch();
209                                }
210                            }
211                        }
212                    }
213                }
214
215                if (preferences != null) {
216                    List<RoleAssignment> assignments = (List<RoleAssignment>) preferences.get("roleAssignments");
217                    for(RoleAssignment ra : Util.safeIterable(assignments)) {
218                        if (!excludeRoles.contains(ra.getRoleName())) {
219                            Bundle role = cachedRoles.get(ra.getRoleName());
220
221                            if (role == null) {
222                                logger.warn("Identity " + id + " appears to have non-real RoleAssignment to " + ra.getRoleName());
223                            } else if (!excludeRoleTypes.contains(role.getType())) {
224                                insertRolesStatement.setString("id", id);
225                                insertRolesStatement.setString("roleName", role.getName());
226                                insertRolesStatement.setString("roleType", role.getType());
227                                insertRolesStatement.setDate("assignedOn", ra.getDate());
228                                insertRolesStatement.addBatch();
229                            }
230                        }
231                    }
232
233                    List<RoleDetection> detections = (List<RoleDetection>) preferences.get("roleDetections");
234                    for(RoleDetection rd : Util.safeIterable(detections)) {
235                        if (!excludeRoles.contains(rd.getRoleName())) {
236                            Bundle role = cachedRoles.get(rd.getRoleName());
237
238                            if (role == null) {
239                                logger.warn("Identity " + id + " appears to have non-real RoleDetection of " + rd.getRoleName());
240                            } else if (!excludeRoleTypes.contains(role.getType())) {
241                                insertRolesStatement.setString("id", id);
242                                insertRolesStatement.setString("roleName", role.getName());
243                                insertRolesStatement.setString("roleType", role.getType());
244                                insertRolesStatement.setDate("assignedOn", rd.getDate());
245                                insertRolesStatement.addBatch();
246                            }
247                        }
248                    }
249                }
250
251                if (batchCount++ > IDENTITY_BATCH_SIZE) {
252                    deleteAttrs.executeBatch();
253                    deleteRolesStatement.executeBatch();
254                    deleteIdentity.executeBatch();
255                    insertIdentityStatement.executeBatch();
256                    insertAttributeStatement.executeBatch();
257                    insertRolesStatement.executeBatch();
258
259                    connection.commit();
260                    batchCount = 0;
261                }
262
263                int currentCount = totalCount.incrementAndGet();
264                if ((currentCount % 100) == 0) {
265                    TaskUtil.withLockedPartitionResult(monitor, (partitionResult) -> {
266                        monitor.updateProgress(partitionResult, "Processed " + currentCount + " of " + count + " identities", -1);
267                        partitionResult.setInt("exportedIdentities", currentCount);
268                    });
269                }
270            }
271
272            deleteAttrs.executeBatch();
273            deleteRolesStatement.executeBatch();
274            deleteIdentity.executeBatch();
275            insertIdentityStatement.executeBatch();
276            insertAttributeStatement.executeBatch();
277            insertRolesStatement.executeBatch();
278
279            connection.commit();
280
281        } catch(SQLException e) {
282            throw new GeneralException(e);
283        }
284    }
285
286}