Source code

001package com.identityworksllc.iiq.common.task;
002
003import com.identityworksllc.iiq.common.Functions;
004import com.identityworksllc.iiq.common.iterators.TransformingIterator;
005import com.identityworksllc.iiq.common.query.ContextConnectionWrapper;
006import org.apache.commons.logging.Log;
007import org.apache.commons.logging.LogFactory;
008import sailpoint.api.IncrementalObjectIterator;
009import sailpoint.api.ObjectUtil;
010import sailpoint.api.SailPointContext;
011import sailpoint.connector.Connector;
012import sailpoint.connector.ConnectorException;
013import sailpoint.connector.ConnectorFactory;
014import sailpoint.object.*;
015import sailpoint.tools.CloseableIterator;
016import sailpoint.tools.GeneralException;
017import sailpoint.tools.RFC4180LineParser;
018import sailpoint.tools.Util;
019
020import java.io.BufferedReader;
021import java.io.File;
022import java.io.FileInputStream;
023import java.io.IOException;
024import java.io.InputStreamReader;
025import java.sql.Connection;
026import java.sql.PreparedStatement;
027import java.sql.ResultSet;
028import java.sql.SQLException;
029import java.util.ArrayList;
030import java.util.Collections;
031import java.util.HashMap;
032import java.util.Iterator;
033import java.util.List;
034import java.util.Map;
035import java.util.Objects;
036import java.util.function.Consumer;
037import java.util.function.Function;
038
039/**
040 * Implements a reusable implementation of {@link ObjectRetriever} for use with any
041 * number of scheduled tasks.
042 *
043 * This class takes the following input arguments in its constructor:
044 *
045 *  - values: If the input type is 'provided', these values will be interpreted as the raw input.
046 *  - retrievalSql: A single-column SQL query that will produce a list of strings. The first column must be a string. Other columns will be ignored.
047 *  - retrievalSqlClass: If specified, the single-column result from the query will be used as an ID look up the object with this type
048 *  - retrievalConnector: The name of an Application; the iteration will be over a list of ResourceObjects as though an aggregation is running
049 *  - retrievalFile: The fully qualified name of an existing, readable text file. Each line will be passed as a List of Strings to convertObject. If a delimiter is specified, the List will contain each delimited element. If not, the list will contain the entire line.
050 *  - retrievalFileDelimiter: If specified, the file will be interpreted as a CSV; each line as a List will be passed to threadExecute
051 *  - retrievalFileClass: If specified, the file will be interpreted as a CSV and the first column will be interpreted as an ID or name of this class type.
052 *  - retrievalRule: The name of a Rule that will produce an Iterator over the desired objects
053 *  - retrievalScript: The source of a Script that will produce an Iterator over the desired objects
054 *  - retrievalFilter: A Filter string combined with retrievalFilterClass to do a search in SailPoint
055 *  - retrievalFilterClass: The class name to search with the retrievalFilter
056 *
057 * You are expected to provide a {@link TransformingIterator} supplier that will convert
058 * from the output type listed below to the desired type. If your TransformingIterator
059 * does not produce the correct output, it is likely that you will run into ClassCastExceptions.
060 * That is beyond the scope of this class.
061 *
062 * Default object types if you do not convert them afterwards are:
063 *
064 *  'provided':
065 *    String
066 *
067 *  'sql':
068 *    If a class is specified, instances of that class. If not, strings.
069 *
070 *  'connector':
071 *    ResourceObject
072 *
073 *  'file':
074 *    List of Strings
075 *
076 *  'rule' and 'script':
077 *    Arbitrary. You are expected to return the right data type from your rule.
078 *    You may return an IncrementalObjectIterator, an Iterator, an Iterable, or
079 *    any other object (which will be wrapped in a singleton list).
080 *
081 *  'filter':
082 *    An object of the specified filter class, or Identity if one is not specified.
083 *    The iterator will wrap an IncrementalObjectIterator so that not all objects
084 *    are loaded into memory at once.
085 *
086 * If you are using this class in a context that can be interrupted (e.g., in a job),
087 * you will need to provide a way for this class to register a "termination handler"
088 * by calling {@link #setTerminationRegistrar(Consumer)}. Your termination logic
089 * MUST invoke any callbacks registered with your consumer.
090 *
091 * * @param <ItemType>
092 */
093public class BasicObjectRetriever<ItemType> implements ObjectRetriever<ItemType> {
094    /**
095     * The list of retrieval types
096     */
097    enum RetrievalType {
098        rule,
099        script,
100        sql,
101        filter,
102        connector,
103        file,
104        provided
105    }
106
107    /**
108     * The class logger
109     */
110    private final Log log;
111    /**
112     * The list of provided values
113     */
114    private List<String> providedValues;
115    /**
116     * The application used to retrieve any inputs
117     */
118    private Application retrievalApplication;
119    /**
120     * The file used to retrieve any inputs
121     */
122    private File retrievalFile;
123    /**
124     * Retrieval file delimiter
125     */
126    private String retrievalFileDelimiter;
127    /**
128     * The class to retrieve, assuming the default retriever is used
129     */
130    private Class<? extends SailPointObject> retrievalFileClass;
131    /**
132     * The filter used to retrieve any objects, assuming the default retriever is used
133     */
134    private Filter retrievalFilter;
135    /**
136     * The class to retrieve, assuming the default retriever is used
137     */
138    private Class<? extends SailPointObject> retrievalFilterClass;
139    /**
140     * A rule that returns objects to be iterated over in parallel
141     */
142    private Rule retrievalRule;
143    /**
144     * A script that returns objects to be iterated over in parallel
145     */
146    private Script retrievalScript;
147    /**
148     * A SQL query that returns IDs of objects to be iterated in parallel
149     */
150    private String retrievalSql;
151    /**
152     * The class represented by the SQL output
153     */
154    private Class<? extends SailPointObject> retrievalSqlClass;
155    /**
156     * The type of retrieval, which must be one of the defined values
157     */
158    private RetrievalType retrievalType;
159
160    /**
161     * The TaskResult of the running task
162     */
163    private final TaskResult taskResult;
164
165    /**
166     * The callback for registering termination handlers, set by setTerminationRegistrar
167     */
168    private Consumer<Functions.GenericCallback> terminationRegistrar;
169
170    /**
171     * The callback used to create the output iterator to the given type
172     */
173    private final Function<Iterator<?>, TransformingIterator<Object, ItemType>> transformerConstructor;
174
175    /**
176     * Constructs a new Basic object retriever
177     * @param context The Sailpoint context to use for searching
178     * @param arguments The arguments to the task (or other)
179     * @param transformerConstructor The callback that creates a transforming iterator
180     * @param taskResult The task result, or null
181     * @throws GeneralException if any failures occur
182     */
183    public BasicObjectRetriever(SailPointContext context, Attributes<String, Object> arguments, Function<Iterator<?>, TransformingIterator<Object, ItemType>> transformerConstructor, TaskResult taskResult) throws GeneralException {
184        Objects.requireNonNull(context);
185        Objects.requireNonNull(arguments);
186
187        initialize(context, arguments);
188
189        this.log = LogFactory.getLog(BasicObjectRetriever.class);
190        this.transformerConstructor = transformerConstructor;
191        this.taskResult = taskResult;
192    }
193
194
195    /**
196     * Retrieves the contents of an input file
197     * @return An iterator over the contents of the file
198     * @throws IOException if there is an error reading the file
199     * @throws GeneralException if there is an error doing anything Sailpoint-y
200     */
201    private Iterator<List<String>> getFileContents() throws IOException, GeneralException {
202        Iterator<List<String>> items;
203        List<List<String>> itemList = new ArrayList<>();
204        try (BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(retrievalFile)))) {
205            RFC4180LineParser parser = null;
206            if (Util.isNotNullOrEmpty(retrievalFileDelimiter)) {
207                parser = new RFC4180LineParser(retrievalFileDelimiter);
208            }
209            String line;
210            while((line = reader.readLine()) != null) {
211                if (Util.isNotNullOrEmpty(line.trim())) {
212                    if (parser != null) {
213                        List<String> item = parser.parseLine(line);
214                        itemList.add(item);
215                    } else {
216                        List<String> lineList = Collections.singletonList(line);
217                        itemList.add(lineList);
218                    }
219                }
220            }
221        }
222        items = itemList.iterator();
223        return items;
224    }
225
226    /**
227     * Gets the object iterator, which will be an instance of {@link TransformingIterator} to
228     * convert whatever are the inputs. See the class Javadoc for information about what the
229     * default return types from each query method are.
230     *
231     * @param context The context to use during querying
232     * @param arguments Any additional query arguments
233     * @return An iterator over the retrieved objects
234     * @throws GeneralException if any failures occur
235     */
236    @Override
237    @SuppressWarnings("unchecked")
238    public Iterator<ItemType> getObjectIterator(SailPointContext context, Attributes<String, Object> arguments) throws GeneralException {
239        Objects.requireNonNull(context);
240
241        Map<String, Object> retrievalParams = new HashMap<>();
242        retrievalParams.put("log", log);
243        retrievalParams.put("context", context);
244        retrievalParams.put("taskResult", taskResult);
245        TransformingIterator<?, ItemType> items = null;
246        if (retrievalType == RetrievalType.provided) {
247            items = transformerConstructor.apply(providedValues.iterator());
248        } else if (retrievalType == RetrievalType.rule) {
249            if (retrievalRule == null) {
250                throw new IllegalArgumentException("Retrieval rule must exist");
251            }
252            Object output = context.runRule(retrievalRule, retrievalParams);
253            if (output instanceof IncrementalObjectIterator) {
254                items = transformerConstructor.apply((IncrementalObjectIterator)output);
255            } else if (output instanceof Iterator) {
256                items = transformerConstructor.apply((Iterator)output);
257            } else if (output instanceof Iterable) {
258                items = transformerConstructor.apply(((Iterable)output).iterator());
259            } else {
260                items = transformerConstructor.apply(Collections.singletonList(output).iterator());
261            }
262        } else if (retrievalType == RetrievalType.script) {
263            if (retrievalScript == null) {
264                throw new IllegalArgumentException("A retrieval script must be defined with 'script' retrieval type");
265            }
266            if (log.isDebugEnabled()) {
267                log.debug("Running retrieval script: " + retrievalScript);
268            }
269            Object output = context.runScript(retrievalScript, retrievalParams);
270            if (output instanceof IncrementalObjectIterator) {
271                items = transformerConstructor.apply((IncrementalObjectIterator)output);
272            } else if (output instanceof Iterator) {
273                items = transformerConstructor.apply((Iterator)output);
274            } else if (output instanceof Iterable) {
275                items = transformerConstructor.apply(((Iterable)output).iterator());
276            } else {
277                items = transformerConstructor.apply(Collections.singletonList(output).iterator());
278            }
279        } else if (retrievalType == RetrievalType.filter) {
280            if (retrievalFilter == null || retrievalFilterClass == null) {
281                throw new IllegalArgumentException("A retrieval filter and class name must be specified for 'filter' retrieval type");
282            }
283            QueryOptions options = new QueryOptions();
284            options.addFilter(retrievalFilter);
285            items = transformerConstructor.apply(new IncrementalObjectIterator<>(context, retrievalFilterClass, options));
286        } else if (retrievalType == RetrievalType.file) {
287            try {
288                Iterator<List<String>> fileContents = getFileContents();
289                if (retrievalFileClass != null) {
290                    items = transformerConstructor.apply(new TransformingIterator<>(fileContents, (id) -> Util.isEmpty(id) ? null : (ItemType) context.getObject(retrievalFileClass, Util.otoa(id.get(0)))));
291                } else {
292                    items = transformerConstructor.apply(fileContents);
293                }
294            } catch (IOException e) {
295                throw new GeneralException(e);
296            }
297        } else if (retrievalType == RetrievalType.sql) {
298            List<String> objectNames = new ArrayList<>();
299            // There is probably a more efficient way to do this, but it's not clear
300            // how to automatically clean up the SQL resources afterwards...
301            try (Connection connection = ContextConnectionWrapper.getConnection(context)) {
302                try (PreparedStatement stmt = connection.prepareStatement(retrievalSql)) {
303                    try (ResultSet results = stmt.executeQuery()) {
304                        while(results.next()) {
305                            String output = results.getString(1);
306                            if (Util.isNotNullOrEmpty(output)) {
307                                objectNames.add(output);
308                            }
309                        }
310                    }
311                }
312            } catch (SQLException e) {
313                throw new GeneralException(e);
314            }
315            if (retrievalSqlClass != null) {
316                items = transformerConstructor.apply(new TransformingIterator<>(objectNames.iterator(), (id) -> (ItemType) context.getObject(retrievalSqlClass, id)));
317            } else {
318                items = transformerConstructor.apply(objectNames.iterator());
319            }
320        } else if (retrievalType == RetrievalType.connector) {
321            if (retrievalApplication == null) {
322                throw new IllegalArgumentException("You must specify an application for 'connector' retrieval type");
323            }
324            if (retrievalApplication.supportsFeature(Application.Feature.NO_AGGREGATION)) {
325                throw new IllegalArgumentException("The application " + retrievalApplication.getName() + " does not support aggregation");
326            }
327            try {
328                Connector connector = ConnectorFactory.getConnector(retrievalApplication, null);
329                CloseableIterator<ResourceObject> objectIterator = connector.iterateObjects(retrievalApplication.getAccountSchema().getName(), null, new HashMap<>());
330                if (terminationRegistrar != null) {
331                    terminationRegistrar.accept((terminationContext) -> objectIterator.close());
332                }
333                return transformerConstructor.apply(new AbstractThreadedObjectIteratorTask.CloseableIteratorWrapper(objectIterator));
334            } catch(ConnectorException e) {
335                throw new GeneralException(e);
336            }
337        } else {
338            throw new IllegalStateException("You must specify a rule, script, filter, sql, file, connector, or list to retrieve target objects");
339        }
340        items.ignoreNulls();
341        return items;
342    }
343
344    /**
345     * Returns true if the retriever is configured with a retrieval class appropriate
346     * for the retrieval type, otherwise returns false.
347     * @return True if a retrieval class is present
348     */
349    public boolean hasRetrievalClass() {
350        if (getRetrievalType() == RetrievalType.file) {
351            return this.retrievalFileClass != null;
352        } else if (getRetrievalType() == RetrievalType.filter) {
353            return this.retrievalFilterClass != null;
354        } else if (getRetrievalType() == RetrievalType.sql) {
355            return this.retrievalSqlClass != null;
356        }
357        return false;
358    }
359
360    /**
361     * Gets the retrieval type
362     * @return The retrieval type
363     */
364    public RetrievalType getRetrievalType() {
365        return retrievalType;
366    }
367
368    /**
369     * Initializes this retriever from the input parameters
370     * @param context The context
371     * @param args The input arguments, usually from a Task
372     * @throws GeneralException if any Sailpoint-related failures occur
373     * @throws IllegalArgumentException if any arguments are incorrect or missing
374     */
375    @SuppressWarnings("unchecked")
376    private void initialize(SailPointContext context, Attributes<String, Object> args) throws GeneralException {
377        this.retrievalType = RetrievalType.valueOf(args.getString("retrievalType"));
378
379        if (retrievalType == RetrievalType.provided) {
380            List<String> values = args.getStringList("values");
381            if (values == null) {
382                throw new IllegalArgumentException("For input type = provided, a CSV in 'values' is required");
383            }
384
385            this.providedValues = values;
386        }
387
388        if (retrievalType == RetrievalType.sql) {
389            if (Util.isNullOrEmpty(args.getString("retrievalSql"))) {
390                throw new IllegalArgumentException("For input type = sql, a retrievalSql is required");
391            }
392
393            String className = args.getString("retrievalSqlClass");
394            if (Util.isNotNullOrEmpty(className)) {
395                this.retrievalSqlClass = (Class<? extends SailPointObject>) ObjectUtil.getSailPointClass(className);
396            }
397            this.retrievalSql = args.getString("retrievalSql");
398        }
399
400        if (retrievalType == RetrievalType.connector) {
401            String applicationName = args.getString("applicationName");
402            this.retrievalApplication = context.getObject(Application.class, applicationName);
403            if (this.retrievalApplication == null) {
404                throw new IllegalArgumentException("The application " + applicationName + " does not exist");
405            }
406        }
407
408        if (retrievalType == RetrievalType.file && Util.isNotNullOrEmpty(args.getString("retrievalFile"))) {
409            String filename = args.getString("retrievalFile");
410            this.retrievalFile = new File(filename);
411            this.retrievalFileDelimiter = args.getString("retrievalFileDelimiter");
412            String className = args.getString("retrievalFileClass");
413            if (Util.isNotNullOrEmpty(className)) {
414                this.retrievalFileClass = (Class<? extends SailPointObject>) ObjectUtil.getSailPointClass(className);
415            }
416            if (!this.retrievalFile.exists()) {
417                throw new IllegalArgumentException("Input file " + filename + " does not exist");
418            }
419            if (!this.retrievalFile.isFile()) {
420                throw new IllegalArgumentException("Input file " + filename + " does not appear to be a file (is it a directory?)");
421            }
422            if (!this.retrievalFile.canRead()) {
423                throw new IllegalArgumentException("Input file " + filename + " is not readable by the IIQ process");
424            }
425        }
426
427        if (retrievalType == RetrievalType.rule && Util.isNotNullOrEmpty(args.getString("retrievalRule"))) {
428            this.retrievalRule = context.getObjectByName(Rule.class, args.getString("retrievalRule"));
429        }
430
431        if (retrievalType == RetrievalType.script && Util.isNotNullOrEmpty(args.getString("retrievalScript"))) {
432            this.retrievalScript = new Script();
433            retrievalScript.setSource(args.getString("retrievalScript"));
434        }
435
436        if (retrievalType == RetrievalType.filter && Util.isNotNullOrEmpty(args.getString("retrievalFilter"))) {
437            this.retrievalFilter = Filter.compile(args.getString("retrievalFilter"));
438            String className = args.getString("retrievalFilterClass");
439            if (Util.isNotNullOrEmpty(className)) {
440                this.retrievalFilterClass = (Class<? extends SailPointObject>) ObjectUtil.getSailPointClass(className);
441            } else {
442                log.warn("No retrieval filter class given; assuming Identity");
443                this.retrievalFilterClass = Identity.class;
444            }
445        }
446    }
447
448    /**
449     * Sets the termination registrar object
450     * @param registar The termination registrar object
451     */
452    @Override
453    public void setTerminationRegistrar(Consumer<Functions.GenericCallback> registar) {
454        this.terminationRegistrar = registar;
455    }
456}