001package com.identityworksllc.iiq.common.task;
002
003import com.identityworksllc.iiq.common.Functions;
004import com.identityworksllc.iiq.common.iterators.TransformingIterator;
005import com.identityworksllc.iiq.common.query.ContextConnectionWrapper;
006import org.apache.commons.logging.Log;
007import org.apache.commons.logging.LogFactory;
008import sailpoint.api.IncrementalObjectIterator;
009import sailpoint.api.ObjectUtil;
010import sailpoint.api.SailPointContext;
011import sailpoint.connector.Connector;
012import sailpoint.connector.ConnectorException;
013import sailpoint.connector.ConnectorFactory;
014import sailpoint.object.*;
015import sailpoint.tools.CloseableIterator;
016import sailpoint.tools.GeneralException;
017import sailpoint.tools.RFC4180LineParser;
018import sailpoint.tools.Util;
019
020import java.io.BufferedReader;
021import java.io.File;
022import java.io.FileInputStream;
023import java.io.IOException;
024import java.io.InputStreamReader;
025import java.sql.Connection;
026import java.sql.PreparedStatement;
027import java.sql.ResultSet;
028import java.sql.SQLException;
029import java.util.ArrayList;
030import java.util.Collections;
031import java.util.HashMap;
032import java.util.Iterator;
033import java.util.List;
034import java.util.Map;
035import java.util.Objects;
036import java.util.function.Consumer;
037import java.util.function.Function;
038
039/**
040 * Implements a reusable implementation of {@link ObjectRetriever} for use with any
041 * number of scheduled tasks.
042 *
043 * This class takes the following input arguments in its constructor:
044 *
045 *  - values: If the input type is 'provided', these values will be interpreted as the raw input.
046 *  - retrievalSql: A single-column SQL query that will produce a list of strings. The first column must be a string. Other columns will be ignored.
047 *  - retrievalSqlClass: If specified, the single-column result from the query will be used as an ID look up the object with this type
048 *  - retrievalConnector: The name of an Application; the iteration will be over a list of ResourceObjects as though an aggregation is running
049 *  - retrievalFile: The fully qualified name of an existing, readable text file. Each line will be passed as a List of Strings to convertObject. If a delimiter is specified, the List will contain each delimited element. If not, the list will contain the entire line.
050 *  - retrievalFileDelimiter: If specified, the file will be interpreted as a CSV; each line as a List will be passed to threadExecute
051 *  - retrievalFileClass: If specified, the file will be interpreted as a CSV and the first column will be interpreted as an ID or name of this class type.
052 *  - retrievalRule: The name of a Rule that will produce an Iterator over the desired objects
053 *  - retrievalScript: The source of a Script that will produce an Iterator over the desired objects
054 *  - retrievalFilter: A Filter string combined with retrievalFilterClass to do a search in SailPoint
055 *  - retrievalFilterClass: The class name to search with the retrievalFilter
056 *
057 * You are expected to provide a {@link TransformingIterator} supplier that will convert
058 * from the output type listed below to the desired type. If your TransformingIterator
059 * does not produce the correct output, it is likely that you will run into ClassCastExceptions.
060 * That is beyond the scope of this class.
061 *
062 * Default object types if you do not convert them afterwards are:
063 *
064 *  'provided':
065 *    String
066 *
067 *  'sql':
068 *    If a class is specified, instances of that class. If not, strings.
069 *
070 *  'connector':
071 *    ResourceObject
072 *
073 *  'file':
074 *    List of Strings
075 *
076 *  'rule' and 'script':
077 *    Arbitrary. You are expected to return the right data type from your rule.
078 *    You may return an IncrementalObjectIterator, an Iterator, an Iterable, or
079 *    any other object (which will be wrapped in a singleton list).
080 *
081 *  'filter':
082 *    An object of the specified filter class, or Identity if one is not specified.
083 *    The iterator will wrap an IncrementalObjectIterator so that not all objects
084 *    are loaded into memory at once.
085 *
086 * If you are using this class in a context that can be interrupted (e.g., in a job),
087 * you will need to provide a way for this class to register a "termination handler"
088 * by calling {@link #setTerminationRegistrar(Consumer)}. Your termination logic
089 * MUST invoke any callbacks registered with your consumer.
090 *
091 * @param <ItemType>
092 *
093 * @author Devin Rosenbauer
094 * @author Instrumental Identity
095 */
096public class BasicObjectRetriever<ItemType> implements ObjectRetriever<ItemType> {
097    /**
098     * The list of retrieval types
099     */
100    enum RetrievalType {
101        rule,
102        script,
103        sql,
104        filter,
105        connector,
106        file,
107        provided
108    }
109
110    /**
111     * The class logger
112     */
113    private final Log log;
114    /**
115     * The list of provided values
116     */
117    private List<String> providedValues;
118    /**
119     * The application used to retrieve any inputs
120     */
121    private Application retrievalApplication;
122    /**
123     * The file used to retrieve any inputs
124     */
125    private File retrievalFile;
126    /**
127     * Retrieval file delimiter
128     */
129    private String retrievalFileDelimiter;
130    /**
131     * The class to retrieve, assuming the default retriever is used
132     */
133    private Class<? extends SailPointObject> retrievalFileClass;
134    /**
135     * The filter used to retrieve any objects, assuming the default retriever is used
136     */
137    private Filter retrievalFilter;
138    /**
139     * The class to retrieve, assuming the default retriever is used
140     */
141    private Class<? extends SailPointObject> retrievalFilterClass;
142    /**
143     * A rule that returns objects to be iterated over in parallel
144     */
145    private Rule retrievalRule;
146    /**
147     * A script that returns objects to be iterated over in parallel
148     */
149    private Script retrievalScript;
150    /**
151     * A SQL query that returns IDs of objects to be iterated in parallel
152     */
153    private String retrievalSql;
154    /**
155     * The class represented by the SQL output
156     */
157    private Class<? extends SailPointObject> retrievalSqlClass;
158    /**
159     * The type of retrieval, which must be one of the defined values
160     */
161    private RetrievalType retrievalType;
162
163    /**
164     * The TaskResult of the running task
165     */
166    private final TaskResult taskResult;
167
168    /**
169     * The callback for registering termination handlers, set by setTerminationRegistrar
170     */
171    private Consumer<Functions.GenericCallback> terminationRegistrar;
172
173    /**
174     * The callback used to create the output iterator to the given type
175     */
176    private final Function<Iterator<?>, TransformingIterator<Object, ItemType>> transformerConstructor;
177
178    /**
179     * Constructs a new Basic object retriever
180     * @param context The Sailpoint context to use for searching
181     * @param arguments The arguments to the task (or other)
182     * @param transformerConstructor The callback that creates a transforming iterator
183     * @param taskResult The task result, or null
184     * @throws GeneralException if any failures occur
185     */
186    public BasicObjectRetriever(SailPointContext context, Attributes<String, Object> arguments, Function<Iterator<?>, TransformingIterator<Object, ItemType>> transformerConstructor, TaskResult taskResult) throws GeneralException {
187        Objects.requireNonNull(context);
188        Objects.requireNonNull(arguments);
189
190        initialize(context, arguments);
191
192        this.log = LogFactory.getLog(BasicObjectRetriever.class);
193        this.transformerConstructor = transformerConstructor;
194        this.taskResult = taskResult;
195    }
196
197
198    /**
199     * Retrieves the contents of an input file
200     * @return An iterator over the contents of the file
201     * @throws IOException if there is an error reading the file
202     * @throws GeneralException if there is an error doing anything Sailpoint-y
203     */
204    private Iterator<List<String>> getFileContents() throws IOException, GeneralException {
205        Iterator<List<String>> items;
206        List<List<String>> itemList = new ArrayList<>();
207        try (BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(retrievalFile)))) {
208            RFC4180LineParser parser = null;
209            if (Util.isNotNullOrEmpty(retrievalFileDelimiter)) {
210                parser = new RFC4180LineParser(retrievalFileDelimiter);
211            }
212            String line;
213            while((line = reader.readLine()) != null) {
214                if (Util.isNotNullOrEmpty(line.trim())) {
215                    if (parser != null) {
216                        List<String> item = parser.parseLine(line);
217                        itemList.add(item);
218                    } else {
219                        List<String> lineList = Collections.singletonList(line);
220                        itemList.add(lineList);
221                    }
222                }
223            }
224        }
225        items = itemList.iterator();
226        return items;
227    }
228
229    /**
230     * Gets the object iterator, which will be an instance of {@link TransformingIterator} to
231     * convert whatever are the inputs. See the class Javadoc for information about what the
232     * default return types from each query method are.
233     *
234     * @param context The context to use during querying
235     * @param arguments Any additional query arguments
236     * @return An iterator over the retrieved objects
237     * @throws GeneralException if any failures occur
238     */
239    @Override
240    @SuppressWarnings("unchecked")
241    public Iterator<ItemType> getObjectIterator(SailPointContext context, Attributes<String, Object> arguments) throws GeneralException {
242        Objects.requireNonNull(context);
243
244        Map<String, Object> retrievalParams = new HashMap<>();
245        retrievalParams.put("log", log);
246        retrievalParams.put("context", context);
247        retrievalParams.put("taskResult", taskResult);
248        TransformingIterator<?, ItemType> items = null;
249        if (retrievalType == RetrievalType.provided) {
250            items = transformerConstructor.apply(providedValues.iterator());
251        } else if (retrievalType == RetrievalType.rule) {
252            if (retrievalRule == null) {
253                throw new IllegalArgumentException("Retrieval rule must exist");
254            }
255            Object output = context.runRule(retrievalRule, retrievalParams);
256            if (output instanceof IncrementalObjectIterator) {
257                items = transformerConstructor.apply((IncrementalObjectIterator)output);
258            } else if (output instanceof Iterator) {
259                items = transformerConstructor.apply((Iterator)output);
260            } else if (output instanceof Iterable) {
261                items = transformerConstructor.apply(((Iterable)output).iterator());
262            } else {
263                items = transformerConstructor.apply(Collections.singletonList(output).iterator());
264            }
265        } else if (retrievalType == RetrievalType.script) {
266            if (retrievalScript == null) {
267                throw new IllegalArgumentException("A retrieval script must be defined with 'script' retrieval type");
268            }
269            if (log.isDebugEnabled()) {
270                log.debug("Running retrieval script: " + retrievalScript);
271            }
272            Object output = context.runScript(retrievalScript, retrievalParams);
273            if (output instanceof IncrementalObjectIterator) {
274                items = transformerConstructor.apply((IncrementalObjectIterator)output);
275            } else if (output instanceof Iterator) {
276                items = transformerConstructor.apply((Iterator)output);
277            } else if (output instanceof Iterable) {
278                items = transformerConstructor.apply(((Iterable)output).iterator());
279            } else {
280                items = transformerConstructor.apply(Collections.singletonList(output).iterator());
281            }
282        } else if (retrievalType == RetrievalType.filter) {
283            if (retrievalFilter == null || retrievalFilterClass == null) {
284                throw new IllegalArgumentException("A retrieval filter and class name must be specified for 'filter' retrieval type");
285            }
286            QueryOptions options = new QueryOptions();
287            options.addFilter(retrievalFilter);
288            items = transformerConstructor.apply(new IncrementalObjectIterator<>(context, retrievalFilterClass, options));
289        } else if (retrievalType == RetrievalType.file) {
290            try {
291                Iterator<List<String>> fileContents = getFileContents();
292                if (retrievalFileClass != null) {
293                    items = transformerConstructor.apply(new TransformingIterator<>(fileContents, (id) -> Util.isEmpty(id) ? null : (ItemType) context.getObject(retrievalFileClass, Util.otoa(id.get(0)))));
294                } else {
295                    items = transformerConstructor.apply(fileContents);
296                }
297            } catch (IOException e) {
298                throw new GeneralException(e);
299            }
300        } else if (retrievalType == RetrievalType.sql) {
301            List<String> objectNames = new ArrayList<>();
302            // There is probably a more efficient way to do this, but it's not clear
303            // how to automatically clean up the SQL resources afterwards...
304            try (Connection connection = ContextConnectionWrapper.getConnection(context)) {
305                try (PreparedStatement stmt = connection.prepareStatement(retrievalSql)) {
306                    try (ResultSet results = stmt.executeQuery()) {
307                        while(results.next()) {
308                            String output = results.getString(1);
309                            if (Util.isNotNullOrEmpty(output)) {
310                                objectNames.add(output);
311                            }
312                        }
313                    }
314                }
315            } catch (SQLException e) {
316                throw new GeneralException(e);
317            }
318            if (retrievalSqlClass != null) {
319                items = transformerConstructor.apply(new TransformingIterator<>(objectNames.iterator(), (id) -> (ItemType) context.getObject(retrievalSqlClass, id)));
320            } else {
321                items = transformerConstructor.apply(objectNames.iterator());
322            }
323        } else if (retrievalType == RetrievalType.connector) {
324            if (retrievalApplication == null) {
325                throw new IllegalArgumentException("You must specify an application for 'connector' retrieval type");
326            }
327            if (retrievalApplication.supportsFeature(Application.Feature.NO_AGGREGATION)) {
328                throw new IllegalArgumentException("The application " + retrievalApplication.getName() + " does not support aggregation");
329            }
330            try {
331                Connector connector = ConnectorFactory.getConnector(retrievalApplication, null);
332                CloseableIterator<ResourceObject> objectIterator = connector.iterateObjects(retrievalApplication.getAccountSchema().getName(), null, new HashMap<>());
333                if (terminationRegistrar != null) {
334                    terminationRegistrar.accept((terminationContext) -> objectIterator.close());
335                }
336                return transformerConstructor.apply(new AbstractThreadedObjectIteratorTask.CloseableIteratorWrapper(objectIterator));
337            } catch(ConnectorException e) {
338                throw new GeneralException(e);
339            }
340        } else {
341            throw new IllegalStateException("You must specify a rule, script, filter, sql, file, connector, or list to retrieve target objects");
342        }
343        items.ignoreNulls();
344        return items;
345    }
346
347    /**
348     * Returns true if the retriever is configured with a retrieval class appropriate
349     * for the retrieval type, otherwise returns false.
350     * @return True if a retrieval class is present
351     */
352    public boolean hasRetrievalClass() {
353        if (getRetrievalType() == RetrievalType.file) {
354            return this.retrievalFileClass != null;
355        } else if (getRetrievalType() == RetrievalType.filter) {
356            return this.retrievalFilterClass != null;
357        } else if (getRetrievalType() == RetrievalType.sql) {
358            return this.retrievalSqlClass != null;
359        }
360        return false;
361    }
362
363    /**
364     * Gets the retrieval type
365     * @return The retrieval type
366     */
367    public RetrievalType getRetrievalType() {
368        return retrievalType;
369    }
370
371    /**
372     * Initializes this retriever from the input parameters
373     * @param context The context
374     * @param args The input arguments, usually from a Task
375     * @throws GeneralException if any Sailpoint-related failures occur
376     * @throws IllegalArgumentException if any arguments are incorrect or missing
377     */
378    @SuppressWarnings("unchecked")
379    private void initialize(SailPointContext context, Attributes<String, Object> args) throws GeneralException {
380        this.retrievalType = RetrievalType.valueOf(args.getString("retrievalType"));
381
382        if (retrievalType == RetrievalType.provided) {
383            List<String> values = args.getStringList("values");
384            if (values == null) {
385                throw new IllegalArgumentException("For input type = provided, a CSV in 'values' is required");
386            }
387
388            this.providedValues = values;
389        }
390
391        if (retrievalType == RetrievalType.sql) {
392            if (Util.isNullOrEmpty(args.getString("retrievalSql"))) {
393                throw new IllegalArgumentException("For input type = sql, a retrievalSql is required");
394            }
395
396            String className = args.getString("retrievalSqlClass");
397            if (Util.isNotNullOrEmpty(className)) {
398                this.retrievalSqlClass = (Class<? extends SailPointObject>) ObjectUtil.getSailPointClass(className);
399            }
400            this.retrievalSql = args.getString("retrievalSql");
401        }
402
403        if (retrievalType == RetrievalType.connector) {
404            String applicationName = args.getString("applicationName");
405            this.retrievalApplication = context.getObject(Application.class, applicationName);
406            if (this.retrievalApplication == null) {
407                throw new IllegalArgumentException("The application " + applicationName + " does not exist");
408            }
409        }
410
411        if (retrievalType == RetrievalType.file && Util.isNotNullOrEmpty(args.getString("retrievalFile"))) {
412            String filename = args.getString("retrievalFile");
413            this.retrievalFile = new File(filename);
414            this.retrievalFileDelimiter = args.getString("retrievalFileDelimiter");
415            String className = args.getString("retrievalFileClass");
416            if (Util.isNotNullOrEmpty(className)) {
417                this.retrievalFileClass = (Class<? extends SailPointObject>) ObjectUtil.getSailPointClass(className);
418            }
419            if (!this.retrievalFile.exists()) {
420                throw new IllegalArgumentException("Input file " + filename + " does not exist");
421            }
422            if (!this.retrievalFile.isFile()) {
423                throw new IllegalArgumentException("Input file " + filename + " does not appear to be a file (is it a directory?)");
424            }
425            if (!this.retrievalFile.canRead()) {
426                throw new IllegalArgumentException("Input file " + filename + " is not readable by the IIQ process");
427            }
428        }
429
430        if (retrievalType == RetrievalType.rule && Util.isNotNullOrEmpty(args.getString("retrievalRule"))) {
431            this.retrievalRule = context.getObjectByName(Rule.class, args.getString("retrievalRule"));
432        }
433
434        if (retrievalType == RetrievalType.script && Util.isNotNullOrEmpty(args.getString("retrievalScript"))) {
435            this.retrievalScript = new Script();
436            retrievalScript.setSource(args.getString("retrievalScript"));
437        }
438
439        if (retrievalType == RetrievalType.filter && Util.isNotNullOrEmpty(args.getString("retrievalFilter"))) {
440            this.retrievalFilter = Filter.compile(args.getString("retrievalFilter"));
441            String className = args.getString("retrievalFilterClass");
442            if (Util.isNotNullOrEmpty(className)) {
443                this.retrievalFilterClass = (Class<? extends SailPointObject>) ObjectUtil.getSailPointClass(className);
444            } else {
445                log.warn("No retrieval filter class given; assuming Identity");
446                this.retrievalFilterClass = Identity.class;
447            }
448        }
449    }
450
451    /**
452     * Sets the termination registrar object
453     * @param registar The termination registrar object
454     */
455    @Override
456    public void setTerminationRegistrar(Consumer<Functions.GenericCallback> registar) {
457        this.terminationRegistrar = registar;
458    }
459}