001package com.identityworksllc.iiq.common.task; 002 003import com.identityworksllc.iiq.common.Functions; 004import com.identityworksllc.iiq.common.iterators.TransformingIterator; 005import com.identityworksllc.iiq.common.query.ContextConnectionWrapper; 006import org.apache.commons.logging.Log; 007import org.apache.commons.logging.LogFactory; 008import sailpoint.api.IncrementalObjectIterator; 009import sailpoint.api.ObjectUtil; 010import sailpoint.api.SailPointContext; 011import sailpoint.connector.Connector; 012import sailpoint.connector.ConnectorException; 013import sailpoint.connector.ConnectorFactory; 014import sailpoint.object.*; 015import sailpoint.tools.CloseableIterator; 016import sailpoint.tools.GeneralException; 017import sailpoint.tools.RFC4180LineParser; 018import sailpoint.tools.Util; 019 020import java.io.BufferedReader; 021import java.io.File; 022import java.io.FileInputStream; 023import java.io.IOException; 024import java.io.InputStreamReader; 025import java.sql.Connection; 026import java.sql.PreparedStatement; 027import java.sql.ResultSet; 028import java.sql.SQLException; 029import java.util.ArrayList; 030import java.util.Collections; 031import java.util.HashMap; 032import java.util.Iterator; 033import java.util.List; 034import java.util.Map; 035import java.util.Objects; 036import java.util.function.Consumer; 037import java.util.function.Function; 038 039/** 040 * Implements a reusable implementation of {@link ObjectRetriever} for use with any 041 * number of scheduled tasks. 042 * 043 * This class takes the following input arguments in its constructor: 044 * 045 * - values: If the input type is 'provided', these values will be interpreted as the raw input. 046 * - retrievalSql: A single-column SQL query that will produce a list of strings. The first column must be a string. Other columns will be ignored. 047 * - retrievalSqlClass: If specified, the single-column result from the query will be used as an ID look up the object with this type 048 * - retrievalConnector: The name of an Application; the iteration will be over a list of ResourceObjects as though an aggregation is running 049 * - retrievalFile: The fully qualified name of an existing, readable text file. Each line will be passed as a List of Strings to convertObject. If a delimiter is specified, the List will contain each delimited element. If not, the list will contain the entire line. 050 * - retrievalFileDelimiter: If specified, the file will be interpreted as a CSV; each line as a List will be passed to threadExecute 051 * - retrievalFileClass: If specified, the file will be interpreted as a CSV and the first column will be interpreted as an ID or name of this class type. 052 * - retrievalRule: The name of a Rule that will produce an Iterator over the desired objects 053 * - retrievalScript: The source of a Script that will produce an Iterator over the desired objects 054 * - retrievalFilter: A Filter string combined with retrievalFilterClass to do a search in SailPoint 055 * - retrievalFilterClass: The class name to search with the retrievalFilter 056 * 057 * You are expected to provide a {@link TransformingIterator} supplier that will convert 058 * from the output type listed below to the desired type. If your TransformingIterator 059 * does not produce the correct output, it is likely that you will run into ClassCastExceptions. 060 * That is beyond the scope of this class. 061 * 062 * Default object types if you do not convert them afterwards are: 063 * 064 * 'provided': 065 * String 066 * 067 * 'sql': 068 * If a class is specified, instances of that class. If not, strings. 069 * 070 * 'connector': 071 * ResourceObject 072 * 073 * 'file': 074 * List of Strings 075 * 076 * 'rule' and 'script': 077 * Arbitrary. You are expected to return the right data type from your rule. 078 * You may return an IncrementalObjectIterator, an Iterator, an Iterable, or 079 * any other object (which will be wrapped in a singleton list). 080 * 081 * 'filter': 082 * An object of the specified filter class, or Identity if one is not specified. 083 * The iterator will wrap an IncrementalObjectIterator so that not all objects 084 * are loaded into memory at once. 085 * 086 * If you are using this class in a context that can be interrupted (e.g., in a job), 087 * you will need to provide a way for this class to register a "termination handler" 088 * by calling {@link #setTerminationRegistrar(Consumer)}. Your termination logic 089 * MUST invoke any callbacks registered with your consumer. 090 * 091 * @param <ItemType> 092 * 093 * @author Devin Rosenbauer 094 * @author Instrumental Identity 095 */ 096public class BasicObjectRetriever<ItemType> implements ObjectRetriever<ItemType> { 097 /** 098 * The list of retrieval types 099 */ 100 enum RetrievalType { 101 rule, 102 script, 103 sql, 104 filter, 105 connector, 106 file, 107 provided 108 } 109 110 /** 111 * The class logger 112 */ 113 private final Log log; 114 /** 115 * The list of provided values 116 */ 117 private List<String> providedValues; 118 /** 119 * The application used to retrieve any inputs 120 */ 121 private Application retrievalApplication; 122 /** 123 * The file used to retrieve any inputs 124 */ 125 private File retrievalFile; 126 /** 127 * Retrieval file delimiter 128 */ 129 private String retrievalFileDelimiter; 130 /** 131 * The class to retrieve, assuming the default retriever is used 132 */ 133 private Class<? extends SailPointObject> retrievalFileClass; 134 /** 135 * The filter used to retrieve any objects, assuming the default retriever is used 136 */ 137 private Filter retrievalFilter; 138 /** 139 * The class to retrieve, assuming the default retriever is used 140 */ 141 private Class<? extends SailPointObject> retrievalFilterClass; 142 /** 143 * A rule that returns objects to be iterated over in parallel 144 */ 145 private Rule retrievalRule; 146 /** 147 * A script that returns objects to be iterated over in parallel 148 */ 149 private Script retrievalScript; 150 /** 151 * A SQL query that returns IDs of objects to be iterated in parallel 152 */ 153 private String retrievalSql; 154 /** 155 * The class represented by the SQL output 156 */ 157 private Class<? extends SailPointObject> retrievalSqlClass; 158 /** 159 * The type of retrieval, which must be one of the defined values 160 */ 161 private RetrievalType retrievalType; 162 163 /** 164 * The TaskResult of the running task 165 */ 166 private final TaskResult taskResult; 167 168 /** 169 * The callback for registering termination handlers, set by setTerminationRegistrar 170 */ 171 private Consumer<Functions.GenericCallback> terminationRegistrar; 172 173 /** 174 * The callback used to create the output iterator to the given type 175 */ 176 private final Function<Iterator<?>, TransformingIterator<Object, ItemType>> transformerConstructor; 177 178 /** 179 * Constructs a new Basic object retriever 180 * @param context The Sailpoint context to use for searching 181 * @param arguments The arguments to the task (or other) 182 * @param transformerConstructor The callback that creates a transforming iterator 183 * @param taskResult The task result, or null 184 * @throws GeneralException if any failures occur 185 */ 186 public BasicObjectRetriever(SailPointContext context, Attributes<String, Object> arguments, Function<Iterator<?>, TransformingIterator<Object, ItemType>> transformerConstructor, TaskResult taskResult) throws GeneralException { 187 Objects.requireNonNull(context); 188 Objects.requireNonNull(arguments); 189 190 initialize(context, arguments); 191 192 this.log = LogFactory.getLog(BasicObjectRetriever.class); 193 this.transformerConstructor = transformerConstructor; 194 this.taskResult = taskResult; 195 } 196 197 198 /** 199 * Retrieves the contents of an input file 200 * @return An iterator over the contents of the file 201 * @throws IOException if there is an error reading the file 202 * @throws GeneralException if there is an error doing anything Sailpoint-y 203 */ 204 private Iterator<List<String>> getFileContents() throws IOException, GeneralException { 205 Iterator<List<String>> items; 206 List<List<String>> itemList = new ArrayList<>(); 207 try (BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(retrievalFile)))) { 208 RFC4180LineParser parser = null; 209 if (Util.isNotNullOrEmpty(retrievalFileDelimiter)) { 210 parser = new RFC4180LineParser(retrievalFileDelimiter); 211 } 212 String line; 213 while((line = reader.readLine()) != null) { 214 if (Util.isNotNullOrEmpty(line.trim())) { 215 if (parser != null) { 216 List<String> item = parser.parseLine(line); 217 itemList.add(item); 218 } else { 219 List<String> lineList = Collections.singletonList(line); 220 itemList.add(lineList); 221 } 222 } 223 } 224 } 225 items = itemList.iterator(); 226 return items; 227 } 228 229 /** 230 * Gets the object iterator, which will be an instance of {@link TransformingIterator} to 231 * convert whatever are the inputs. See the class Javadoc for information about what the 232 * default return types from each query method are. 233 * 234 * @param context The context to use during querying 235 * @param arguments Any additional query arguments 236 * @return An iterator over the retrieved objects 237 * @throws GeneralException if any failures occur 238 */ 239 @Override 240 @SuppressWarnings("unchecked") 241 public Iterator<ItemType> getObjectIterator(SailPointContext context, Attributes<String, Object> arguments) throws GeneralException { 242 Objects.requireNonNull(context); 243 244 Map<String, Object> retrievalParams = new HashMap<>(); 245 retrievalParams.put("log", log); 246 retrievalParams.put("context", context); 247 retrievalParams.put("taskResult", taskResult); 248 TransformingIterator<?, ItemType> items = null; 249 if (retrievalType == RetrievalType.provided) { 250 items = transformerConstructor.apply(providedValues.iterator()); 251 } else if (retrievalType == RetrievalType.rule) { 252 if (retrievalRule == null) { 253 throw new IllegalArgumentException("Retrieval rule must exist"); 254 } 255 Object output = context.runRule(retrievalRule, retrievalParams); 256 if (output instanceof IncrementalObjectIterator) { 257 items = transformerConstructor.apply((IncrementalObjectIterator)output); 258 } else if (output instanceof Iterator) { 259 items = transformerConstructor.apply((Iterator)output); 260 } else if (output instanceof Iterable) { 261 items = transformerConstructor.apply(((Iterable)output).iterator()); 262 } else { 263 items = transformerConstructor.apply(Collections.singletonList(output).iterator()); 264 } 265 } else if (retrievalType == RetrievalType.script) { 266 if (retrievalScript == null) { 267 throw new IllegalArgumentException("A retrieval script must be defined with 'script' retrieval type"); 268 } 269 if (log.isDebugEnabled()) { 270 log.debug("Running retrieval script: " + retrievalScript); 271 } 272 Object output = context.runScript(retrievalScript, retrievalParams); 273 if (output instanceof IncrementalObjectIterator) { 274 items = transformerConstructor.apply((IncrementalObjectIterator)output); 275 } else if (output instanceof Iterator) { 276 items = transformerConstructor.apply((Iterator)output); 277 } else if (output instanceof Iterable) { 278 items = transformerConstructor.apply(((Iterable)output).iterator()); 279 } else { 280 items = transformerConstructor.apply(Collections.singletonList(output).iterator()); 281 } 282 } else if (retrievalType == RetrievalType.filter) { 283 if (retrievalFilter == null || retrievalFilterClass == null) { 284 throw new IllegalArgumentException("A retrieval filter and class name must be specified for 'filter' retrieval type"); 285 } 286 QueryOptions options = new QueryOptions(); 287 options.addFilter(retrievalFilter); 288 items = transformerConstructor.apply(new IncrementalObjectIterator<>(context, retrievalFilterClass, options)); 289 } else if (retrievalType == RetrievalType.file) { 290 try { 291 Iterator<List<String>> fileContents = getFileContents(); 292 if (retrievalFileClass != null) { 293 items = transformerConstructor.apply(new TransformingIterator<>(fileContents, (id) -> Util.isEmpty(id) ? null : (ItemType) context.getObject(retrievalFileClass, Util.otoa(id.get(0))))); 294 } else { 295 items = transformerConstructor.apply(fileContents); 296 } 297 } catch (IOException e) { 298 throw new GeneralException(e); 299 } 300 } else if (retrievalType == RetrievalType.sql) { 301 List<String> objectNames = new ArrayList<>(); 302 // There is probably a more efficient way to do this, but it's not clear 303 // how to automatically clean up the SQL resources afterwards... 304 try (Connection connection = ContextConnectionWrapper.getConnection(context)) { 305 try (PreparedStatement stmt = connection.prepareStatement(retrievalSql)) { 306 try (ResultSet results = stmt.executeQuery()) { 307 while(results.next()) { 308 String output = results.getString(1); 309 if (Util.isNotNullOrEmpty(output)) { 310 objectNames.add(output); 311 } 312 } 313 } 314 } 315 } catch (SQLException e) { 316 throw new GeneralException(e); 317 } 318 if (retrievalSqlClass != null) { 319 items = transformerConstructor.apply(new TransformingIterator<>(objectNames.iterator(), (id) -> (ItemType) context.getObject(retrievalSqlClass, id))); 320 } else { 321 items = transformerConstructor.apply(objectNames.iterator()); 322 } 323 } else if (retrievalType == RetrievalType.connector) { 324 if (retrievalApplication == null) { 325 throw new IllegalArgumentException("You must specify an application for 'connector' retrieval type"); 326 } 327 if (retrievalApplication.supportsFeature(Application.Feature.NO_AGGREGATION)) { 328 throw new IllegalArgumentException("The application " + retrievalApplication.getName() + " does not support aggregation"); 329 } 330 try { 331 Connector connector = ConnectorFactory.getConnector(retrievalApplication, null); 332 CloseableIterator<ResourceObject> objectIterator = connector.iterateObjects(retrievalApplication.getAccountSchema().getName(), null, new HashMap<>()); 333 if (terminationRegistrar != null) { 334 terminationRegistrar.accept((terminationContext) -> objectIterator.close()); 335 } 336 return transformerConstructor.apply(new AbstractThreadedObjectIteratorTask.CloseableIteratorWrapper(objectIterator)); 337 } catch(ConnectorException e) { 338 throw new GeneralException(e); 339 } 340 } else { 341 throw new IllegalStateException("You must specify a rule, script, filter, sql, file, connector, or list to retrieve target objects"); 342 } 343 items.ignoreNulls(); 344 return items; 345 } 346 347 /** 348 * Returns true if the retriever is configured with a retrieval class appropriate 349 * for the retrieval type, otherwise returns false. 350 * @return True if a retrieval class is present 351 */ 352 public boolean hasRetrievalClass() { 353 if (getRetrievalType() == RetrievalType.file) { 354 return this.retrievalFileClass != null; 355 } else if (getRetrievalType() == RetrievalType.filter) { 356 return this.retrievalFilterClass != null; 357 } else if (getRetrievalType() == RetrievalType.sql) { 358 return this.retrievalSqlClass != null; 359 } 360 return false; 361 } 362 363 /** 364 * Gets the retrieval type 365 * @return The retrieval type 366 */ 367 public RetrievalType getRetrievalType() { 368 return retrievalType; 369 } 370 371 /** 372 * Initializes this retriever from the input parameters 373 * @param context The context 374 * @param args The input arguments, usually from a Task 375 * @throws GeneralException if any Sailpoint-related failures occur 376 * @throws IllegalArgumentException if any arguments are incorrect or missing 377 */ 378 @SuppressWarnings("unchecked") 379 private void initialize(SailPointContext context, Attributes<String, Object> args) throws GeneralException { 380 this.retrievalType = RetrievalType.valueOf(args.getString("retrievalType")); 381 382 if (retrievalType == RetrievalType.provided) { 383 List<String> values = args.getStringList("values"); 384 if (values == null) { 385 throw new IllegalArgumentException("For input type = provided, a CSV in 'values' is required"); 386 } 387 388 this.providedValues = values; 389 } 390 391 if (retrievalType == RetrievalType.sql) { 392 if (Util.isNullOrEmpty(args.getString("retrievalSql"))) { 393 throw new IllegalArgumentException("For input type = sql, a retrievalSql is required"); 394 } 395 396 String className = args.getString("retrievalSqlClass"); 397 if (Util.isNotNullOrEmpty(className)) { 398 this.retrievalSqlClass = (Class<? extends SailPointObject>) ObjectUtil.getSailPointClass(className); 399 } 400 this.retrievalSql = args.getString("retrievalSql"); 401 } 402 403 if (retrievalType == RetrievalType.connector) { 404 String applicationName = args.getString("applicationName"); 405 this.retrievalApplication = context.getObject(Application.class, applicationName); 406 if (this.retrievalApplication == null) { 407 throw new IllegalArgumentException("The application " + applicationName + " does not exist"); 408 } 409 } 410 411 if (retrievalType == RetrievalType.file && Util.isNotNullOrEmpty(args.getString("retrievalFile"))) { 412 String filename = args.getString("retrievalFile"); 413 this.retrievalFile = new File(filename); 414 this.retrievalFileDelimiter = args.getString("retrievalFileDelimiter"); 415 String className = args.getString("retrievalFileClass"); 416 if (Util.isNotNullOrEmpty(className)) { 417 this.retrievalFileClass = (Class<? extends SailPointObject>) ObjectUtil.getSailPointClass(className); 418 } 419 if (!this.retrievalFile.exists()) { 420 throw new IllegalArgumentException("Input file " + filename + " does not exist"); 421 } 422 if (!this.retrievalFile.isFile()) { 423 throw new IllegalArgumentException("Input file " + filename + " does not appear to be a file (is it a directory?)"); 424 } 425 if (!this.retrievalFile.canRead()) { 426 throw new IllegalArgumentException("Input file " + filename + " is not readable by the IIQ process"); 427 } 428 } 429 430 if (retrievalType == RetrievalType.rule && Util.isNotNullOrEmpty(args.getString("retrievalRule"))) { 431 this.retrievalRule = context.getObjectByName(Rule.class, args.getString("retrievalRule")); 432 } 433 434 if (retrievalType == RetrievalType.script && Util.isNotNullOrEmpty(args.getString("retrievalScript"))) { 435 this.retrievalScript = new Script(); 436 retrievalScript.setSource(args.getString("retrievalScript")); 437 } 438 439 if (retrievalType == RetrievalType.filter && Util.isNotNullOrEmpty(args.getString("retrievalFilter"))) { 440 this.retrievalFilter = Filter.compile(args.getString("retrievalFilter")); 441 String className = args.getString("retrievalFilterClass"); 442 if (Util.isNotNullOrEmpty(className)) { 443 this.retrievalFilterClass = (Class<? extends SailPointObject>) ObjectUtil.getSailPointClass(className); 444 } else { 445 log.warn("No retrieval filter class given; assuming Identity"); 446 this.retrievalFilterClass = Identity.class; 447 } 448 } 449 } 450 451 /** 452 * Sets the termination registrar object 453 * @param registar The termination registrar object 454 */ 455 @Override 456 public void setTerminationRegistrar(Consumer<Functions.GenericCallback> registar) { 457 this.terminationRegistrar = registar; 458 } 459}